From 91d319e849ca912e1ff77046cb277985db5844d3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 14:06:25 -0400 Subject: [PATCH 0001/2835] moved from my doc repo --- git-annex.mdwn | 165 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 git-annex.mdwn diff --git a/git-annex.mdwn b/git-annex.mdwn new file mode 100644 index 0000000000..bc3550398c --- /dev/null +++ b/git-annex.mdwn @@ -0,0 +1,165 @@ +git-annex allows managing files with git, without checking the file +contents into git. This is useful when dealing with files larger than git +can currently easily handle, whether due to limitations in memory, +checksumming time, or disk space (only one copy need be stored of an +annexed file). + +Even without file content tracking, being able to manage file metadata with +git, move files around and delete files with versioned directory trees, and use +branches and distributed clone, are all very handy reasons to use git. And +annexed files can co-exist in the same git repository with regularly versioned +files, which is convenient for maintaining code, Makefiles, etc that are +associated with annexed files but that benefit from full revision control. + +Enough broad picture, here's how it actually looks: + +* `git annex --add $file` moves the file into `.git/annex/`, and replaces + it with a symlink pointing at the annexed file, and then calls `git add` + to version the *symlink*. (If the file has already been annexed, it does + nothing.) +* You can move the symlink around, copy it, delete it, etc, and commit changes + as desired using git. Reading the symlink will always get you the annexed + file content, or the link may be broken if the content is not currently + available. +* If you use normal git push/pull commands, the annexed file contents + won't be sent, but the symlinks will be. So different clones of a repository + can have different sets of annexed files available. +* `git annex --push $repository` pushes *all* annexed files to the specified + repository. +* `git annex --pull $repository` pulls *all* annexed files from the specified + repository. +* `git annex --want $file` indicates that you want access to a file's + content, without immediatly transferring it. +* `git annex --get $file` is used to transfer a specified file, and/or + files previously indicated with --want. If a configured repository has it, + or it is available from other key/value storage, it will be immediatly + downloaded. +* `git annex --drop $file` indicates that you no longer want the file's + content to be available in this repository. +* `git annex $file` is a shorthand for either --add or --get. If the file + is already known, it does --get, otherwise it does --add. + +## copies + +git-annex can be configured to try to keep N copies of a file's content +available across all repositories. By default, N is 1 (configured by +annex.numcopies). + +`git annex --drop` attempts to communicate with all other configured +repositories, to check that N copies of the file exist. If enough +repositories cannot be contacted, it will retain the file content. +You can later use `git annex --drop --retry` to retry pending drops. +Or you can use `git annex --drop --force $file` to force dropping of +file content. + +For example, consider three repositories: Server, Laptop, and USB. Both Server +and USB have a copy of a file, and N=1. If on Laptop, you `git annex --get +$file`, this will transfer it from either Server or USB (depending on which +is available), and there are now 3 copies of the file. + +Suppose you want to free up space on laptop again, and you --drop the file +there. If USB is connected, or Server can be contacted, git-annex can check +that it still has a copy of the file, and the content is removed from +Laptop. But if USB is currently disconnected, and Server also cannot be +contacted, it can't check that and will retain the file content. + +With N=2, in order to drop the file content from Laptop, it would need access +to both USB and Server. + +Note that different repositories can be configured with different values of +N. So just because Laptop has N=2, this does not prevent the number of +copies falling to 1, when USB and Server have N=1, and of they have the +only copies of a file. + +## the .git-annex directory + +The `.git-annex` directory at the top of the repository, is used to store +git-annex information that should be propigated between repositories. + +Data is stored here in files that are arranged to avoid conflicts in most +cases. A conflict could occur if a file with the same name but different +content was added to multiple clones. + +## key/value storage + +git-annex uses a key/value abstraction layer to allow files contents to be +stored in different ways. In theory, any key/value storage system could be +used to store the file contents, and git-annex would then retrieve them +as needed and put them in `.git/annex/`. + +When a file is annexed, a key is generated from its content and/or metadata. +This key can later be used to retrieve the file's content (its value). This +key generation must be stable for a given file content, name, and size. + +The mapping from filename to its key is stored in the .git-annex directory, +in a file named `$filename.$backend` + +Multiple pluggable backends are supported, and more than one can be used +to store different files' contents in a given repository. + +* `file` -- This backend stores the file's content in + `.git/annex/`, and assumes that any file with the same basename + has the same content. So with this backend, files can be moved around, + but should never be added to or changed. This is the default, and + the least expensive backend. +* `sha1sum` -- This backend stores the file's content in + `.git/annex/`, with a name based on its sha1 checksum. This backend allows + modifications of files to be tracked. Its need to generate checksums + can make it slow for large files. +* `url` -- This backend downloads the file's content from an external URL. + +## location tracking + +git-annex keeps track of on which repository it last saw a file's content. +This can be useful when using it for archiving with offline storage. When +you indicate you --want a file, git-annex will tell you which repositories +have the file's content. + +Location tracking information is stored in `.git-annex/$filename.log`. +Repositories record their name and the date when they --get or --drop +a file's content. (Git is configured to use a union merge for this file, +so the lines may be in arbitrary order, but it will never conflict.) + +## configuration + +* `annex.numcopies` -- number of copies of files to keep +* `annex.backend` -- name of the default key/value backend to use to + store new files +* `annex.name` -- allows specifying a unique name for this repository. + If not specified, the name is derived from its directory's location and + the hostname. When a repository is on removable media it is useful to give + it a more stable name. Typically the name of a repository is the same + name configured as a git remote to allow pulling from that repository. +* `remote..annex-cost` -- When determining which repository to + transfer annexed files from or to, ones with lower costs are preferred. + The default cost is 50. Note that other factors may be configured + when pushing files to repositories, in particular, whether the repository + is on a filesystem with sufficient free space. + +## issues + +### symlinks + +If the symlink to annexed content is relative, moving it to a subdir will +break it. But it it's absolute, moving the git repo (or mounting its drive +elsewhere) will break it. Either: + +* Use relative links and need `git annex mv` to move (or post-commit + hook that caches moves and updates links). +* Use absolute links and need `git annex fixlinks` when location changes; + note that would also mean that git would see the symlink targets changed + and want to commit the change. + +### free space determination + +Need a way to tell how much free space is available on the disk containing +a given repository. The repository may be remote, so ssh may need to be +used. + +Similarly, need a way to tell the size of a file before downloading it from +remote, to check local disk space. + +### auto-drop files on rm + +When git-rm removed a file, it should get dropped too. Of course, it may +not be dropped right away, depending on number of copies available. From a667d99cd1aa90691ded4fc110a1e11e965fc3a8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 19:22:40 -0400 Subject: [PATCH 0002/2835] first module --- LocationLog.hs | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 LocationLog.hs diff --git a/LocationLog.hs b/LocationLog.hs new file mode 100644 index 0000000000..c756a17b05 --- /dev/null +++ b/LocationLog.hs @@ -0,0 +1,81 @@ +{- git-annex location log + - + - git-annex keeps track of on which repository it last saw a file's content. + - This can be useful when using it for archiving with offline storage. + - When you indicate you --want a file, git-annex will tell you which + - repositories have the file's content. + - + - Location tracking information is stored in `.git-annex/$filename.log`. + - Repositories record their name and the date when they --get or --drop + - a file's content. (Git is configured to use a union merge for this file, + - so the lines may be in arbitrary order, but it will never conflict.) + - + - A line of the log will look like: "date reponame filename" + - + -} + +module LocationLog where + +import Data.DateTime +import System.IO +import System.Posix.IO + +data LogLine = LogLine { + date :: DateTime, + repo :: String, + file :: String +} deriving (Eq) + +-- a special value representing a log file line that could not be parsed +unparsable = (LogLine (fromSeconds 0) "" "") + +instance Show LogLine where + show (LogLine date repo file) = unwords + [(show (toSeconds date)), repo, file] + +instance Read LogLine where + -- this parser is robust in that even unparsable log lines are + -- read without an exception being thrown + readsPrec _ string = if (length w >= 3) + then [((LogLine time repo file), "")] + else [(unparsable, "")] + where + time = fromSeconds $ read $ w !! 0 + repo = w !! 1 + file = unwords $ rest w + w = words string + rest (_:_:l) = l + +{- Reads a log file -} +readLog :: String -> IO [LogLine] +readLog file = do + h <- openLocked file ReadMode + s <- hGetContents h + -- hClose handle' -- TODO disabled due to lazy IO issue + -- filter out any unparsable lines + return $ filter ( /= unparsable ) $ map read $ lines s + +{- Adds a LogLine to a log file -} +writeLog :: String -> LogLine -> IO () +writeLog file line = do + h <- openLocked file AppendMode + hPutStrLn h $ show line + hClose h + +{- Let's just say that Haskell makes reading/writing a file with + - file locking excessively difficult. -} +openLocked file mode = do + handle <- openFile file mode + lockfd <- handleToFd handle -- closes handle + waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) + handle' <- fdToHandle lockfd + return handle' + where + lockType ReadMode = ReadLock + lockType _ = WriteLock + +{- Generates a new log line with the current date. -} +logNow :: String -> String -> IO LogLine +logNow repo file = do + now <- getCurrentTime + return $ LogLine now repo file From 6b54817f2688cffc8751b3b1552dca0a34744e61 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 21:06:46 -0400 Subject: [PATCH 0003/2835] second module! --- GitRepo.hs | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 GitRepo.hs diff --git a/GitRepo.hs b/GitRepo.hs new file mode 100644 index 0000000000..fece797858 --- /dev/null +++ b/GitRepo.hs @@ -0,0 +1,57 @@ +{- git repository handling -} + +module GitRepo where + +import Directory +import System.Directory +import Data.String.Utils + +{- Returns the path to the current repository's .git directory. + - (For a bare repository, that is the root of the repository.) -} +gitDir :: IO String +gitDir = do + repo <- repoTop + bare <- isBareRepo repo + if (bare) + then return repo + else return $ repo ++ "/.git" + +{- Finds the top of the current git repository, which may be in a parent + - directory. -} +repoTop :: IO String +repoTop = do + dir <- getCurrentDirectory + top <- seekUp dir isRepoTop + case top of + (Just dir) -> return dir + Nothing -> error "Not in a git repository." + +seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) +seekUp dir want = do + ok <- want dir + if ok + then return (Just dir) + else case (parentDir dir) of + (Just d) -> seekUp d want + Nothing -> return Nothing + +parentDir :: String -> Maybe String +parentDir dir = + if length dirs > 0 + then Just ("/" ++ (join "/" $ take ((length dirs) - 1) dirs)) + else Nothing + where + dirs = filter (\x -> length x > 0) $ split "/" dir + +isRepoTop dir = do + r <- isGitRepo dir + b <- isBareRepo dir + return (r || b) + +isGitRepo dir = gitSignature dir ".git" ".git/config" +isBareRepo dir = gitSignature dir "objects" "config" + +gitSignature dir subdir file = do + s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) + f <- (doesFileExist (dir ++ "/" ++ file)) + return (s && f) From c920505a52ab3c42b7892b7f7a1c5244c39e916f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 22:09:10 -0400 Subject: [PATCH 0004/2835] add gitRelative function --- GitRepo.hs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index fece797858..f1372bf3a1 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -4,8 +4,25 @@ module GitRepo where import Directory import System.Directory +import System.Path import Data.String.Utils +{- Given a relative or absolute filename, calculates the name to use + - relative to a git repository directory (which must be absolute). + - This is the same form displayed and used by git. -} +gitRelative :: String -> String -> String +gitRelative file repo = drop (length absrepo) absfile + where + -- normalize both repo and file, so that repo + -- will be substring of file + absrepo = case (absNormPath "/" repo) of + Just f -> f ++ "/" + Nothing -> error $ "bad repo" ++ repo + absfile = case (secureAbsNormPath absrepo file) of + Just f -> f + Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo + + {- Returns the path to the current repository's .git directory. - (For a bare repository, that is the root of the repository.) -} gitDir :: IO String @@ -20,8 +37,8 @@ gitDir = do - directory. -} repoTop :: IO String repoTop = do - dir <- getCurrentDirectory - top <- seekUp dir isRepoTop + cwd <- getCurrentDirectory + top <- seekUp cwd isRepoTop case top of (Just dir) -> return dir Nothing -> error "Not in a git repository." From dcfb24e5b5764c8a7bde0a1410022a903ba3c99b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 22:14:13 -0400 Subject: [PATCH 0005/2835] add logFile --- GitRepo.hs | 2 +- LocationLog.hs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index f1372bf3a1..01e6746aef 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -11,7 +11,7 @@ import Data.String.Utils - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} gitRelative :: String -> String -> String -gitRelative file repo = drop (length absrepo) absfile +gitRelative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file diff --git a/LocationLog.hs b/LocationLog.hs index c756a17b05..ab109460ad 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -19,6 +19,7 @@ module LocationLog where import Data.DateTime import System.IO import System.Posix.IO +import GitRepo data LogLine = LogLine { date :: DateTime, @@ -79,3 +80,10 @@ logNow :: String -> String -> IO LogLine logNow repo file = do now <- getCurrentTime return $ LogLine now repo file + +{- Returns the filename of the log file for a given annexed file. -} +logFile :: String -> IO String +logFile annexedFile = do + repo <- repoTop + return $ repo ++ "/.git-annex/" ++ + (gitRelative repo annexedFile) ++ ".log" From 9ae522bb7689842e1d0251d486c22d26bb6461da Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 22:29:16 -0400 Subject: [PATCH 0006/2835] add status field to log --- LocationLog.hs | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index ab109460ad..ff357aaecd 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -10,7 +10,8 @@ - a file's content. (Git is configured to use a union merge for this file, - so the lines may be in arbitrary order, but it will never conflict.) - - - A line of the log will look like: "date reponame filename" + - A line of the log will look like: "date N reponame filename" + - Where N=1 when the repo has the file, and 0 otherwise. - -} @@ -21,31 +22,44 @@ import System.IO import System.Posix.IO import GitRepo +data LogStatus = FilePresent | FileMissing | Undefined + deriving (Eq) + +instance Show LogStatus where + show FilePresent = "1" + show FileMissing = "0" + show Undefined = "undefined" + +instance Read LogStatus where + readsPrec _ "1" = [(FilePresent, "")] + readsPrec _ "0" = [(FileMissing, "")] + readsPrec _ _ = [(Undefined, "")] + data LogLine = LogLine { date :: DateTime, + status :: LogStatus, repo :: String, file :: String } deriving (Eq) --- a special value representing a log file line that could not be parsed -unparsable = (LogLine (fromSeconds 0) "" "") - instance Show LogLine where - show (LogLine date repo file) = unwords - [(show (toSeconds date)), repo, file] + show (LogLine date status repo file) = unwords + [(show (toSeconds date)), (show status), repo, file] instance Read LogLine where - -- this parser is robust in that even unparsable log lines are - -- read without an exception being thrown + -- This parser is robust in that even unparsable log lines are + -- read without an exception being thrown. + -- Such lines have a status of Undefined. readsPrec _ string = if (length w >= 3) - then [((LogLine time repo file), "")] - else [(unparsable, "")] + then [((LogLine date status repo file), "")] + else [((LogLine (fromSeconds 0) Undefined "" ""), "")] where - time = fromSeconds $ read $ w !! 0 - repo = w !! 1 + date = fromSeconds $ read $ w !! 0 + status = read $ w !! 1 + repo = w !! 2 file = unwords $ rest w w = words string - rest (_:_:l) = l + rest (_:_:_:l) = l {- Reads a log file -} readLog :: String -> IO [LogLine] @@ -54,7 +68,7 @@ readLog file = do s <- hGetContents h -- hClose handle' -- TODO disabled due to lazy IO issue -- filter out any unparsable lines - return $ filter ( /= unparsable ) $ map read $ lines s + return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s {- Adds a LogLine to a log file -} writeLog :: String -> LogLine -> IO () @@ -76,10 +90,10 @@ openLocked file mode = do lockType _ = WriteLock {- Generates a new log line with the current date. -} -logNow :: String -> String -> IO LogLine -logNow repo file = do +logNow :: LogStatus -> String -> String -> IO LogLine +logNow status repo file = do now <- getCurrentTime - return $ LogLine now repo file + return $ LogLine now status repo file {- Returns the filename of the log file for a given annexed file. -} logFile :: String -> IO String From 011118dbdff84458a5f9eea05547d79fbf7e88ac Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 22:46:35 -0400 Subject: [PATCH 0007/2835] adding file presence calculation code --- LocationLog.hs | 55 +++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index ff357aaecd..911e4765b4 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -5,12 +5,12 @@ - When you indicate you --want a file, git-annex will tell you which - repositories have the file's content. - - - Location tracking information is stored in `.git-annex/$filename.log`. + - Location tracking information is stored in `.git-annex/filename.log`. - Repositories record their name and the date when they --get or --drop - a file's content. (Git is configured to use a union merge for this file, - so the lines may be in arbitrary order, but it will never conflict.) - - - A line of the log will look like: "date N reponame filename" + - A line of the log will look like: "date N reponame" - Where N=1 when the repo has the file, and 0 otherwise. - -} @@ -19,8 +19,8 @@ module LocationLog where import Data.DateTime import System.IO -import System.Posix.IO import GitRepo +import Utility data LogStatus = FilePresent | FileMissing | Undefined deriving (Eq) @@ -38,28 +38,26 @@ instance Read LogStatus where data LogLine = LogLine { date :: DateTime, status :: LogStatus, - repo :: String, - file :: String + repo :: String } deriving (Eq) instance Show LogLine where - show (LogLine date status repo file) = unwords - [(show (toSeconds date)), (show status), repo, file] + show (LogLine date status repo) = unwords + [(show (toSeconds date)), (show status), repo] instance Read LogLine where -- This parser is robust in that even unparsable log lines are -- read without an exception being thrown. -- Such lines have a status of Undefined. readsPrec _ string = if (length w >= 3) - then [((LogLine date status repo file), "")] - else [((LogLine (fromSeconds 0) Undefined "" ""), "")] + then [((LogLine date status repo), "")] + else [((LogLine (fromSeconds 0) Undefined ""), "")] where date = fromSeconds $ read $ w !! 0 status = read $ w !! 1 - repo = w !! 2 - file = unwords $ rest w + repo = unwords $ rest w w = words string - rest (_:_:_:l) = l + rest (_:_:l) = l {- Reads a log file -} readLog :: String -> IO [LogLine] @@ -77,23 +75,11 @@ writeLog file line = do hPutStrLn h $ show line hClose h -{- Let's just say that Haskell makes reading/writing a file with - - file locking excessively difficult. -} -openLocked file mode = do - handle <- openFile file mode - lockfd <- handleToFd handle -- closes handle - waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) - handle' <- fdToHandle lockfd - return handle' - where - lockType ReadMode = ReadLock - lockType _ = WriteLock - -{- Generates a new log line with the current date. -} -logNow :: LogStatus -> String -> String -> IO LogLine -logNow status repo file = do +{- Generates a new LogLine with the current date. -} +logNow :: LogStatus -> String -> IO LogLine +logNow status repo = do now <- getCurrentTime - return $ LogLine now status repo file + return $ LogLine now status repo {- Returns the filename of the log file for a given annexed file. -} logFile :: String -> IO String @@ -101,3 +87,16 @@ logFile annexedFile = do repo <- repoTop return $ repo ++ "/.git-annex/" ++ (gitRelative repo annexedFile) ++ ".log" + +{- Returns a list of repositories that, according to the log, have + - the content of a file -} +fileLocations :: String -> IO [String] +fileLocations file = do + log <- logFile file + lines <- readLog log + return $ map repo (filterPresent lines) + +{- Filters the list of LogLines to find repositories where the file + - is (or should still be) present. -} +filterPresent :: [LogLine] -> [LogLine] +filterPresent lines = From 381e6f84e5f4ddc64ed86f08064ebaf2313b18db Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 23:35:05 -0400 Subject: [PATCH 0008/2835] robustness --- GitRepo.hs | 13 +++---------- LocationLog.hs | 26 +++++++++++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 01e6746aef..2e7fff22ea 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -6,6 +6,7 @@ import Directory import System.Directory import System.Path import Data.String.Utils +import Utility {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). @@ -49,16 +50,8 @@ seekUp dir want = do if ok then return (Just dir) else case (parentDir dir) of - (Just d) -> seekUp d want - Nothing -> return Nothing - -parentDir :: String -> Maybe String -parentDir dir = - if length dirs > 0 - then Just ("/" ++ (join "/" $ take ((length dirs) - 1) dirs)) - else Nothing - where - dirs = filter (\x -> length x > 0) $ split "/" dir + "" -> return Nothing + d -> seekUp d want isRepoTop dir = do r <- isGitRepo dir diff --git a/LocationLog.hs b/LocationLog.hs index 911e4765b4..b6c85113d8 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -19,6 +19,7 @@ module LocationLog where import Data.DateTime import System.IO +import System.Directory import GitRepo import Utility @@ -55,22 +56,29 @@ instance Read LogLine where where date = fromSeconds $ read $ w !! 0 status = read $ w !! 1 - repo = unwords $ rest w + repo = unwords $ drop 2 w w = words string - rest (_:_:l) = l -{- Reads a log file -} +{- Reads a log file. + - Note that the LogLines returned may be in any order. -} readLog :: String -> IO [LogLine] readLog file = do - h <- openLocked file ReadMode - s <- hGetContents h - -- hClose handle' -- TODO disabled due to lazy IO issue - -- filter out any unparsable lines - return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s + exists <- doesFileExist file + if exists + then do + h <- openLocked file ReadMode + s <- hGetContents h + -- hClose handle' -- TODO disabled due to lazy IO issue + -- filter out any unparsable lines + return $ filter (\l -> (status l) /= Undefined ) + $ map read $ lines s + else do + return [] {- Adds a LogLine to a log file -} writeLog :: String -> LogLine -> IO () writeLog file line = do + createDirectoryIfMissing True (parentDir file) h <- openLocked file AppendMode hPutStrLn h $ show line hClose h @@ -99,4 +107,4 @@ fileLocations file = do {- Filters the list of LogLines to find repositories where the file - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] -filterPresent lines = +filterPresent lines = error "unimplimented" -- TODO From e64d1becf429489f8c6ded230e6e17b63a89c483 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 00:02:07 -0400 Subject: [PATCH 0009/2835] robustness fix avoid crash if the seconds field is not numeric --- LocationLog.hs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index b6c85113d8..db1fac1444 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -20,6 +20,7 @@ module LocationLog where import Data.DateTime import System.IO import System.Directory +import Data.Char import GitRepo import Utility @@ -50,14 +51,15 @@ instance Read LogLine where -- This parser is robust in that even unparsable log lines are -- read without an exception being thrown. -- Such lines have a status of Undefined. - readsPrec _ string = if (length w >= 3) - then [((LogLine date status repo), "")] - else [((LogLine (fromSeconds 0) Undefined ""), "")] + readsPrec _ string = + if (length w >= 3 && all isDigit date) + then [((LogLine (fromSeconds $ read date) status repo), "")] + else [((LogLine (fromSeconds 0) Undefined ""), "")] where - date = fromSeconds $ read $ w !! 0 + w = words string + date = w !! 0 status = read $ w !! 1 repo = unwords $ drop 2 w - w = words string {- Reads a log file. - Note that the LogLines returned may be in any order. -} From d0e82d0b9218a9ff3a693e066c4320c08d4d1c47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 00:18:10 -0400 Subject: [PATCH 0010/2835] add --- Utility.hs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Utility.hs diff --git a/Utility.hs b/Utility.hs new file mode 100644 index 0000000000..05b06dea79 --- /dev/null +++ b/Utility.hs @@ -0,0 +1,29 @@ +{- git-annex utility functions + -} + +module Utility where + +import System.IO +import System.Posix.IO +import Data.String.Utils + +{- Let's just say that Haskell makes reading/writing a file with + - file locking excessively difficult. -} +openLocked file mode = do + handle <- openFile file mode + lockfd <- handleToFd handle -- closes handle + waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) + handle' <- fdToHandle lockfd + return handle' + where + lockType ReadMode = ReadLock + lockType _ = WriteLock + +{- Returns the parent directory of a path. Parent of / is "" -} +parentDir :: String -> String +parentDir dir = + if length dirs > 0 + then "/" ++ (join "/" $ take ((length dirs) - 1) dirs) + else "" + where + dirs = filter (\x -> length x > 0) $ split "/" dir From c67521741abd9a49ebf43d6c649fe0356fa68fb3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 00:18:16 -0400 Subject: [PATCH 0011/2835] add --- .gitignore | 5 +++++ Makefile | 7 +++++++ demo.log | 8 ++++++++ git-annex.hs | 8 ++++++++ 4 files changed, 28 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 demo.log create mode 100644 git-annex.hs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..7dd8869b16 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.o +*.hi +*.ho +*.a +git-annex diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..8b7c9d3a0e --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +git-annex: + ghc --make git-annex + +clean: + rm -f git-annex *.o *.hi *.ho *.a + +.PHONY: git-annex diff --git a/demo.log b/demo.log new file mode 100644 index 0000000000..251a84c520 --- /dev/null +++ b/demo.log @@ -0,0 +1,8 @@ +1286654242 1 repo +1286652724 0 foo +a a a +a 1 a +-1 a a +1286652724 1 foo +1286656282 1 foo +1286656282 0 repo diff --git a/git-annex.hs b/git-annex.hs new file mode 100644 index 0000000000..a57e9e2dbc --- /dev/null +++ b/git-annex.hs @@ -0,0 +1,8 @@ +{- git-annex main program + - -} + +import LocationLog + +main = do + l <- readLog "demo.log" + putStrLn "hi" From 60c672e444decf59c20beb70b89f030ad9d62b3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 02:22:35 -0400 Subject: [PATCH 0012/2835] strictness and handle closing --- LocationLog.hs | 7 +++---- Utility.hs | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index db1fac1444..c921a20052 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -12,7 +12,6 @@ - - A line of the log will look like: "date N reponame" - Where N=1 when the repo has the file, and 0 otherwise. - - -} module LocationLog where @@ -69,8 +68,8 @@ readLog file = do if exists then do h <- openLocked file ReadMode - s <- hGetContents h - -- hClose handle' -- TODO disabled due to lazy IO issue + s <- hGetContentsStrict h + hClose h -- filter out any unparsable lines return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s @@ -95,7 +94,7 @@ logNow status repo = do logFile :: String -> IO String logFile annexedFile = do repo <- repoTop - return $ repo ++ "/.git-annex/" ++ + return $ (gitStateDir repo) ++ (gitRelative repo annexedFile) ++ ".log" {- Returns a list of repositories that, according to the log, have diff --git a/Utility.hs b/Utility.hs index 05b06dea79..ab9ce04f36 100644 --- a/Utility.hs +++ b/Utility.hs @@ -19,6 +19,10 @@ openLocked file mode = do lockType ReadMode = ReadLock lockType _ = WriteLock +{- A version of hgetContents that is not lazy. Ensures file is + - all read before it gets closed. -} +hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s + {- Returns the parent directory of a path. Parent of / is "" -} parentDir :: String -> String parentDir dir = From 852ead470756744cd6663ee2d537f3d281f1e7c8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 02:22:47 -0400 Subject: [PATCH 0013/2835] add gitPrep to handle .gitattributes --- GitRepo.hs | 27 ++++++++++++++++++++++++++- git-annex.hs | 4 ++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 2e7fff22ea..21c683bd27 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -8,6 +8,12 @@ import System.Path import Data.String.Utils import Utility +{- Long-term state is stored in files inside the .git-annex directory + - in the git repository. -} +stateLoc = ".git-annex" +gitStateDir :: String -> String +gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" + {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} @@ -23,9 +29,28 @@ gitRelative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo +{- Sets up the current git repo for git-annex. May be called repeatedly. -} +gitPrep :: IO () +gitPrep = do + repo <- repoTop + bare <- isBareRepo repo + -- configure git to use union merge driver on state files + let attributes = repo ++ "/.gitattributes" + let attrLine = stateLoc ++ "/* merge=union" + exists <- doesFileExist attributes + if (not bare) + then if (not exists) + then writeFile attributes $ attrLine ++ "\n" + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then appendFile attributes $ attrLine ++ "\n" + else return () + else return () {- Returns the path to the current repository's .git directory. - - (For a bare repository, that is the root of the repository.) -} + - (For a bare repository, that is the root of the repository.) + - TODO: support GIT_DIR -} gitDir :: IO String gitDir = do repo <- repoTop diff --git a/git-annex.hs b/git-annex.hs index a57e9e2dbc..e8032a1325 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -2,7 +2,7 @@ - -} import LocationLog +import GitRepo main = do - l <- readLog "demo.log" - putStrLn "hi" + gitPrep From 058cef945023843219e09d4cec80bb7e137b9876 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 02:27:14 -0400 Subject: [PATCH 0014/2835] handle bare repo right for gitattributes also simplere code! --- GitRepo.hs | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 21c683bd27..2e2c1d52b8 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -32,21 +32,26 @@ gitRelative repo file = drop (length absrepo) absfile {- Sets up the current git repo for git-annex. May be called repeatedly. -} gitPrep :: IO () gitPrep = do + -- configure git to use union merge driver on state files + let attrLine = stateLoc ++ "/* merge=union" + attributes <- gitAttributes + exists <- doesFileExist attributes + if (not exists) + then writeFile attributes $ attrLine ++ "\n" + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then appendFile attributes $ attrLine ++ "\n" + else return () + +{- Returns the path to the current repository's gitattributes file. -} +gitAttributes :: IO String +gitAttributes = do repo <- repoTop bare <- isBareRepo repo - -- configure git to use union merge driver on state files - let attributes = repo ++ "/.gitattributes" - let attrLine = stateLoc ++ "/* merge=union" - exists <- doesFileExist attributes - if (not bare) - then if (not exists) - then writeFile attributes $ attrLine ++ "\n" - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then appendFile attributes $ attrLine ++ "\n" - else return () - else return () + if (bare) + then return $ repo ++ "/info/.gitattributes" + else return $ repo ++ "/.gitattributes" {- Returns the path to the current repository's .git directory. - (For a bare repository, that is the root of the repository.) @@ -56,7 +61,7 @@ gitDir = do repo <- repoTop bare <- isBareRepo repo if (bare) - then return repo + then return $ repo else return $ repo ++ "/.git" {- Finds the top of the current git repository, which may be in a parent From 11ad93f023fa5e867b5b7bd47f45393caceb401a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 02:29:58 -0400 Subject: [PATCH 0015/2835] reorg --- GitRepo.hs | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 2e2c1d52b8..8737d82512 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -14,6 +14,26 @@ stateLoc = ".git-annex" gitStateDir :: String -> String gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" +{- Path to the current repository's gitattributes file. -} +gitAttributes :: IO String +gitAttributes = do + repo <- repoTop + bare <- isBareRepo repo + if (bare) + then return $ repo ++ "/info/.gitattributes" + else return $ repo ++ "/.gitattributes" + +{- Path to the current repository's .git directory. + - (For a bare repository, that is the root of the repository.) + - TODO: support GIT_DIR -} +gitDir :: IO String +gitDir = do + repo <- repoTop + bare <- isBareRepo repo + if (bare) + then return $ repo + else return $ repo ++ "/.git" + {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} @@ -41,29 +61,11 @@ gitPrep = do else do content <- readFile attributes if (all (/= attrLine) (lines content)) - then appendFile attributes $ attrLine ++ "\n" + then do + appendFile attributes $ attrLine ++ "\n" + -- TODO check attributes file into git? else return () -{- Returns the path to the current repository's gitattributes file. -} -gitAttributes :: IO String -gitAttributes = do - repo <- repoTop - bare <- isBareRepo repo - if (bare) - then return $ repo ++ "/info/.gitattributes" - else return $ repo ++ "/.gitattributes" - -{- Returns the path to the current repository's .git directory. - - (For a bare repository, that is the root of the repository.) - - TODO: support GIT_DIR -} -gitDir :: IO String -gitDir = do - repo <- repoTop - bare <- isBareRepo repo - if (bare) - then return $ repo - else return $ repo ++ "/.git" - {- Finds the top of the current git repository, which may be in a parent - directory. -} repoTop :: IO String From f98fa53d7f6d851b8a1ae804c02780769c98e07c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 11:08:40 -0400 Subject: [PATCH 0016/2835] fixed close after locking --- LocationLog.hs | 15 +++++++++------ Utility.hs | 7 +++++-- git-annex.hs | 2 ++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index c921a20052..1523901df7 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -12,6 +12,11 @@ - - A line of the log will look like: "date N reponame" - Where N=1 when the repo has the file, and 0 otherwise. + - + - TOOD: compact logs, by storing only current presence infomation when + - writing them. + - + - TODO: use ByteString -} module LocationLog where @@ -67,9 +72,8 @@ readLog file = do exists <- doesFileExist file if exists then do - h <- openLocked file ReadMode - s <- hGetContentsStrict h - hClose h + s <- withFileLocked file ReadMode $ \h -> + hGetContentsStrict h -- filter out any unparsable lines return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s @@ -80,9 +84,8 @@ readLog file = do writeLog :: String -> LogLine -> IO () writeLog file line = do createDirectoryIfMissing True (parentDir file) - h <- openLocked file AppendMode - hPutStrLn h $ show line - hClose h + withFileLocked file AppendMode $ \h -> + hPutStrLn h $ show line {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> IO LogLine diff --git a/Utility.hs b/Utility.hs index ab9ce04f36..d1eb247d3c 100644 --- a/Utility.hs +++ b/Utility.hs @@ -9,12 +9,15 @@ import Data.String.Utils {- Let's just say that Haskell makes reading/writing a file with - file locking excessively difficult. -} -openLocked file mode = do +withFileLocked file mode action = do + -- TODO: find a way to use bracket here handle <- openFile file mode lockfd <- handleToFd handle -- closes handle waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) handle' <- fdToHandle lockfd - return handle' + ret <- action handle' + hClose handle' + return ret where lockType ReadMode = ReadLock lockType _ = WriteLock diff --git a/git-annex.hs b/git-annex.hs index e8032a1325..66b9491bde 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,3 +6,5 @@ import GitRepo main = do gitPrep + l <- readLog "demo.log" + writeLog "demo2.log" $ l !! 0 From 7ad4a0bb7d4beb469f0aba017fae1ac48060e862 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 12:31:14 -0400 Subject: [PATCH 0017/2835] log compaction --- LocationLog.hs | 44 +++++++++++++++++++++++++++++++++++--------- demo.log | 7 +++++-- git-annex.hs | 2 +- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 1523901df7..028ceed5fb 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -7,21 +7,19 @@ - - Location tracking information is stored in `.git-annex/filename.log`. - Repositories record their name and the date when they --get or --drop - - a file's content. (Git is configured to use a union merge for this file, - - so the lines may be in arbitrary order, but it will never conflict.) + - a file's content. - - A line of the log will look like: "date N reponame" - Where N=1 when the repo has the file, and 0 otherwise. - - - - TOOD: compact logs, by storing only current presence infomation when - - writing them. - - - - TODO: use ByteString + - + - Git is configured to use a union merge for this file, + - so the lines may be in arbitrary order, but it will never conflict. -} module LocationLog where import Data.DateTime +import qualified Data.Map as Map import System.IO import System.Directory import Data.Char @@ -81,12 +79,19 @@ readLog file = do return [] {- Adds a LogLine to a log file -} -writeLog :: String -> LogLine -> IO () -writeLog file line = do +appendLog :: String -> LogLine -> IO () +appendLog file line = do createDirectoryIfMissing True (parentDir file) withFileLocked file AppendMode $ \h -> hPutStrLn h $ show line +{- Writes a set of lines to a log file -} +writeLog :: String -> [LogLine] -> IO () +writeLog file lines = do + createDirectoryIfMissing True (parentDir file) + withFileLocked file WriteMode $ \h -> + hPutStr h $ unlines $ map show lines + {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> IO LogLine logNow status repo = do @@ -112,3 +117,24 @@ fileLocations file = do - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] filterPresent lines = error "unimplimented" -- TODO + +{- Compacts a set of logs, returning a subset that contains the current + - status. -} +compactLog :: [LogLine] -> [LogLine] +compactLog lines = compactLog' Map.empty lines +compactLog' map [] = Map.elems map +compactLog' map (l:ls) = compactLog' (mapLog map l) ls + +{- Inserts a log into a map of logs, if the log has better (ie, newer) + - information about a repo than the other logs in the map -} +mapLog map log = + if (better) + then Map.insert (repo log) log map + else map + where + better = case (Map.lookup (repo log) map) of + -- <= used because two log entries could + -- have the same timestamp; if so the one that + -- is seen last should win. + Just l -> (date l <= date log) + Nothing -> True diff --git a/demo.log b/demo.log index 251a84c520..7a42630566 100644 --- a/demo.log +++ b/demo.log @@ -1,8 +1,11 @@ 1286654242 1 repo 1286652724 0 foo +1286656282 1 foo +1286656282 0 repo +1286656281 0 foo +# some garbage, should be ignored a a a + a 1 a -1 a a 1286652724 1 foo -1286656282 1 foo -1286656282 0 repo diff --git a/git-annex.hs b/git-annex.hs index 66b9491bde..cae72f00db 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -7,4 +7,4 @@ import GitRepo main = do gitPrep l <- readLog "demo.log" - writeLog "demo2.log" $ l !! 0 + writeLog "demo2.log" $ compactLog l From a745043e7db87ef43dbbb3f93cdf5807ff9958ec Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 12:35:28 -0400 Subject: [PATCH 0018/2835] don't repeatedly call repoTop, it's a bit expensive --- GitRepo.hs | 24 +++++++++++------------- git-annex.hs | 3 ++- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 8737d82512..140fb628a7 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -14,21 +14,19 @@ stateLoc = ".git-annex" gitStateDir :: String -> String gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" -{- Path to the current repository's gitattributes file. -} -gitAttributes :: IO String -gitAttributes = do - repo <- repoTop +{- Path to a repository's gitattributes file. -} +gitAttributes :: FilePath -> IO String +gitAttributes repo = do bare <- isBareRepo repo if (bare) then return $ repo ++ "/info/.gitattributes" else return $ repo ++ "/.gitattributes" -{- Path to the current repository's .git directory. +{- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} -gitDir :: IO String -gitDir = do - repo <- repoTop +gitDir :: FilePath -> IO String +gitDir repo = do bare <- isBareRepo repo if (bare) then return $ repo @@ -37,7 +35,7 @@ gitDir = do {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} -gitRelative :: String -> String -> String +gitRelative :: FilePath -> String -> String gitRelative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo @@ -49,12 +47,12 @@ gitRelative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -{- Sets up the current git repo for git-annex. May be called repeatedly. -} -gitPrep :: IO () -gitPrep = do +{- Sets up a git repo for git-annex. May be called repeatedly. -} +gitPrep :: FilePath -> IO () +gitPrep repo = do -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/* merge=union" - attributes <- gitAttributes + attributes <- gitAttributes repo exists <- doesFileExist attributes if (not exists) then writeFile attributes $ attrLine ++ "\n" diff --git a/git-annex.hs b/git-annex.hs index cae72f00db..0f274e674b 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -5,6 +5,7 @@ import LocationLog import GitRepo main = do - gitPrep + repo <- repoTop + gitPrep repo l <- readLog "demo.log" writeLog "demo2.log" $ compactLog l From a55f49efb6c05c5ddb031f077690e90ed7358642 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 12:41:20 -0400 Subject: [PATCH 0019/2835] update --- LocationLog.hs | 14 +++++++------- git-annex.hs | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 028ceed5fb..f9421cd9a0 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -65,7 +65,7 @@ instance Read LogLine where {- Reads a log file. - Note that the LogLines returned may be in any order. -} -readLog :: String -> IO [LogLine] +readLog :: FilePath -> IO [LogLine] readLog file = do exists <- doesFileExist file if exists @@ -79,14 +79,14 @@ readLog file = do return [] {- Adds a LogLine to a log file -} -appendLog :: String -> LogLine -> IO () +appendLog :: FilePath -> LogLine -> IO () appendLog file line = do createDirectoryIfMissing True (parentDir file) withFileLocked file AppendMode $ \h -> hPutStrLn h $ show line {- Writes a set of lines to a log file -} -writeLog :: String -> [LogLine] -> IO () +writeLog :: FilePath -> [LogLine] -> IO () writeLog file lines = do createDirectoryIfMissing True (parentDir file) withFileLocked file WriteMode $ \h -> @@ -99,7 +99,7 @@ logNow status repo = do return $ LogLine now status repo {- Returns the filename of the log file for a given annexed file. -} -logFile :: String -> IO String +logFile :: FilePath -> IO String logFile annexedFile = do repo <- repoTop return $ (gitStateDir repo) ++ @@ -107,16 +107,16 @@ logFile annexedFile = do {- Returns a list of repositories that, according to the log, have - the content of a file -} -fileLocations :: String -> IO [String] +fileLocations :: FilePath -> IO [String] fileLocations file = do log <- logFile file lines <- readLog log return $ map repo (filterPresent lines) -{- Filters the list of LogLines to find repositories where the file +{- Filters the list of LogLines to find ones where the file - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] -filterPresent lines = error "unimplimented" -- TODO +filterPresent lines = filter (\l -> FilePresent == status l) $ compactLog lines {- Compacts a set of logs, returning a subset that contains the current - status. -} diff --git a/git-annex.hs b/git-annex.hs index 0f274e674b..8944b50f55 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -7,5 +7,6 @@ import GitRepo main = do repo <- repoTop gitPrep repo + l <- readLog "demo.log" writeLog "demo2.log" $ compactLog l From 80ce5f90db1de10a5fa42583efcb7390cf185662 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 13:47:04 -0400 Subject: [PATCH 0020/2835] update --- Backend.hs | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ BackendFile.hs | 17 +++++++++++++++ BackendUrl.hs | 17 +++++++++++++++ GitRepo.hs | 14 ++++++------ git-annex.hs | 6 ++++++ 5 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 Backend.hs create mode 100644 BackendFile.hs create mode 100644 BackendUrl.hs diff --git a/Backend.hs b/Backend.hs new file mode 100644 index 0000000000..cb91325c61 --- /dev/null +++ b/Backend.hs @@ -0,0 +1,58 @@ +{- git-annex key/value storage backends + - + - git-annex uses a key/value abstraction layer to allow files contents to be + - stored in different ways. In theory, any key/value storage system could be + - used to store the file contents, and git-annex would then retrieve them + - as needed and put them in `.git/annex/`. + - + - When a file is annexed, a key is generated from its content and/or metadata. + - This key can later be used to retrieve the file's content (its value). This + - key generation must be stable for a given file content, name, and size. + - + - The mapping from filename to its key is stored in the .git-annex directory, + - in a file named `$filename.$backend` + - + - Multiple pluggable backends are supported, and more than one can be used + - to store different files' contents in a given repository. + - -} + +module Backend where + +import GitRepo +import System.Directory + +data Backend = Backend { + name :: String, -- name of this backend + keyvalue :: FilePath -> Maybe String, -- maps from key to value + retrievekey :: IO String -> IO (Bool) -- retrieves value given key +} + +{- Name of state file that holds the key for an annexed file, + - using a given backend. -} +backendFile :: Backend -> GitRepo -> FilePath -> String +backendFile backend repo file = gitStateDir repo ++ + (gitRelative repo file) ++ "." ++ (name backend) + +{- Looks up the backend used for an already annexed file. -} +lookupBackend :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Backend) +lookupBackend [] repo file = return Nothing +lookupBackend (b:bs) repo file = do + present <- checkBackend b repo file + if present + then + return $ Just b + else + lookupBackend bs repo file + +{- Checks if a file is available via a given backend. -} +checkBackend :: Backend -> GitRepo -> FilePath -> IO (Bool) +checkBackend backend repo file = doesFileExist $ backendFile backend repo file + +{- Attempts to retrieve an annexed file from one of the backends. -} +retrieveFile :: [Backend] -> GitRepo -> FilePath -> IO (Bool) +retrieveFile backends repo file = do + result <- lookupBackend backends repo file + case (result) of + Nothing -> return False + Just b -> (retrievekey b) key + where key = readFile (backendFile b repo file) diff --git a/BackendFile.hs b/BackendFile.hs new file mode 100644 index 0000000000..b1a3be58a6 --- /dev/null +++ b/BackendFile.hs @@ -0,0 +1,17 @@ +{- git-annex "file" backend + - -} + +module BackendFile (backend) where + +import Backend + +backend = Backend { + name = "file", + keyvalue = keyValue, + retrievekey = copyFile +} + +-- direct mapping from filename to key +keyValue k = Just $ id k + +copyFile f = error "unimplemented" diff --git a/BackendUrl.hs b/BackendUrl.hs new file mode 100644 index 0000000000..f95c53bbfc --- /dev/null +++ b/BackendUrl.hs @@ -0,0 +1,17 @@ +{- git-annex "url" backend + - -} + +module BackendUrl (backend) where + +import Backend + +backend = Backend { + name = "url", + keyvalue = keyValue, + retrievekey = downloadUrl +} + +-- cannot generate url from filename +keyValue k = Nothing + +downloadUrl k = error "unimplemented" diff --git a/GitRepo.hs b/GitRepo.hs index 140fb628a7..8974d9db6c 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -8,14 +8,16 @@ import System.Path import Data.String.Utils import Utility +type GitRepo = FilePath + {- Long-term state is stored in files inside the .git-annex directory - in the git repository. -} stateLoc = ".git-annex" -gitStateDir :: String -> String +gitStateDir :: GitRepo -> FilePath gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" {- Path to a repository's gitattributes file. -} -gitAttributes :: FilePath -> IO String +gitAttributes :: GitRepo -> IO String gitAttributes repo = do bare <- isBareRepo repo if (bare) @@ -25,7 +27,7 @@ gitAttributes repo = do {- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} -gitDir :: FilePath -> IO String +gitDir :: GitRepo -> IO String gitDir repo = do bare <- isBareRepo repo if (bare) @@ -35,7 +37,7 @@ gitDir repo = do {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} -gitRelative :: FilePath -> String -> String +gitRelative :: GitRepo -> String -> String gitRelative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo @@ -48,7 +50,7 @@ gitRelative repo file = drop (length absrepo) absfile Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo {- Sets up a git repo for git-annex. May be called repeatedly. -} -gitPrep :: FilePath -> IO () +gitPrep :: GitRepo -> IO () gitPrep repo = do -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/* merge=union" @@ -66,7 +68,7 @@ gitPrep repo = do {- Finds the top of the current git repository, which may be in a parent - directory. -} -repoTop :: IO String +repoTop :: IO GitRepo repoTop = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop diff --git a/git-annex.hs b/git-annex.hs index 8944b50f55..77faea2b78 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -3,6 +3,12 @@ import LocationLog import GitRepo +import Backend + +-- When adding a new backend, import it here and add it to the backends list. +import qualified BackendFile +import qualified BackendUrl +backends = [BackendFile.backend, BackendUrl.backend] main = do repo <- repoTop From f4d2a05e86df464790fb183148717e7ac7f49cda Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:04:07 -0400 Subject: [PATCH 0021/2835] got annexing working --- Annex.hs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Annex.hs diff --git a/Annex.hs b/Annex.hs new file mode 100644 index 0000000000..7d89c882aa --- /dev/null +++ b/Annex.hs @@ -0,0 +1,36 @@ +{- git-annex + -} + +module Annex where + +import Backend +import System.Posix.Files +import System.Directory +import GitRepo +import Utility + +{- An annexed file's content is stored in .git/annex/. -} +annexedFileLocation repo file = do + dir <- gitDir repo + return $ dir ++ "/annex/" ++ (gitRelative repo file) + +{- Annexes a file, storing it in a backend, and then moving it into + - the annex directory and setting up the symlink pointing to its + - content. -} +annexFile :: [Backend] -> GitRepo -> FilePath -> IO () +annexFile backends repo file = do + alreadyannexed <- lookupBackend backends repo file + case (alreadyannexed) of + Just _ -> error $ "already annexed " ++ file + Nothing -> do + stored <- storeFile backends repo file + if (not stored) + then error $ "no backend could store " ++ file + else symlink + where + symlink = do + dest <- annexedFileLocation repo file + createDirectoryIfMissing True (parentDir dest) + renameFile file dest + createSymbolicLink dest file + gitAdd repo file From cc235192353561a374c431485c6c3834659e0fa6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:04:18 -0400 Subject: [PATCH 0022/2835] update --- Backend.hs | 55 +++++++++++++++++++++++++++++++++++++------------- BackendFile.hs | 15 ++++++++++---- BackendUrl.hs | 15 ++++++++++---- GitRepo.hs | 11 ++++++++-- LocationLog.hs | 12 +++++------ git-annex.hs | 1 + 6 files changed, 79 insertions(+), 30 deletions(-) diff --git a/Backend.hs b/Backend.hs index cb91325c61..c55634a681 100644 --- a/Backend.hs +++ b/Backend.hs @@ -18,24 +18,60 @@ module Backend where -import GitRepo import System.Directory +import GitRepo +import Utility + +type Key = String data Backend = Backend { - name :: String, -- name of this backend - keyvalue :: FilePath -> Maybe String, -- maps from key to value - retrievekey :: IO String -> IO (Bool) -- retrieves value given key + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: FilePath -> Key -> IO (Bool), + -- retrieves a key's contents to a file + retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) } +instance Show Backend where + show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" + {- Name of state file that holds the key for an annexed file, - using a given backend. -} backendFile :: Backend -> GitRepo -> FilePath -> String backendFile backend repo file = gitStateDir repo ++ (gitRelative repo file) ++ "." ++ (name backend) +{- Attempts to Stores a file in one of the backends. -} +storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Bool) +storeFile [] _ _ = return False +storeFile (b:bs) repo file = do + try <- (getKey b) (gitRelative repo file) + case (try) of + Nothing -> storeFile bs repo file + Just key -> do + (storeFileKey b) file key + createDirectoryIfMissing True (parentDir backendfile) + writeFile backendfile key + return True + where backendfile = backendFile b repo file + +{- Attempts to retrieve an file from one of the backends, saving it to + - a specified location. -} +retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO (Bool) +retrieveFile backends repo file dest = do + result <- lookupBackend backends repo file + case (result) of + Nothing -> return False + Just b -> (retrieveKeyFile b) key dest + where + key = readFile (backendFile b repo file) + {- Looks up the backend used for an already annexed file. -} lookupBackend :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Backend) -lookupBackend [] repo file = return Nothing +lookupBackend [] _ _ = return Nothing lookupBackend (b:bs) repo file = do present <- checkBackend b repo file if present @@ -47,12 +83,3 @@ lookupBackend (b:bs) repo file = do {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> GitRepo -> FilePath -> IO (Bool) checkBackend backend repo file = doesFileExist $ backendFile backend repo file - -{- Attempts to retrieve an annexed file from one of the backends. -} -retrieveFile :: [Backend] -> GitRepo -> FilePath -> IO (Bool) -retrieveFile backends repo file = do - result <- lookupBackend backends repo file - case (result) of - Nothing -> return False - Just b -> (retrievekey b) key - where key = readFile (backendFile b repo file) diff --git a/BackendFile.hs b/BackendFile.hs index b1a3be58a6..324a4d8cd9 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -7,11 +7,18 @@ import Backend backend = Backend { name = "file", - keyvalue = keyValue, - retrievekey = copyFile + getKey = keyValue, + storeFileKey = moveToAnnex, + retrieveKeyFile = copyFromOtherRepo } -- direct mapping from filename to key -keyValue k = Just $ id k +keyValue :: FilePath -> IO (Maybe Key) +keyValue k = return $ Just $ id k + +moveToAnnex :: FilePath -> Key -> IO (Bool) +moveToAnnex file key = return False + +copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) +copyFromOtherRepo key file = return False -copyFile f = error "unimplemented" diff --git a/BackendUrl.hs b/BackendUrl.hs index f95c53bbfc..9b4c83d618 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -7,11 +7,18 @@ import Backend backend = Backend { name = "url", - keyvalue = keyValue, - retrievekey = downloadUrl + getKey = keyValue, + storeFileKey = dummyStore, + retrieveKeyFile = downloadUrl } -- cannot generate url from filename -keyValue k = Nothing +keyValue :: FilePath -> IO (Maybe Key) +keyValue k = return Nothing -downloadUrl k = error "unimplemented" +-- cannot store to urls +dummyStore :: FilePath -> Key -> IO (Bool) +dummyStore file url = return False + +downloadUrl :: IO Key -> FilePath -> IO (Bool) +downloadUrl url file = error "unimplemented" diff --git a/GitRepo.hs b/GitRepo.hs index 8974d9db6c..690782f0dc 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -57,15 +57,22 @@ gitPrep repo = do attributes <- gitAttributes repo exists <- doesFileExist attributes if (not exists) - then writeFile attributes $ attrLine ++ "\n" + then do + writeFile attributes $ attrLine ++ "\n" + gitAdd repo attributes else do content <- readFile attributes if (all (/= attrLine) (lines content)) then do appendFile attributes $ attrLine ++ "\n" - -- TODO check attributes file into git? + gitAdd repo attributes else return () +{- Stages a changed file in git's index. -} +gitAdd repo file = do + -- TODO + return () + {- Finds the top of the current git repository, which may be in a parent - directory. -} repoTop :: IO GitRepo diff --git a/LocationLog.hs b/LocationLog.hs index f9421cd9a0..32af824612 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -84,6 +84,7 @@ appendLog file line = do createDirectoryIfMissing True (parentDir file) withFileLocked file AppendMode $ \h -> hPutStrLn h $ show line + -- TODO git add log {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () @@ -99,17 +100,16 @@ logNow status repo = do return $ LogLine now status repo {- Returns the filename of the log file for a given annexed file. -} -logFile :: FilePath -> IO String -logFile annexedFile = do - repo <- repoTop +logFile :: GitRepo -> FilePath -> IO String +logFile repo annexedFile = do return $ (gitStateDir repo) ++ (gitRelative repo annexedFile) ++ ".log" {- Returns a list of repositories that, according to the log, have - the content of a file -} -fileLocations :: FilePath -> IO [String] -fileLocations file = do - log <- logFile file +fileLocations :: GitRepo -> FilePath -> IO [String] +fileLocations thisrepo file = do + log <- logFile thisrepo file lines <- readLog log return $ map repo (filterPresent lines) diff --git a/git-annex.hs b/git-annex.hs index 77faea2b78..556e0607e8 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -4,6 +4,7 @@ import LocationLog import GitRepo import Backend +import Annex -- When adding a new backend, import it here and add it to the backends list. import qualified BackendFile From 4631927a5c7b14605725f1c6f272fee19d8b4318 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:21:17 -0400 Subject: [PATCH 0023/2835] fix storing files in .git/annex by key --- Annex.hs | 19 +++++++++---------- Backend.hs | 24 ++++++++++++++++-------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Annex.hs b/Annex.hs index 7d89c882aa..bd9ce92a4e 100644 --- a/Annex.hs +++ b/Annex.hs @@ -9,14 +9,13 @@ import System.Directory import GitRepo import Utility -{- An annexed file's content is stored in .git/annex/. -} -annexedFileLocation repo file = do +{- An annexed file's content is stored somewhere under .git/annex/ -} +annexLoc repo key = do dir <- gitDir repo - return $ dir ++ "/annex/" ++ (gitRelative repo file) + return $ dir ++ "/annex/" ++ key {- Annexes a file, storing it in a backend, and then moving it into - - the annex directory and setting up the symlink pointing to its - - content. -} + - the annex directory and setting up the symlink pointing to its content. -} annexFile :: [Backend] -> GitRepo -> FilePath -> IO () annexFile backends repo file = do alreadyannexed <- lookupBackend backends repo file @@ -24,12 +23,12 @@ annexFile backends repo file = do Just _ -> error $ "already annexed " ++ file Nothing -> do stored <- storeFile backends repo file - if (not stored) - then error $ "no backend could store " ++ file - else symlink + case (stored) of + Nothing -> error $ "no backend could store " ++ file + Just key -> symlink key where - symlink = do - dest <- annexedFileLocation repo file + symlink key = do + dest <- annexLoc repo key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file diff --git a/Backend.hs b/Backend.hs index c55634a681..d6b4339890 100644 --- a/Backend.hs +++ b/Backend.hs @@ -44,21 +44,29 @@ backendFile :: Backend -> GitRepo -> FilePath -> String backendFile backend repo file = gitStateDir repo ++ (gitRelative repo file) ++ "." ++ (name backend) -{- Attempts to Stores a file in one of the backends. -} -storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Bool) -storeFile [] _ _ = return False +{- Attempts to store a file in one of the backends, and returns + - its key. -} +storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) +storeFile [] _ _ = return Nothing storeFile (b:bs) repo file = do try <- (getKey b) (gitRelative repo file) case (try) of - Nothing -> storeFile bs repo file + Nothing -> nextbackend Just key -> do - (storeFileKey b) file key + stored <- (storeFileKey b) file key + if (not stored) + then nextbackend + else do + bookkeeping key + return $ Just key + where + nextbackend = storeFile bs repo file + backendfile = backendFile b repo file + bookkeeping key = do createDirectoryIfMissing True (parentDir backendfile) writeFile backendfile key - return True - where backendfile = backendFile b repo file -{- Attempts to retrieve an file from one of the backends, saving it to +{- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO (Bool) retrieveFile backends repo file dest = do From eb577ee37ff1d631aa3580a235b9954043d0fb27 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:27:49 -0400 Subject: [PATCH 0024/2835] stub checksum backend --- BackendChecksum.hs | 18 ++++++++++++++++++ BackendFile.hs | 1 - git-annex.hs | 3 ++- 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 BackendChecksum.hs diff --git a/BackendChecksum.hs b/BackendChecksum.hs new file mode 100644 index 0000000000..267f8099c7 --- /dev/null +++ b/BackendChecksum.hs @@ -0,0 +1,18 @@ +{- git-annex "checksum" backend + - -} + +module BackendChecksum (backend) where + +import Backend +import qualified BackendFile +import Data.Digest.Pure.SHA + +-- based on BackendFile just with a different key type +backend = BackendFile.backend { + name = "checksum", + getKey = keyValue +} + +-- +keyValue :: FilePath -> IO (Maybe Key) +keyValue k = error "unimplemented" -- TODO diff --git a/BackendFile.hs b/BackendFile.hs index 324a4d8cd9..dd6ff595aa 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -21,4 +21,3 @@ moveToAnnex file key = return False copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) copyFromOtherRepo key file = return False - diff --git a/git-annex.hs b/git-annex.hs index 556e0607e8..cce49050b3 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -8,8 +8,9 @@ import Annex -- When adding a new backend, import it here and add it to the backends list. import qualified BackendFile +import qualified BackendChecksum import qualified BackendUrl -backends = [BackendFile.backend, BackendUrl.backend] +backends = [BackendFile.backend, BackendChecksum.backend, BackendUrl.backend] main = do repo <- repoTop From 7880dc16fef81bb6a8812c6b4e9578a6ae2b2879 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:41:35 -0400 Subject: [PATCH 0025/2835] update --- Backend.hs | 8 ++++---- BackendChecksum.hs | 7 ++++--- BackendFile.hs | 16 ++++++++++------ BackendUrl.hs | 11 ++++++----- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/Backend.hs b/Backend.hs index d6b4339890..40279866fd 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,9 +28,9 @@ data Backend = Backend { -- name of this backend name :: String, -- converts a filename to a key - getKey :: FilePath -> IO (Maybe Key), + getKey :: GitRepo -> FilePath -> IO (Maybe Key), -- stores a file's contents to a key - storeFileKey :: FilePath -> Key -> IO (Bool), + storeFileKey :: GitRepo -> FilePath -> Key -> IO (Bool), -- retrieves a key's contents to a file retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) } @@ -49,11 +49,11 @@ backendFile backend repo file = gitStateDir repo ++ storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) storeFile [] _ _ = return Nothing storeFile (b:bs) repo file = do - try <- (getKey b) (gitRelative repo file) + try <- (getKey b) repo (gitRelative repo file) case (try) of Nothing -> nextbackend Just key -> do - stored <- (storeFileKey b) file key + stored <- (storeFileKey b) repo file key if (not stored) then nextbackend else do diff --git a/BackendChecksum.hs b/BackendChecksum.hs index 267f8099c7..7b8d2c281e 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -4,6 +4,7 @@ module BackendChecksum (backend) where import Backend +import GitRepo import qualified BackendFile import Data.Digest.Pure.SHA @@ -13,6 +14,6 @@ backend = BackendFile.backend { getKey = keyValue } --- -keyValue :: FilePath -> IO (Maybe Key) -keyValue k = error "unimplemented" -- TODO +-- checksum the file to get its key +keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue k = error "checksum keyValue unimplemented" -- TODO diff --git a/BackendFile.hs b/BackendFile.hs index dd6ff595aa..6caf30f65d 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -4,20 +4,24 @@ module BackendFile (backend) where import Backend +import GitRepo backend = Backend { name = "file", getKey = keyValue, - storeFileKey = moveToAnnex, + storeFileKey = dummyStore, retrieveKeyFile = copyFromOtherRepo } -- direct mapping from filename to key -keyValue :: FilePath -> IO (Maybe Key) -keyValue k = return $ Just $ id k +keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue repo file = return $ Just file -moveToAnnex :: FilePath -> Key -> IO (Bool) -moveToAnnex file key = return False +-- This backend does not really do any independant data storage, +-- it relies on the file contents in .git/annex/ in this repo, +-- and other accessible repos. So storing a file is a no-op. +dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) +dummyStore repo file key = return True copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) -copyFromOtherRepo key file = return False +copyFromOtherRepo key file = error "copyFromOtherRepo unimplemented" -- TODO diff --git a/BackendUrl.hs b/BackendUrl.hs index 9b4c83d618..1aa5224b58 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -4,6 +4,7 @@ module BackendUrl (backend) where import Backend +import GitRepo backend = Backend { name = "url", @@ -13,12 +14,12 @@ backend = Backend { } -- cannot generate url from filename -keyValue :: FilePath -> IO (Maybe Key) -keyValue k = return Nothing +keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue repo file = return Nothing -- cannot store to urls -dummyStore :: FilePath -> Key -> IO (Bool) -dummyStore file url = return False +dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) +dummyStore repo file url = return False downloadUrl :: IO Key -> FilePath -> IO (Bool) -downloadUrl url file = error "unimplemented" +downloadUrl url file = error "downloadUrl unimplemented" From dce9c2e0804d2c94f46dcac8c9884766bb22dcc7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:54:02 -0400 Subject: [PATCH 0026/2835] convert GitRepo to struct with constructor --- Annex.hs | 30 +++++++++++++++++++------ Backend.hs | 1 + BackendFile.hs | 8 ++++--- GitRepo.hs | 61 ++++++++++++++++++-------------------------------- LocationLog.hs | 1 + Locations.hs | 18 +++++++++++++++ git-annex.hs | 2 +- 7 files changed, 71 insertions(+), 50 deletions(-) create mode 100644 Locations.hs diff --git a/Annex.hs b/Annex.hs index bd9ce92a4e..f23358bf63 100644 --- a/Annex.hs +++ b/Annex.hs @@ -3,16 +3,12 @@ module Annex where -import Backend import System.Posix.Files import System.Directory import GitRepo import Utility - -{- An annexed file's content is stored somewhere under .git/annex/ -} -annexLoc repo key = do - dir <- gitDir repo - return $ dir ++ "/annex/" ++ key +import Locations +import Backend {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} @@ -28,8 +24,28 @@ annexFile backends repo file = do Just key -> symlink key where symlink key = do - dest <- annexLoc repo key + dest <- annexDir repo key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file gitAdd repo file + +{- Sets up a git repo for git-annex. May be called repeatedly. -} +gitPrep :: GitRepo -> IO () +gitPrep repo = do + -- configure git to use union merge driver on state files + let attrLine = stateLoc ++ "/* merge=union" + attributes <- gitAttributes repo + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + gitAdd repo attributes + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then do + appendFile attributes $ attrLine ++ "\n" + gitAdd repo attributes + else return () + diff --git a/Backend.hs b/Backend.hs index 40279866fd..e01f122395 100644 --- a/Backend.hs +++ b/Backend.hs @@ -19,6 +19,7 @@ module Backend where import System.Directory +import Locations import GitRepo import Utility diff --git a/BackendFile.hs b/BackendFile.hs index 6caf30f65d..92f708696b 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -17,11 +17,13 @@ backend = Backend { keyValue :: GitRepo -> FilePath -> IO (Maybe Key) keyValue repo file = return $ Just file --- This backend does not really do any independant data storage, --- it relies on the file contents in .git/annex/ in this repo, --- and other accessible repos. So storing a file is a no-op. +{- This backend does not really do any independant data storage, + - it relies on the file contents in .git/annex/ in this repo, + - and other accessible repos. So storing a file is a no-op. -} dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) dummyStore repo file key = return True +{- Try to find a copy of the file in one of the other repos, + - and copy it over to this one. -} copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) copyFromOtherRepo key file = error "copyFromOtherRepo unimplemented" -- TODO diff --git a/GitRepo.hs b/GitRepo.hs index 690782f0dc..fda83f7d8e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -8,79 +8,62 @@ import System.Path import Data.String.Utils import Utility -type GitRepo = FilePath +data GitRepo = GitRepo { + top :: FilePath, + remotes :: [GitRepo] +} deriving (Eq, Show, Read) -{- Long-term state is stored in files inside the .git-annex directory - - in the git repository. -} -stateLoc = ".git-annex" -gitStateDir :: GitRepo -> FilePath -gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" +{- GitRepo constructor -} +gitRepo :: FilePath -> IO GitRepo +gitRepo dir = do + -- TOOD query repo for configuration settings; other repositories; etc + return GitRepo { top = dir, remotes = [] } {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String gitAttributes repo = do - bare <- isBareRepo repo + bare <- isBareRepo (top repo) if (bare) - then return $ repo ++ "/info/.gitattributes" - else return $ repo ++ "/.gitattributes" + then return $ (top repo) ++ "/info/.gitattributes" + else return $ (top repo) ++ "/.gitattributes" {- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} gitDir :: GitRepo -> IO String gitDir repo = do - bare <- isBareRepo repo + bare <- isBareRepo (top repo) if (bare) - then return $ repo - else return $ repo ++ "/.git" + then return $ (top repo) + else return $ (top repo) ++ "/.git" {- Given a relative or absolute filename, calculates the name to use - - relative to a git repository directory (which must be absolute). + - to refer to the file relative to a git repository directory. - This is the same form displayed and used by git. -} gitRelative :: GitRepo -> String -> String gitRelative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file - absrepo = case (absNormPath "/" repo) of + absrepo = case (absNormPath "/" (top repo)) of Just f -> f ++ "/" - Nothing -> error $ "bad repo" ++ repo + Nothing -> error $ "bad repo" ++ (top repo) absfile = case (secureAbsNormPath absrepo file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -{- Sets up a git repo for git-annex. May be called repeatedly. -} -gitPrep :: GitRepo -> IO () -gitPrep repo = do - -- configure git to use union merge driver on state files - let attrLine = stateLoc ++ "/* merge=union" - attributes <- gitAttributes repo - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - gitAdd repo attributes - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - gitAdd repo attributes - else return () - {- Stages a changed file in git's index. -} gitAdd repo file = do -- TODO return () -{- Finds the top of the current git repository, which may be in a parent - - directory. -} -repoTop :: IO GitRepo -repoTop = do +{- Finds the current git repository, which may be in a parent directory. -} +currentRepo :: IO GitRepo +currentRepo = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> return dir + (Just dir) -> gitRepo dir Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) diff --git a/LocationLog.hs b/LocationLog.hs index 32af824612..73e9f1c6d0 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -25,6 +25,7 @@ import System.Directory import Data.Char import GitRepo import Utility +import Locations data LogStatus = FilePresent | FileMissing | Undefined deriving (Eq) diff --git a/Locations.hs b/Locations.hs new file mode 100644 index 0000000000..7273797ef9 --- /dev/null +++ b/Locations.hs @@ -0,0 +1,18 @@ +{- git-annex file locations + -} + +module Locations where + +import GitRepo + +{- An annexed file's content is stored somewhere under .git/annex/ -} +annexDir :: GitRepo -> String -> IO FilePath +annexDir repo key = do + dir <- gitDir repo + return $ dir ++ "/annex/" ++ key + +{- Long-term state is stored in files inside the .git-annex directory + - in the git repository. -} +stateLoc = ".git-annex" +gitStateDir :: GitRepo -> FilePath +gitStateDir repo = (top repo) ++ "/" ++ stateLoc ++ "/" diff --git a/git-annex.hs b/git-annex.hs index cce49050b3..f8c67b1fdd 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -13,7 +13,7 @@ import qualified BackendUrl backends = [BackendFile.backend, BackendChecksum.backend, BackendUrl.backend] main = do - repo <- repoTop + repo <- currentRepo gitPrep repo l <- readLog "demo.log" From e5514e0cb0809848645814e8c1f67cd89cb16c4f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 18:05:37 -0400 Subject: [PATCH 0027/2835] update --- Annex.hs | 1 + Backend.hs | 14 +------------- BackendChecksum.hs | 3 +-- BackendFile.hs | 3 +-- BackendList.hs | 14 ++++++++++++++ BackendUrl.hs | 3 +-- CmdLine.hs | 41 +++++++++++++++++++++++++++++++++++++++++ GitRepo.hs | 13 +++++++------ LocationLog.hs | 1 + Locations.hs | 3 ++- Types.hs | 24 ++++++++++++++++++++++++ git-annex.hs | 17 +++++++---------- 12 files changed, 101 insertions(+), 36 deletions(-) create mode 100644 BackendList.hs create mode 100644 CmdLine.hs create mode 100644 Types.hs diff --git a/Annex.hs b/Annex.hs index f23358bf63..bddff1e13d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,6 +8,7 @@ import System.Directory import GitRepo import Utility import Locations +import Types import Backend {- Annexes a file, storing it in a backend, and then moving it into diff --git a/Backend.hs b/Backend.hs index e01f122395..93ceee234a 100644 --- a/Backend.hs +++ b/Backend.hs @@ -22,19 +22,7 @@ import System.Directory import Locations import GitRepo import Utility - -type Key = String - -data Backend = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: GitRepo -> FilePath -> IO (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO (Bool), - -- retrieves a key's contents to a file - retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) -} +import Types instance Show Backend where show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" diff --git a/BackendChecksum.hs b/BackendChecksum.hs index 7b8d2c281e..18ff0cb57c 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -3,8 +3,7 @@ module BackendChecksum (backend) where -import Backend -import GitRepo +import Types import qualified BackendFile import Data.Digest.Pure.SHA diff --git a/BackendFile.hs b/BackendFile.hs index 92f708696b..deb4bce7e8 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,8 +3,7 @@ module BackendFile (backend) where -import Backend -import GitRepo +import Types backend = Backend { name = "file", diff --git a/BackendList.hs b/BackendList.hs new file mode 100644 index 0000000000..c744949b6c --- /dev/null +++ b/BackendList.hs @@ -0,0 +1,14 @@ +{- git-annex backend list + - -} + +module BackendList where + +-- When adding a new backend, import it here and add it to the list. +import qualified BackendFile +import qualified BackendChecksum +import qualified BackendUrl +supportedBackends = + [ BackendFile.backend + , BackendChecksum.backend + , BackendUrl.backend + ] diff --git a/BackendUrl.hs b/BackendUrl.hs index 1aa5224b58..2bc34434be 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,8 +3,7 @@ module BackendUrl (backend) where -import Backend -import GitRepo +import Types backend = Backend { name = "url", diff --git a/CmdLine.hs b/CmdLine.hs new file mode 100644 index 0000000000..79bd55cd9e --- /dev/null +++ b/CmdLine.hs @@ -0,0 +1,41 @@ +{- git-annex command line + - + - TODO: This is very rough and stupid; I would like to use + - System.Console.CmdArgs.Implicit but it is not yet packaged in Debian. + -} + +module CmdLine where + +import System.Console.GetOpt +import Types +import Annex + +data Flag = Add FilePath | Push String | Pull String | + Want FilePath | Get (Maybe FilePath) | Drop FilePath + deriving Show + +options :: [OptDescr Flag] +options = + [ Option ['a'] ["add"] (ReqArg Add "FILE") "add file to annex" + , Option ['p'] ["push"] (ReqArg Push "REPO") "push annex to repo" + , Option ['P'] ["pull"] (ReqArg Pull "REPO") "pull annex from repo" + , Option ['w'] ["want"] (ReqArg Want "FILE") "request file contents" + , Option ['g'] ["get"] (OptArg Get "FILE") "transfer file contents" + , Option ['d'] ["drop"] (ReqArg Drop "FILE") "indicate file content not needed" + ] + +argvToFlags argv = do + case getOpt Permute options argv of + -- no options? add listed files + ([],p,[] ) -> return $ map (\f -> Add f) p + -- all options parsed, return flags + (o,[],[] ) -> return o + -- error case + (_,n,errs) -> ioError (userError (concat errs ++ usageInfo header options)) + where header = "Usage: git-annex [option] file" + +dispatch :: Flag -> [Backend] -> GitRepo -> IO () +dispatch flag backends repo = do + case (flag) of + Add f -> annexFile backends repo f + _ -> error "not implemented" diff --git a/GitRepo.hs b/GitRepo.hs index fda83f7d8e..a0909d5ecd 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -7,17 +7,18 @@ import System.Directory import System.Path import Data.String.Utils import Utility - -data GitRepo = GitRepo { - top :: FilePath, - remotes :: [GitRepo] -} deriving (Eq, Show, Read) +import Types +import BackendList {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo gitRepo dir = do -- TOOD query repo for configuration settings; other repositories; etc - return GitRepo { top = dir, remotes = [] } + return GitRepo { + top = dir, + remotes = [], + backends = supportedBackends + } {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String diff --git a/LocationLog.hs b/LocationLog.hs index 73e9f1c6d0..a5e9a2679a 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -26,6 +26,7 @@ import Data.Char import GitRepo import Utility import Locations +import Types data LogStatus = FilePresent | FileMissing | Undefined deriving (Eq) diff --git a/Locations.hs b/Locations.hs index 7273797ef9..50f94a727d 100644 --- a/Locations.hs +++ b/Locations.hs @@ -3,10 +3,11 @@ module Locations where +import Types import GitRepo {- An annexed file's content is stored somewhere under .git/annex/ -} -annexDir :: GitRepo -> String -> IO FilePath +annexDir :: GitRepo -> Key -> IO FilePath annexDir repo key = do dir <- gitDir repo return $ dir ++ "/annex/" ++ key diff --git a/Types.hs b/Types.hs new file mode 100644 index 0000000000..2308b6fde9 --- /dev/null +++ b/Types.hs @@ -0,0 +1,24 @@ +{- git-annex data types + - -} + +module Types where + +type Key = String + +data Backend = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: GitRepo -> FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: GitRepo -> FilePath -> Key -> IO (Bool), + -- retrieves a key's contents to a file + retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) +} + +data GitRepo = GitRepo { + top :: FilePath, + remotes :: [GitRepo], + backends :: [Backend] +} + diff --git a/git-annex.hs b/git-annex.hs index f8c67b1fdd..590a7c0519 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,20 +1,17 @@ {- git-annex main program - -} -import LocationLog +import System.Environment import GitRepo -import Backend +import CmdLine import Annex - --- When adding a new backend, import it here and add it to the backends list. -import qualified BackendFile -import qualified BackendChecksum -import qualified BackendUrl -backends = [BackendFile.backend, BackendChecksum.backend, BackendUrl.backend] +import BackendList main = do + args <- getArgs + flags <- argvToFlags args + repo <- currentRepo gitPrep repo - l <- readLog "demo.log" - writeLog "demo2.log" $ compactLog l + mapM (\f -> dispatch f supportedBackends repo) flags From 026adce5a01381e9a802747f2ddf4ca5635468c9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 18:25:31 -0400 Subject: [PATCH 0028/2835] update --- Annex.hs | 18 ++++++++++++------ CmdLine.hs | 6 +++--- GitRepo.hs | 4 +--- Types.hs | 11 +++++++++-- git-annex.hs | 5 ++--- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/Annex.hs b/Annex.hs index bddff1e13d..bd3cade583 100644 --- a/Annex.hs +++ b/Annex.hs @@ -10,26 +10,32 @@ import Utility import Locations import Types import Backend +import BackendList + +startAnnex :: IO State +startAnnex = do + r <- currentRepo + return State { repo = r, backends = supportedBackends } {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} -annexFile :: [Backend] -> GitRepo -> FilePath -> IO () -annexFile backends repo file = do - alreadyannexed <- lookupBackend backends repo file +annexFile :: State -> FilePath -> IO () +annexFile state file = do + alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of Just _ -> error $ "already annexed " ++ file Nothing -> do - stored <- storeFile backends repo file + stored <- storeFile (backends state) (repo state) file case (stored) of Nothing -> error $ "no backend could store " ++ file Just key -> symlink key where symlink key = do - dest <- annexDir repo key + dest <- annexDir (repo state) key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file - gitAdd repo file + gitAdd (repo state) file {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () diff --git a/CmdLine.hs b/CmdLine.hs index 79bd55cd9e..d848ee8f9c 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -34,8 +34,8 @@ argvToFlags argv = do (_,n,errs) -> ioError (userError (concat errs ++ usageInfo header options)) where header = "Usage: git-annex [option] file" -dispatch :: Flag -> [Backend] -> GitRepo -> IO () -dispatch flag backends repo = do +dispatch :: Flag -> State -> IO () +dispatch flag state = do case (flag) of - Add f -> annexFile backends repo f + Add f -> annexFile state f _ -> error "not implemented" diff --git a/GitRepo.hs b/GitRepo.hs index a0909d5ecd..06da2ff883 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -8,7 +8,6 @@ import System.Path import Data.String.Utils import Utility import Types -import BackendList {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo @@ -16,8 +15,7 @@ gitRepo dir = do -- TOOD query repo for configuration settings; other repositories; etc return GitRepo { top = dir, - remotes = [], - backends = supportedBackends + remotes = [] } {- Path to a repository's gitattributes file. -} diff --git a/Types.hs b/Types.hs index 2308b6fde9..cab4b2016e 100644 --- a/Types.hs +++ b/Types.hs @@ -3,8 +3,10 @@ module Types where +-- annexed filenames are mapped into keys type Key = String +-- this structure represents a key/value backend data Backend = Backend { -- name of this backend name :: String, @@ -16,9 +18,14 @@ data Backend = Backend { retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) } +-- a git repository data GitRepo = GitRepo { top :: FilePath, - remotes :: [GitRepo], - backends :: [Backend] + remotes :: [GitRepo] } +-- git-annex's runtime state +data State = State { + repo :: GitRepo, + backends :: [Backend] +} diff --git a/git-annex.hs b/git-annex.hs index 590a7c0519..2c9b1315fe 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -11,7 +11,6 @@ main = do args <- getArgs flags <- argvToFlags args - repo <- currentRepo - gitPrep repo + state <- startAnnex - mapM (\f -> dispatch f supportedBackends repo) flags + mapM (\f -> dispatch f state) flags From 586266e444f72808101a055323887fe08ae7fce3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 19:00:08 -0400 Subject: [PATCH 0029/2835] robustness --- Annex.hs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index bd3cade583..402c767daa 100644 --- a/Annex.hs +++ b/Annex.hs @@ -1,4 +1,4 @@ -{- git-annex +{- git-annex toplevel code -} module Annex where @@ -12,15 +12,21 @@ import Types import Backend import BackendList +{- On startup, examine the git repo, prepare it, and record state for + - later. -} startAnnex :: IO State startAnnex = do r <- currentRepo + gitPrep r + -- TODO query git repo for configuration return State { repo = r, backends = supportedBackends } {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do + checkExists file + checkLegal file alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of Just _ -> error $ "already annexed " ++ file @@ -36,6 +42,16 @@ annexFile state file = do renameFile file dest createSymbolicLink dest file gitAdd (repo state) file + checkExists file = do + exists <- doesFileExist file + case (exists) of + False -> error $ "does not exist: " ++ file + True -> return () + checkLegal file = do + s <- getFileStatus file + case (not (isSymbolicLink s) && not (isRegularFile s)) of + False -> error $ "not a regular file: " ++ file + True -> return () {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () From 93d2dc0d6878ccb1067376d2a03193c222429d3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 19:14:32 -0400 Subject: [PATCH 0030/2835] cache whether a repo is bare --- GitRepo.hs | 9 +++++---- Types.hs | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 06da2ff883..ef76fb9766 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -12,17 +12,19 @@ import Types {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo gitRepo dir = do + b <- isBareRepo dir + -- TOOD query repo for configuration settings; other repositories; etc return GitRepo { top = dir, + bare = b, remotes = [] } {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String gitAttributes repo = do - bare <- isBareRepo (top repo) - if (bare) + if (bare repo) then return $ (top repo) ++ "/info/.gitattributes" else return $ (top repo) ++ "/.gitattributes" @@ -31,8 +33,7 @@ gitAttributes repo = do - TODO: support GIT_DIR -} gitDir :: GitRepo -> IO String gitDir repo = do - bare <- isBareRepo (top repo) - if (bare) + if (bare repo) then return $ (top repo) else return $ (top repo) ++ "/.git" diff --git a/Types.hs b/Types.hs index cab4b2016e..e1f598f0f5 100644 --- a/Types.hs +++ b/Types.hs @@ -21,6 +21,7 @@ data Backend = Backend { -- a git repository data GitRepo = GitRepo { top :: FilePath, + bare :: Bool, remotes :: [GitRepo] } From 344f13394fe5b12cbdd5eeb99bb63892c7096bfd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 19:53:31 -0400 Subject: [PATCH 0031/2835] update --- Annex.hs | 30 +++++++++++++++++++++++------- Backend.hs | 23 +++++++++++++++++++---- BackendFile.hs | 10 +++++++--- BackendUrl.hs | 11 +++++++---- CmdLine.hs | 6 ++++-- Types.hs | 6 ++++-- git-annex.mdwn | 1 + 7 files changed, 65 insertions(+), 22 deletions(-) diff --git a/Annex.hs b/Annex.hs index 402c767daa..964532f3f2 100644 --- a/Annex.hs +++ b/Annex.hs @@ -44,14 +44,30 @@ annexFile state file = do gitAdd (repo state) file checkExists file = do exists <- doesFileExist file - case (exists) of - False -> error $ "does not exist: " ++ file - True -> return () + if (not exists) + then error $ "does not exist: " ++ file + else return () checkLegal file = do - s <- getFileStatus file - case (not (isSymbolicLink s) && not (isRegularFile s)) of - False -> error $ "not a regular file: " ++ file - True -> return () + s <- getSymbolicLinkStatus file + if ((isSymbolicLink s) || (not $ isRegularFile s)) + then error $ "not a regular file: " ++ file + else return () + +{- Inverse of annexFile. -} +unannexFile :: State -> FilePath -> IO () +unannexFile state file = do + alreadyannexed <- lookupBackend (backends state) (repo state) file + case (alreadyannexed) of + Nothing -> error $ "not annexed " ++ file + Just _ -> do + mkey <- dropFile (backends state) (repo state) file + case (mkey) of + Nothing -> return () + Just key -> do + src <- annexDir (repo state) key + removeFile file + renameFile src file + return () {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () diff --git a/Backend.hs b/Backend.hs index 93ceee234a..5ddd3aac65 100644 --- a/Backend.hs +++ b/Backend.hs @@ -57,14 +57,29 @@ storeFile (b:bs) repo file = do {- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} -retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO (Bool) +retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO Bool retrieveFile backends repo file dest = do result <- lookupBackend backends repo file case (result) of Nothing -> return False - Just b -> (retrieveKeyFile b) key dest - where - key = readFile (backendFile b repo file) + Just b -> do + key <- lookupKey b repo file + (retrieveKeyFile b) key dest + +{- Drops the key for a file from the backend that has it. -} +dropFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) +dropFile backends repo file = do + result <- lookupBackend backends repo file + case (result) of + Nothing -> return Nothing + Just b -> do + key <- lookupKey b repo file + (removeKey b) key + return $ Just key + +{- Looks up the key a backend uses for an already annexed file. -} +lookupKey :: Backend -> GitRepo -> FilePath -> IO Key +lookupKey backend repo file = readFile (backendFile backend repo file) {- Looks up the backend used for an already annexed file. -} lookupBackend :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Backend) diff --git a/BackendFile.hs b/BackendFile.hs index deb4bce7e8..de60803c36 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -9,7 +9,8 @@ backend = Backend { name = "file", getKey = keyValue, storeFileKey = dummyStore, - retrieveKeyFile = copyFromOtherRepo + retrieveKeyFile = copyFromOtherRepo, + removeKey = dummyRemove } -- direct mapping from filename to key @@ -18,11 +19,14 @@ keyValue repo file = return $ Just file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - - and other accessible repos. So storing a file is a no-op. -} + - and other accessible repos. So storing or removing a key is + - a no-op. -} dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) dummyStore repo file key = return True +dummyRemove :: Key -> IO Bool +dummyRemove url = return False {- Try to find a copy of the file in one of the other repos, - and copy it over to this one. -} -copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) +copyFromOtherRepo :: Key -> FilePath -> IO (Bool) copyFromOtherRepo key file = error "copyFromOtherRepo unimplemented" -- TODO diff --git a/BackendUrl.hs b/BackendUrl.hs index 2bc34434be..ddeab9e042 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -9,16 +9,19 @@ backend = Backend { name = "url", getKey = keyValue, storeFileKey = dummyStore, - retrieveKeyFile = downloadUrl + retrieveKeyFile = downloadUrl, + removeKey = dummyRemove } -- cannot generate url from filename keyValue :: GitRepo -> FilePath -> IO (Maybe Key) keyValue repo file = return Nothing --- cannot store to urls -dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) +-- cannot change urls +dummyStore :: GitRepo -> FilePath -> Key -> IO Bool dummyStore repo file url = return False +dummyRemove :: Key -> IO Bool +dummyRemove url = return False -downloadUrl :: IO Key -> FilePath -> IO (Bool) +downloadUrl :: Key -> FilePath -> IO Bool downloadUrl url file = error "downloadUrl unimplemented" diff --git a/CmdLine.hs b/CmdLine.hs index d848ee8f9c..3709f836bd 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,8 +10,8 @@ import System.Console.GetOpt import Types import Annex -data Flag = Add FilePath | Push String | Pull String | - Want FilePath | Get (Maybe FilePath) | Drop FilePath +data Flag = Add FilePath | Push String | Pull String | Want FilePath | + Get (Maybe FilePath) | Drop FilePath | Unannex FilePath deriving Show options :: [OptDescr Flag] @@ -22,6 +22,7 @@ options = , Option ['w'] ["want"] (ReqArg Want "FILE") "request file contents" , Option ['g'] ["get"] (OptArg Get "FILE") "transfer file contents" , Option ['d'] ["drop"] (ReqArg Drop "FILE") "indicate file content not needed" + , Option ['u'] ["unannex"] (ReqArg Unannex "FILE") "undo --add" ] argvToFlags argv = do @@ -38,4 +39,5 @@ dispatch :: Flag -> State -> IO () dispatch flag state = do case (flag) of Add f -> annexFile state f + Unannex f -> unannexFile state f _ -> error "not implemented" diff --git a/Types.hs b/Types.hs index e1f598f0f5..6e3727e25a 100644 --- a/Types.hs +++ b/Types.hs @@ -13,9 +13,11 @@ data Backend = Backend { -- converts a filename to a key getKey :: GitRepo -> FilePath -> IO (Maybe Key), -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO (Bool), + storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) + retrieveKeyFile :: Key -> FilePath -> IO Bool, + -- removes a key + removeKey :: Key -> IO Bool } -- a git repository diff --git a/git-annex.mdwn b/git-annex.mdwn index bc3550398c..2996a90b51 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -36,6 +36,7 @@ Enough broad picture, here's how it actually looks: downloaded. * `git annex --drop $file` indicates that you no longer want the file's content to be available in this repository. +* `git annex --unannex $file` undoes a `git annex --add`. * `git annex $file` is a shorthand for either --add or --get. If the file is already known, it does --get, otherwise it does --add. From 200bc6fdb84658593bfb02f34f984531b6710d26 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 21:00:42 -0400 Subject: [PATCH 0032/2835] better option handling multiple-file support for all modes --- Annex.hs | 12 +++--------- CmdLine.hs | 45 +++++++++++++++++++++++---------------------- git-annex.hs | 29 +++++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/Annex.hs b/Annex.hs index 964532f3f2..ee94a98097 100644 --- a/Annex.hs +++ b/Annex.hs @@ -25,15 +25,14 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - checkExists file - checkLegal file alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of - Just _ -> error $ "already annexed " ++ file + Just _ -> error $ "already annexed: " ++ file Nothing -> do + checkLegal file stored <- storeFile (backends state) (repo state) file case (stored) of - Nothing -> error $ "no backend could store " ++ file + Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key where symlink key = do @@ -42,11 +41,6 @@ annexFile state file = do renameFile file dest createSymbolicLink dest file gitAdd (repo state) file - checkExists file = do - exists <- doesFileExist file - if (not exists) - then error $ "does not exist: " ++ file - else return () checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) diff --git a/CmdLine.hs b/CmdLine.hs index 3709f836bd..c956f29a55 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,34 +10,35 @@ import System.Console.GetOpt import Types import Annex -data Flag = Add FilePath | Push String | Pull String | Want FilePath | - Get (Maybe FilePath) | Drop FilePath | Unannex FilePath +data Mode = Add | Push | Pull | Want | Get | Drop | Unannex deriving Show -options :: [OptDescr Flag] +options :: [OptDescr Mode] options = - [ Option ['a'] ["add"] (ReqArg Add "FILE") "add file to annex" - , Option ['p'] ["push"] (ReqArg Push "REPO") "push annex to repo" - , Option ['P'] ["pull"] (ReqArg Pull "REPO") "pull annex from repo" - , Option ['w'] ["want"] (ReqArg Want "FILE") "request file contents" - , Option ['g'] ["get"] (OptArg Get "FILE") "transfer file contents" - , Option ['d'] ["drop"] (ReqArg Drop "FILE") "indicate file content not needed" - , Option ['u'] ["unannex"] (ReqArg Unannex "FILE") "undo --add" + [ Option ['a'] ["add"] (NoArg Add) "add files to annex" + , Option ['p'] ["push"] (NoArg Push) "push annex to repos" + , Option ['P'] ["pull"] (NoArg Pull) "pull annex from repos" + , Option ['w'] ["want"] (NoArg Want) "request file contents" + , Option ['g'] ["get"] (NoArg Get) "transfer file contents" + , Option ['d'] ["drop"] (NoArg Drop) "indicate file contents not needed" + , Option ['u'] ["unannex"] (NoArg Unannex) "undo --add" ] -argvToFlags argv = do +argvToMode argv = do case getOpt Permute options argv of - -- no options? add listed files - ([],p,[] ) -> return $ map (\f -> Add f) p - -- all options parsed, return flags - (o,[],[] ) -> return o + -- default mode is Add + ([],files,[]) -> return (Add, files) + -- one mode is normal case + (m:[],files,[]) -> return (m, files) + -- multiple modes is an error + (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) -- error case - (_,n,errs) -> ioError (userError (concat errs ++ usageInfo header options)) - where header = "Usage: git-annex [option] file" + (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) + where header = "Usage: git-annex [mode] file" -dispatch :: Flag -> State -> IO () -dispatch flag state = do - case (flag) of - Add f -> annexFile state f - Unannex f -> unannexFile state f +dispatch :: State -> Mode -> FilePath -> IO () +dispatch state mode file = do + case (mode) of + Add -> annexFile state file + Unannex -> unannexFile state file _ -> error "not implemented" diff --git a/git-annex.hs b/git-annex.hs index 2c9b1315fe..22fbe60ca3 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,16 +1,37 @@ {- git-annex main program - -} +import System.IO import System.Environment -import GitRepo +import Control.Exception import CmdLine import Annex -import BackendList main = do args <- getArgs - flags <- argvToFlags args + (mode, files) <- argvToMode args state <- startAnnex - mapM (\f -> dispatch f state) flags + tryRun 0 $ map (\f -> dispatch state mode f) files + +{- Tries to run a series of actions, not stopping if some error out, + - and propigating an overall error status at the end. -} +tryRun errflag [] = do + if (errflag > 0) + then error "unsuccessful" + else return () +tryRun errflag (a:as) = do + result <- try (a)::IO (Either SomeException ()) + case (result) of + Left err -> do + showErr err + tryRun 1 as + Right _ -> tryRun errflag as + +{- Exception pretty-printing. -} +showErr :: SomeException -> IO () +showErr e = do + let err = show e + hPutStrLn stderr $ "git-annex: " ++ err + return () From dc5e8853f3b0857f2023df6cb23e57bf42b5b858 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 21:04:25 -0400 Subject: [PATCH 0033/2835] missed a file --- Backend.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Backend.hs b/Backend.hs index 5ddd3aac65..eb4a948c4e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -75,6 +75,7 @@ dropFile backends repo file = do Just b -> do key <- lookupKey b repo file (removeKey b) key + removeFile $ backendFile b repo file return $ Just key {- Looks up the key a backend uses for an already annexed file. -} From c5d7ca0a5a2c6837d394e23d1a18a1005ee6f1b6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 22:20:52 -0400 Subject: [PATCH 0034/2835] use Data.Time instead of Data.DateTime The latter has shady rounding. The new module is a bit harder to use, but worth it, it adds subsecond timestamps too. --- Annex.hs | 1 + LocationLog.hs | 50 ++++++++++++++++++++++++++++---------------------- demo.log | 12 ++++++------ 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/Annex.hs b/Annex.hs index ee94a98097..ad94758c50 100644 --- a/Annex.hs +++ b/Annex.hs @@ -11,6 +11,7 @@ import Locations import Types import Backend import BackendList +import LocationLog {- On startup, examine the git repo, prepare it, and record state for - later. -} diff --git a/LocationLog.hs b/LocationLog.hs index a5e9a2679a..195596bda3 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -18,7 +18,9 @@ module LocationLog where -import Data.DateTime +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale import qualified Data.Map as Map import System.IO import System.Directory @@ -28,6 +30,12 @@ import Utility import Locations import Types +data LogLine = LogLine { + date :: POSIXTime, + status :: LogStatus, + reponame :: String +} deriving (Eq) + data LogStatus = FilePresent | FileMissing | Undefined deriving (Eq) @@ -41,29 +49,30 @@ instance Read LogStatus where readsPrec _ "0" = [(FileMissing, "")] readsPrec _ _ = [(Undefined, "")] -data LogLine = LogLine { - date :: DateTime, - status :: LogStatus, - repo :: String -} deriving (Eq) - instance Show LogLine where - show (LogLine date status repo) = unwords - [(show (toSeconds date)), (show status), repo] + show (LogLine date status reponame) = unwords + [(show date), (show status), reponame] instance Read LogLine where -- This parser is robust in that even unparsable log lines are -- read without an exception being thrown. -- Such lines have a status of Undefined. readsPrec _ string = - if (length w >= 3 && all isDigit date) - then [((LogLine (fromSeconds $ read date) status repo), "")] - else [((LogLine (fromSeconds 0) Undefined ""), "")] + if (length w >= 3) + then case (pdate) of + Just v -> good v + Nothing -> undefined + else undefined where w = words string date = w !! 0 status = read $ w !! 1 - repo = unwords $ drop 2 w + reponame = unwords $ drop 2 w + pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime + + good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status reponame + undefined = ret $ LogLine (0) Undefined "" + ret v = [(v, "")] {- Reads a log file. - Note that the LogLines returned may be in any order. -} @@ -97,9 +106,9 @@ writeLog file lines = do {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> IO LogLine -logNow status repo = do - now <- getCurrentTime - return $ LogLine now status repo +logNow status reponame = do + now <- getPOSIXTime + return $ LogLine now status reponame {- Returns the filename of the log file for a given annexed file. -} logFile :: GitRepo -> FilePath -> IO String @@ -113,7 +122,7 @@ fileLocations :: GitRepo -> FilePath -> IO [String] fileLocations thisrepo file = do log <- logFile thisrepo file lines <- readLog log - return $ map repo (filterPresent lines) + return $ map reponame (filterPresent lines) {- Filters the list of LogLines to find ones where the file - is (or should still be) present. -} @@ -131,12 +140,9 @@ compactLog' map (l:ls) = compactLog' (mapLog map l) ls - information about a repo than the other logs in the map -} mapLog map log = if (better) - then Map.insert (repo log) log map + then Map.insert (reponame log) log map else map where - better = case (Map.lookup (repo log) map) of - -- <= used because two log entries could - -- have the same timestamp; if so the one that - -- is seen last should win. + better = case (Map.lookup (reponame log) map) of Just l -> (date l <= date log) Nothing -> True diff --git a/demo.log b/demo.log index 7a42630566..bdecb7d401 100644 --- a/demo.log +++ b/demo.log @@ -1,11 +1,11 @@ -1286654242 1 repo -1286652724 0 foo -1286656282 1 foo -1286656282 0 repo -1286656281 0 foo +1286654242s 1 repo +1286652724s 0 foo +1286656282s 1 foo +1286656282s 0 repo +1286656281s 0 foo # some garbage, should be ignored a a a a 1 a -1 a a -1286652724 1 foo +1286652724.0001s 1 foo From 2bd3eea0318fe52452fa7077fe94ae3f224ae9c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 00:19:38 -0400 Subject: [PATCH 0035/2835] add git config lookups for annex.name, annex.backends, etc --- Annex.hs | 33 +++++++++++++++++++++++++++------ BackendList.hs | 18 ++++++++++++++++++ GitRepo.hs | 16 +++++++++++++--- Types.hs | 11 ++++++++--- git-annex.mdwn | 13 +++++++++---- 5 files changed, 75 insertions(+), 16 deletions(-) diff --git a/Annex.hs b/Annex.hs index ad94758c50..882ed2761b 100644 --- a/Annex.hs +++ b/Annex.hs @@ -18,20 +18,38 @@ import LocationLog startAnnex :: IO State startAnnex = do r <- currentRepo + config <- getConfig r gitPrep r - -- TODO query git repo for configuration - return State { repo = r, backends = supportedBackends } + return State { + repo = r, + gitconfig = config + } + +{- Query the git repo for relevant configuration settings. -} +getConfig :: GitRepo -> IO GitConfig +getConfig repo = do + -- a name can be configured, if none is, use the repository path + name <- gitConfigGet "annex.name" (top repo) + -- default number of copies to keep of file contents is 1 + numcopies <- gitConfigGet "annex.numcopies" "1" + backends <- gitConfigGet "annex.backends" "" + + return GitConfig { + annex_name = name, + annex_numcopies = read numcopies, + annex_backends = parseBackendList backends + } {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend backends (repo state) file case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile (backends state) (repo state) file + stored <- storeFile (annex_backends $ gitconfig state) (repo state) file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key @@ -47,15 +65,16 @@ annexFile state file = do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () + backends = annex_backends $ gitconfig state {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend backends (repo state) file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile (backends state) (repo state) file + mkey <- dropFile backends (repo state) file case (mkey) of Nothing -> return () Just key -> do @@ -63,6 +82,8 @@ unannexFile state file = do removeFile file renameFile src file return () + where + backends = annex_backends $ gitconfig state {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () diff --git a/BackendList.hs b/BackendList.hs index c744949b6c..77e4bd817f 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -4,6 +4,7 @@ module BackendList where -- When adding a new backend, import it here and add it to the list. +import Types import qualified BackendFile import qualified BackendChecksum import qualified BackendUrl @@ -12,3 +13,20 @@ supportedBackends = , BackendChecksum.backend , BackendUrl.backend ] + +{- Parses a string with a list of backend names into + - a list of Backend objects. If the list is empty, + - defaults to supportedBackends. -} +parseBackendList :: String -> [Backend] +parseBackendList s = + if (length s == 0) + then supportedBackends + else map (lookupBackendName) $ words s + +{- Looks up a supported backed by name. -} +lookupBackendName :: String -> Backend +lookupBackendName s = + if ((length matches) /= 1) + then error $ "unknown backend " ++ s + else matches !! 0 + where matches = filter (\b -> s == name b) supportedBackends diff --git a/GitRepo.hs b/GitRepo.hs index ef76fb9766..3a8a8110dd 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -5,7 +5,10 @@ module GitRepo where import Directory import System.Directory import System.Path +import System.Cmd.Utils +import System.IO import Data.String.Utils +import Control.Exception import Utility import Types @@ -14,11 +17,9 @@ gitRepo :: FilePath -> IO GitRepo gitRepo dir = do b <- isBareRepo dir - -- TOOD query repo for configuration settings; other repositories; etc return GitRepo { top = dir, - bare = b, - remotes = [] + bare = b } {- Path to a repository's gitattributes file. -} @@ -53,10 +54,19 @@ gitRelative repo file = drop (length absrepo) absfile Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo {- Stages a changed file in git's index. -} +gitAdd :: GitRepo -> FilePath -> IO () gitAdd repo file = do -- TODO return () +{- Queries git-config. -} +gitConfigGet :: String -> String -> IO String +gitConfigGet name defaultValue = + handle ((\_ -> return defaultValue)::SomeException -> IO String) $ + pOpen ReadFromPipe "git" ["config", "--get", name] $ \h -> do + ret <- hGetLine h + return ret + {- Finds the current git repository, which may be in a parent directory. -} currentRepo :: IO GitRepo currentRepo = do diff --git a/Types.hs b/Types.hs index 6e3727e25a..5c5a428d59 100644 --- a/Types.hs +++ b/Types.hs @@ -23,12 +23,17 @@ data Backend = Backend { -- a git repository data GitRepo = GitRepo { top :: FilePath, - bare :: Bool, - remotes :: [GitRepo] + bare :: Bool } -- git-annex's runtime state data State = State { repo :: GitRepo, - backends :: [Backend] + gitconfig :: GitConfig +} + +data GitConfig = GitConfig { + annex_name :: String, + annex_numcopies :: Int, + annex_backends :: [Backend] } diff --git a/git-annex.mdwn b/git-annex.mdwn index 2996a90b51..6bfdd57c7f 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -124,8 +124,9 @@ so the lines may be in arbitrary order, but it will never conflict.) ## configuration * `annex.numcopies` -- number of copies of files to keep -* `annex.backend` -- name of the default key/value backend to use to - store new files +* `annex.backends` -- space-separated list of names of + the key/value backends to use. The first listed is used to store + new files. * `annex.name` -- allows specifying a unique name for this repository. If not specified, the name is derived from its directory's location and the hostname. When a repository is on removable media it is useful to give @@ -145,11 +146,15 @@ If the symlink to annexed content is relative, moving it to a subdir will break it. But it it's absolute, moving the git repo (or mounting its drive elsewhere) will break it. Either: -* Use relative links and need `git annex mv` to move (or post-commit +* Use relative links and need `git annex --mv` to move (or post-commit hook that caches moves and updates links). * Use absolute links and need `git annex fixlinks` when location changes; note that would also mean that git would see the symlink targets changed - and want to commit the change. + and want to commit the change. And, other clones of the repo would + diverge and there would be conflicts on the symlink text. Ugh. + +Hard links are not an option, because git would then happily commit the +file content. Amoung other reasons.. ### free space determination From 779ebba96153e712803c8284a0502d7080c609bf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 00:22:50 -0400 Subject: [PATCH 0036/2835] adjust merge config *.log will merge, but foo.$backend files will not --- Annex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index 882ed2761b..e3956cbf20 100644 --- a/Annex.hs +++ b/Annex.hs @@ -89,7 +89,7 @@ unannexFile state file = do gitPrep :: GitRepo -> IO () gitPrep repo = do -- configure git to use union merge driver on state files - let attrLine = stateLoc ++ "/* merge=union" + let attrLine = stateLoc ++ "/*.log merge=union" attributes <- gitAttributes repo exists <- doesFileExist attributes if (not exists) From ecf19abf76d449d2e69d89518566a7cce899708d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 00:23:49 -0400 Subject: [PATCH 0037/2835] foo --- Locations.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Locations.hs b/Locations.hs index 50f94a727d..b25e99197a 100644 --- a/Locations.hs +++ b/Locations.hs @@ -12,8 +12,8 @@ annexDir repo key = do dir <- gitDir repo return $ dir ++ "/annex/" ++ key -{- Long-term state is stored in files inside the .git-annex directory - - in the git repository. -} +{- Long-term, cross-repo state is stored in files inside the .git-annex + - directory, in the git repository. -} stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath gitStateDir repo = (top repo) ++ "/" ++ stateLoc ++ "/" From de3dafae80f45af8db56dc95e11863f049cf3cb2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 16:35:28 -0400 Subject: [PATCH 0038/2835] update --- TODO | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 TODO diff --git a/TODO b/TODO new file mode 100644 index 0000000000..ea930f8034 --- /dev/null +++ b/TODO @@ -0,0 +1,9 @@ +* bug when annexing files in a subdir of a git repo +* how to handle git mv file? + +* query remotes for their annex.name settings + +* hook up LocationLog +* --push/--pull/--get/--want/--drop + +* finish BackendUrl and BackendChecksum From af82586adff96f18fe768e432f501c647401262f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 17:19:55 -0400 Subject: [PATCH 0039/2835] split up Types --- Annex.hs | 19 ++++++++++++++++++- Backend.hs | 18 +++++++++++++++++- BackendChecksum.hs | 3 ++- BackendFile.hs | 3 ++- BackendList.hs | 3 ++- BackendUrl.hs | 3 ++- CmdLine.hs | 1 - GitRepo.hs | 7 ++++++- LocationLog.hs | 1 - Locations.hs | 7 ------- Types.hs | 39 --------------------------------------- 11 files changed, 49 insertions(+), 55 deletions(-) delete mode 100644 Types.hs diff --git a/Annex.hs b/Annex.hs index e3956cbf20..cedc478a43 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,11 +8,28 @@ import System.Directory import GitRepo import Utility import Locations -import Types import Backend import BackendList import LocationLog +-- git-annex's runtime state +data State = State { + repo :: GitRepo, + gitconfig :: GitConfig +} + +data GitConfig = GitConfig { + annex_name :: String, + annex_numcopies :: Int, + annex_backends :: [Backend] +} + +{- An annexed file's content is stored somewhere under .git/annex/ -} +annexDir :: GitRepo -> Key -> IO FilePath +annexDir repo key = do + dir <- gitDir repo + return $ dir ++ "/annex/" ++ key + {- On startup, examine the git repo, prepare it, and record state for - later. -} startAnnex :: IO State diff --git a/Backend.hs b/Backend.hs index eb4a948c4e..fb7d5666f3 100644 --- a/Backend.hs +++ b/Backend.hs @@ -22,7 +22,23 @@ import System.Directory import Locations import GitRepo import Utility -import Types + +-- annexed filenames are mapped into keys +type Key = FilePath + +-- this structure represents a key/value backend +data Backend = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: GitRepo -> FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> FilePath -> IO Bool, + -- removes a key + removeKey :: Key -> IO Bool +} instance Show Backend where show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" diff --git a/BackendChecksum.hs b/BackendChecksum.hs index 18ff0cb57c..e262962cab 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -3,9 +3,10 @@ module BackendChecksum (backend) where -import Types import qualified BackendFile import Data.Digest.Pure.SHA +import Backend +import GitRepo -- based on BackendFile just with a different key type backend = BackendFile.backend { diff --git a/BackendFile.hs b/BackendFile.hs index de60803c36..831f804173 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,7 +3,8 @@ module BackendFile (backend) where -import Types +import Backend +import GitRepo backend = Backend { name = "file", diff --git a/BackendList.hs b/BackendList.hs index 77e4bd817f..c3a1b13a1b 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -3,8 +3,9 @@ module BackendList where +import Backend + -- When adding a new backend, import it here and add it to the list. -import Types import qualified BackendFile import qualified BackendChecksum import qualified BackendUrl diff --git a/BackendUrl.hs b/BackendUrl.hs index ddeab9e042..f08f0bdb46 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,7 +3,8 @@ module BackendUrl (backend) where -import Types +import Backend +import GitRepo backend = Backend { name = "url", diff --git a/CmdLine.hs b/CmdLine.hs index c956f29a55..53707cd716 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -7,7 +7,6 @@ module CmdLine where import System.Console.GetOpt -import Types import Annex data Mode = Add | Push | Pull | Want | Get | Drop | Unannex diff --git a/GitRepo.hs b/GitRepo.hs index 3a8a8110dd..c26f752ef8 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -10,7 +10,12 @@ import System.IO import Data.String.Utils import Control.Exception import Utility -import Types + +-- a git repository +data GitRepo = GitRepo { + top :: FilePath, + bare :: Bool +} {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo diff --git a/LocationLog.hs b/LocationLog.hs index 195596bda3..8e6b56fe8e 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -28,7 +28,6 @@ import Data.Char import GitRepo import Utility import Locations -import Types data LogLine = LogLine { date :: POSIXTime, diff --git a/Locations.hs b/Locations.hs index b25e99197a..22a0db8e2c 100644 --- a/Locations.hs +++ b/Locations.hs @@ -3,15 +3,8 @@ module Locations where -import Types import GitRepo -{- An annexed file's content is stored somewhere under .git/annex/ -} -annexDir :: GitRepo -> Key -> IO FilePath -annexDir repo key = do - dir <- gitDir repo - return $ dir ++ "/annex/" ++ key - {- Long-term, cross-repo state is stored in files inside the .git-annex - directory, in the git repository. -} stateLoc = ".git-annex" diff --git a/Types.hs b/Types.hs deleted file mode 100644 index 5c5a428d59..0000000000 --- a/Types.hs +++ /dev/null @@ -1,39 +0,0 @@ -{- git-annex data types - - -} - -module Types where - --- annexed filenames are mapped into keys -type Key = String - --- this structure represents a key/value backend -data Backend = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: GitRepo -> FilePath -> IO (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, - -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> IO Bool, - -- removes a key - removeKey :: Key -> IO Bool -} - --- a git repository -data GitRepo = GitRepo { - top :: FilePath, - bare :: Bool -} - --- git-annex's runtime state -data State = State { - repo :: GitRepo, - gitconfig :: GitConfig -} - -data GitConfig = GitConfig { - annex_name :: String, - annex_numcopies :: Int, - annex_backends :: [Backend] -} From ebc3fbe9ae2c5cc52332c77a92697c2517ce8263 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 17:52:46 -0400 Subject: [PATCH 0040/2835] explicit exports --- Annex.hs | 11 ++++++++--- Backend.hs | 29 ++++++++--------------------- BackendChecksum.hs | 2 +- BackendFile.hs | 2 +- BackendList.hs | 8 ++++++-- BackendUrl.hs | 2 +- CmdLine.hs | 5 ++++- GitRepo.hs | 22 +++++++++++++++++----- LocationLog.hs | 3 ++- Locations.hs | 7 +++++-- Utility.hs | 6 +++++- 11 files changed, 58 insertions(+), 39 deletions(-) diff --git a/Annex.hs b/Annex.hs index cedc478a43..8660febd57 100644 --- a/Annex.hs +++ b/Annex.hs @@ -1,7 +1,12 @@ {- git-annex toplevel code -} -module Annex where +module Annex ( + State, + startAnnex, + annexFile, + unannexFile +) where import System.Posix.Files import System.Directory @@ -34,7 +39,7 @@ annexDir repo key = do - later. -} startAnnex :: IO State startAnnex = do - r <- currentRepo + r <- gitRepoCurrent config <- getConfig r gitPrep r return State { @@ -46,7 +51,7 @@ startAnnex = do getConfig :: GitRepo -> IO GitConfig getConfig repo = do -- a name can be configured, if none is, use the repository path - name <- gitConfigGet "annex.name" (top repo) + name <- gitConfigGet "annex.name" (gitRepoTop repo) -- default number of copies to keep of file contents is 1 numcopies <- gitConfigGet "annex.numcopies" "1" backends <- gitConfigGet "annex.backends" "" diff --git a/Backend.hs b/Backend.hs index fb7d5666f3..2d3ea42d64 100644 --- a/Backend.hs +++ b/Backend.hs @@ -16,32 +16,19 @@ - to store different files' contents in a given repository. - -} -module Backend where +module Backend ( + Key, + Backend, -- note only data type is exported, not destructors + lookupBackend, + storeFile, + dropFile +) where import System.Directory import Locations import GitRepo import Utility - --- annexed filenames are mapped into keys -type Key = FilePath - --- this structure represents a key/value backend -data Backend = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: GitRepo -> FilePath -> IO (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, - -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> IO Bool, - -- removes a key - removeKey :: Key -> IO Bool -} - -instance Show Backend where - show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" +import BackendType {- Name of state file that holds the key for an annexed file, - using a given backend. -} diff --git a/BackendChecksum.hs b/BackendChecksum.hs index e262962cab..e80dbe793c 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -5,7 +5,7 @@ module BackendChecksum (backend) where import qualified BackendFile import Data.Digest.Pure.SHA -import Backend +import BackendType import GitRepo -- based on BackendFile just with a different key type diff --git a/BackendFile.hs b/BackendFile.hs index 831f804173..ae53f460f1 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,7 +3,7 @@ module BackendFile (backend) where -import Backend +import BackendType import GitRepo backend = Backend { diff --git a/BackendList.hs b/BackendList.hs index c3a1b13a1b..f733a44be9 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -1,9 +1,13 @@ {- git-annex backend list - -} -module BackendList where +module BackendList ( + supportedBackends, + parseBackendList, + lookupBackendName +) where -import Backend +import BackendType -- When adding a new backend, import it here and add it to the list. import qualified BackendFile diff --git a/BackendUrl.hs b/BackendUrl.hs index f08f0bdb46..4ba1dbadb5 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,7 +3,7 @@ module BackendUrl (backend) where -import Backend +import BackendType import GitRepo backend = Backend { diff --git a/CmdLine.hs b/CmdLine.hs index 53707cd716..cc87088898 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -4,7 +4,10 @@ - System.Console.CmdArgs.Implicit but it is not yet packaged in Debian. -} -module CmdLine where +module CmdLine ( + argvToMode, + dispatch +) where import System.Console.GetOpt import Annex diff --git a/GitRepo.hs b/GitRepo.hs index c26f752ef8..d01ba642b0 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -1,6 +1,15 @@ {- git repository handling -} -module GitRepo where +module GitRepo ( + GitRepo, + gitRepoCurrent, + gitRepoTop, + gitDir, + gitRelative, + gitConfigGet, + gitAdd, + gitAttributes +) where import Directory import System.Directory @@ -13,7 +22,7 @@ import Utility -- a git repository data GitRepo = GitRepo { - top :: FilePath, + gitRepoTop :: FilePath, bare :: Bool } @@ -23,10 +32,13 @@ gitRepo dir = do b <- isBareRepo dir return GitRepo { - top = dir, + gitRepoTop = dir, bare = b } +{- Short name used in here for top of repo. -} +top = gitRepoTop + {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String gitAttributes repo = do @@ -73,8 +85,8 @@ gitConfigGet name defaultValue = return ret {- Finds the current git repository, which may be in a parent directory. -} -currentRepo :: IO GitRepo -currentRepo = do +gitRepoCurrent :: IO GitRepo +gitRepoCurrent = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of diff --git a/LocationLog.hs b/LocationLog.hs index 8e6b56fe8e..31d454f103 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -16,7 +16,8 @@ - so the lines may be in arbitrary order, but it will never conflict. -} -module LocationLog where +module LocationLog ( +) where import Data.Time.Clock.POSIX import Data.Time diff --git a/Locations.hs b/Locations.hs index 22a0db8e2c..31bb3d9de3 100644 --- a/Locations.hs +++ b/Locations.hs @@ -1,7 +1,10 @@ {- git-annex file locations -} -module Locations where +module Locations ( + gitStateDir, + stateLoc +) where import GitRepo @@ -9,4 +12,4 @@ import GitRepo - directory, in the git repository. -} stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath -gitStateDir repo = (top repo) ++ "/" ++ stateLoc ++ "/" +gitStateDir repo = (gitRepoTop repo) ++ "/" ++ stateLoc ++ "/" diff --git a/Utility.hs b/Utility.hs index d1eb247d3c..dea53967fc 100644 --- a/Utility.hs +++ b/Utility.hs @@ -1,7 +1,11 @@ {- git-annex utility functions -} -module Utility where +module Utility ( + withFileLocked, + hGetContentsStrict, + parentDir +) where import System.IO import System.Posix.IO From 104fe9132af2553a29631b1cd38cc79169e9d9f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:15:05 -0400 Subject: [PATCH 0041/2835] cleanup --- Annex.hs | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/Annex.hs b/Annex.hs index 8660febd57..972cb3e0f0 100644 --- a/Annex.hs +++ b/Annex.hs @@ -20,10 +20,10 @@ import LocationLog -- git-annex's runtime state data State = State { repo :: GitRepo, - gitconfig :: GitConfig + config :: Config } -data GitConfig = GitConfig { +data Config = Config { annex_name :: String, annex_numcopies :: Int, annex_backends :: [Backend] @@ -40,26 +40,11 @@ annexDir repo key = do startAnnex :: IO State startAnnex = do r <- gitRepoCurrent - config <- getConfig r + config <- queryConfig r gitPrep r return State { repo = r, - gitconfig = config - } - -{- Query the git repo for relevant configuration settings. -} -getConfig :: GitRepo -> IO GitConfig -getConfig repo = do - -- a name can be configured, if none is, use the repository path - name <- gitConfigGet "annex.name" (gitRepoTop repo) - -- default number of copies to keep of file contents is 1 - numcopies <- gitConfigGet "annex.numcopies" "1" - backends <- gitConfigGet "annex.backends" "" - - return GitConfig { - annex_name = name, - annex_numcopies = read numcopies, - annex_backends = parseBackendList backends + config = config } {- Annexes a file, storing it in a backend, and then moving it into @@ -71,7 +56,7 @@ annexFile state file = do Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile (annex_backends $ gitconfig state) (repo state) file + stored <- storeFile backends (repo state) file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key @@ -87,7 +72,7 @@ annexFile state file = do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - backends = annex_backends $ gitconfig state + backends = getConfig state annex_backends {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () @@ -105,7 +90,22 @@ unannexFile state file = do renameFile src file return () where - backends = annex_backends $ gitconfig state + backends = getConfig state annex_backends + +{- Query the git repo for relevant configuration settings. -} +queryConfig :: GitRepo -> IO Config +queryConfig repo = do + -- a name can be configured, if none is, use the repository path + name <- gitConfigGet "annex.name" (gitRepoTop repo) + -- default number of copies to keep of file contents is 1 + numcopies <- gitConfigGet "annex.numcopies" "1" + backends <- gitConfigGet "annex.backends" "" + + return Config { + annex_name = name, + annex_numcopies = read numcopies, + annex_backends = parseBackendList backends + } {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () @@ -126,3 +126,6 @@ gitPrep repo = do gitAdd repo attributes else return () +{- Looks up a key in a State's Config -} +getConfig :: State -> (Config -> b) -> b +getConfig state key = key $ config state From f516b820caa702ee76c85b005fef285b8372c4da Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:15:14 -0400 Subject: [PATCH 0042/2835] add --- BackendType.hs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 BackendType.hs diff --git a/BackendType.hs b/BackendType.hs new file mode 100644 index 0000000000..3bc822f329 --- /dev/null +++ b/BackendType.hs @@ -0,0 +1,31 @@ +{- git-annex backend data types + - -} + +module BackendType ( + -- the entire types are exported, for use in backend implementations + Key(..), + Backend(..) +) where + +import GitRepo + +-- annexed filenames are mapped into keys +type Key = FilePath + +-- this structure represents a key/value backend +data Backend = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: GitRepo -> FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> FilePath -> IO Bool, + -- removes a key + removeKey :: Key -> IO Bool +} + +instance Show Backend where + show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" + From 8f99409518d343ded6a1355b4366bd21ee4cf66d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:31:41 -0400 Subject: [PATCH 0043/2835] simpler exception handling --- GitRepo.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index d01ba642b0..de54f6dca6 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -16,8 +16,8 @@ import System.Directory import System.Path import System.Cmd.Utils import System.IO +import System.IO.Error import Data.String.Utils -import Control.Exception import Utility -- a git repository @@ -79,7 +79,7 @@ gitAdd repo file = do {- Queries git-config. -} gitConfigGet :: String -> String -> IO String gitConfigGet name defaultValue = - handle ((\_ -> return defaultValue)::SomeException -> IO String) $ + flip catch (\_ -> return defaultValue) $ pOpen ReadFromPipe "git" ["config", "--get", name] $ \h -> do ret <- hGetLine h return ret From 530f16b980bbfe70b49d5112ad9c48a9754e69c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:39:09 -0400 Subject: [PATCH 0044/2835] better result summary --- git-annex.hs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/git-annex.hs b/git-annex.hs index 22fbe60ca3..ec80359b6c 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -13,21 +13,21 @@ main = do state <- startAnnex - tryRun 0 $ map (\f -> dispatch state mode f) files + tryRun 0 0 $ map (\f -> dispatch state mode f) files {- Tries to run a series of actions, not stopping if some error out, - and propigating an overall error status at the end. -} -tryRun errflag [] = do - if (errflag > 0) - then error "unsuccessful" +tryRun errnum oknum [] = do + if (errnum > 0) + then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " succeeded" else return () -tryRun errflag (a:as) = do +tryRun errnum oknum (a:as) = do result <- try (a)::IO (Either SomeException ()) case (result) of Left err -> do showErr err - tryRun 1 as - Right _ -> tryRun errflag as + tryRun (errnum + 1) oknum as + Right _ -> tryRun errnum (oknum + 1) as {- Exception pretty-printing. -} showErr :: SomeException -> IO () From f6306bc301af7db3da7afa6e095014de37e2bce3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:39:36 -0400 Subject: [PATCH 0045/2835] wording --- git-annex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-annex.hs b/git-annex.hs index ec80359b6c..7bcd4de226 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -19,7 +19,7 @@ main = do - and propigating an overall error status at the end. -} tryRun errnum oknum [] = do if (errnum > 0) - then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " succeeded" + then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" else return () tryRun errnum oknum (a:as) = do result <- try (a)::IO (Either SomeException ()) From cd1e39b127e96298685906e455ff186312d08029 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 23:22:38 -0400 Subject: [PATCH 0046/2835] moved config reading into GitRepo --- Annex.hs | 52 +++++++------------------------ GitRepo.hs | 89 +++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 76 insertions(+), 65 deletions(-) diff --git a/Annex.hs b/Annex.hs index 972cb3e0f0..abb7bff6e8 100644 --- a/Annex.hs +++ b/Annex.hs @@ -20,49 +20,41 @@ import LocationLog -- git-annex's runtime state data State = State { repo :: GitRepo, - config :: Config -} - -data Config = Config { - annex_name :: String, - annex_numcopies :: Int, - annex_backends :: [Backend] + backends :: [Backend] } {- An annexed file's content is stored somewhere under .git/annex/ -} -annexDir :: GitRepo -> Key -> IO FilePath -annexDir repo key = do - dir <- gitDir repo - return $ dir ++ "/annex/" ++ key +annexDir :: GitRepo -> Key -> FilePath +annexDir repo key = gitDir repo ++ "/annex/" ++ key {- On startup, examine the git repo, prepare it, and record state for - later. -} startAnnex :: IO State startAnnex = do r <- gitRepoCurrent - config <- queryConfig r gitPrep r + return State { repo = r, - config = config + backends = parseBackendList $ gitConfig r "annex.backends" "" } {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend backends (repo state) file + alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile backends (repo state) file + stored <- storeFile (backends state) (repo state) file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key where symlink key = do - dest <- annexDir (repo state) key + let dest = annexDir (repo state) key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file @@ -72,40 +64,22 @@ annexFile state file = do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - backends = getConfig state annex_backends {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend backends (repo state) file + alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile backends (repo state) file + mkey <- dropFile (backends state) (repo state) file case (mkey) of Nothing -> return () Just key -> do - src <- annexDir (repo state) key + let src = annexDir (repo state) key removeFile file renameFile src file return () - where - backends = getConfig state annex_backends - -{- Query the git repo for relevant configuration settings. -} -queryConfig :: GitRepo -> IO Config -queryConfig repo = do - -- a name can be configured, if none is, use the repository path - name <- gitConfigGet "annex.name" (gitRepoTop repo) - -- default number of copies to keep of file contents is 1 - numcopies <- gitConfigGet "annex.numcopies" "1" - backends <- gitConfigGet "annex.backends" "" - - return Config { - annex_name = name, - annex_numcopies = read numcopies, - annex_backends = parseBackendList backends - } {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () @@ -125,7 +99,3 @@ gitPrep repo = do appendFile attributes $ attrLine ++ "\n" gitAdd repo attributes else return () - -{- Looks up a key in a State's Config -} -getConfig :: State -> (Config -> b) -> b -getConfig state key = key $ config state diff --git a/GitRepo.hs b/GitRepo.hs index de54f6dca6..7ae6584dd4 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -1,4 +1,9 @@ -{- git repository handling -} +{- git repository handling + - + - This is written to be completely independant of git-annex and should be + - suitable for other uses. + - + - -} module GitRepo ( GitRepo, @@ -6,38 +11,46 @@ module GitRepo ( gitRepoTop, gitDir, gitRelative, - gitConfigGet, + gitConfig, gitAdd, gitAttributes ) where import Directory +import System import System.Directory import System.Path import System.Cmd.Utils import System.IO -import System.IO.Error +import System.Posix.Process import Data.String.Utils +import Data.Map as Map (fromList, empty, lookup, Map) import Utility -- a git repository data GitRepo = GitRepo { - gitRepoTop :: FilePath, - bare :: Bool -} + top :: FilePath, + bare :: Bool, + config :: Map String String +} deriving (Show, Read, Eq) {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo gitRepo dir = do b <- isBareRepo dir - return GitRepo { - gitRepoTop = dir, - bare = b + let r = GitRepo { + top = dir, + bare = b, + config = Map.empty } + r' <- gitConfigRead r -{- Short name used in here for top of repo. -} -top = gitRepoTop + return r' + +{- Field accessor. -} +gitRepoTop :: GitRepo -> FilePath +gitRepoTop repo = top repo {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String @@ -49,11 +62,11 @@ gitAttributes repo = do {- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} -gitDir :: GitRepo -> IO String -gitDir repo = do +gitDir :: GitRepo -> String +gitDir repo = if (bare repo) - then return $ (top repo) - else return $ (top repo) ++ "/.git" + then top repo + else top repo ++ "/.git" {- Given a relative or absolute filename, calculates the name to use - to refer to the file relative to a git repository directory. @@ -72,17 +85,45 @@ gitRelative repo file = drop (length absrepo) absfile {- Stages a changed file in git's index. -} gitAdd :: GitRepo -> FilePath -> IO () -gitAdd repo file = do - -- TODO +gitAdd repo file = runGit repo ["add", file] + +{- Constructs a git command line operating on the specified repo. -} +gitCommandLine :: GitRepo -> [String] -> [String] +gitCommandLine repo params = + -- force use of specified repo via --git-dir and --work-tree + ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + +{- Runs git in the specified repo. -} +runGit :: GitRepo -> [String] -> IO () +runGit repo params = do + r <- executeFile "git" True (gitCommandLine repo params) Nothing return () -{- Queries git-config. -} -gitConfigGet :: String -> String -> IO String -gitConfigGet name defaultValue = - flip catch (\_ -> return defaultValue) $ - pOpen ReadFromPipe "git" ["config", "--get", name] $ \h -> do - ret <- hGetLine h - return ret +{- Runs a git subcommand and returns its output. -} +gitPipeRead :: GitRepo -> [String] -> IO String +gitPipeRead repo params = + pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do + ret <- hGetContentsStrict h + return ret + +{- Runs git config and populates a repo with its settings. -} +gitConfigRead :: GitRepo -> IO GitRepo +gitConfigRead repo = do + c <- gitPipeRead repo ["config", "--list"] + return repo { config = Map.fromList $ parse c } + where + parse s = map ( \l -> (key l, val l) ) $ lines s + keyval l = split sep l :: [String] + key l = (keyval l) !! 0 + val l = join sep $ drop 1 $ keyval l + sep = "=" + +{- Returns a single git config setting, or a default value if not set. -} +gitConfig :: GitRepo -> String -> String -> String +gitConfig repo key defaultValue = + case (Map.lookup key $ config repo) of + Just value -> value + Nothing -> defaultValue {- Finds the current git repository, which may be in a parent directory. -} gitRepoCurrent :: IO GitRepo From 16b551726d9d846a51656b7b1d4736a3b1b438f4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 23:27:48 -0400 Subject: [PATCH 0047/2835] minor --- GitRepo.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 7ae6584dd4..2f9084ff7c 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -112,10 +112,11 @@ gitConfigRead repo = do c <- gitPipeRead repo ["config", "--list"] return repo { config = Map.fromList $ parse c } where - parse s = map ( \l -> (key l, val l) ) $ lines s - keyval l = split sep l :: [String] + parse s = map pair $ lines s + pair l = (key l, val l) key l = (keyval l) !! 0 val l = join sep $ drop 1 $ keyval l + keyval l = split sep l :: [String] sep = "=" {- Returns a single git config setting, or a default value if not set. -} From 107074d6623a687d046615a5034af10be7ff1756 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 23:41:12 -0400 Subject: [PATCH 0048/2835] fiddle --- GitRepo.hs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 2f9084ff7c..2a97e60704 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -7,6 +7,7 @@ module GitRepo ( GitRepo, + gitRepoFromPath, gitRepoCurrent, gitRepoTop, gitDir, @@ -35,8 +36,8 @@ data GitRepo = GitRepo { } deriving (Show, Read, Eq) {- GitRepo constructor -} -gitRepo :: FilePath -> IO GitRepo -gitRepo dir = do +gitRepoFromPath :: FilePath -> IO GitRepo +gitRepoFromPath dir = do b <- isBareRepo dir let r = GitRepo { @@ -110,14 +111,17 @@ gitPipeRead repo params = gitConfigRead :: GitRepo -> IO GitRepo gitConfigRead repo = do c <- gitPipeRead repo ["config", "--list"] - return repo { config = Map.fromList $ parse c } - where - parse s = map pair $ lines s - pair l = (key l, val l) - key l = (keyval l) !! 0 - val l = join sep $ drop 1 $ keyval l - keyval l = split sep l :: [String] - sep = "=" + return repo { config = gitConfigParse c } + +{- Parses git config --list output into a config map. -} +gitConfigParse :: String -> Map.Map String String +gitConfigParse s = Map.fromList $ map pair $ lines s + where + pair l = (key l, val l) + key l = (keyval l) !! 0 + val l = join sep $ drop 1 $ keyval l + keyval l = split sep l :: [String] + sep = "=" {- Returns a single git config setting, or a default value if not set. -} gitConfig :: GitRepo -> String -> String -> String @@ -132,7 +136,7 @@ gitRepoCurrent = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> gitRepo dir + (Just dir) -> gitRepoFromPath dir Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) From 92bf408c664e4fa66132a7095e1f856312ce667c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 00:53:42 -0400 Subject: [PATCH 0049/2835] faddle --- Annex.hs | 2 +- GitRepo.hs | 99 ++++++++++++++++++++++++++++++++++++---------------- Locations.hs | 4 +-- 3 files changed, 72 insertions(+), 33 deletions(-) diff --git a/Annex.hs b/Annex.hs index abb7bff6e8..5a7274ca15 100644 --- a/Annex.hs +++ b/Annex.hs @@ -86,7 +86,7 @@ gitPrep :: GitRepo -> IO () gitPrep repo = do -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/*.log merge=union" - attributes <- gitAttributes repo + let attributes = gitAttributes repo exists <- doesFileExist attributes if (not exists) then do diff --git a/GitRepo.hs b/GitRepo.hs index 2a97e60704..dc1c52b478 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -7,9 +7,10 @@ module GitRepo ( GitRepo, - gitRepoFromPath, gitRepoCurrent, - gitRepoTop, + gitRepoFromPath, + gitRepoFromUrl, + gitWorkTree, gitDir, gitRelative, gitConfig, @@ -26,21 +27,28 @@ import System.IO import System.Posix.Process import Data.String.Utils import Data.Map as Map (fromList, empty, lookup, Map) +import Network.URI +import Maybe import Utility --- a git repository -data GitRepo = GitRepo { - top :: FilePath, - bare :: Bool, - config :: Map String String -} deriving (Show, Read, Eq) +{- A git repository can be local or remote. -} +data GitRepo = + LocalGitRepo { + top :: FilePath, + bare :: Bool, + config :: Map String String + } | RemoteGitRepo { + url :: String, + top :: FilePath, + config :: Map String String + } deriving (Show, Read, Eq) -{- GitRepo constructor -} +{- Local GitRepo constructor. -} gitRepoFromPath :: FilePath -> IO GitRepo gitRepoFromPath dir = do b <- isBareRepo dir - let r = GitRepo { + let r = LocalGitRepo { top = dir, bare = b, config = Map.empty @@ -49,28 +57,49 @@ gitRepoFromPath dir = do return r' -{- Field accessor. -} -gitRepoTop :: GitRepo -> FilePath -gitRepoTop repo = top repo +{- Remote GitRepo constructor. Note that remote repo config is not read. + - Throws exception on invalid url. -} +gitRepoFromUrl :: String -> IO GitRepo +gitRepoFromUrl url = do + return RemoteGitRepo { + url = url, + top = path url, + config = Map.empty + } + where path url = uriPath $ fromJust $ parseURI url + +{- Some code needs to vary between remote and local repos. -} +local repo = case (repo) of + LocalGitRepo {} -> True + RemoteGitRepo {} -> False +remote repo = not $ local repo +assertlocal repo action = + if (local repo) + then action + else error "acting on remote git repo not supported" {- Path to a repository's gitattributes file. -} -gitAttributes :: GitRepo -> IO String -gitAttributes repo = do +gitAttributes :: GitRepo -> String +gitAttributes repo = assertlocal repo $ do if (bare repo) - then return $ (top repo) ++ "/info/.gitattributes" - else return $ (top repo) ++ "/.gitattributes" + then (top repo) ++ "/info/.gitattributes" + else (top repo) ++ "/.gitattributes" {- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} gitDir :: GitRepo -> String -gitDir repo = +gitDir repo = assertlocal repo $ if (bare repo) then top repo else top repo ++ "/.git" -{- Given a relative or absolute filename, calculates the name to use - - to refer to the file relative to a git repository directory. +{- Path to a repository's --work-tree. -} +gitWorkTree :: GitRepo -> FilePath +gitWorkTree repo = top repo + +{- Given a relative or absolute filename in a repository, calculates the + - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} gitRelative :: GitRepo -> String -> String gitRelative repo file = drop (length absrepo) absfile @@ -92,26 +121,36 @@ gitAdd repo file = runGit repo ["add", file] gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + if (local repo) + then ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + else error "gitCommandLine not implemented for remote repo" {- Runs git in the specified repo. -} runGit :: GitRepo -> [String] -> IO () -runGit repo params = do - r <- executeFile "git" True (gitCommandLine repo params) Nothing - return () +runGit repo params = + if (local repo) + then do + r <- executeFile "git" True (gitCommandLine repo params) Nothing + return () + else error "runGit not implemented for remote repo" {- Runs a git subcommand and returns its output. -} gitPipeRead :: GitRepo -> [String] -> IO String gitPipeRead repo params = - pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - ret <- hGetContentsStrict h - return ret + if (local repo) + then pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do + ret <- hGetContentsStrict h + return ret + else error "gitPipeRead not implemented for remote repo" {- Runs git config and populates a repo with its settings. -} gitConfigRead :: GitRepo -> IO GitRepo -gitConfigRead repo = do - c <- gitPipeRead repo ["config", "--list"] - return repo { config = gitConfigParse c } +gitConfigRead repo = + if (local repo) + then do + c <- gitPipeRead repo ["config", "--list"] + return repo { config = gitConfigParse c } + else error "gitConfigRead not implemented for remote repo" {- Parses git config --list output into a config map. -} gitConfigParse :: String -> Map.Map String String diff --git a/Locations.hs b/Locations.hs index 31bb3d9de3..300f443f7f 100644 --- a/Locations.hs +++ b/Locations.hs @@ -9,7 +9,7 @@ module Locations ( import GitRepo {- Long-term, cross-repo state is stored in files inside the .git-annex - - directory, in the git repository. -} + - directory, in the git repository's working tree. -} stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath -gitStateDir repo = (gitRepoTop repo) ++ "/" ++ stateLoc ++ "/" +gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" From eea55856e9db85884a7fb28ce1b408fdbc05f90f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 01:35:32 -0400 Subject: [PATCH 0050/2835] remotes lookup --- GitRepo.hs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index dc1c52b478..fb3ddbaf85 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -26,7 +26,7 @@ import System.Cmd.Utils import System.IO import System.Posix.Process import Data.String.Utils -import Data.Map as Map (fromList, empty, lookup, Map) +import Data.Map as Map hiding (map, split) import Network.URI import Maybe import Utility @@ -61,14 +61,15 @@ gitRepoFromPath dir = do - Throws exception on invalid url. -} gitRepoFromUrl :: String -> IO GitRepo gitRepoFromUrl url = do - return RemoteGitRepo { + return $ RemoteGitRepo { url = url, top = path url, config = Map.empty } where path url = uriPath $ fromJust $ parseURI url -{- Some code needs to vary between remote and local repos. -} +{- Some code needs to vary between remote and local repos, these functions + - help with that. -} local repo = case (repo) of LocalGitRepo {} -> True RemoteGitRepo {} -> False @@ -165,9 +166,19 @@ gitConfigParse s = Map.fromList $ map pair $ lines s {- Returns a single git config setting, or a default value if not set. -} gitConfig :: GitRepo -> String -> String -> String gitConfig repo key defaultValue = - case (Map.lookup key $ config repo) of - Just value -> value - Nothing -> defaultValue + Map.findWithDefault key defaultValue (config repo) + +{- Returns a list of a repo's configured remotes. -} +gitConfigRemotes :: GitRepo -> IO [GitRepo] +gitConfigRemotes repo = mapM construct remotes + where + remotes = elems $ filter $ config repo + filter = filterWithKey (\k _ -> isremote k) + isremote k = (startswith "remote." k) && (endswith ".url" k) + construct r = + if (isURI r) + then gitRepoFromUrl r + else gitRepoFromPath r {- Finds the current git repository, which may be in a parent directory. -} gitRepoCurrent :: IO GitRepo From c8002bd91b03b66c195014ecaa9111c50fa5e716 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 02:00:29 -0400 Subject: [PATCH 0051/2835] update --- GitRepo.hs | 4 +++- TODO | 34 +++++++++++++++++++++++++++++++++- git-annex.mdwn | 24 +++++++++++++++--------- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index fb3ddbaf85..27fc0632c6 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -31,7 +31,9 @@ import Network.URI import Maybe import Utility -{- A git repository can be local or remote. -} +{- A git repository can be on local disk or remote. Not to be confused + - with a git repo's configured remotes, some of which may be on local + - disk. -} data GitRepo = LocalGitRepo { top :: FilePath, diff --git a/TODO b/TODO index ea930f8034..b08784ec24 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,39 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? -* query remotes for their annex.name settings +* query remotes for their annex.name settings, or figure out a different + solution to nameing problem? + + - querying network remotes all the time will be slow. local caching in + .git/config? + - having a git annex name and a git remote name that are distinct + will be confusing + - but git remote names are repo-local, I want a global name + - really, I don't want a name at all, I want a per-repo UUID + + So, each repo has a UUID, stored in annex.uuid. + + And also, the last seen UUID for each remote is listed: + + remote.origin.annex-uuid=d3d2474c-d5c3-11df-80a9-002170d25c55 + + Then when it need to find a repo by UUID, it can see if a known remote + has it -- and then query the remote to confirm the repo there still has + that UUID (a different repo may have been mounted there). + + Failing that, it can force a refresh of all uuids, updating .git/config, + and check again. + + - Only downside for this is that if I put a repo on a usb disk, + and it is disconnected and I have no remote for it, + git-annex will have to say: + + "You asked for a file that is only present on a git repo with + UUID d3d2474c-d5c3-11df-80a9-002170d25c55 -- and I don't know + where it is." + + To solve that, let .git-annex/uuid.map map between uuids and descriptions, + like "d3d2474c-d5c3-11df-80a9-002170d25c55 SATA drive labeled '* arch-2'" * hook up LocationLog * --push/--pull/--get/--want/--drop diff --git a/git-annex.mdwn b/git-annex.mdwn index 6bfdd57c7f..1348886f2b 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -79,7 +79,7 @@ git-annex information that should be propigated between repositories. Data is stored here in files that are arranged to avoid conflicts in most cases. A conflict could occur if a file with the same name but different -content was added to multiple clones. +content was added to multiple repositories. ## key/value storage @@ -117,26 +117,32 @@ you indicate you --want a file, git-annex will tell you which repositories have the file's content. Location tracking information is stored in `.git-annex/$filename.log`. -Repositories record their name and the date when they --get or --drop +Repositories record their UUID and the date when they --get or --drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) +The optional file `.git-annex/uuid.map` can be created to add a description +to a UUID. If git-annex needs a file from a repository and it cannot find +the repository amoung the remotes, it will use the description from this +file when asking for the repository to be made available. The file format +is a UUID, a space, and the rest of the line is its description. For +example: + + UUID d3d2474c-d5c3-11df-80a9-002170d25c55 USB drive in red enclosure + ## configuration -* `annex.numcopies` -- number of copies of files to keep +* `annex.uuid` -- a unique UUID for this repository +* `annex.numcopies` -- number of copies of files to keep (default: 1) * `annex.backends` -- space-separated list of names of the key/value backends to use. The first listed is used to store - new files. -* `annex.name` -- allows specifying a unique name for this repository. - If not specified, the name is derived from its directory's location and - the hostname. When a repository is on removable media it is useful to give - it a more stable name. Typically the name of a repository is the same - name configured as a git remote to allow pulling from that repository. + new files. (default: file, checksum, url) * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 50. Note that other factors may be configured when pushing files to repositories, in particular, whether the repository is on a filesystem with sufficient free space. +* `remote..annex-uuid` -- git-annex caches UUIDs of remotes here ## issues From 97b31a525e31abf1db95154d09c7efa368d3f59c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 02:05:33 -0400 Subject: [PATCH 0052/2835] foo --- git-annex.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/git-annex.mdwn b/git-annex.mdwn index 1348886f2b..9dd2d44ef7 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -129,6 +129,7 @@ is a UUID, a space, and the rest of the line is its description. For example: UUID d3d2474c-d5c3-11df-80a9-002170d25c55 USB drive in red enclosure + UUID 60cf39c8-d5c6-11df-aa8b-93fda39008d6 my colocated server ## configuration From 8f069bd2875022cfceb0c50cb9a5667a9bae88d8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 02:40:09 -0400 Subject: [PATCH 0053/2835] tweak --- Annex.hs | 2 +- GitRepo.hs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Annex.hs b/Annex.hs index 5a7274ca15..3461a92cde 100644 --- a/Annex.hs +++ b/Annex.hs @@ -31,7 +31,7 @@ annexDir repo key = gitDir repo ++ "/annex/" ++ key - later. -} startAnnex :: IO State startAnnex = do - r <- gitRepoCurrent + r <- gitRepoFromCwd gitPrep r return State { diff --git a/GitRepo.hs b/GitRepo.hs index 27fc0632c6..643b725e67 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -7,7 +7,7 @@ module GitRepo ( GitRepo, - gitRepoCurrent, + gitRepoFromCwd, gitRepoFromPath, gitRepoFromUrl, gitWorkTree, @@ -183,8 +183,8 @@ gitConfigRemotes repo = mapM construct remotes else gitRepoFromPath r {- Finds the current git repository, which may be in a parent directory. -} -gitRepoCurrent :: IO GitRepo -gitRepoCurrent = do +gitRepoFromCwd :: IO GitRepo +gitRepoFromCwd = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of From b430f55b80e0c4efba352817d8eecded586d0726 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 02:51:44 -0400 Subject: [PATCH 0054/2835] tweak --- GitRepo.hs | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 643b725e67..241dd4009b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -15,6 +15,7 @@ module GitRepo ( gitRelative, gitConfig, gitAdd, + gitRm, gitAttributes ) where @@ -79,7 +80,8 @@ remote repo = not $ local repo assertlocal repo action = if (local repo) then action - else error "acting on remote git repo not supported" + else error $ "acting on remote git repo " ++ (url repo) ++ + " not supported" {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> String @@ -116,44 +118,38 @@ gitRelative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -{- Stages a changed file in git's index. -} +{- Stages a changed/new file in git's index. -} gitAdd :: GitRepo -> FilePath -> IO () gitAdd repo file = runGit repo ["add", file] +{- Removes a file. -} +gitRm :: GitRepo -> FilePath -> IO () +gitRm repo file = runGit repo ["rm", file] + {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: GitRepo -> [String] -> [String] -gitCommandLine repo params = +gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - if (local repo) - then ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params - else error "gitCommandLine not implemented for remote repo" + ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} runGit :: GitRepo -> [String] -> IO () -runGit repo params = - if (local repo) - then do - r <- executeFile "git" True (gitCommandLine repo params) Nothing - return () - else error "runGit not implemented for remote repo" +runGit repo params = assertlocal repo $ do + r <- executeFile "git" True (gitCommandLine repo params) Nothing + return () {- Runs a git subcommand and returns its output. -} gitPipeRead :: GitRepo -> [String] -> IO String -gitPipeRead repo params = - if (local repo) - then pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - ret <- hGetContentsStrict h - return ret - else error "gitPipeRead not implemented for remote repo" +gitPipeRead repo params = assertlocal repo $ do + pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do + ret <- hGetContentsStrict h + return ret {- Runs git config and populates a repo with its settings. -} gitConfigRead :: GitRepo -> IO GitRepo -gitConfigRead repo = - if (local repo) - then do - c <- gitPipeRead repo ["config", "--list"] - return repo { config = gitConfigParse c } - else error "gitConfigRead not implemented for remote repo" +gitConfigRead repo = assertlocal repo $ do + c <- gitPipeRead repo ["config", "--list"] + return repo { config = gitConfigParse c } {- Parses git config --list output into a config map. -} gitConfigParse :: String -> Map.Map String String From 10b7c405fa427b5657d2336974a7e0a19ed098ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 12:23:34 -0400 Subject: [PATCH 0055/2835] better git repo querying and bare repo detection --- GitRepo.hs | 67 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 241dd4009b..21b37519bd 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -38,7 +38,6 @@ import Utility data GitRepo = LocalGitRepo { top :: FilePath, - bare :: Bool, config :: Map String String } | RemoteGitRepo { url :: String, @@ -46,24 +45,20 @@ data GitRepo = config :: Map String String } deriving (Show, Read, Eq) -{- Local GitRepo constructor. -} -gitRepoFromPath :: FilePath -> IO GitRepo -gitRepoFromPath dir = do - b <- isBareRepo dir - +{- Local GitRepo constructor. Can optionally query the repo for its config. -} +gitRepoFromPath :: FilePath -> Bool -> IO GitRepo +gitRepoFromPath dir query = do let r = LocalGitRepo { top = dir, - bare = b, config = Map.empty } - r' <- gitConfigRead r + if (query) + then gitConfigRead r + else return r - return r' - -{- Remote GitRepo constructor. Note that remote repo config is not read. - - Throws exception on invalid url. -} -gitRepoFromUrl :: String -> IO GitRepo -gitRepoFromUrl url = do +{- Remote GitRepo constructor. Throws exception on invalid url. -} +gitRepoFromUrl :: String -> Bool -> IO GitRepo +gitRepoFromUrl url query = do return $ RemoteGitRepo { url = url, top = path url, @@ -71,8 +66,11 @@ gitRepoFromUrl url = do } where path url = uriPath $ fromJust $ parseURI url -{- Some code needs to vary between remote and local repos, these functions - - help with that. -} +{- User-visible description of a git repo by path or url -} +describe repo = if (local repo) then top repo else url repo + +{- Some code needs to vary between remote and local repos, or bare and + - non-bare, these functions help with that. -} local repo = case (repo) of LocalGitRepo {} -> True RemoteGitRepo {} -> False @@ -80,8 +78,16 @@ remote repo = not $ local repo assertlocal repo action = if (local repo) then action - else error $ "acting on remote git repo " ++ (url repo) ++ + else error $ "acting on remote git repo " ++ (describe repo) ++ " not supported" +bare :: GitRepo -> Bool +bare repo = + if (member b (config repo)) + then ("true" == fromJust (Map.lookup b (config repo))) + else error $ "it is not known if git repo " ++ (describe repo) ++ + " is a bare repository; config not read" + where + b = "core.bare" {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> String @@ -130,7 +136,11 @@ gitRm repo file = runGit repo ["rm", file] gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + -- gitDir cannot be used for --git-dir because the config may + -- not have been read (and gitConfigRead relies on this function). + -- So this relies on git doing the right thing when told that + -- --git-dir is the top of a work tree. + ["--git-dir="++(top repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} runGit :: GitRepo -> [String] -> IO () @@ -175,8 +185,8 @@ gitConfigRemotes repo = mapM construct remotes isremote k = (startswith "remote." k) && (endswith ".url" k) construct r = if (isURI r) - then gitRepoFromUrl r - else gitRepoFromPath r + then gitRepoFromUrl r False + else gitRepoFromPath r False {- Finds the current git repository, which may be in a parent directory. -} gitRepoFromCwd :: IO GitRepo @@ -184,7 +194,7 @@ gitRepoFromCwd = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> gitRepoFromPath dir + (Just dir) -> gitRepoFromPath dir True Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) @@ -200,11 +210,10 @@ isRepoTop dir = do r <- isGitRepo dir b <- isBareRepo dir return (r || b) - -isGitRepo dir = gitSignature dir ".git" ".git/config" -isBareRepo dir = gitSignature dir "objects" "config" - -gitSignature dir subdir file = do - s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) - f <- (doesFileExist (dir ++ "/" ++ file)) - return (s && f) + where + isGitRepo dir = gitSignature dir ".git" ".git/config" + isBareRepo dir = gitSignature dir "objects" "config" + gitSignature dir subdir file = do + s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) + f <- (doesFileExist (dir ++ "/" ++ file)) + return (s && f) From 8a3ea4edcbf1cc7059d5382ca84d0033cd9152c6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 12:31:19 -0400 Subject: [PATCH 0056/2835] typo --- GitRepo.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 21b37519bd..f3c959bec4 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -174,7 +174,7 @@ gitConfigParse s = Map.fromList $ map pair $ lines s {- Returns a single git config setting, or a default value if not set. -} gitConfig :: GitRepo -> String -> String -> String gitConfig repo key defaultValue = - Map.findWithDefault key defaultValue (config repo) + Map.findWithDefault defaultValue key (config repo) {- Returns a list of a repo's configured remotes. -} gitConfigRemotes :: GitRepo -> IO [GitRepo] From e4bc7e599a799d758c4d948dce65a7fa05dd50cb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 12:35:05 -0400 Subject: [PATCH 0057/2835] revert bad change I was wrong about git-config's level of smarts --- GitRepo.hs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index f3c959bec4..5657803119 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -136,11 +136,7 @@ gitRm repo file = runGit repo ["rm", file] gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - -- gitDir cannot be used for --git-dir because the config may - -- not have been read (and gitConfigRead relies on this function). - -- So this relies on git doing the right thing when told that - -- --git-dir is the top of a work tree. - ["--git-dir="++(top repo), "--work-tree="++(top repo)] ++ params + ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} runGit :: GitRepo -> [String] -> IO () From 31b24348d25f5aec7ff521b7452fab6833a1d051 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 12:47:11 -0400 Subject: [PATCH 0058/2835] new git config read method --- GitRepo.hs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 5657803119..c87bd355e0 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -22,9 +22,11 @@ module GitRepo ( import Directory import System import System.Directory +import System.Posix.Directory import System.Path import System.Cmd.Utils import System.IO +import IO (bracket_) import System.Posix.Process import Data.String.Utils import Data.Map as Map hiding (map, split) @@ -151,11 +153,17 @@ gitPipeRead repo params = assertlocal repo $ do ret <- hGetContentsStrict h return ret -{- Runs git config and populates a repo with its settings. -} +{- Runs git config and populates a repo with its config. -} gitConfigRead :: GitRepo -> IO GitRepo gitConfigRead repo = assertlocal repo $ do - c <- gitPipeRead repo ["config", "--list"] - return repo { config = gitConfigParse c } + {- Cannot use gitPipeRead because it relies on the config having + been already read. Instead, chdir to the repo. -} + cwd <- getCurrentDirectory + bracket_ (changeWorkingDirectory (top repo)) + (\_ -> changeWorkingDirectory cwd) $ do + pOpen ReadFromPipe "git" ["config", "--list"] $ \h -> do + val <- hGetContentsStrict h + return repo { config = gitConfigParse val } {- Parses git config --list output into a config map. -} gitConfigParse :: String -> Map.Map String String From ea5d7fe07a5c40349e66848fc9cd06a9f748b724 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 13:02:41 -0400 Subject: [PATCH 0059/2835] add uuid --- .gitattributes | 1 + Annex.hs | 15 +++++++++++++++ GitRepo.hs | 9 +++++---- 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..b98b07d7d2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git-annex/*.log merge=union diff --git a/Annex.hs b/Annex.hs index 3461a92cde..5adc739964 100644 --- a/Annex.hs +++ b/Annex.hs @@ -10,6 +10,8 @@ module Annex ( import System.Posix.Files import System.Directory +import System.Cmd.Utils +import System.IO import GitRepo import Utility import Locations @@ -84,6 +86,13 @@ unannexFile state file = do {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () gitPrep repo = do + -- Make sure that the repo has an annex.uuid setting. + if ("" == gitConfig repo "annex.uuid" "") + then do + uuid <- genUUID + gitRun repo ["config", "annex.uuid", uuid] + else return () + -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/*.log merge=union" let attributes = gitAttributes repo @@ -99,3 +108,9 @@ gitPrep repo = do appendFile attributes $ attrLine ++ "\n" gitAdd repo attributes else return () + +{- Generates a UUID. There is a library for this, but it's not packaged, + - so use the command line tool. -} +genUUID :: IO String +genUUID = do + pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h diff --git a/GitRepo.hs b/GitRepo.hs index c87bd355e0..b166e32814 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -16,6 +16,7 @@ module GitRepo ( gitConfig, gitAdd, gitRm, + gitRun, gitAttributes ) where @@ -128,11 +129,11 @@ gitRelative repo file = drop (length absrepo) absfile {- Stages a changed/new file in git's index. -} gitAdd :: GitRepo -> FilePath -> IO () -gitAdd repo file = runGit repo ["add", file] +gitAdd repo file = gitRun repo ["add", file] {- Removes a file. -} gitRm :: GitRepo -> FilePath -> IO () -gitRm repo file = runGit repo ["rm", file] +gitRm repo file = gitRun repo ["rm", file] {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: GitRepo -> [String] -> [String] @@ -141,8 +142,8 @@ gitCommandLine repo params = assertlocal repo $ ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} -runGit :: GitRepo -> [String] -> IO () -runGit repo params = assertlocal repo $ do +gitRun :: GitRepo -> [String] -> IO () +gitRun repo params = assertlocal repo $ do r <- executeFile "git" True (gitCommandLine repo params) Nothing return () From dc1d5e68317b85043c8c30a82f53f78b0a9a9f51 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 13:10:07 -0400 Subject: [PATCH 0060/2835] update --- Annex.hs | 16 ++-------------- TODO | 34 +--------------------------------- UUID.hs | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 47 deletions(-) create mode 100644 UUID.hs diff --git a/Annex.hs b/Annex.hs index 5adc739964..31897479d4 100644 --- a/Annex.hs +++ b/Annex.hs @@ -10,13 +10,12 @@ module Annex ( import System.Posix.Files import System.Directory -import System.Cmd.Utils -import System.IO import GitRepo import Utility import Locations import Backend import BackendList +import UUID import LocationLog -- git-annex's runtime state @@ -86,12 +85,7 @@ unannexFile state file = do {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () gitPrep repo = do - -- Make sure that the repo has an annex.uuid setting. - if ("" == gitConfig repo "annex.uuid" "") - then do - uuid <- genUUID - gitRun repo ["config", "annex.uuid", uuid] - else return () + prepUUID repo -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/*.log merge=union" @@ -108,9 +102,3 @@ gitPrep repo = do appendFile attributes $ attrLine ++ "\n" gitAdd repo attributes else return () - -{- Generates a UUID. There is a library for this, but it's not packaged, - - so use the command line tool. -} -genUUID :: IO String -genUUID = do - pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h diff --git a/TODO b/TODO index b08784ec24..c951eb3f1e 100644 --- a/TODO +++ b/TODO @@ -1,39 +1,7 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? -* query remotes for their annex.name settings, or figure out a different - solution to nameing problem? - - - querying network remotes all the time will be slow. local caching in - .git/config? - - having a git annex name and a git remote name that are distinct - will be confusing - - but git remote names are repo-local, I want a global name - - really, I don't want a name at all, I want a per-repo UUID - - So, each repo has a UUID, stored in annex.uuid. - - And also, the last seen UUID for each remote is listed: - - remote.origin.annex-uuid=d3d2474c-d5c3-11df-80a9-002170d25c55 - - Then when it need to find a repo by UUID, it can see if a known remote - has it -- and then query the remote to confirm the repo there still has - that UUID (a different repo may have been mounted there). - - Failing that, it can force a refresh of all uuids, updating .git/config, - and check again. - - - Only downside for this is that if I put a repo on a usb disk, - and it is disconnected and I have no remote for it, - git-annex will have to say: - - "You asked for a file that is only present on a git repo with - UUID d3d2474c-d5c3-11df-80a9-002170d25c55 -- and I don't know - where it is." - - To solve that, let .git-annex/uuid.map map between uuids and descriptions, - like "d3d2474c-d5c3-11df-80a9-002170d25c55 SATA drive labeled '* arch-2'" +* query remotes for their annex.uuid settings * hook up LocationLog * --push/--pull/--get/--want/--drop diff --git a/UUID.hs b/UUID.hs new file mode 100644 index 0000000000..a0e0784821 --- /dev/null +++ b/UUID.hs @@ -0,0 +1,36 @@ +{- git-annex uuids + - + - Each git repository used by git-annex has an annex.uuid setting that + - uniquely identifies that repository. + - + -} + +module UUID ( + getUUID, + prepUUID, + genUUID +) where + +import System.Cmd.Utils +import System.IO +import GitRepo + +configkey="annex.uuid" + +{- Generates a UUID. There is a library for this, but it's not packaged, + - so use the command line tool. -} +genUUID :: IO String +genUUID = do + pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h + +getUUID :: GitRepo -> String +getUUID repo = gitConfig repo "annex.uuid" "" + +{- Make sure that the repo has an annex.uuid setting. -} +prepUUID :: GitRepo -> IO () +prepUUID repo = + if ("" == getUUID repo) + then do + uuid <- genUUID + gitRun repo ["config", configkey, uuid] + else return () From 4fbdb197d524720d1ea77795b33cb5d24152bce9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 13:12:47 -0400 Subject: [PATCH 0061/2835] correctness --- Annex.hs | 9 ++++----- GitRepo.hs | 1 + UUID.hs | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Annex.hs b/Annex.hs index 31897479d4..ed3e4e33ad 100644 --- a/Annex.hs +++ b/Annex.hs @@ -33,11 +33,12 @@ annexDir repo key = gitDir repo ++ "/annex/" ++ key startAnnex :: IO State startAnnex = do r <- gitRepoFromCwd - gitPrep r + r' <- prepUUID r + gitPrep r' return State { - repo = r, - backends = parseBackendList $ gitConfig r "annex.backends" "" + repo = r', + backends = parseBackendList $ gitConfig r' "annex.backends" "" } {- Annexes a file, storing it in a backend, and then moving it into @@ -85,8 +86,6 @@ unannexFile state file = do {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () gitPrep repo = do - prepUUID repo - -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/*.log merge=union" let attributes = gitAttributes repo diff --git a/GitRepo.hs b/GitRepo.hs index b166e32814..9a919128e6 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -14,6 +14,7 @@ module GitRepo ( gitDir, gitRelative, gitConfig, + gitConfigRead, gitAdd, gitRm, gitRun, diff --git a/UUID.hs b/UUID.hs index a0e0784821..0b795b8c10 100644 --- a/UUID.hs +++ b/UUID.hs @@ -23,14 +23,16 @@ genUUID :: IO String genUUID = do pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h +{- Looks up a repo's UUID -} getUUID :: GitRepo -> String getUUID repo = gitConfig repo "annex.uuid" "" {- Make sure that the repo has an annex.uuid setting. -} -prepUUID :: GitRepo -> IO () +prepUUID :: GitRepo -> IO GitRepo prepUUID repo = if ("" == getUUID repo) then do uuid <- genUUID gitRun repo ["config", configkey, uuid] - else return () + gitConfigRead repo -- return new repo with updated config + else return repo From e53900a54597437818d785aa6b1683b3b49d2afb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 15:44:54 -0400 Subject: [PATCH 0062/2835] stub --- Annex.hs | 33 ++++++++++++++++++++++++++++++++- CmdLine.hs | 12 ++++++++---- UUID.hs | 3 ++- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/Annex.hs b/Annex.hs index ed3e4e33ad..af25697198 100644 --- a/Annex.hs +++ b/Annex.hs @@ -5,7 +5,12 @@ module Annex ( State, startAnnex, annexFile, - unannexFile + unannexFile, + annexGetFile, + annexWantFile, + annexDropFile, + annexPushRepo, + annexPullRepo ) where import System.Posix.Files @@ -83,6 +88,32 @@ unannexFile state file = do renameFile src file return () +{- Transfers the file from a remote. -} +annexGetFile :: State -> FilePath -> IO () +annexGetFile state file = do + alreadyannexed <- lookupBackend (backends state) (repo state) file + case (alreadyannexed) of + Nothing -> error $ "not annexed " ++ file + Just _ -> do error "not implemented" -- TODO + -- 1. find remote with file + -- 2. copy file from remote + +{- Indicates a file is wanted. -} +annexWantFile :: State -> FilePath -> IO () +annexWantFile state file = do error "not implemented" -- TODO + +{- Indicates a file is now wanted. -} +annexDropFile :: State -> FilePath -> IO () +annexDropFile state file = do error "not implemented" -- TODO + +{- Pushes all files to a remote repository. -} +annexPushRepo :: State -> String -> IO () +annexPushRepo state reponame = do error "not implemented" -- TODO + +{- Pulls all files from a remote repository. -} +annexPullRepo :: State -> String -> IO () +annexPullRepo state reponame = do error "not implemented" -- TODO + {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () gitPrep repo = do diff --git a/CmdLine.hs b/CmdLine.hs index cc87088898..60ba81d30b 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -39,8 +39,12 @@ argvToMode argv = do where header = "Usage: git-annex [mode] file" dispatch :: State -> Mode -> FilePath -> IO () -dispatch state mode file = do +dispatch state mode item = do case (mode) of - Add -> annexFile state file - Unannex -> unannexFile state file - _ -> error "not implemented" + Add -> annexFile state item + Push -> annexPushRepo state item + Pull -> annexPullRepo state item + Want -> annexWantFile state item + Get -> annexGetFile state item + Drop -> annexDropFile state item + Unannex -> unannexFile state item diff --git a/UUID.hs b/UUID.hs index 0b795b8c10..40c2330ee9 100644 --- a/UUID.hs +++ b/UUID.hs @@ -34,5 +34,6 @@ prepUUID repo = then do uuid <- genUUID gitRun repo ["config", configkey, uuid] - gitConfigRead repo -- return new repo with updated config + -- return new repo with updated config + gitConfigRead repo else return repo From d257bad93c4ae6f8e6ef6a9c848e63d0f46eb225 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 15:48:00 -0400 Subject: [PATCH 0063/2835] uuid type --- Annex.hs | 2 +- UUID.hs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Annex.hs b/Annex.hs index af25697198..f695985e1f 100644 --- a/Annex.hs +++ b/Annex.hs @@ -102,7 +102,7 @@ annexGetFile state file = do annexWantFile :: State -> FilePath -> IO () annexWantFile state file = do error "not implemented" -- TODO -{- Indicates a file is now wanted. -} +{- Indicates a file is not wanted. -} annexDropFile :: State -> FilePath -> IO () annexDropFile state file = do error "not implemented" -- TODO diff --git a/UUID.hs b/UUID.hs index 40c2330ee9..4364e20700 100644 --- a/UUID.hs +++ b/UUID.hs @@ -15,16 +15,18 @@ import System.Cmd.Utils import System.IO import GitRepo +type UUID = String + configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} -genUUID :: IO String +genUUID :: IO UUID genUUID = do pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h {- Looks up a repo's UUID -} -getUUID :: GitRepo -> String +getUUID :: GitRepo -> UUID getUUID repo = gitConfig repo "annex.uuid" "" {- Make sure that the repo has an annex.uuid setting. -} From 759f146d0fd5857cbbb796367c3dd8c695550b46 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 15:52:18 -0400 Subject: [PATCH 0064/2835] update --- Annex.hs | 7 +------ Types.hs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 Types.hs diff --git a/Annex.hs b/Annex.hs index f695985e1f..830f619197 100644 --- a/Annex.hs +++ b/Annex.hs @@ -22,12 +22,7 @@ import Backend import BackendList import UUID import LocationLog - --- git-annex's runtime state -data State = State { - repo :: GitRepo, - backends :: [Backend] -} +import Types {- An annexed file's content is stored somewhere under .git/annex/ -} annexDir :: GitRepo -> Key -> FilePath diff --git a/Types.hs b/Types.hs new file mode 100644 index 0000000000..df95880274 --- /dev/null +++ b/Types.hs @@ -0,0 +1,14 @@ +{- git-annex core data types -} + +module Types ( + State(..) +) where + +import BackendType +import GitRepo + +-- git-annex's runtime state +data State = State { + repo :: GitRepo, + backends :: [Backend] +} From 2ac47a3a59b2b9b8980b4a9d3277bcb653bcb026 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:06:10 -0400 Subject: [PATCH 0065/2835] thread State thru to backends --- Annex.hs | 10 ++++----- Backend.hs | 55 +++++++++++++++++++++++----------------------- BackendChecksum.hs | 4 ++-- BackendFile.hs | 10 ++++----- BackendList.hs | 2 +- BackendType.hs | 31 -------------------------- BackendUrl.hs | 7 +++--- Types.hs | 25 +++++++++++++++++++-- 8 files changed, 67 insertions(+), 77 deletions(-) delete mode 100644 BackendType.hs diff --git a/Annex.hs b/Annex.hs index 830f619197..78d990eacb 100644 --- a/Annex.hs +++ b/Annex.hs @@ -45,12 +45,12 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend (backends state) state file case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile (backends state) (repo state) file + stored <- storeFile (backends state) state file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key @@ -70,11 +70,11 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend (backends state) state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile (backends state) (repo state) file + mkey <- dropFile (backends state) state file case (mkey) of Nothing -> return () Just key -> do @@ -86,7 +86,7 @@ unannexFile state file = do {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend (backends state) state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do error "not implemented" -- TODO diff --git a/Backend.hs b/Backend.hs index 2d3ea42d64..ddfd8b19d9 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,74 +28,75 @@ import System.Directory import Locations import GitRepo import Utility -import BackendType +import Types {- Name of state file that holds the key for an annexed file, - using a given backend. -} -backendFile :: Backend -> GitRepo -> FilePath -> String -backendFile backend repo file = gitStateDir repo ++ - (gitRelative repo file) ++ "." ++ (name backend) +backendFile :: Backend -> State -> FilePath -> String +backendFile backend state file = + gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ + "." ++ (name backend) {- Attempts to store a file in one of the backends, and returns - its key. -} -storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) +storeFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) storeFile [] _ _ = return Nothing -storeFile (b:bs) repo file = do - try <- (getKey b) repo (gitRelative repo file) +storeFile (b:bs) state file = do + try <- (getKey b) state (gitRelative (repo state) file) case (try) of Nothing -> nextbackend Just key -> do - stored <- (storeFileKey b) repo file key + stored <- (storeFileKey b) state file key if (not stored) then nextbackend else do bookkeeping key return $ Just key where - nextbackend = storeFile bs repo file - backendfile = backendFile b repo file + nextbackend = storeFile bs state file + backendfile = backendFile b state file bookkeeping key = do createDirectoryIfMissing True (parentDir backendfile) writeFile backendfile key {- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} -retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO Bool -retrieveFile backends repo file dest = do - result <- lookupBackend backends repo file +retrieveFile :: [Backend] -> State -> FilePath -> FilePath -> IO Bool +retrieveFile backends state file dest = do + result <- lookupBackend backends state file case (result) of Nothing -> return False Just b -> do - key <- lookupKey b repo file + key <- lookupKey b state file (retrieveKeyFile b) key dest {- Drops the key for a file from the backend that has it. -} -dropFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) -dropFile backends repo file = do - result <- lookupBackend backends repo file +dropFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) +dropFile backends state file = do + result <- lookupBackend backends state file case (result) of Nothing -> return Nothing Just b -> do - key <- lookupKey b repo file + key <- lookupKey b state file (removeKey b) key - removeFile $ backendFile b repo file + removeFile $ backendFile b state file return $ Just key {- Looks up the key a backend uses for an already annexed file. -} -lookupKey :: Backend -> GitRepo -> FilePath -> IO Key -lookupKey backend repo file = readFile (backendFile backend repo file) +lookupKey :: Backend -> State -> FilePath -> IO Key +lookupKey backend state file = readFile (backendFile backend state file) {- Looks up the backend used for an already annexed file. -} -lookupBackend :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Backend) +lookupBackend :: [Backend] -> State -> FilePath -> IO (Maybe Backend) lookupBackend [] _ _ = return Nothing -lookupBackend (b:bs) repo file = do - present <- checkBackend b repo file +lookupBackend (b:bs) state file = do + present <- checkBackend b state file if present then return $ Just b else - lookupBackend bs repo file + lookupBackend bs state file {- Checks if a file is available via a given backend. -} -checkBackend :: Backend -> GitRepo -> FilePath -> IO (Bool) -checkBackend backend repo file = doesFileExist $ backendFile backend repo file +checkBackend :: Backend -> State -> FilePath -> IO (Bool) +checkBackend backend state file = doesFileExist $ backendFile backend state file diff --git a/BackendChecksum.hs b/BackendChecksum.hs index e80dbe793c..72b4744e31 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -5,7 +5,7 @@ module BackendChecksum (backend) where import qualified BackendFile import Data.Digest.Pure.SHA -import BackendType +import Types import GitRepo -- based on BackendFile just with a different key type @@ -15,5 +15,5 @@ backend = BackendFile.backend { } -- checksum the file to get its key -keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue :: State -> FilePath -> IO (Maybe Key) keyValue k = error "checksum keyValue unimplemented" -- TODO diff --git a/BackendFile.hs b/BackendFile.hs index ae53f460f1..33c2985bca 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,7 +3,7 @@ module BackendFile (backend) where -import BackendType +import Types import GitRepo backend = Backend { @@ -15,15 +15,15 @@ backend = Backend { } -- direct mapping from filename to key -keyValue :: GitRepo -> FilePath -> IO (Maybe Key) -keyValue repo file = return $ Just file +keyValue :: State -> FilePath -> IO (Maybe Key) +keyValue state file = return $ Just file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. So storing or removing a key is - a no-op. -} -dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) -dummyStore repo file key = return True +dummyStore :: State -> FilePath -> Key -> IO (Bool) +dummyStore state file key = return True dummyRemove :: Key -> IO Bool dummyRemove url = return False diff --git a/BackendList.hs b/BackendList.hs index f733a44be9..104444dc20 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -7,7 +7,7 @@ module BackendList ( lookupBackendName ) where -import BackendType +import Types -- When adding a new backend, import it here and add it to the list. import qualified BackendFile diff --git a/BackendType.hs b/BackendType.hs deleted file mode 100644 index 3bc822f329..0000000000 --- a/BackendType.hs +++ /dev/null @@ -1,31 +0,0 @@ -{- git-annex backend data types - - -} - -module BackendType ( - -- the entire types are exported, for use in backend implementations - Key(..), - Backend(..) -) where - -import GitRepo - --- annexed filenames are mapped into keys -type Key = FilePath - --- this structure represents a key/value backend -data Backend = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: GitRepo -> FilePath -> IO (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, - -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> IO Bool, - -- removes a key - removeKey :: Key -> IO Bool -} - -instance Show Backend where - show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" - diff --git a/BackendUrl.hs b/BackendUrl.hs index 4ba1dbadb5..aad6477443 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,8 +3,7 @@ module BackendUrl (backend) where -import BackendType -import GitRepo +import Types backend = Backend { name = "url", @@ -15,11 +14,11 @@ backend = Backend { } -- cannot generate url from filename -keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue :: State -> FilePath -> IO (Maybe Key) keyValue repo file = return Nothing -- cannot change urls -dummyStore :: GitRepo -> FilePath -> Key -> IO Bool +dummyStore :: State -> FilePath -> Key -> IO Bool dummyStore repo file url = return False dummyRemove :: Key -> IO Bool dummyRemove url = return False diff --git a/Types.hs b/Types.hs index df95880274..de6bff9ff6 100644 --- a/Types.hs +++ b/Types.hs @@ -1,14 +1,35 @@ {- git-annex core data types -} module Types ( - State(..) + State(..), + Key(..), + Backend(..) ) where -import BackendType import GitRepo -- git-annex's runtime state data State = State { repo :: GitRepo, backends :: [Backend] +} deriving (Show) + +-- annexed filenames are mapped into keys +type Key = FilePath + +-- this structure represents a key/value backend +data Backend = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: State -> FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: State -> FilePath -> Key -> IO Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> FilePath -> IO Bool, + -- removes a key + removeKey :: Key -> IO Bool } + +instance Show Backend where + show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" From 20acda0423b1a00eae64296835679887ca79ea2f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:10:15 -0400 Subject: [PATCH 0066/2835] more state --- Backend.hs | 4 ++-- BackendChecksum.hs | 1 - BackendFile.hs | 9 ++++----- BackendUrl.hs | 8 ++++---- Types.hs | 4 ++-- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Backend.hs b/Backend.hs index ddfd8b19d9..622d558e31 100644 --- a/Backend.hs +++ b/Backend.hs @@ -68,7 +68,7 @@ retrieveFile backends state file dest = do Nothing -> return False Just b -> do key <- lookupKey b state file - (retrieveKeyFile b) key dest + (retrieveKeyFile b) state key dest {- Drops the key for a file from the backend that has it. -} dropFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) @@ -78,7 +78,7 @@ dropFile backends state file = do Nothing -> return Nothing Just b -> do key <- lookupKey b state file - (removeKey b) key + (removeKey b) state key removeFile $ backendFile b state file return $ Just key diff --git a/BackendChecksum.hs b/BackendChecksum.hs index 72b4744e31..efa2244125 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -6,7 +6,6 @@ module BackendChecksum (backend) where import qualified BackendFile import Data.Digest.Pure.SHA import Types -import GitRepo -- based on BackendFile just with a different key type backend = BackendFile.backend { diff --git a/BackendFile.hs b/BackendFile.hs index 33c2985bca..c59cbcbaae 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -4,7 +4,6 @@ module BackendFile (backend) where import Types -import GitRepo backend = Backend { name = "file", @@ -24,10 +23,10 @@ keyValue state file = return $ Just file - a no-op. -} dummyStore :: State -> FilePath -> Key -> IO (Bool) dummyStore state file key = return True -dummyRemove :: Key -> IO Bool -dummyRemove url = return False +dummyRemove :: State -> Key -> IO Bool +dummyRemove state url = return False {- Try to find a copy of the file in one of the other repos, - and copy it over to this one. -} -copyFromOtherRepo :: Key -> FilePath -> IO (Bool) -copyFromOtherRepo key file = error "copyFromOtherRepo unimplemented" -- TODO +copyFromOtherRepo :: State -> Key -> FilePath -> IO (Bool) +copyFromOtherRepo state key file = error "copyFromOtherRepo unimplemented" -- TODO diff --git a/BackendUrl.hs b/BackendUrl.hs index aad6477443..71503c5c13 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -20,8 +20,8 @@ keyValue repo file = return Nothing -- cannot change urls dummyStore :: State -> FilePath -> Key -> IO Bool dummyStore repo file url = return False -dummyRemove :: Key -> IO Bool -dummyRemove url = return False +dummyRemove :: State -> Key -> IO Bool +dummyRemove state url = return False -downloadUrl :: Key -> FilePath -> IO Bool -downloadUrl url file = error "downloadUrl unimplemented" +downloadUrl :: State -> Key -> FilePath -> IO Bool +downloadUrl state url file = error "downloadUrl unimplemented" diff --git a/Types.hs b/Types.hs index de6bff9ff6..26ba2a9043 100644 --- a/Types.hs +++ b/Types.hs @@ -26,9 +26,9 @@ data Backend = Backend { -- stores a file's contents to a key storeFileKey :: State -> FilePath -> Key -> IO Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> IO Bool, + retrieveKeyFile :: State -> Key -> FilePath -> IO Bool, -- removes a key - removeKey :: Key -> IO Bool + removeKey :: State -> Key -> IO Bool } instance Show Backend where From 603e01e96ce8d76f4b689b4503c3f4528c39957f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:20:41 -0400 Subject: [PATCH 0067/2835] simplify some function signatures using state --- Annex.hs | 11 +++++------ Backend.hs | 44 ++++++++++++++++++++++---------------------- CmdLine.hs | 1 + 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Annex.hs b/Annex.hs index 78d990eacb..1752cabffc 100644 --- a/Annex.hs +++ b/Annex.hs @@ -2,7 +2,6 @@ -} module Annex ( - State, startAnnex, annexFile, unannexFile, @@ -45,12 +44,12 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend (backends state) state file + alreadyannexed <- lookupBackend state file case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile (backends state) state file + stored <- storeFile state file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key @@ -70,11 +69,11 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend (backends state) state file + alreadyannexed <- lookupBackend state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile (backends state) state file + mkey <- dropFile state file case (mkey) of Nothing -> return () Just key -> do @@ -86,7 +85,7 @@ unannexFile state file = do {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = do - alreadyannexed <- lookupBackend (backends state) state file + alreadyannexed <- lookupBackend state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do error "not implemented" -- TODO diff --git a/Backend.hs b/Backend.hs index 622d558e31..2e5275ba68 100644 --- a/Backend.hs +++ b/Backend.hs @@ -17,8 +17,6 @@ - -} module Backend ( - Key, - Backend, -- note only data type is exported, not destructors lookupBackend, storeFile, dropFile @@ -32,16 +30,17 @@ import Types {- Name of state file that holds the key for an annexed file, - using a given backend. -} -backendFile :: Backend -> State -> FilePath -> String -backendFile backend state file = +backendFile :: State -> Backend -> FilePath -> String +backendFile state backend file = gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ "." ++ (name backend) {- Attempts to store a file in one of the backends, and returns - its key. -} -storeFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) -storeFile [] _ _ = return Nothing -storeFile (b:bs) state file = do +storeFile :: State -> FilePath -> IO (Maybe Key) +storeFile state file = storeFile' (backends state) state file +storeFile' [] _ _ = return Nothing +storeFile' (b:bs) state file = do try <- (getKey b) state (gitRelative (repo state) file) case (try) of Nothing -> nextbackend @@ -53,17 +52,17 @@ storeFile (b:bs) state file = do bookkeeping key return $ Just key where - nextbackend = storeFile bs state file - backendfile = backendFile b state file + nextbackend = storeFile' bs state file + backendfile = backendFile state b file bookkeeping key = do createDirectoryIfMissing True (parentDir backendfile) writeFile backendfile key {- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} -retrieveFile :: [Backend] -> State -> FilePath -> FilePath -> IO Bool -retrieveFile backends state file dest = do - result <- lookupBackend backends state file +retrieveFile :: State -> FilePath -> FilePath -> IO Bool +retrieveFile state file dest = do + result <- lookupBackend state file case (result) of Nothing -> return False Just b -> do @@ -71,32 +70,33 @@ retrieveFile backends state file dest = do (retrieveKeyFile b) state key dest {- Drops the key for a file from the backend that has it. -} -dropFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) -dropFile backends state file = do - result <- lookupBackend backends state file +dropFile :: State -> FilePath -> IO (Maybe Key) +dropFile state file = do + result <- lookupBackend state file case (result) of Nothing -> return Nothing Just b -> do key <- lookupKey b state file (removeKey b) state key - removeFile $ backendFile b state file + removeFile $ backendFile state b file return $ Just key {- Looks up the key a backend uses for an already annexed file. -} lookupKey :: Backend -> State -> FilePath -> IO Key -lookupKey backend state file = readFile (backendFile backend state file) +lookupKey backend state file = readFile (backendFile state backend file) {- Looks up the backend used for an already annexed file. -} -lookupBackend :: [Backend] -> State -> FilePath -> IO (Maybe Backend) -lookupBackend [] _ _ = return Nothing -lookupBackend (b:bs) state file = do +lookupBackend :: State -> FilePath -> IO (Maybe Backend) +lookupBackend state file = lookupBackend' (backends state) state file +lookupBackend' [] _ _ = return Nothing +lookupBackend' (b:bs) state file = do present <- checkBackend b state file if present then return $ Just b else - lookupBackend bs state file + lookupBackend' bs state file {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> State -> FilePath -> IO (Bool) -checkBackend backend state file = doesFileExist $ backendFile backend state file +checkBackend backend state file = doesFileExist $ backendFile state backend file diff --git a/CmdLine.hs b/CmdLine.hs index 60ba81d30b..9da2b6493c 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,6 +10,7 @@ module CmdLine ( ) where import System.Console.GetOpt +import Types import Annex data Mode = Add | Push | Pull | Want | Get | Drop | Unannex From 570899ed0c16121705ad5db1cb7aa96181a306a5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:39:10 -0400 Subject: [PATCH 0068/2835] handle newlines on keys --- Backend.hs | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/Backend.hs b/Backend.hs index 2e5275ba68..fab82a0935 100644 --- a/Backend.hs +++ b/Backend.hs @@ -23,18 +23,12 @@ module Backend ( ) where import System.Directory +import Data.String.Utils import Locations import GitRepo import Utility import Types -{- Name of state file that holds the key for an annexed file, - - using a given backend. -} -backendFile :: State -> Backend -> FilePath -> String -backendFile state backend file = - gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ - "." ++ (name backend) - {- Attempts to store a file in one of the backends, and returns - its key. -} storeFile :: State -> FilePath -> IO (Maybe Key) @@ -49,14 +43,10 @@ storeFile' (b:bs) state file = do if (not stored) then nextbackend else do - bookkeeping key + recordKey state b file key return $ Just key where nextbackend = storeFile' bs state file - backendfile = backendFile state b file - bookkeeping key = do - createDirectoryIfMissing True (parentDir backendfile) - writeFile backendfile key {- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} @@ -81,10 +71,6 @@ dropFile state file = do removeFile $ backendFile state b file return $ Just key -{- Looks up the key a backend uses for an already annexed file. -} -lookupKey :: Backend -> State -> FilePath -> IO Key -lookupKey backend state file = readFile (backendFile state backend file) - {- Looks up the backend used for an already annexed file. -} lookupBackend :: State -> FilePath -> IO (Maybe Backend) lookupBackend state file = lookupBackend' (backends state) state file @@ -97,6 +83,31 @@ lookupBackend' (b:bs) state file = do else lookupBackend' bs state file +{- Name of state file that holds the key for an annexed file, + - using a given backend. -} +backendFile :: State -> Backend -> FilePath -> String +backendFile state backend file = + gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ + "." ++ (name backend) + {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> State -> FilePath -> IO (Bool) checkBackend backend state file = doesFileExist $ backendFile state backend file + +{- Looks up the key a backend uses for an already annexed file. -} +lookupKey :: Backend -> State -> FilePath -> IO Key +lookupKey backend state file = do + k <- readFile (backendFile state backend file) + return $ chomp k + where + chomp s = if (endswith s "\n") + then (reverse . (drop 1) . reverse) s + else s + +{- Records the key a backend uses for an annexed file. -} +recordKey :: State -> Backend -> FilePath -> Key -> IO () +recordKey state backend file key = do + createDirectoryIfMissing True (parentDir record) + writeFile record $ key ++ "\n" + where + record = backendFile state backend file From 921313bcc706f054c33c3eb923c47955710cd0a3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:40:17 -0400 Subject: [PATCH 0069/2835] consistency --- Backend.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Backend.hs b/Backend.hs index fab82a0935..f03f098cf1 100644 --- a/Backend.hs +++ b/Backend.hs @@ -56,7 +56,7 @@ retrieveFile state file dest = do case (result) of Nothing -> return False Just b -> do - key <- lookupKey b state file + key <- lookupKey state b file (retrieveKeyFile b) state key dest {- Drops the key for a file from the backend that has it. -} @@ -66,7 +66,7 @@ dropFile state file = do case (result) of Nothing -> return Nothing Just b -> do - key <- lookupKey b state file + key <- lookupKey state b file (removeKey b) state key removeFile $ backendFile state b file return $ Just key @@ -95,8 +95,8 @@ checkBackend :: Backend -> State -> FilePath -> IO (Bool) checkBackend backend state file = doesFileExist $ backendFile state backend file {- Looks up the key a backend uses for an already annexed file. -} -lookupKey :: Backend -> State -> FilePath -> IO Key -lookupKey backend state file = do +lookupKey :: State -> Backend -> FilePath -> IO Key +lookupKey state backend file = do k <- readFile (backendFile state backend file) return $ chomp k where From cad916d92695c7c04d3cdacbcd333a2dcd109d53 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:52:01 -0400 Subject: [PATCH 0070/2835] hookup annexgetfile --- Annex.hs | 19 ++++++++++++------- Backend.hs | 2 ++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Annex.hs b/Annex.hs index 1752cabffc..7ac9932f1d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,8 +24,8 @@ import LocationLog import Types {- An annexed file's content is stored somewhere under .git/annex/ -} -annexDir :: GitRepo -> Key -> FilePath -annexDir repo key = gitDir repo ++ "/annex/" ++ key +annexLocation :: GitRepo -> Key -> FilePath +annexLocation repo key = gitDir repo ++ "/annex/" ++ key {- On startup, examine the git repo, prepare it, and record state for - later. -} @@ -55,7 +55,7 @@ annexFile state file = do Just key -> symlink key where symlink key = do - let dest = annexDir (repo state) key + let dest = annexLocation (repo state) key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file @@ -77,7 +77,7 @@ unannexFile state file = do case (mkey) of Nothing -> return () Just key -> do - let src = annexDir (repo state) key + let src = annexLocation (repo state) key removeFile file renameFile src file return () @@ -88,9 +88,14 @@ annexGetFile state file = do alreadyannexed <- lookupBackend state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file - Just _ -> do error "not implemented" -- TODO - -- 1. find remote with file - -- 2. copy file from remote + Just backend -> do + key <- lookupKey state backend file + let dest = annexLocation (repo state) key + createDirectoryIfMissing True (parentDir dest) + success <- retrieveFile state file dest + if (success) + then return () + else error $ "failed to get " ++ file {- Indicates a file is wanted. -} annexWantFile :: State -> FilePath -> IO () diff --git a/Backend.hs b/Backend.hs index f03f098cf1..9d1b0cdbee 100644 --- a/Backend.hs +++ b/Backend.hs @@ -19,6 +19,8 @@ module Backend ( lookupBackend, storeFile, + retrieveFile, + lookupKey, dropFile ) where From a36c39ad0af168259948a360087d2ff05df2857e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:26:34 -0400 Subject: [PATCH 0071/2835] getting files via http working! --- Annex.hs | 10 ++++++++-- Backend.hs | 2 +- BackendUrl.hs | 11 +++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Annex.hs b/Annex.hs index 7ac9932f1d..68379cf203 100644 --- a/Annex.hs +++ b/Annex.hs @@ -14,6 +14,7 @@ module Annex ( import System.Posix.Files import System.Directory +import Data.String.Utils import GitRepo import Utility import Locations @@ -23,9 +24,14 @@ import UUID import LocationLog import Types -{- An annexed file's content is stored somewhere under .git/annex/ -} +{- An annexed file's content is stored somewhere under .git/annex/, + - based on the key. Since the symlink is user-visible, the filename + - used should be as close to the key as possible, in case the key is a + - filename or url. Just escape "/" in the key name, to keep a flat + - tree of files and avoid issues with files ending with "/" etc. -} annexLocation :: GitRepo -> Key -> FilePath -annexLocation repo key = gitDir repo ++ "/annex/" ++ key +annexLocation repo key = gitDir repo ++ "/annex/" ++ (transform key) + where transform s = replace "/" "%" $ replace "%" "%%" s {- On startup, examine the git repo, prepare it, and record state for - later. -} diff --git a/Backend.hs b/Backend.hs index 9d1b0cdbee..a16dfab6a3 100644 --- a/Backend.hs +++ b/Backend.hs @@ -102,7 +102,7 @@ lookupKey state backend file = do k <- readFile (backendFile state backend file) return $ chomp k where - chomp s = if (endswith s "\n") + chomp s = if (endswith "\n" s) then (reverse . (drop 1) . reverse) s else s diff --git a/BackendUrl.hs b/BackendUrl.hs index 71503c5c13..ca44a5c37b 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,6 +3,8 @@ module BackendUrl (backend) where +import System.Posix.Process +import IO import Types backend = Backend { @@ -17,11 +19,16 @@ backend = Backend { keyValue :: State -> FilePath -> IO (Maybe Key) keyValue repo file = return Nothing --- cannot change urls +-- cannot change url contents dummyStore :: State -> FilePath -> Key -> IO Bool dummyStore repo file url = return False dummyRemove :: State -> Key -> IO Bool dummyRemove state url = return False downloadUrl :: State -> Key -> FilePath -> IO Bool -downloadUrl state url file = error "downloadUrl unimplemented" +downloadUrl state url file = do + putStrLn $ "download: " ++ url + result <- try $ executeFile "curl" True ["-o", file, url] Nothing + case (result) of + Left _ -> return False + Right _ -> return True From 0561ea1b2873962199aca5ba6529254aa5b2632b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:43:54 -0400 Subject: [PATCH 0072/2835] oops, wrong system --- BackendUrl.hs | 4 ++-- GitRepo.hs | 4 ++-- TODO | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/BackendUrl.hs b/BackendUrl.hs index ca44a5c37b..f18c800e96 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,7 +3,7 @@ module BackendUrl (backend) where -import System.Posix.Process +import System.Cmd import IO import Types @@ -28,7 +28,7 @@ dummyRemove state url = return False downloadUrl :: State -> Key -> FilePath -> IO Bool downloadUrl state url file = do putStrLn $ "download: " ++ url - result <- try $ executeFile "curl" True ["-o", file, url] Nothing + result <- try $ rawSystem "curl" ["-o", file, url] case (result) of Left _ -> return False Right _ -> return True diff --git a/GitRepo.hs b/GitRepo.hs index 9a919128e6..068b2569c0 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -26,10 +26,10 @@ import System import System.Directory import System.Posix.Directory import System.Path +import System.Cmd import System.Cmd.Utils import System.IO import IO (bracket_) -import System.Posix.Process import Data.String.Utils import Data.Map as Map hiding (map, split) import Network.URI @@ -145,7 +145,7 @@ gitCommandLine repo params = assertlocal repo $ {- Runs git in the specified repo. -} gitRun :: GitRepo -> [String] -> IO () gitRun repo params = assertlocal repo $ do - r <- executeFile "git" True (gitCommandLine repo params) Nothing + r <- rawSystem "git" (gitCommandLine repo params) return () {- Runs a git subcommand and returns its output. -} diff --git a/TODO b/TODO index c951eb3f1e..ad0389f850 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,7 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? +* if curl fails to download, git-annex crashes and does not complete + further actions.. exception seems to somehow not get caught * query remotes for their annex.uuid settings From 3d2b44ffe58ddc2f235f71cb548ba4a43b6fe641 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:51:41 -0400 Subject: [PATCH 0073/2835] better progress --- BackendUrl.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BackendUrl.hs b/BackendUrl.hs index f18c800e96..3f0846885d 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -28,7 +28,7 @@ dummyRemove state url = return False downloadUrl :: State -> Key -> FilePath -> IO Bool downloadUrl state url file = do putStrLn $ "download: " ++ url - result <- try $ rawSystem "curl" ["-o", file, url] + result <- try $ rawSystem "curl" ["-#", "-o", file, url] case (result) of Left _ -> return False Right _ -> return True From 10992b90c97e8c6abfd26da3d6cb50011b4230b6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:56:29 -0400 Subject: [PATCH 0074/2835] avoid redownload --- Annex.hs | 26 +++++++++++++++++--------- TODO | 2 -- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Annex.hs b/Annex.hs index 68379cf203..725009fd2c 100644 --- a/Annex.hs +++ b/Annex.hs @@ -29,10 +29,14 @@ import Types - used should be as close to the key as possible, in case the key is a - filename or url. Just escape "/" in the key name, to keep a flat - tree of files and avoid issues with files ending with "/" etc. -} -annexLocation :: GitRepo -> Key -> FilePath -annexLocation repo key = gitDir repo ++ "/annex/" ++ (transform key) +annexLocation :: State -> Key -> FilePath +annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (transform key) where transform s = replace "/" "%" $ replace "%" "%%" s +{- Checks if a given key is currently present in the annexLocation -} +inAnnex :: State -> Key -> IO Bool +inAnnex state key = doesFileExist $ annexLocation state key + {- On startup, examine the git repo, prepare it, and record state for - later. -} startAnnex :: IO State @@ -61,7 +65,7 @@ annexFile state file = do Just key -> symlink key where symlink key = do - let dest = annexLocation (repo state) key + let dest = annexLocation state key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file @@ -83,7 +87,7 @@ unannexFile state file = do case (mkey) of Nothing -> return () Just key -> do - let src = annexLocation (repo state) key + let src = annexLocation state key removeFile file renameFile src file return () @@ -96,12 +100,16 @@ annexGetFile state file = do Nothing -> error $ "not annexed " ++ file Just backend -> do key <- lookupKey state backend file - let dest = annexLocation (repo state) key - createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state file dest - if (success) + inannex <- inAnnex state key + if (inannex) then return () - else error $ "failed to get " ++ file + else do + let dest = annexLocation state key + createDirectoryIfMissing True (parentDir dest) + success <- retrieveFile state file dest + if (success) + then return () + else error $ "failed to get " ++ file {- Indicates a file is wanted. -} annexWantFile :: State -> FilePath -> IO () diff --git a/TODO b/TODO index ad0389f850..c951eb3f1e 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,5 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? -* if curl fails to download, git-annex crashes and does not complete - further actions.. exception seems to somehow not get caught * query remotes for their annex.uuid settings From f1eb4fef99aa553899256f3542e5bf30523e3512 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:57:23 -0400 Subject: [PATCH 0075/2835] todo --- TODO | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index c951eb3f1e..36ec65c2f5 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,11 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? +* implement retrieval for backendfile + * query remotes for their annex.uuid settings * hook up LocationLog -* --push/--pull/--get/--want/--drop +* --push/--pull/--want/--drop * finish BackendUrl and BackendChecksum From b882fe8410f33b2c8b170e6a60b55d156e336d47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 18:06:34 -0400 Subject: [PATCH 0076/2835] locationlog will use uuids --- BackendFile.hs | 7 ++++++- LocationLog.hs | 25 +++++++++++++------------ UUID.hs | 1 + 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/BackendFile.hs b/BackendFile.hs index c59cbcbaae..43ca2191ce 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -29,4 +29,9 @@ dummyRemove state url = return False {- Try to find a copy of the file in one of the other repos, - and copy it over to this one. -} copyFromOtherRepo :: State -> Key -> FilePath -> IO (Bool) -copyFromOtherRepo state key file = error "copyFromOtherRepo unimplemented" -- TODO +copyFromOtherRepo state key file = + -- 1. get ordered list of remotes (local repos, then remote repos) + -- 2. read locationlog for file + -- 3. filter remotes list to ones that have file + -- 4. attempt to transfer from each remote until success + error "copyFromOtherRepo unimplemented" -- TODO diff --git a/LocationLog.hs b/LocationLog.hs index 31d454f103..2cd84db1f7 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -6,10 +6,10 @@ - repositories have the file's content. - - Location tracking information is stored in `.git-annex/filename.log`. - - Repositories record their name and the date when they --get or --drop + - Repositories record their UUID and the date when they --get or --drop - a file's content. - - - A line of the log will look like: "date N reponame" + - A line of the log will look like: "date N UUID" - Where N=1 when the repo has the file, and 0 otherwise. - - Git is configured to use a union merge for this file, @@ -28,12 +28,13 @@ import System.Directory import Data.Char import GitRepo import Utility +import UUID import Locations data LogLine = LogLine { date :: POSIXTime, status :: LogStatus, - reponame :: String + uuid :: UUID } deriving (Eq) data LogStatus = FilePresent | FileMissing | Undefined @@ -50,8 +51,8 @@ instance Read LogStatus where readsPrec _ _ = [(Undefined, "")] instance Show LogLine where - show (LogLine date status reponame) = unwords - [(show date), (show status), reponame] + show (LogLine date status uuid) = unwords + [(show date), (show status), uuid] instance Read LogLine where -- This parser is robust in that even unparsable log lines are @@ -67,10 +68,10 @@ instance Read LogLine where w = words string date = w !! 0 status = read $ w !! 1 - reponame = unwords $ drop 2 w + uuid = unwords $ drop 2 w pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime - good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status reponame + good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status uuid undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] @@ -106,9 +107,9 @@ writeLog file lines = do {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> IO LogLine -logNow status reponame = do +logNow status uuid = do now <- getPOSIXTime - return $ LogLine now status reponame + return $ LogLine now status uuid {- Returns the filename of the log file for a given annexed file. -} logFile :: GitRepo -> FilePath -> IO String @@ -122,7 +123,7 @@ fileLocations :: GitRepo -> FilePath -> IO [String] fileLocations thisrepo file = do log <- logFile thisrepo file lines <- readLog log - return $ map reponame (filterPresent lines) + return $ map uuid (filterPresent lines) {- Filters the list of LogLines to find ones where the file - is (or should still be) present. -} @@ -140,9 +141,9 @@ compactLog' map (l:ls) = compactLog' (mapLog map l) ls - information about a repo than the other logs in the map -} mapLog map log = if (better) - then Map.insert (reponame log) log map + then Map.insert (uuid log) log map else map where - better = case (Map.lookup (reponame log) map) of + better = case (Map.lookup (uuid log) map) of Just l -> (date l <= date log) Nothing -> True diff --git a/UUID.hs b/UUID.hs index 4364e20700..e2b624d69c 100644 --- a/UUID.hs +++ b/UUID.hs @@ -6,6 +6,7 @@ -} module UUID ( + UUID, getUUID, prepUUID, genUUID From 3b89924f53cd88c0b5c21767dfd03b65d9d32f09 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 18:25:41 -0400 Subject: [PATCH 0077/2835] record annexed files in log --- Annex.hs | 1 + LocationLog.hs | 32 ++++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/Annex.hs b/Annex.hs index 725009fd2c..29cd7b0fda 100644 --- a/Annex.hs +++ b/Annex.hs @@ -68,6 +68,7 @@ annexFile state file = do let dest = annexLocation state key createDirectoryIfMissing True (parentDir dest) renameFile file dest + logChange (repo state) file (getUUID (repo state)) FilePresent createSymbolicLink dest file gitAdd (repo state) file checkLegal file = do diff --git a/LocationLog.hs b/LocationLog.hs index 2cd84db1f7..d3dd07a4e2 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -17,6 +17,8 @@ -} module LocationLog ( + LogStatus(..), + logChange ) where import Data.Time.Clock.POSIX @@ -75,6 +77,22 @@ instance Read LogLine where undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] +{- Log a change in the presence of a file in a repository, + - and add the log to git so it will propigate to other repos. -} +logChange :: GitRepo -> FilePath -> UUID -> LogStatus -> IO () +logChange repo file uuid status = do + log <- logNow status uuid + if (status == FilePresent) + -- file added; just append to log + then appendLog logfile log + -- file removed; compact log + else do + ls <- readLog logfile + writeLog logfile (log:ls) + gitAdd repo logfile + where + logfile = logFile repo file + {- Reads a log file. - Note that the LogLines returned may be in any order. -} readLog :: FilePath -> IO [LogLine] @@ -106,23 +124,21 @@ writeLog file lines = do hPutStr h $ unlines $ map show lines {- Generates a new LogLine with the current date. -} -logNow :: LogStatus -> String -> IO LogLine +logNow :: LogStatus -> UUID -> IO LogLine logNow status uuid = do now <- getPOSIXTime return $ LogLine now status uuid {- Returns the filename of the log file for a given annexed file. -} -logFile :: GitRepo -> FilePath -> IO String -logFile repo annexedFile = do - return $ (gitStateDir repo) ++ +logFile :: GitRepo -> FilePath -> String +logFile repo annexedFile = (gitStateDir repo) ++ (gitRelative repo annexedFile) ++ ".log" -{- Returns a list of repositories that, according to the log, have +{- Returns a list of repository UUIDs that, according to the log, have - the content of a file -} -fileLocations :: GitRepo -> FilePath -> IO [String] +fileLocations :: GitRepo -> FilePath -> IO [UUID] fileLocations thisrepo file = do - log <- logFile thisrepo file - lines <- readLog log + lines <- readLog $ logFile thisrepo file return $ map uuid (filterPresent lines) {- Filters the list of LogLines to find ones where the file From 476f66abb99ad2baa18b699c26ac9ee7250eca76 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 18:31:02 -0400 Subject: [PATCH 0078/2835] now that a uuid is used, don't need to rejoin --- LocationLog.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LocationLog.hs b/LocationLog.hs index d3dd07a4e2..da702d6503 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -70,7 +70,7 @@ instance Read LogLine where w = words string date = w !! 0 status = read $ w !! 1 - uuid = unwords $ drop 2 w + uuid = w !! 3 pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status uuid From b7858ada038084c8455cdf9d3598382308dc52b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 20:04:36 -0400 Subject: [PATCH 0079/2835] bugfixes --- Annex.hs | 51 +++++++++++++++++++++++++--------------- Backend.hs | 20 +++++----------- GitRepo.hs | 10 -------- LocationLog.hs | 64 +++++++++++++++++++++++--------------------------- Locations.hs | 25 +++++++++++++++++++- Types.hs | 1 + 6 files changed, 93 insertions(+), 78 deletions(-) diff --git a/Annex.hs b/Annex.hs index 29cd7b0fda..b8e70e6c8d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,15 +24,6 @@ import UUID import LocationLog import Types -{- An annexed file's content is stored somewhere under .git/annex/, - - based on the key. Since the symlink is user-visible, the filename - - used should be as close to the key as possible, in case the key is a - - filename or url. Just escape "/" in the key name, to keep a flat - - tree of files and avoid issues with files ending with "/" etc. -} -annexLocation :: State -> Key -> FilePath -annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (transform key) - where transform s = replace "/" "%" $ replace "%" "%%" s - {- Checks if a given key is currently present in the annexLocation -} inAnnex :: State -> Key -> IO Bool inAnnex state key = doesFileExist $ annexLocation state key @@ -62,15 +53,18 @@ annexFile state file = do stored <- storeFile state file case (stored) of Nothing -> error $ "no backend could store: " ++ file - Just key -> symlink key + Just (key, backend) -> setup key backend where - symlink key = do + setup key backend = do let dest = annexLocation state key createDirectoryIfMissing True (parentDir dest) renameFile file dest - logChange (repo state) file (getUUID (repo state)) FilePresent createSymbolicLink dest file - gitAdd (repo state) file + gitRun (repo state) ["add", file, bfile] + gitRun (repo state) ["commit", "-m", + ("git-annex annexed " ++ file), file, bfile] + logStatus state key ValuePresent + where bfile = backendFile state backend file checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -87,11 +81,17 @@ unannexFile state file = do mkey <- dropFile state file case (mkey) of Nothing -> return () - Just key -> do + Just (key, backend) -> do let src = annexLocation state key removeFile file + gitRun (repo state) ["rm", file, bfile] + gitRun (repo state) ["commit", "-m", + ("git-annex unannexed " ++ file), + file, bfile] renameFile src file + logStatus state key ValueMissing return () + where bfile = backendFile state backend file {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () @@ -109,7 +109,9 @@ annexGetFile state file = do createDirectoryIfMissing True (parentDir dest) success <- retrieveFile state file dest if (success) - then return () + then do + logStatus state key ValuePresent + return () else error $ "failed to get " ++ file {- Indicates a file is wanted. -} @@ -132,17 +134,28 @@ annexPullRepo state reponame = do error "not implemented" -- TODO gitPrep :: GitRepo -> IO () gitPrep repo = do -- configure git to use union merge driver on state files - let attrLine = stateLoc ++ "/*.log merge=union" - let attributes = gitAttributes repo exists <- doesFileExist attributes if (not exists) then do writeFile attributes $ attrLine ++ "\n" - gitAdd repo attributes + commit else do content <- readFile attributes if (all (/= attrLine) (lines content)) then do appendFile attributes $ attrLine ++ "\n" - gitAdd repo attributes + commit else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = gitAttributes repo + commit = do + gitRun repo ["add", attributes] + gitRun repo ["commit", "-m", "git-annex setup", + attributes] + +{- Updates the LocationLog when a key's presence changes. -} +logStatus state key status = do + f <- logChange (repo state) key (getUUID (repo state)) status + gitRun (repo state) ["add", f] + gitRun (repo state) ["commit", "-m", "git-annex log update", f] diff --git a/Backend.hs b/Backend.hs index a16dfab6a3..d7bde241a3 100644 --- a/Backend.hs +++ b/Backend.hs @@ -31,9 +31,8 @@ import GitRepo import Utility import Types -{- Attempts to store a file in one of the backends, and returns - - its key. -} -storeFile :: State -> FilePath -> IO (Maybe Key) +{- Attempts to store a file in one of the backends. -} +storeFile :: State -> FilePath -> IO (Maybe (Key, Backend)) storeFile state file = storeFile' (backends state) state file storeFile' [] _ _ = return Nothing storeFile' (b:bs) state file = do @@ -46,7 +45,7 @@ storeFile' (b:bs) state file = do then nextbackend else do recordKey state b file key - return $ Just key + return $ Just (key, b) where nextbackend = storeFile' bs state file @@ -62,7 +61,7 @@ retrieveFile state file dest = do (retrieveKeyFile b) state key dest {- Drops the key for a file from the backend that has it. -} -dropFile :: State -> FilePath -> IO (Maybe Key) +dropFile :: State -> FilePath -> IO (Maybe (Key, Backend)) dropFile state file = do result <- lookupBackend state file case (result) of @@ -71,7 +70,7 @@ dropFile state file = do key <- lookupKey state b file (removeKey b) state key removeFile $ backendFile state b file - return $ Just key + return $ Just (key, b) {- Looks up the backend used for an already annexed file. -} lookupBackend :: State -> FilePath -> IO (Maybe Backend) @@ -85,13 +84,6 @@ lookupBackend' (b:bs) state file = do else lookupBackend' bs state file -{- Name of state file that holds the key for an annexed file, - - using a given backend. -} -backendFile :: State -> Backend -> FilePath -> String -backendFile state backend file = - gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ - "." ++ (name backend) - {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> State -> FilePath -> IO (Bool) checkBackend backend state file = doesFileExist $ backendFile state backend file @@ -106,7 +98,7 @@ lookupKey state backend file = do then (reverse . (drop 1) . reverse) s else s -{- Records the key a backend uses for an annexed file. -} +{- Records the key used for an annexed file. -} recordKey :: State -> Backend -> FilePath -> Key -> IO () recordKey state backend file key = do createDirectoryIfMissing True (parentDir record) diff --git a/GitRepo.hs b/GitRepo.hs index 068b2569c0..fcaae1253f 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -15,8 +15,6 @@ module GitRepo ( gitRelative, gitConfig, gitConfigRead, - gitAdd, - gitRm, gitRun, gitAttributes ) where @@ -128,14 +126,6 @@ gitRelative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -{- Stages a changed/new file in git's index. -} -gitAdd :: GitRepo -> FilePath -> IO () -gitAdd repo file = gitRun repo ["add", file] - -{- Removes a file. -} -gitRm :: GitRepo -> FilePath -> IO () -gitRm repo file = gitRun repo ["rm", file] - {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ diff --git a/LocationLog.hs b/LocationLog.hs index da702d6503..2eab4815ed 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -1,13 +1,13 @@ {- git-annex location log - - - git-annex keeps track of on which repository it last saw a file's content. + - git-annex keeps track of on which repository it last saw a value. - This can be useful when using it for archiving with offline storage. - When you indicate you --want a file, git-annex will tell you which - - repositories have the file's content. + - repositories have the value. - - - Location tracking information is stored in `.git-annex/filename.log`. + - Location tracking information is stored in `.git-annex/key.log`. - Repositories record their UUID and the date when they --get or --drop - - a file's content. + - a value. - - A line of the log will look like: "date N UUID" - Where N=1 when the repo has the file, and 0 otherwise. @@ -31,6 +31,7 @@ import Data.Char import GitRepo import Utility import UUID +import Types import Locations data LogLine = LogLine { @@ -39,17 +40,17 @@ data LogLine = LogLine { uuid :: UUID } deriving (Eq) -data LogStatus = FilePresent | FileMissing | Undefined +data LogStatus = ValuePresent | ValueMissing | Undefined deriving (Eq) instance Show LogStatus where - show FilePresent = "1" - show FileMissing = "0" + show ValuePresent = "1" + show ValueMissing = "0" show Undefined = "undefined" instance Read LogStatus where - readsPrec _ "1" = [(FilePresent, "")] - readsPrec _ "0" = [(FileMissing, "")] + readsPrec _ "1" = [(ValuePresent, "")] + readsPrec _ "0" = [(ValueMissing, "")] readsPrec _ _ = [(Undefined, "")] instance Show LogLine where @@ -61,7 +62,7 @@ instance Read LogLine where -- read without an exception being thrown. -- Such lines have a status of Undefined. readsPrec _ string = - if (length w >= 3) + if (length w == 3) then case (pdate) of Just v -> good v Nothing -> undefined @@ -70,28 +71,23 @@ instance Read LogLine where w = words string date = w !! 0 status = read $ w !! 1 - uuid = w !! 3 + uuid = w !! 2 pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status uuid undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] -{- Log a change in the presence of a file in a repository, - - and add the log to git so it will propigate to other repos. -} -logChange :: GitRepo -> FilePath -> UUID -> LogStatus -> IO () -logChange repo file uuid status = do +{- Log a change in the presence of a key's value in a repository, + - and return the log filename. -} +logChange :: GitRepo -> Key -> UUID -> LogStatus -> IO FilePath +logChange repo key uuid status = do log <- logNow status uuid - if (status == FilePresent) - -- file added; just append to log - then appendLog logfile log - -- file removed; compact log - else do - ls <- readLog logfile - writeLog logfile (log:ls) - gitAdd repo logfile + ls <- readLog logfile + writeLog logfile (compactLog $ log:ls) + return logfile where - logfile = logFile repo file + logfile = logFile repo key {- Reads a log file. - Note that the LogLines returned may be in any order. -} @@ -129,22 +125,22 @@ logNow status uuid = do now <- getPOSIXTime return $ LogLine now status uuid -{- Returns the filename of the log file for a given annexed file. -} -logFile :: GitRepo -> FilePath -> String -logFile repo annexedFile = (gitStateDir repo) ++ - (gitRelative repo annexedFile) ++ ".log" +{- Returns the filename of the log file for a given key. -} +logFile :: GitRepo -> Key -> String +logFile repo key = + (gitStateDir repo) ++ (gitRelative repo (keyFile key)) ++ ".log" {- Returns a list of repository UUIDs that, according to the log, have - - the content of a file -} -fileLocations :: GitRepo -> FilePath -> IO [UUID] -fileLocations thisrepo file = do - lines <- readLog $ logFile thisrepo file + - the value of a key. -} +keyLocations :: GitRepo -> Key -> IO [UUID] +keyLocations thisrepo key = do + lines <- readLog $ logFile thisrepo key return $ map uuid (filterPresent lines) -{- Filters the list of LogLines to find ones where the file +{- Filters the list of LogLines to find ones where the value - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] -filterPresent lines = filter (\l -> FilePresent == status l) $ compactLog lines +filterPresent lines = filter (\l -> ValuePresent == status l) $ compactLog lines {- Compacts a set of logs, returning a subset that contains the current - status. -} diff --git a/Locations.hs b/Locations.hs index 300f443f7f..59f9df727a 100644 --- a/Locations.hs +++ b/Locations.hs @@ -3,9 +3,14 @@ module Locations ( gitStateDir, - stateLoc + stateLoc, + keyFile, + annexLocation, + backendFile ) where +import Data.String.Utils +import Types import GitRepo {- Long-term, cross-repo state is stored in files inside the .git-annex @@ -13,3 +18,21 @@ import GitRepo stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" + +{- Generates a filename that can be used to record a key somewhere to disk. + - Just escape "/" in the key name, to keep a flat + - tree of files and avoid issues with files ending with "/" etc. -} +keyFile :: Key -> FilePath +keyFile key = replace "/" "%" $ replace "%" "%%" key + +{- An annexed file's content is stored somewhere under .git/annex/, + - based on the key. -} +annexLocation :: State -> Key -> FilePath +annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (keyFile key) + +{- Name of state file that holds the key for an annexed file, + - using a given backend. -} +backendFile :: State -> Backend -> FilePath -> String +backendFile state backend file = + gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ + "." ++ (name backend) diff --git a/Types.hs b/Types.hs index 26ba2a9043..73492dfc38 100644 --- a/Types.hs +++ b/Types.hs @@ -6,6 +6,7 @@ module Types ( Backend(..) ) where +import Data.String.Utils import GitRepo -- git-annex's runtime state From 490eb66be40d4e9e6a5e4d89f67610e073e7574f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 20:20:59 -0400 Subject: [PATCH 0080/2835] update --- Backend.hs | 3 --- Locations.hs | 7 ++++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Backend.hs b/Backend.hs index d7bde241a3..525a52beea 100644 --- a/Backend.hs +++ b/Backend.hs @@ -9,9 +9,6 @@ - This key can later be used to retrieve the file's content (its value). This - key generation must be stable for a given file content, name, and size. - - - The mapping from filename to its key is stored in the .git-annex directory, - - in a file named `$filename.$backend` - - - Multiple pluggable backends are supported, and more than one can be used - to store different files' contents in a given repository. - -} diff --git a/Locations.hs b/Locations.hs index 59f9df727a..925aa39e56 100644 --- a/Locations.hs +++ b/Locations.hs @@ -30,9 +30,10 @@ keyFile key = replace "/" "%" $ replace "%" "%%" key annexLocation :: State -> Key -> FilePath annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (keyFile key) -{- Name of state file that holds the key for an annexed file, - - using a given backend. -} +{- The mapping from filename to its key is stored in the .git-annex directory, + - in a file named `key/$filename.$backend` -} backendFile :: State -> Backend -> FilePath -> String backendFile state backend file = - gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ + gitStateDir (repo state) ++ "key/" ++ + (gitRelative (repo state) file) ++ "." ++ (name backend) From d029c48d0ae5485329b4e8757b45f320bb8bfea9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 20:26:02 -0400 Subject: [PATCH 0081/2835] docs --- git-annex.mdwn | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index 9dd2d44ef7..84030bfcaa 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -36,7 +36,9 @@ Enough broad picture, here's how it actually looks: downloaded. * `git annex --drop $file` indicates that you no longer want the file's content to be available in this repository. -* `git annex --unannex $file` undoes a `git annex --add`. +* `git annex --unannex $file` undoes a `git annex --add`. But use `--drop` + if you're just done with a file; only use `--unannex` if you + accidentially added a file. * `git annex $file` is a shorthand for either --add or --get. If the file is already known, it does --get, otherwise it does --add. @@ -74,12 +76,12 @@ only copies of a file. ## the .git-annex directory -The `.git-annex` directory at the top of the repository, is used to store +The `.git-annex` directory at the top of the repository is used to store git-annex information that should be propigated between repositories. Data is stored here in files that are arranged to avoid conflicts in most cases. A conflict could occur if a file with the same name but different -content was added to multiple repositories. +content was added to different repositories. ## key/value storage @@ -93,7 +95,7 @@ This key can later be used to retrieve the file's content (its value). This key generation must be stable for a given file content, name, and size. The mapping from filename to its key is stored in the .git-annex directory, -in a file named `$filename.$backend` +in a file named `key/$filename.$backend` Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. @@ -116,7 +118,7 @@ This can be useful when using it for archiving with offline storage. When you indicate you --want a file, git-annex will tell you which repositories have the file's content. -Location tracking information is stored in `.git-annex/$filename.log`. +Location tracking information is stored in `.git-annex/$key.log`. Repositories record their UUID and the date when they --get or --drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) @@ -140,10 +142,12 @@ example: new files. (default: file, checksum, url) * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. - The default cost is 50. Note that other factors may be configured - when pushing files to repositories, in particular, whether the repository - is on a filesystem with sufficient free space. -* `remote..annex-uuid` -- git-annex caches UUIDs of remotes here + The default cost is 100 for local repositories, and 200 for remote + repositories. Note that other factors may be configured when pushing + files to repositories, in particular, whether the repository is on + a filesystem with sufficient free space. +* `remote..annex-uuid` -- git-annex caches UUIDs of repositories + here. ## issues From fe612bac0361b42af2cdff590797e929cebbb53d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 20:35:20 -0400 Subject: [PATCH 0082/2835] thought --- git-annex.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/git-annex.mdwn b/git-annex.mdwn index 84030bfcaa..3a8db6dcfb 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -180,3 +180,11 @@ remote, to check local disk space. When git-rm removed a file, it should get dropped too. Of course, it may not be dropped right away, depending on number of copies available. + +### branching + +The use of `.git-annex` to store state means that if a repo has branches +and the user switched between them, git-annex will see different state in +the different branches. Whether that is a bug or a feature may depend on +point of view -- call it Too Be Determined. An alternative would be to +store data directly in the git repo as `pristine-tar` does. From 9926fe5c8a1479f734c0a5b68c7c4e6ddfc2f8cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 00:21:34 -0400 Subject: [PATCH 0083/2835] bugfix in escaping --- Locations.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Locations.hs b/Locations.hs index 925aa39e56..c12b9fadce 100644 --- a/Locations.hs +++ b/Locations.hs @@ -19,11 +19,11 @@ stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" -{- Generates a filename that can be used to record a key somewhere to disk. +{- Converts a key into a filename fragment. - Just escape "/" in the key name, to keep a flat - tree of files and avoid issues with files ending with "/" etc. -} keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "%%" key +keyFile key = replace "/" "&s" $ replace "&" "&a" key {- An annexed file's content is stored somewhere under .git/annex/, - based on the key. -} From 208bba8d3062133733d27a5db521013e3a2ead57 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 00:42:46 -0400 Subject: [PATCH 0084/2835] got rid of the .git-annex/key.backend files --- Annex.hs | 24 +++++++++++------------- Backend.hs | 46 ++++++++++++++++++---------------------------- Locations.hs | 22 ++++++++-------------- TODO | 7 +++++++ git-annex.mdwn | 18 ++++++++---------- 5 files changed, 52 insertions(+), 65 deletions(-) diff --git a/Annex.hs b/Annex.hs index b8e70e6c8d..82efd543d4 100644 --- a/Annex.hs +++ b/Annex.hs @@ -25,8 +25,8 @@ import LocationLog import Types {- Checks if a given key is currently present in the annexLocation -} -inAnnex :: State -> Key -> IO Bool -inAnnex state key = doesFileExist $ annexLocation state key +inAnnex :: State -> Backend -> Key -> IO Bool +inAnnex state backend key = doesFileExist $ annexLocation state backend key {- On startup, examine the git repo, prepare it, and record state for - later. -} @@ -56,15 +56,14 @@ annexFile state file = do Just (key, backend) -> setup key backend where setup key backend = do - let dest = annexLocation state key + let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file - gitRun (repo state) ["add", file, bfile] + gitRun (repo state) ["add", file] gitRun (repo state) ["commit", "-m", - ("git-annex annexed " ++ file), file, bfile] + ("git-annex annexed " ++ file), file] logStatus state key ValuePresent - where bfile = backendFile state backend file checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -82,16 +81,15 @@ unannexFile state file = do case (mkey) of Nothing -> return () Just (key, backend) -> do - let src = annexLocation state key + let src = annexLocation state backend key removeFile file - gitRun (repo state) ["rm", file, bfile] + gitRun (repo state) ["rm", file] gitRun (repo state) ["commit", "-m", ("git-annex unannexed " ++ file), - file, bfile] + file] renameFile src file logStatus state key ValueMissing return () - where bfile = backendFile state backend file {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () @@ -100,12 +98,12 @@ annexGetFile state file = do case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just backend -> do - key <- lookupKey state backend file - inannex <- inAnnex state key + key <- fileKey file + inannex <- inAnnex state backend key if (inannex) then return () else do - let dest = annexLocation state key + let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) success <- retrieveFile state file dest if (success) diff --git a/Backend.hs b/Backend.hs index 525a52beea..68d70feec7 100644 --- a/Backend.hs +++ b/Backend.hs @@ -17,12 +17,14 @@ module Backend ( lookupBackend, storeFile, retrieveFile, - lookupKey, + fileKey, dropFile ) where import System.Directory +import System.FilePath import Data.String.Utils +import System.Posix.Files import Locations import GitRepo import Utility @@ -41,7 +43,6 @@ storeFile' (b:bs) state file = do if (not stored) then nextbackend else do - recordKey state b file key return $ Just (key, b) where nextbackend = storeFile' bs state file @@ -53,9 +54,9 @@ retrieveFile state file dest = do result <- lookupBackend state file case (result) of Nothing -> return False - Just b -> do - key <- lookupKey state b file - (retrieveKeyFile b) state key dest + Just backend -> do + key <- fileKey file + (retrieveKeyFile backend) state key dest {- Drops the key for a file from the backend that has it. -} dropFile :: State -> FilePath -> IO (Maybe (Key, Backend)) @@ -63,11 +64,10 @@ dropFile state file = do result <- lookupBackend state file case (result) of Nothing -> return Nothing - Just b -> do - key <- lookupKey state b file - (removeKey b) state key - removeFile $ backendFile state b file - return $ Just (key, b) + Just backend -> do + key <- fileKey file + (removeKey backend) state key + return $ Just (key, backend) {- Looks up the backend used for an already annexed file. -} lookupBackend :: State -> FilePath -> IO (Maybe Backend) @@ -83,22 +83,12 @@ lookupBackend' (b:bs) state file = do {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> State -> FilePath -> IO (Bool) -checkBackend backend state file = doesFileExist $ backendFile state backend file +checkBackend backend state file = + doesFileExist $ annexLocation state backend file -{- Looks up the key a backend uses for an already annexed file. -} -lookupKey :: State -> Backend -> FilePath -> IO Key -lookupKey state backend file = do - k <- readFile (backendFile state backend file) - return $ chomp k - where - chomp s = if (endswith "\n" s) - then (reverse . (drop 1) . reverse) s - else s - -{- Records the key used for an annexed file. -} -recordKey :: State -> Backend -> FilePath -> Key -> IO () -recordKey state backend file key = do - createDirectoryIfMissing True (parentDir record) - writeFile record $ key ++ "\n" - where - record = backendFile state backend file +{- Looks up the key corresponding to an annexed file, + - by examining what the file symlinks to. -} +fileKey :: FilePath -> IO Key +fileKey file = do + l <- readSymbolicLink (file) + return $ takeFileName $ l diff --git a/Locations.hs b/Locations.hs index c12b9fadce..b859fd2f27 100644 --- a/Locations.hs +++ b/Locations.hs @@ -5,8 +5,7 @@ module Locations ( gitStateDir, stateLoc, keyFile, - annexLocation, - backendFile + annexLocation ) where import Data.String.Utils @@ -25,15 +24,10 @@ gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" keyFile :: Key -> FilePath keyFile key = replace "/" "&s" $ replace "&" "&a" key -{- An annexed file's content is stored somewhere under .git/annex/, - - based on the key. -} -annexLocation :: State -> Key -> FilePath -annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (keyFile key) - -{- The mapping from filename to its key is stored in the .git-annex directory, - - in a file named `key/$filename.$backend` -} -backendFile :: State -> Backend -> FilePath -> String -backendFile state backend file = - gitStateDir (repo state) ++ "key/" ++ - (gitRelative (repo state) file) ++ - "." ++ (name backend) +{- An annexed file's content is stored in + - .git/annex// ; this allows deriving the key and backend + - by looking at the symlink to it. -} +annexLocation :: State -> Backend -> Key -> FilePath +annexLocation state backend key = + gitDir (repo state) ++ "/annex/" ++ (name backend) ++ + "/" ++ (keyFile key) diff --git a/TODO b/TODO index 36ec65c2f5..392ec4990e 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,13 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? +* if the annexed files were in .git/annex//key, and + files in the repo symlink to that, the .git-annex/key/. + would be redundant, and not needed + + -- no separate merge problem with it + -- want to add an url? `ln -s .git/annex//http:%%kitenet.net%foo myfile` + * implement retrieval for backendfile * query remotes for their annex.uuid settings diff --git a/git-annex.mdwn b/git-annex.mdwn index 3a8db6dcfb..dd0b3bc07a 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -91,11 +91,9 @@ used to store the file contents, and git-annex would then retrieve them as needed and put them in `.git/annex/`. When a file is annexed, a key is generated from its content and/or metadata. -This key can later be used to retrieve the file's content (its value). This -key generation must be stable for a given file content, name, and size. - -The mapping from filename to its key is stored in the .git-annex directory, -in a file named `key/$filename.$backend` +The file checked into git symlinks to the key. This key can later be used +to retrieve the file's content (its value). This key generation must be +stable for a given file content, name, and size. Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. @@ -183,8 +181,8 @@ not be dropped right away, depending on number of copies available. ### branching -The use of `.git-annex` to store state means that if a repo has branches -and the user switched between them, git-annex will see different state in -the different branches. Whether that is a bug or a feature may depend on -point of view -- call it Too Be Determined. An alternative would be to -store data directly in the git repo as `pristine-tar` does. +The use of `.git-annex` to store logs means that if a repo has branches +and the user switched between them, git-annex will see different logs in +the different branches, and so may miss info about what remotes have which +files (though it can re-learn). An alternative would be to +store the log data directly in the git repo as `pristine-tar` does. From 14d7b2ac13318ba513bbab4f08b98434741f0e12 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 00:45:09 -0400 Subject: [PATCH 0085/2835] update --- Annex.hs | 6 +++--- demo.log | 11 ----------- git-annex.mdwn | 4 ---- 3 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 demo.log diff --git a/Annex.hs b/Annex.hs index 82efd543d4..e5eb1894f8 100644 --- a/Annex.hs +++ b/Annex.hs @@ -34,7 +34,7 @@ startAnnex :: IO State startAnnex = do r <- gitRepoFromCwd r' <- prepUUID r - gitPrep r' + gitSetup r' return State { repo = r', @@ -129,8 +129,8 @@ annexPullRepo :: State -> String -> IO () annexPullRepo state reponame = do error "not implemented" -- TODO {- Sets up a git repo for git-annex. May be called repeatedly. -} -gitPrep :: GitRepo -> IO () -gitPrep repo = do +gitSetup :: GitRepo -> IO () +gitSetup repo = do -- configure git to use union merge driver on state files exists <- doesFileExist attributes if (not exists) diff --git a/demo.log b/demo.log deleted file mode 100644 index bdecb7d401..0000000000 --- a/demo.log +++ /dev/null @@ -1,11 +0,0 @@ -1286654242s 1 repo -1286652724s 0 foo -1286656282s 1 foo -1286656282s 0 repo -1286656281s 0 foo -# some garbage, should be ignored -a a a - -a 1 a --1 a a -1286652724.0001s 1 foo diff --git a/git-annex.mdwn b/git-annex.mdwn index dd0b3bc07a..6852ed0080 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -79,10 +79,6 @@ only copies of a file. The `.git-annex` directory at the top of the repository is used to store git-annex information that should be propigated between repositories. -Data is stored here in files that are arranged to avoid conflicts in most -cases. A conflict could occur if a file with the same name but different -content was added to different repositories. - ## key/value storage git-annex uses a key/value abstraction layer to allow files contents to be From 67ae9d7fa109503e4b798e2b7703282b92ce3deb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 00:58:59 -0400 Subject: [PATCH 0086/2835] relative symlink to annexed file --- Annex.hs | 2 +- GitRepo.hs | 10 ++++------ Locations.hs | 9 +++++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Annex.hs b/Annex.hs index e5eb1894f8..1c369be926 100644 --- a/Annex.hs +++ b/Annex.hs @@ -59,7 +59,7 @@ annexFile state file = do let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) renameFile file dest - createSymbolicLink dest file + createSymbolicLink (annexLocationRelative state backend key) file gitRun (repo state) ["add", file] gitRun (repo state) ["commit", "-m", ("git-annex annexed " ++ file), file] diff --git a/GitRepo.hs b/GitRepo.hs index fcaae1253f..f0686ff203 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -98,14 +98,12 @@ gitAttributes repo = assertlocal repo $ do then (top repo) ++ "/info/.gitattributes" else (top repo) ++ "/.gitattributes" -{- Path to a repository's .git directory. - - (For a bare repository, that is the root of the repository.) - - TODO: support GIT_DIR -} +{- Path to a repository's .git directory, relative to its topdir. -} gitDir :: GitRepo -> String gitDir repo = assertlocal repo $ if (bare repo) - then top repo - else top repo ++ "/.git" + then "" + else ".git" {- Path to a repository's --work-tree. -} gitWorkTree :: GitRepo -> FilePath @@ -130,7 +128,7 @@ gitRelative repo file = drop (length absrepo) absfile gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + ["--git-dir="++(top repo)++"/"++(gitDir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} gitRun :: GitRepo -> [String] -> IO () diff --git a/Locations.hs b/Locations.hs index b859fd2f27..faf29235f6 100644 --- a/Locations.hs +++ b/Locations.hs @@ -5,7 +5,8 @@ module Locations ( gitStateDir, stateLoc, keyFile, - annexLocation + annexLocation, + annexLocationRelative ) where import Data.String.Utils @@ -28,6 +29,10 @@ keyFile key = replace "/" "&s" $ replace "&" "&a" key - .git/annex// ; this allows deriving the key and backend - by looking at the symlink to it. -} annexLocation :: State -> Backend -> Key -> FilePath -annexLocation state backend key = +annexLocation state backend key = + (gitWorkTree $ repo state) ++ "/" ++ + (annexLocationRelative state backend key) +annexLocationRelative :: State -> Backend -> Key -> FilePath +annexLocationRelative state backend key = gitDir (repo state) ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) From 16cd682290b065fee59575b077525d20713e4b4b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:04:06 -0400 Subject: [PATCH 0087/2835] better key to file mapping --- Locations.hs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Locations.hs b/Locations.hs index faf29235f6..6aba0ed1a2 100644 --- a/Locations.hs +++ b/Locations.hs @@ -20,10 +20,17 @@ gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" {- Converts a key into a filename fragment. - - Just escape "/" in the key name, to keep a flat - - tree of files and avoid issues with files ending with "/" etc. -} + - + - Escape "/" in the key name, to keep a flat tree of files and avoid + - issues with keys containing "/../" or ending with "/" etc. + - + - "/" is escaped to "%" because it's short and rarely used, and resembles + - a slash + - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping + - is one to one. + - -} keyFile :: Key -> FilePath -keyFile key = replace "/" "&s" $ replace "&" "&a" key +keyFile key = replace "/" "%" $ replace "%" "%s" $ replace "&" "&a" key {- An annexed file's content is stored in - .git/annex// ; this allows deriving the key and backend @@ -32,6 +39,7 @@ annexLocation :: State -> Backend -> Key -> FilePath annexLocation state backend key = (gitWorkTree $ repo state) ++ "/" ++ (annexLocationRelative state backend key) + annexLocationRelative :: State -> Backend -> Key -> FilePath annexLocationRelative state backend key = gitDir (repo state) ++ "/annex/" ++ (name backend) ++ From 3e65384f06400e06f78173d64b13da07c5d024d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:36:20 -0400 Subject: [PATCH 0088/2835] fix relative symlink 2 --- Annex.hs | 28 +++++++++++++++++++--------- Locations.hs | 3 ++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Annex.hs b/Annex.hs index 1c369be926..1ad9569f97 100644 --- a/Annex.hs +++ b/Annex.hs @@ -55,20 +55,30 @@ annexFile state file = do Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> setup key backend where - setup key backend = do - let dest = annexLocation state backend key - createDirectoryIfMissing True (parentDir dest) - renameFile file dest - createSymbolicLink (annexLocationRelative state backend key) file - gitRun (repo state) ["add", file] - gitRun (repo state) ["commit", "-m", - ("git-annex annexed " ++ file), file] - logStatus state key ValuePresent checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () + setup key backend = do + let dest = annexLocation state backend key + let reldest = annexLocationRelative state backend key + createDirectoryIfMissing True (parentDir dest) + renameFile file dest + createSymbolicLink ((linkTarget file) ++ reldest) file + gitRun (repo state) ["add", file] + gitRun (repo state) ["commit", "-m", + ("git-annex annexed " ++ file), file] + logStatus state key ValuePresent + linkTarget file = + -- relies on file being relative to the top of the + -- git repo; just replace each subdirectory with ".." + if (subdirs > 0) + then (join "/" $ take subdirs $ repeat "..") ++ "/" + else "" + where + subdirs = (length $ split "/" file) - 1 + {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () diff --git a/Locations.hs b/Locations.hs index 6aba0ed1a2..72f4c451fd 100644 --- a/Locations.hs +++ b/Locations.hs @@ -40,7 +40,8 @@ annexLocation state backend key = (gitWorkTree $ repo state) ++ "/" ++ (annexLocationRelative state backend key) +{- Annexed file's location relative to the gitWorkTree -} annexLocationRelative :: State -> Backend -> Key -> FilePath -annexLocationRelative state backend key = +annexLocationRelative state backend key = gitDir (repo state) ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) From 3a18b6d2ae95f8b536640f2438437d1d0a99082e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:42:39 -0400 Subject: [PATCH 0089/2835] bugfix --- Annex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index 1ad9569f97..dd6912d852 100644 --- a/Annex.hs +++ b/Annex.hs @@ -91,7 +91,7 @@ unannexFile state file = do case (mkey) of Nothing -> return () Just (key, backend) -> do - let src = annexLocation state backend key + let src = annexLocation state backend file removeFile file gitRun (repo state) ["rm", file] gitRun (repo state) ["commit", "-m", From 4ecebfb218e58fb85a8e4484af93b5178a8046e7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:43:24 -0400 Subject: [PATCH 0090/2835] bugfx --- Annex.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Annex.hs b/Annex.hs index dd6912d852..63cf0d2fb1 100644 --- a/Annex.hs +++ b/Annex.hs @@ -97,6 +97,9 @@ unannexFile state file = do gitRun (repo state) ["commit", "-m", ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) renameFile src file logStatus state key ValueMissing return () From 77d052af3c527b3ebe349329305d80c9c5a2bf36 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:49:21 -0400 Subject: [PATCH 0091/2835] fix parentDir to work for relative too --- Utility.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index dea53967fc..349dd9355f 100644 --- a/Utility.hs +++ b/Utility.hs @@ -34,7 +34,8 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s parentDir :: String -> String parentDir dir = if length dirs > 0 - then "/" ++ (join "/" $ take ((length dirs) - 1) dirs) + then absolute ++ (join "/" $ take ((length dirs) - 1) dirs) else "" where dirs = filter (\x -> length x > 0) $ split "/" dir + absolute = if ((dir !! 0) == '/') then "/" else "" From 3f2ce326fac36cef36a2ab9667d20d921ab91a6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:53:16 -0400 Subject: [PATCH 0092/2835] update --- TODO | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/TODO b/TODO index 392ec4990e..0bba79f102 100644 --- a/TODO +++ b/TODO @@ -1,12 +1,5 @@ -* bug when annexing files in a subdir of a git repo -* how to handle git mv file? - -* if the annexed files were in .git/annex//key, and - files in the repo symlink to that, the .git-annex/key/. - would be redundant, and not needed - - -- no separate merge problem with it - -- want to add an url? `ln -s .git/annex//http:%%kitenet.net%foo myfile` +* bug when annexing files while in a subdir of a git repo +* bug when specifying full path to files when annexing * implement retrieval for backendfile @@ -15,4 +8,6 @@ * hook up LocationLog * --push/--pull/--want/--drop -* finish BackendUrl and BackendChecksum +* how to handle git mv file? + +* finish BackendChecksum From 490a3a828cbb5a4046178b36fc0f9fe0696d0e9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:55:44 -0400 Subject: [PATCH 0093/2835] update --- TODO | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/TODO b/TODO index 0bba79f102..a0f7c8b5fa 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,10 @@ * bug when annexing files while in a subdir of a git repo -* bug when specifying full path to files when annexing +* bug when specifying absolute path to files when annexing * implement retrieval for backendfile * query remotes for their annex.uuid settings -* hook up LocationLog * --push/--pull/--want/--drop * how to handle git mv file? From d1071bd1fe879abb3ebb229f9347f7855a697b8c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 02:31:24 -0400 Subject: [PATCH 0094/2835] autobugfixing! Converted Key to a real data type and caught all the places where I used an unconverted filename as a key. Had to loose some sanity checks around whether something is already annexed, but I guess I can add those back other ways. --- Annex.hs | 21 ++++++++++++-------- Backend.hs | 54 ++++++++++++++++++++++++++++---------------------- BackendFile.hs | 5 +++-- BackendUrl.hs | 4 ++-- Locations.hs | 2 +- Types.hs | 6 +++++- 6 files changed, 54 insertions(+), 38 deletions(-) diff --git a/Annex.hs b/Annex.hs index 63cf0d2fb1..8a7b8d860c 100644 --- a/Annex.hs +++ b/Annex.hs @@ -45,7 +45,8 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend state file + -- TODO check if already annexed + let alreadyannexed = Nothing case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do @@ -83,15 +84,17 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend state file + -- TODO check if already annexed + let alreadyannexed = Just 1 case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile state file - case (mkey) of + key <- fileKey file + dropped <- dropFile state key + case (dropped) of Nothing -> return () Just (key, backend) -> do - let src = annexLocation state backend file + let src = annexLocation state backend key removeFile file gitRun (repo state) ["rm", file] gitRun (repo state) ["commit", "-m", @@ -107,18 +110,20 @@ unannexFile state file = do {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = do - alreadyannexed <- lookupBackend state file + -- TODO check if already annexed + let alreadyannexed = Just 1 case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file - Just backend -> do + Just _ -> do key <- fileKey file + backend <- fileBackend file inannex <- inAnnex state backend key if (inannex) then return () else do let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state file dest + success <- retrieveFile state key dest if (success) then do logStatus state key ValuePresent diff --git a/Backend.hs b/Backend.hs index 68d70feec7..dbb0064a51 100644 --- a/Backend.hs +++ b/Backend.hs @@ -16,15 +16,17 @@ module Backend ( lookupBackend, storeFile, + dropFile, retrieveFile, fileKey, - dropFile + fileBackend ) where import System.Directory import System.FilePath import Data.String.Utils import System.Posix.Files +import BackendList import Locations import GitRepo import Utility @@ -47,48 +49,52 @@ storeFile' (b:bs) state file = do where nextbackend = storeFile' bs state file -{- Attempts to retrieve an file from one of the backends, saving it to +{- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveFile :: State -> FilePath -> FilePath -> IO Bool -retrieveFile state file dest = do - result <- lookupBackend state file +retrieveFile :: State -> Key -> FilePath -> IO Bool +retrieveFile state key dest = do + result <- lookupBackend state key case (result) of Nothing -> return False - Just backend -> do - key <- fileKey file - (retrieveKeyFile backend) state key dest + Just backend -> (retrieveKeyFile backend) state key dest -{- Drops the key for a file from the backend that has it. -} -dropFile :: State -> FilePath -> IO (Maybe (Key, Backend)) -dropFile state file = do - result <- lookupBackend state file +{- Drops a key from the backend that has it. -} +dropFile :: State -> Key -> IO (Maybe (Key, Backend)) +dropFile state key = do + result <- lookupBackend state key case (result) of Nothing -> return Nothing Just backend -> do - key <- fileKey file (removeKey backend) state key return $ Just (key, backend) -{- Looks up the backend used for an already annexed file. -} -lookupBackend :: State -> FilePath -> IO (Maybe Backend) -lookupBackend state file = lookupBackend' (backends state) state file +{- Looks up the backend that has a key. -} +lookupBackend :: State -> Key -> IO (Maybe Backend) +lookupBackend state key = lookupBackend' (backends state) state key lookupBackend' [] _ _ = return Nothing -lookupBackend' (b:bs) state file = do - present <- checkBackend b state file +lookupBackend' (b:bs) state key = do + present <- checkBackend b state key if present then return $ Just b else - lookupBackend' bs state file + lookupBackend' bs state key -{- Checks if a file is available via a given backend. -} -checkBackend :: Backend -> State -> FilePath -> IO (Bool) -checkBackend backend state file = - doesFileExist $ annexLocation state backend file +{- Checks if a key is available via a given backend. -} +checkBackend :: Backend -> State -> Key -> IO (Bool) +checkBackend backend state key = + doesFileExist $ annexLocation state backend key {- Looks up the key corresponding to an annexed file, - by examining what the file symlinks to. -} fileKey :: FilePath -> IO Key fileKey file = do l <- readSymbolicLink (file) - return $ takeFileName $ l + return $ Key $ takeFileName $ l + +{- Looks up the backend corresponding to an annexed file, + - by examining what the file symlinks to. -} +fileBackend :: FilePath -> IO Backend +fileBackend file = do + l <- readSymbolicLink (file) + return $ lookupBackendName $ takeFileName $ parentDir $ l diff --git a/BackendFile.hs b/BackendFile.hs index 43ca2191ce..15b23536bf 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -15,12 +15,13 @@ backend = Backend { -- direct mapping from filename to key keyValue :: State -> FilePath -> IO (Maybe Key) -keyValue state file = return $ Just file +keyValue state file = return $ Just $ Key file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. So storing or removing a key is - - a no-op. -} + - a no-op. TODO until support is added for git annex --push otherrepo, + - then these could implement that.. -} dummyStore :: State -> FilePath -> Key -> IO (Bool) dummyStore state file key = return True dummyRemove :: State -> Key -> IO Bool diff --git a/BackendUrl.hs b/BackendUrl.hs index 3f0846885d..5b586497c4 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -27,8 +27,8 @@ dummyRemove state url = return False downloadUrl :: State -> Key -> FilePath -> IO Bool downloadUrl state url file = do - putStrLn $ "download: " ++ url - result <- try $ rawSystem "curl" ["-#", "-o", file, url] + putStrLn $ "download: " ++ (show url) + result <- try $ rawSystem "curl" ["-#", "-o", file, (show url)] case (result) of Left _ -> return False Right _ -> return True diff --git a/Locations.hs b/Locations.hs index 72f4c451fd..a99ad6ec40 100644 --- a/Locations.hs +++ b/Locations.hs @@ -30,7 +30,7 @@ gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" - is one to one. - -} keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "%s" $ replace "&" "&a" key +keyFile key = replace "/" "%" $ replace "%" "%s" $ replace "&" "&a" $ show key {- An annexed file's content is stored in - .git/annex// ; this allows deriving the key and backend diff --git a/Types.hs b/Types.hs index 73492dfc38..9b0bb00fd5 100644 --- a/Types.hs +++ b/Types.hs @@ -16,7 +16,11 @@ data State = State { } deriving (Show) -- annexed filenames are mapped into keys -type Key = FilePath +data Key = Key String deriving (Eq) + +-- show a key to convert it to a string +instance Show Key where + show (Key v) = v -- this structure represents a key/value backend data Backend = Backend { From 4b801b265afa94b1219a1abb6e52e08e0790582a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:20:05 -0400 Subject: [PATCH 0095/2835] error handling --- Annex.hs | 34 ++++++++++++--------------- Backend.hs | 67 ++++++++++++++++-------------------------------------- 2 files changed, 34 insertions(+), 67 deletions(-) diff --git a/Annex.hs b/Annex.hs index 8a7b8d860c..30ec0843a5 100644 --- a/Annex.hs +++ b/Annex.hs @@ -45,10 +45,9 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - -- TODO check if already annexed - let alreadyannexed = Nothing - case (alreadyannexed) of - Just _ -> error $ "already annexed: " ++ file + r <- lookupFile file + case (r) of + Just _ -> error $ "already annexed " ++ file Nothing -> do checkLegal file stored <- storeFile state file @@ -84,16 +83,14 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - -- TODO check if already annexed - let alreadyannexed = Just 1 - case (alreadyannexed) of + r <- lookupFile file + case (r) of Nothing -> error $ "not annexed " ++ file - Just _ -> do - key <- fileKey file - dropped <- dropFile state key - case (dropped) of - Nothing -> return () - Just (key, backend) -> do + Just (key, backend) -> do + dropped <- dropFile state backend key + if (not dropped) + then error $ "backend refused to drop " ++ file + else do let src = annexLocation state backend key removeFile file gitRun (repo state) ["rm", file] @@ -110,20 +107,17 @@ unannexFile state file = do {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = do - -- TODO check if already annexed - let alreadyannexed = Just 1 - case (alreadyannexed) of + r <- lookupFile file + case (r) of Nothing -> error $ "not annexed " ++ file - Just _ -> do - key <- fileKey file - backend <- fileBackend file + Just (key, backend) -> do inannex <- inAnnex state backend key if (inannex) then return () else do let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state key dest + success <- retrieveFile state backend key dest if (success) then do logStatus state key ValuePresent diff --git a/Backend.hs b/Backend.hs index dbb0064a51..2697f43d4a 100644 --- a/Backend.hs +++ b/Backend.hs @@ -14,14 +14,13 @@ - -} module Backend ( - lookupBackend, storeFile, dropFile, retrieveFile, - fileKey, - fileBackend + lookupFile ) where +import Control.Exception import System.Directory import System.FilePath import Data.String.Utils @@ -51,50 +50,24 @@ storeFile' (b:bs) state file = do {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveFile :: State -> Key -> FilePath -> IO Bool -retrieveFile state key dest = do - result <- lookupBackend state key - case (result) of - Nothing -> return False - Just backend -> (retrieveKeyFile backend) state key dest +retrieveFile :: State -> Backend -> Key -> FilePath -> IO Bool +retrieveFile state backend key dest = (retrieveKeyFile backend) state key dest -{- Drops a key from the backend that has it. -} -dropFile :: State -> Key -> IO (Maybe (Key, Backend)) -dropFile state key = do - result <- lookupBackend state key - case (result) of - Nothing -> return Nothing - Just backend -> do - (removeKey backend) state key - return $ Just (key, backend) +{- Drops a key from a backend. -} +dropFile :: State -> Backend -> Key -> IO Bool +dropFile state backend key = (removeKey backend) state key -{- Looks up the backend that has a key. -} -lookupBackend :: State -> Key -> IO (Maybe Backend) -lookupBackend state key = lookupBackend' (backends state) state key -lookupBackend' [] _ _ = return Nothing -lookupBackend' (b:bs) state key = do - present <- checkBackend b state key - if present - then - return $ Just b - else - lookupBackend' bs state key - -{- Checks if a key is available via a given backend. -} -checkBackend :: Backend -> State -> Key -> IO (Bool) -checkBackend backend state key = - doesFileExist $ annexLocation state backend key - -{- Looks up the key corresponding to an annexed file, +{- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} -fileKey :: FilePath -> IO Key -fileKey file = do - l <- readSymbolicLink (file) - return $ Key $ takeFileName $ l - -{- Looks up the backend corresponding to an annexed file, - - by examining what the file symlinks to. -} -fileBackend :: FilePath -> IO Backend -fileBackend file = do - l <- readSymbolicLink (file) - return $ lookupBackendName $ takeFileName $ parentDir $ l +lookupFile :: FilePath -> IO (Maybe (Key, Backend)) +lookupFile file = do + result <- try (lookup)::IO (Either SomeException (Maybe (Key, Backend))) + case (result) of + Left err -> return Nothing + Right succ -> return succ + where + lookup = do + l <- readSymbolicLink file + return $ Just (k l, b l) + k l = Key $ takeFileName $ l + b l = lookupBackendName $ takeFileName $ parentDir $ l From cc5cf0093ea1aacc4c5460dfdd4d35f2963687bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:30:51 -0400 Subject: [PATCH 0096/2835] cleanup --- Annex.hs | 64 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/Annex.hs b/Annex.hs index 30ec0843a5..7cee3c4cba 100644 --- a/Annex.hs +++ b/Annex.hs @@ -41,20 +41,24 @@ startAnnex = do backends = parseBackendList $ gitConfig r' "annex.backends" "" } +inBackend file yes no = do + r <- lookupFile file + case (r) of + Just v -> yes v + Nothing -> no +notinBackend file yes no = inBackend file no yes + {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () -annexFile state file = do - r <- lookupFile file - case (r) of - Just _ -> error $ "already annexed " ++ file - Nothing -> do - checkLegal file - stored <- storeFile state file - case (stored) of - Nothing -> error $ "no backend could store: " ++ file - Just (key, backend) -> setup key backend +annexFile state file = inBackend file err $ do + checkLegal file + stored <- storeFile state file + case (stored) of + Nothing -> error $ "no backend could store: " ++ file + Just (key, backend) -> setup key backend where + err = error $ "already annexed " ++ file checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -82,27 +86,25 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () -unannexFile state file = do - r <- lookupFile file - case (r) of - Nothing -> error $ "not annexed " ++ file - Just (key, backend) -> do - dropped <- dropFile state backend key - if (not dropped) - then error $ "backend refused to drop " ++ file - else do - let src = annexLocation state backend key - removeFile file - gitRun (repo state) ["rm", file] - gitRun (repo state) ["commit", "-m", - ("git-annex unannexed " ++ file), - file] - -- git rm deletes empty directories; - -- put them back - createDirectoryIfMissing True (parentDir file) - renameFile src file - logStatus state key ValueMissing - return () +unannexFile state file = notinBackend file err $ \(key, backend) -> do + dropped <- dropFile state backend key + if (not dropped) + then error $ "backend refused to drop " ++ file + else cleanup key backend + where + err = error $ "not annexed " ++ file + cleanup key backend = do + let src = annexLocation state backend key + removeFile file + gitRun (repo state) ["rm", file] + gitRun (repo state) ["commit", "-m", + ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) + renameFile src file + logStatus state key ValueMissing + return () {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () From 99b2029236248f6b4ce68e126b70fa0855fac37f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:41:12 -0400 Subject: [PATCH 0097/2835] key conversion back from file bugfixes --- Annex.hs | 6 ++---- Backend.hs | 2 +- Locations.hs | 31 ++++++++++++++++++------------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Annex.hs b/Annex.hs index 7cee3c4cba..936e62502d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -87,10 +87,8 @@ annexFile state file = inBackend file err $ do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = notinBackend file err $ \(key, backend) -> do - dropped <- dropFile state backend key - if (not dropped) - then error $ "backend refused to drop " ++ file - else cleanup key backend + dropFile state backend key + cleanup key backend where err = error $ "not annexed " ++ file cleanup key backend = do diff --git a/Backend.hs b/Backend.hs index 2697f43d4a..bc7eb206ff 100644 --- a/Backend.hs +++ b/Backend.hs @@ -69,5 +69,5 @@ lookupFile file = do lookup = do l <- readSymbolicLink file return $ Just (k l, b l) - k l = Key $ takeFileName $ l + k l = fileKey $ takeFileName $ l b l = lookupBackendName $ takeFileName $ parentDir $ l diff --git a/Locations.hs b/Locations.hs index a99ad6ec40..304ca060e9 100644 --- a/Locations.hs +++ b/Locations.hs @@ -5,6 +5,7 @@ module Locations ( gitStateDir, stateLoc, keyFile, + fileKey, annexLocation, annexLocationRelative ) where @@ -19,19 +20,6 @@ stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" -{- Converts a key into a filename fragment. - - - - Escape "/" in the key name, to keep a flat tree of files and avoid - - issues with keys containing "/../" or ending with "/" etc. - - - - "/" is escaped to "%" because it's short and rarely used, and resembles - - a slash - - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping - - is one to one. - - -} -keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "%s" $ replace "&" "&a" $ show key - {- An annexed file's content is stored in - .git/annex// ; this allows deriving the key and backend - by looking at the symlink to it. -} @@ -45,3 +33,20 @@ annexLocationRelative :: State -> Backend -> Key -> FilePath annexLocationRelative state backend key = gitDir (repo state) ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) + +{- Converts a key into a filename fragment. + - + - Escape "/" in the key name, to keep a flat tree of files and avoid + - issues with keys containing "/../" or ending with "/" etc. + - + - "/" is escaped to "%" because it's short and rarely used, and resembles + - a slash + - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping + - is one to one. + - -} +keyFile :: Key -> FilePath +keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key + +{- Reverses keyFile -} +fileKey :: FilePath -> Key +fileKey file = Key $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file From ff998a9a677ba4a4af9e4bd45a651653421760cb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:46:40 -0400 Subject: [PATCH 0098/2835] cleanup2 --- Annex.hs | 58 ++++++++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/Annex.hs b/Annex.hs index 936e62502d..eb57e2d57a 100644 --- a/Annex.hs +++ b/Annex.hs @@ -65,6 +65,7 @@ annexFile state file = inBackend file err $ do then error $ "not a regular file: " ++ file else return () setup key backend = do + logStatus state key ValuePresent let dest = annexLocation state backend key let reldest = annexLocationRelative state backend key createDirectoryIfMissing True (parentDir dest) @@ -73,7 +74,6 @@ annexFile state file = inBackend file err $ do gitRun (repo state) ["add", file] gitRun (repo state) ["commit", "-m", ("git-annex annexed " ++ file), file] - logStatus state key ValuePresent linkTarget file = -- relies on file being relative to the top of the -- git repo; just replace each subdirectory with ".." @@ -88,41 +88,37 @@ annexFile state file = inBackend file err $ do unannexFile :: State -> FilePath -> IO () unannexFile state file = notinBackend file err $ \(key, backend) -> do dropFile state backend key - cleanup key backend + logStatus state key ValueMissing + let src = annexLocation state backend key + removeFile file + gitRun (repo state) ["rm", file] + gitRun (repo state) ["commit", "-m", + ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) + renameFile src file + return () where err = error $ "not annexed " ++ file - cleanup key backend = do - let src = annexLocation state backend key - removeFile file - gitRun (repo state) ["rm", file] - gitRun (repo state) ["commit", "-m", - ("git-annex unannexed " ++ file), file] - -- git rm deletes empty directories; - -- put them back - createDirectoryIfMissing True (parentDir file) - renameFile src file - logStatus state key ValueMissing - return () {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () -annexGetFile state file = do - r <- lookupFile file - case (r) of - Nothing -> error $ "not annexed " ++ file - Just (key, backend) -> do - inannex <- inAnnex state backend key - if (inannex) - then return () - else do - let dest = annexLocation state backend key - createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state backend key dest - if (success) - then do - logStatus state key ValuePresent - return () - else error $ "failed to get " ++ file +annexGetFile state file = notinBackend file err $ \(key, backend) -> do + inannex <- inAnnex state backend key + if (inannex) + then return () + else do + let dest = annexLocation state backend key + createDirectoryIfMissing True (parentDir dest) + success <- retrieveFile state backend key dest + if (success) + then do + logStatus state key ValuePresent + return () + else error $ "failed to get " ++ file + where + err = error $ "not annexed " ++ file {- Indicates a file is wanted. -} annexWantFile :: State -> FilePath -> IO () From 1dbf36bf9a951b7a92770ea0b57bc79c8b465795 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:51:55 -0400 Subject: [PATCH 0099/2835] cleanup3 --- Annex.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Annex.hs b/Annex.hs index eb57e2d57a..012e901997 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,10 +24,6 @@ import UUID import LocationLog import Types -{- Checks if a given key is currently present in the annexLocation -} -inAnnex :: State -> Backend -> Key -> IO Bool -inAnnex state backend key = doesFileExist $ annexLocation state backend key - {- On startup, examine the git repo, prepare it, and record state for - later. -} startAnnex :: IO State @@ -89,7 +85,6 @@ unannexFile :: State -> FilePath -> IO () unannexFile state file = notinBackend file err $ \(key, backend) -> do dropFile state backend key logStatus state key ValueMissing - let src = annexLocation state backend key removeFile file gitRun (repo state) ["rm", file] gitRun (repo state) ["commit", "-m", @@ -97,12 +92,13 @@ unannexFile state file = notinBackend file err $ \(key, backend) -> do -- git rm deletes empty directories; -- put them back createDirectoryIfMissing True (parentDir file) + let src = annexLocation state backend key renameFile src file return () where err = error $ "not annexed " ++ file -{- Transfers the file from a remote. -} +{- Gets an annexed file from one of the backends. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = notinBackend file err $ \(key, backend) -> do inannex <- inAnnex state backend key @@ -165,3 +161,7 @@ logStatus state key status = do f <- logChange (repo state) key (getUUID (repo state)) status gitRun (repo state) ["add", f] gitRun (repo state) ["commit", "-m", "git-annex log update", f] + +{- Checks if a given key is currently present in the annexLocation -} +inAnnex :: State -> Backend -> Key -> IO Bool +inAnnex state backend key = doesFileExist $ annexLocation state backend key From 794d44cf1daf073f05d1a27b2a02c47db37c443a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 14:01:17 -0400 Subject: [PATCH 0100/2835] add remoteName --- GitRepo.hs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index f0686ff203..489c9cf758 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -40,11 +40,14 @@ import Utility data GitRepo = LocalGitRepo { top :: FilePath, - config :: Map String String + config :: Map String String, + -- remoteName holds the name used for this repo in remotes + remoteName :: Maybe String } | RemoteGitRepo { url :: String, top :: FilePath, - config :: Map String String + config :: Map String String, + remoteName :: Maybe String } deriving (Show, Read, Eq) {- Local GitRepo constructor. Can optionally query the repo for its config. -} @@ -52,7 +55,8 @@ gitRepoFromPath :: FilePath -> Bool -> IO GitRepo gitRepoFromPath dir query = do let r = LocalGitRepo { top = dir, - config = Map.empty + config = Map.empty, + remoteName = Nothing } if (query) then gitConfigRead r @@ -64,7 +68,8 @@ gitRepoFromUrl url query = do return $ RemoteGitRepo { url = url, top = path url, - config = Map.empty + config = Map.empty, + remoteName = Nothing } where path url = uriPath $ fromJust $ parseURI url @@ -174,13 +179,15 @@ gitConfig repo key defaultValue = gitConfigRemotes :: GitRepo -> IO [GitRepo] gitConfigRemotes repo = mapM construct remotes where - remotes = elems $ filter $ config repo + remotes = toList $ filter $ config repo filter = filterWithKey (\k _ -> isremote k) isremote k = (startswith "remote." k) && (endswith ".url" k) - construct r = - if (isURI r) - then gitRepoFromUrl r False - else gitRepoFromPath r False + remotename k = (split "." k) !! 1 + construct (k,v) = do + r <- if (isURI v) + then gitRepoFromUrl v False + else gitRepoFromPath v False + return r { remoteName = Just $ remotename k } {- Finds the current git repository, which may be in a parent directory. -} gitRepoFromCwd :: IO GitRepo From 771a6b36e1527571b9a38baacbee6e864f44172a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 14:40:56 -0400 Subject: [PATCH 0101/2835] cost ordering --- Annex.hs | 25 +++++++++++++++++++++++++ GitRepo.hs | 21 ++++++++++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/Annex.hs b/Annex.hs index 012e901997..bd57514ead 100644 --- a/Annex.hs +++ b/Annex.hs @@ -9,12 +9,14 @@ module Annex ( annexWantFile, annexDropFile, annexPushRepo, + repoCost, annexPullRepo ) where import System.Posix.Files import System.Directory import Data.String.Utils +import List import GitRepo import Utility import Locations @@ -165,3 +167,26 @@ logStatus state key status = do {- Checks if a given key is currently present in the annexLocation -} inAnnex :: State -> Backend -> Key -> IO Bool inAnnex state backend key = doesFileExist $ annexLocation state backend key + +{- Orders a list of git repos by cost. -} +reposByCost :: State -> [GitRepo] -> [GitRepo] +reposByCost state l = + fst $ unzip $ sortBy (\(r1, c1) (r2, c2) -> compare c1 c2) $ costpairs l + where + costpairs l = map (\r -> (r, repoCost state r)) l + +{- Calculates cost for a repo. + - + - The default cost is 100 for local repositories, and 200 for remote + - repositories; it can also be configured by remote..annex-cost + -} +repoCost :: State -> GitRepo -> Int +repoCost state r = + if ((length $ config state r) > 0) + then read $ config state r + else if (gitRepoIsLocal r) + then 100 + else 200 + where + config state r = gitConfig (repo state) (configkey r) "" + configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" diff --git a/GitRepo.hs b/GitRepo.hs index 489c9cf758..c4a55863d2 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -10,13 +10,17 @@ module GitRepo ( gitRepoFromCwd, gitRepoFromPath, gitRepoFromUrl, + gitRepoIsLocal, + gitRepoIsRemote, + gitConfigRemotes, gitWorkTree, gitDir, gitRelative, gitConfig, gitConfigRead, gitRun, - gitAttributes + gitAttributes, + gitRepoRemoteName ) where import Directory @@ -74,16 +78,23 @@ gitRepoFromUrl url query = do where path url = uriPath $ fromJust $ parseURI url {- User-visible description of a git repo by path or url -} -describe repo = if (local repo) then top repo else url repo +describe repo = if (gitRepoIsLocal repo) then top repo else url repo + +{- Returns the name of the remote that corresponds to the repo, if + - it is a remote. Otherwise, "" -} +gitRepoRemoteName r = + if (isJust $ remoteName r) + then fromJust $ remoteName r + else "" {- Some code needs to vary between remote and local repos, or bare and - non-bare, these functions help with that. -} -local repo = case (repo) of +gitRepoIsLocal repo = case (repo) of LocalGitRepo {} -> True RemoteGitRepo {} -> False -remote repo = not $ local repo +gitRepoIsRemote repo = not $ gitRepoIsLocal repo assertlocal repo action = - if (local repo) + if (gitRepoIsLocal repo) then action else error $ "acting on remote git repo " ++ (describe repo) ++ " not supported" From 77055f5ff82d2712f599ba77e03d5d2cc022ff65 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 14:51:09 -0400 Subject: [PATCH 0102/2835] move some stuff out of IO --- Annex.hs | 11 ++++++++--- GitRepo.hs | 32 ++++++++++++++------------------ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/Annex.hs b/Annex.hs index bd57514ead..e06bd84bc2 100644 --- a/Annex.hs +++ b/Annex.hs @@ -9,7 +9,7 @@ module Annex ( annexWantFile, annexDropFile, annexPushRepo, - repoCost, + annexRemotes, annexPullRepo ) where @@ -31,8 +31,9 @@ import Types startAnnex :: IO State startAnnex = do r <- gitRepoFromCwd - r' <- prepUUID r - gitSetup r' + r' <- gitConfigRead r + r'' <- prepUUID r' + gitSetup r'' return State { repo = r', @@ -168,6 +169,10 @@ logStatus state key status = do inAnnex :: State -> Backend -> Key -> IO Bool inAnnex state backend key = doesFileExist $ annexLocation state backend key +{- Ordered list of remotes for the annex. -} +annexRemotes :: State -> [GitRepo] +annexRemotes state = reposByCost state $ gitConfigRemotes (repo state) + {- Orders a list of git repos by cost. -} reposByCost :: State -> [GitRepo] -> [GitRepo] reposByCost state l = diff --git a/GitRepo.hs b/GitRepo.hs index c4a55863d2..06e244d6bc 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -54,22 +54,19 @@ data GitRepo = remoteName :: Maybe String } deriving (Show, Read, Eq) -{- Local GitRepo constructor. Can optionally query the repo for its config. -} -gitRepoFromPath :: FilePath -> Bool -> IO GitRepo -gitRepoFromPath dir query = do - let r = LocalGitRepo { +{- Local GitRepo constructor. -} +gitRepoFromPath :: FilePath -> GitRepo +gitRepoFromPath dir = + LocalGitRepo { top = dir, config = Map.empty, remoteName = Nothing } - if (query) - then gitConfigRead r - else return r {- Remote GitRepo constructor. Throws exception on invalid url. -} -gitRepoFromUrl :: String -> Bool -> IO GitRepo -gitRepoFromUrl url query = do - return $ RemoteGitRepo { +gitRepoFromUrl :: String -> GitRepo +gitRepoFromUrl url = + RemoteGitRepo { url = url, top = path url, config = Map.empty, @@ -187,18 +184,17 @@ gitConfig repo key defaultValue = Map.findWithDefault defaultValue key (config repo) {- Returns a list of a repo's configured remotes. -} -gitConfigRemotes :: GitRepo -> IO [GitRepo] -gitConfigRemotes repo = mapM construct remotes +gitConfigRemotes :: GitRepo -> [GitRepo] +gitConfigRemotes repo = map construct remotes where remotes = toList $ filter $ config repo filter = filterWithKey (\k _ -> isremote k) isremote k = (startswith "remote." k) && (endswith ".url" k) remotename k = (split "." k) !! 1 - construct (k,v) = do - r <- if (isURI v) - then gitRepoFromUrl v False - else gitRepoFromPath v False - return r { remoteName = Just $ remotename k } + construct (k,v) = (gen v) { remoteName = Just $ remotename k } + gen v = if (isURI v) + then gitRepoFromUrl v + else gitRepoFromPath v {- Finds the current git repository, which may be in a parent directory. -} gitRepoFromCwd :: IO GitRepo @@ -206,7 +202,7 @@ gitRepoFromCwd = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> gitRepoFromPath dir True + (Just dir) -> return $ gitRepoFromPath dir Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) From e28ff5bdaf7ce56c0c928904ff883c1e2cd093de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 15:55:18 -0400 Subject: [PATCH 0103/2835] almost able to get files from remotes now! --- Annex.hs | 30 +----------------------------- BackendFile.hs | 36 +++++++++++++++++++++++++++--------- GitRepo.hs | 14 ++++++++++---- LocationLog.hs | 3 ++- Remotes.hs | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ UUID.hs | 31 ++++++++++++++++++++++++++----- 6 files changed, 116 insertions(+), 48 deletions(-) create mode 100644 Remotes.hs diff --git a/Annex.hs b/Annex.hs index e06bd84bc2..834c26115b 100644 --- a/Annex.hs +++ b/Annex.hs @@ -9,7 +9,6 @@ module Annex ( annexWantFile, annexDropFile, annexPushRepo, - annexRemotes, annexPullRepo ) where @@ -161,37 +160,10 @@ gitSetup repo = do {- Updates the LocationLog when a key's presence changes. -} logStatus state key status = do - f <- logChange (repo state) key (getUUID (repo state)) status + f <- logChange (repo state) key (getUUID state (repo state)) status gitRun (repo state) ["add", f] gitRun (repo state) ["commit", "-m", "git-annex log update", f] {- Checks if a given key is currently present in the annexLocation -} inAnnex :: State -> Backend -> Key -> IO Bool inAnnex state backend key = doesFileExist $ annexLocation state backend key - -{- Ordered list of remotes for the annex. -} -annexRemotes :: State -> [GitRepo] -annexRemotes state = reposByCost state $ gitConfigRemotes (repo state) - -{- Orders a list of git repos by cost. -} -reposByCost :: State -> [GitRepo] -> [GitRepo] -reposByCost state l = - fst $ unzip $ sortBy (\(r1, c1) (r2, c2) -> compare c1 c2) $ costpairs l - where - costpairs l = map (\r -> (r, repoCost state r)) l - -{- Calculates cost for a repo. - - - - The default cost is 100 for local repositories, and 200 for remote - - repositories; it can also be configured by remote..annex-cost - -} -repoCost :: State -> GitRepo -> Int -repoCost state r = - if ((length $ config state r) > 0) - then read $ config state r - else if (gitRepoIsLocal r) - then 100 - else 200 - where - config state r = gitConfig (repo state) (configkey r) "" - configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" diff --git a/BackendFile.hs b/BackendFile.hs index 15b23536bf..d4d137e53b 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -4,12 +4,16 @@ module BackendFile (backend) where import Types +import LocationLog +import Locations +import Remotes +import GitRepo backend = Backend { name = "file", getKey = keyValue, storeFileKey = dummyStore, - retrieveKeyFile = copyFromOtherRepo, + retrieveKeyFile = copyKeyFile, removeKey = dummyRemove } @@ -27,12 +31,26 @@ dummyStore state file key = return True dummyRemove :: State -> Key -> IO Bool dummyRemove state url = return False -{- Try to find a copy of the file in one of the other repos, +{- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} -copyFromOtherRepo :: State -> Key -> FilePath -> IO (Bool) -copyFromOtherRepo state key file = - -- 1. get ordered list of remotes (local repos, then remote repos) - -- 2. read locationlog for file - -- 3. filter remotes list to ones that have file - -- 4. attempt to transfer from each remote until success - error "copyFromOtherRepo unimplemented" -- TODO +copyKeyFile :: State -> Key -> FilePath -> IO (Bool) +copyKeyFile state key file = do + remotes <- remotesWithKey state key + if (0 == length remotes) + then error $ "no known remotes have: " ++ (keyFile key) ++ "\n" ++ + "(Perhaps you need to git remote add a repository?)" + else trycopy remotes remotes + where + trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ + "To get that file, need access to one of these remotes: " ++ + (remotesList full) + trycopy full (r:rs) = do + ok <- copyFromRemote r key file + if (ok) + then return True + else trycopy full rs + +{- Tries to copy a file from a remote. -} +copyFromRemote :: GitRepo -> Key -> FilePath -> IO (Bool) +copyFromRemote r key file = do + return False -- TODO diff --git a/GitRepo.hs b/GitRepo.hs index 06e244d6bc..e1f086b693 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -13,6 +13,7 @@ module GitRepo ( gitRepoIsLocal, gitRepoIsRemote, gitConfigRemotes, + gitRepoDescribe, gitWorkTree, gitDir, gitRelative, @@ -74,8 +75,13 @@ gitRepoFromUrl url = } where path url = uriPath $ fromJust $ parseURI url -{- User-visible description of a git repo by path or url -} -describe repo = if (gitRepoIsLocal repo) then top repo else url repo +{- User-visible description of a git repo. -} +gitRepoDescribe repo = + if (isJust $ remoteName repo) + then fromJust $ remoteName repo + else if (gitRepoIsLocal repo) + then top repo + else url repo {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} @@ -93,13 +99,13 @@ gitRepoIsRemote repo = not $ gitRepoIsLocal repo assertlocal repo action = if (gitRepoIsLocal repo) then action - else error $ "acting on remote git repo " ++ (describe repo) ++ + else error $ "acting on remote git repo " ++ (gitRepoDescribe repo) ++ " not supported" bare :: GitRepo -> Bool bare repo = if (member b (config repo)) then ("true" == fromJust (Map.lookup b (config repo))) - else error $ "it is not known if git repo " ++ (describe repo) ++ + else error $ "it is not known if git repo " ++ (gitRepoDescribe repo) ++ " is a bare repository; config not read" where b = "core.bare" diff --git a/LocationLog.hs b/LocationLog.hs index 2eab4815ed..28ac46b90c 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -18,7 +18,8 @@ module LocationLog ( LogStatus(..), - logChange + logChange, + keyLocations ) where import Data.Time.Clock.POSIX diff --git a/Remotes.hs b/Remotes.hs new file mode 100644 index 0000000000..ae709a3c23 --- /dev/null +++ b/Remotes.hs @@ -0,0 +1,50 @@ +{- git-annex remote repositories -} + +module Remotes ( + remotesList, + remotesWithKey +) where + +import Types +import GitRepo +import LocationLog +import Data.String.Utils +import UUID +import List + +{- Human visible list of remotes. -} +remotesList :: [GitRepo] -> String +remotesList remotes = join " " $ map gitRepoDescribe remotes + +{- Cost ordered list of remotes that the LocationLog indicate may have a key. -} +remotesWithKey :: State -> Key -> IO [GitRepo] +remotesWithKey state key = do + uuids <- keyLocations (repo state) key + return $ reposByUUID state (remotesByCost state) uuids + +{- Cost Ordered list of remotes. -} +remotesByCost :: State -> [GitRepo] +remotesByCost state = reposByCost state $ gitConfigRemotes (repo state) + +{- Orders a list of git repos by cost. -} +reposByCost :: State -> [GitRepo] -> [GitRepo] +reposByCost state l = + fst $ unzip $ sortBy (\(r1, c1) (r2, c2) -> compare c1 c2) $ costpairs l + where + costpairs l = map (\r -> (r, repoCost state r)) l + +{- Calculates cost for a repo. + - + - The default cost is 100 for local repositories, and 200 for remote + - repositories; it can also be configured by remote..annex-cost + -} +repoCost :: State -> GitRepo -> Int +repoCost state r = + if ((length $ config state r) > 0) + then read $ config state r + else if (gitRepoIsLocal r) + then 100 + else 200 + where + config state r = gitConfig (repo state) (configkey r) "" + configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" diff --git a/UUID.hs b/UUID.hs index e2b624d69c..b4c4c0cc0b 100644 --- a/UUID.hs +++ b/UUID.hs @@ -9,12 +9,16 @@ module UUID ( UUID, getUUID, prepUUID, - genUUID + genUUID, + reposByUUID ) where +import Maybe +import List import System.Cmd.Utils import System.IO import GitRepo +import Types type UUID = String @@ -26,17 +30,34 @@ genUUID :: IO UUID genUUID = do pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h -{- Looks up a repo's UUID -} -getUUID :: GitRepo -> UUID -getUUID repo = gitConfig repo "annex.uuid" "" +{- Looks up a repo's UUID. May return "" if none is known. + - + - UUIDs of remotes are cached in git config, using keys named + - remote..annex-uuid + - + - -} +getUUID :: State -> GitRepo -> UUID +getUUID s r = + if ("" /= getUUID' r) + then getUUID' r + else cached s r + where + cached s r = gitConfig (repo s) (configkey r) "" + configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-uuid" +getUUID' r = gitConfig r "annex.uuid" "" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: GitRepo -> IO GitRepo prepUUID repo = - if ("" == getUUID repo) + if ("" == getUUID' repo) then do uuid <- genUUID gitRun repo ["config", configkey, uuid] -- return new repo with updated config gitConfigRead repo else return repo + +{- Filters a list of repos to ones that have listed UUIDs. -} +reposByUUID :: State -> [GitRepo] -> [UUID] -> [GitRepo] +reposByUUID state repos uuids = + filter (\r -> isJust $ elemIndex (getUUID state r) uuids) repos From f87c5ed9496f50646d9f5e8be540f8bc059db242 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 16:21:50 -0400 Subject: [PATCH 0104/2835] copying almost working --- Annex.hs | 10 +++++----- BackendFile.hs | 35 ++++++++++++++++++++++++++++------- Locations.hs | 20 ++++++++++---------- git-annex.hs | 3 +-- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/Annex.hs b/Annex.hs index 834c26115b..8489c2ca6b 100644 --- a/Annex.hs +++ b/Annex.hs @@ -64,8 +64,8 @@ annexFile state file = inBackend file err $ do else return () setup key backend = do logStatus state key ValuePresent - let dest = annexLocation state backend key - let reldest = annexLocationRelative state backend key + let dest = annexLocation (repo state) backend key + let reldest = annexLocationRelative (repo state) backend key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink ((linkTarget file) ++ reldest) file @@ -94,7 +94,7 @@ unannexFile state file = notinBackend file err $ \(key, backend) -> do -- git rm deletes empty directories; -- put them back createDirectoryIfMissing True (parentDir file) - let src = annexLocation state backend key + let src = annexLocation (repo state) backend key renameFile src file return () where @@ -107,7 +107,7 @@ annexGetFile state file = notinBackend file err $ \(key, backend) -> do if (inannex) then return () else do - let dest = annexLocation state backend key + let dest = annexLocation (repo state) backend key createDirectoryIfMissing True (parentDir dest) success <- retrieveFile state backend key dest if (success) @@ -166,4 +166,4 @@ logStatus state key status = do {- Checks if a given key is currently present in the annexLocation -} inAnnex :: State -> Backend -> Key -> IO Bool -inAnnex state backend key = doesFileExist $ annexLocation state backend key +inAnnex state backend key = doesFileExist $ annexLocation (repo state) backend key diff --git a/BackendFile.hs b/BackendFile.hs index d4d137e53b..adb8da8bd7 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,6 +3,9 @@ module BackendFile (backend) where +import System.IO +import System.Cmd +import Control.Exception import Types import LocationLog import Locations @@ -45,12 +48,30 @@ copyKeyFile state key file = do "To get that file, need access to one of these remotes: " ++ (remotesList full) trycopy full (r:rs) = do - ok <- copyFromRemote r key file - if (ok) - then return True - else trycopy full rs + putStrLn "trying a remote" + result <- try (copyFromRemote r key file)::IO (Either SomeException ()) + case (result) of + Left err -> do + showerr err r + trycopy full rs + Right succ -> return True + showerr err r = do + hPutStrLn stderr $ "git-annex: copy from " ++ + (gitRepoDescribe r ) ++ " failed: " ++ + (show err) -{- Tries to copy a file from a remote. -} -copyFromRemote :: GitRepo -> Key -> FilePath -> IO (Bool) +{- Tries to copy a file from a remote, exception on error. -} +copyFromRemote :: GitRepo -> Key -> FilePath -> IO () copyFromRemote r key file = do - return False -- TODO + r <- if (gitRepoIsLocal r) + then getlocal + else getremote + return () + where + getlocal = do + putStrLn $ "get: " ++ location + rawSystem "cp" ["-a", location, file] + getremote = do + putStrLn $ "get: " ++ location + error "get via network not yet implemented!" + location = annexLocation r backend key diff --git a/Locations.hs b/Locations.hs index 304ca060e9..d6d7d42480 100644 --- a/Locations.hs +++ b/Locations.hs @@ -21,18 +21,18 @@ gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" {- An annexed file's content is stored in - - .git/annex// ; this allows deriving the key and backend - - by looking at the symlink to it. -} -annexLocation :: State -> Backend -> Key -> FilePath -annexLocation state backend key = - (gitWorkTree $ repo state) ++ "/" ++ - (annexLocationRelative state backend key) + - /path/to/repo/.git/annex// + - + - (That allows deriving the key and backend by looking at the symlink to it.) + -} +annexLocation :: GitRepo -> Backend -> Key -> FilePath +annexLocation r backend key = + (gitWorkTree r) ++ "/" ++ (annexLocationRelative r backend key) {- Annexed file's location relative to the gitWorkTree -} -annexLocationRelative :: State -> Backend -> Key -> FilePath -annexLocationRelative state backend key = - gitDir (repo state) ++ "/annex/" ++ (name backend) ++ - "/" ++ (keyFile key) +annexLocationRelative :: GitRepo -> Backend -> Key -> FilePath +annexLocationRelative r backend key = + gitDir r ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) {- Converts a key into a filename fragment. - diff --git a/git-annex.hs b/git-annex.hs index 7bcd4de226..7785e4f2d7 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -32,6 +32,5 @@ tryRun errnum oknum (a:as) = do {- Exception pretty-printing. -} showErr :: SomeException -> IO () showErr e = do - let err = show e - hPutStrLn stderr $ "git-annex: " ++ err + hPutStrLn stderr $ "git-annex: " ++ (show e) return () From e5c1db355f5fa31af14ed8474aee89872b934f1a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 16:32:16 -0400 Subject: [PATCH 0105/2835] it works!! --- BackendFile.hs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/BackendFile.hs b/BackendFile.hs index adb8da8bd7..a31cbfeb1e 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -48,30 +48,31 @@ copyKeyFile state key file = do "To get that file, need access to one of these remotes: " ++ (remotesList full) trycopy full (r:rs) = do - putStrLn "trying a remote" result <- try (copyFromRemote r key file)::IO (Either SomeException ()) case (result) of Left err -> do - showerr err r + hPutStrLn stderr (show err) trycopy full rs Right succ -> return True - showerr err r = do - hPutStrLn stderr $ "git-annex: copy from " ++ - (gitRepoDescribe r ) ++ " failed: " ++ - (show err) {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: GitRepo -> Key -> FilePath -> IO () copyFromRemote r key file = do - r <- if (gitRepoIsLocal r) - then getlocal - else getremote + putStrLn $ "copy from " ++ (gitRepoDescribe r ) ++ " " ++ file + + -- annexLocation needs the git config read for the remote first. + -- FIXME: Having this here means git-config is run repeatedly when + -- copying a series of files; need to use state monad to avoid + -- this. + r' <- gitConfigRead r + + _ <- if (gitRepoIsLocal r') + then getlocal r' + else getremote r' return () where - getlocal = do - putStrLn $ "get: " ++ location - rawSystem "cp" ["-a", location, file] - getremote = do - putStrLn $ "get: " ++ location + getlocal r = do + rawSystem "cp" ["-a", location r, file] + getremote r = do error "get via network not yet implemented!" - location = annexLocation r backend key + location r = annexLocation r backend key From b1607485168e851f69fe3a5b74d73f3c36edf886 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 21:28:47 -0400 Subject: [PATCH 0106/2835] use a state monad enormous reworking --- Annex.hs | 138 ++++++++++++++++++++++++++------------------- Backend.hs | 25 ++++---- BackendChecksum.hs | 2 +- BackendFile.hs | 21 +++---- BackendUrl.hs | 21 +++---- CmdLine.hs | 21 +++---- Remotes.hs | 44 +++++++++------ TODO | 4 +- Types.hs | 51 +++++++++++++++-- UUID.hs | 50 +++++++++------- git-annex.hs | 31 ++++++---- 11 files changed, 251 insertions(+), 157 deletions(-) diff --git a/Annex.hs b/Annex.hs index 8489c2ca6b..f3c8f533a4 100644 --- a/Annex.hs +++ b/Annex.hs @@ -12,6 +12,7 @@ module Annex ( annexPullRepo ) where +import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory import Data.String.Utils @@ -25,22 +26,27 @@ import UUID import LocationLog import Types -{- On startup, examine the git repo, prepare it, and record state for - - later. -} -startAnnex :: IO State +{- Create and returns an Annex state object. + - Examines and prepares the git repo. + -} +startAnnex :: IO AnnexState startAnnex = do - r <- gitRepoFromCwd - r' <- gitConfigRead r - r'' <- prepUUID r' - gitSetup r'' - - return State { - repo = r', - backends = parseBackendList $ gitConfig r' "annex.backends" "" - } + g <- gitRepoFromCwd + let s = makeAnnexState g + (_,s') <- runAnnexState s (prep g) + return s' + where + prep g = do + -- setup git and read its config; update state + liftIO $ gitSetup g + g' <- liftIO $ gitConfigRead g + gitAnnexChange g' + backendsAnnexChange $ parseBackendList $ + gitConfig g' "annex.backends" "" + prepUUID inBackend file yes no = do - r <- lookupFile file + r <- liftIO $ lookupFile file case (r) of Just v -> yes v Nothing -> no @@ -48,13 +54,16 @@ notinBackend file yes no = inBackend file no yes {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} -annexFile :: State -> FilePath -> IO () -annexFile state file = inBackend file err $ do - checkLegal file - stored <- storeFile state file +annexFile :: FilePath -> Annex () +annexFile file = inBackend file err $ do + liftIO $ checkLegal file + stored <- storeFile file + g <- gitAnnex case (stored) of Nothing -> error $ "no backend could store: " ++ file - Just (key, backend) -> setup key backend + Just (key, backend) -> do + logStatus key ValuePresent + liftIO $ setup g key backend where err = error $ "already annexed " ++ file checkLegal file = do @@ -62,15 +71,14 @@ annexFile state file = inBackend file err $ do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - setup key backend = do - logStatus state key ValuePresent - let dest = annexLocation (repo state) backend key - let reldest = annexLocationRelative (repo state) backend key + setup g key backend = do + let dest = annexLocation g backend key + let reldest = annexLocationRelative g backend key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink ((linkTarget file) ++ reldest) file - gitRun (repo state) ["add", file] - gitRun (repo state) ["commit", "-m", + gitRun g ["add", file] + gitRun g ["commit", "-m", ("git-annex annexed " ++ file), file] linkTarget file = -- relies on file being relative to the top of the @@ -83,56 +91,60 @@ annexFile state file = inBackend file err $ do {- Inverse of annexFile. -} -unannexFile :: State -> FilePath -> IO () -unannexFile state file = notinBackend file err $ \(key, backend) -> do - dropFile state backend key - logStatus state key ValueMissing - removeFile file - gitRun (repo state) ["rm", file] - gitRun (repo state) ["commit", "-m", - ("git-annex unannexed " ++ file), file] - -- git rm deletes empty directories; - -- put them back - createDirectoryIfMissing True (parentDir file) - let src = annexLocation (repo state) backend key - renameFile src file - return () +unannexFile :: FilePath -> Annex () +unannexFile file = notinBackend file err $ \(key, backend) -> do + dropFile backend key + logStatus key ValueMissing + g <- gitAnnex + let src = annexLocation g backend key + liftIO $ moveout g src where err = error $ "not annexed " ++ file + moveout g src = do + removeFile file + gitRun g ["rm", file] + gitRun g ["commit", "-m", + ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) + renameFile src file + return () {- Gets an annexed file from one of the backends. -} -annexGetFile :: State -> FilePath -> IO () -annexGetFile state file = notinBackend file err $ \(key, backend) -> do - inannex <- inAnnex state backend key +annexGetFile :: FilePath -> Annex () +annexGetFile file = notinBackend file err $ \(key, backend) -> do + inannex <- inAnnex backend key if (inannex) then return () else do - let dest = annexLocation (repo state) backend key - createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state backend key dest + g <- gitAnnex + let dest = annexLocation g backend key + liftIO $ createDirectoryIfMissing True (parentDir dest) + success <- retrieveFile backend key dest if (success) then do - logStatus state key ValuePresent + logStatus key ValuePresent return () else error $ "failed to get " ++ file where err = error $ "not annexed " ++ file {- Indicates a file is wanted. -} -annexWantFile :: State -> FilePath -> IO () -annexWantFile state file = do error "not implemented" -- TODO +annexWantFile :: FilePath -> Annex () +annexWantFile file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} -annexDropFile :: State -> FilePath -> IO () -annexDropFile state file = do error "not implemented" -- TODO +annexDropFile :: FilePath -> Annex () +annexDropFile file = do error "not implemented" -- TODO {- Pushes all files to a remote repository. -} -annexPushRepo :: State -> String -> IO () -annexPushRepo state reponame = do error "not implemented" -- TODO +annexPushRepo :: String -> Annex () +annexPushRepo reponame = do error "not implemented" -- TODO {- Pulls all files from a remote repository. -} -annexPullRepo :: State -> String -> IO () -annexPullRepo state reponame = do error "not implemented" -- TODO +annexPullRepo :: String -> Annex () +annexPullRepo reponame = do error "not implemented" -- TODO {- Sets up a git repo for git-annex. May be called repeatedly. -} gitSetup :: GitRepo -> IO () @@ -159,11 +171,19 @@ gitSetup repo = do attributes] {- Updates the LocationLog when a key's presence changes. -} -logStatus state key status = do - f <- logChange (repo state) key (getUUID state (repo state)) status - gitRun (repo state) ["add", f] - gitRun (repo state) ["commit", "-m", "git-annex log update", f] +logStatus :: Key -> LogStatus -> Annex () +logStatus key status = do + g <- gitAnnex + u <- getUUID g + f <- liftIO $ logChange g key u status + liftIO $ commit g f + where + commit g f = do + gitRun g ["add", f] + gitRun g ["commit", "-m", "git-annex log update", f] {- Checks if a given key is currently present in the annexLocation -} -inAnnex :: State -> Backend -> Key -> IO Bool -inAnnex state backend key = doesFileExist $ annexLocation (repo state) backend key +inAnnex :: Backend -> Key -> Annex Bool +inAnnex backend key = do + g <- gitAnnex + liftIO $ doesFileExist $ annexLocation g backend key diff --git a/Backend.hs b/Backend.hs index bc7eb206ff..775c4a02f2 100644 --- a/Backend.hs +++ b/Backend.hs @@ -20,6 +20,7 @@ module Backend ( lookupFile ) where +import Control.Monad.State import Control.Exception import System.Directory import System.FilePath @@ -32,30 +33,34 @@ import Utility import Types {- Attempts to store a file in one of the backends. -} -storeFile :: State -> FilePath -> IO (Maybe (Key, Backend)) -storeFile state file = storeFile' (backends state) state file +storeFile :: FilePath -> Annex (Maybe (Key, Backend)) +storeFile file = do + g <- gitAnnex + let relfile = gitRelative g file + b <- backendsAnnex + storeFile' b file relfile storeFile' [] _ _ = return Nothing -storeFile' (b:bs) state file = do - try <- (getKey b) state (gitRelative (repo state) file) +storeFile' (b:bs) file relfile = do + try <- (getKey b) relfile case (try) of Nothing -> nextbackend Just key -> do - stored <- (storeFileKey b) state file key + stored <- (storeFileKey b) file key if (not stored) then nextbackend else do return $ Just (key, b) where - nextbackend = storeFile' bs state file + nextbackend = storeFile' bs file relfile {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveFile :: State -> Backend -> Key -> FilePath -> IO Bool -retrieveFile state backend key dest = (retrieveKeyFile backend) state key dest +retrieveFile :: Backend -> Key -> FilePath -> Annex Bool +retrieveFile backend key dest = (retrieveKeyFile backend) key dest {- Drops a key from a backend. -} -dropFile :: State -> Backend -> Key -> IO Bool -dropFile state backend key = (removeKey backend) state key +dropFile :: Backend -> Key -> Annex Bool +dropFile backend key = (removeKey backend) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/BackendChecksum.hs b/BackendChecksum.hs index efa2244125..c6e68ffed4 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -14,5 +14,5 @@ backend = BackendFile.backend { } -- checksum the file to get its key -keyValue :: State -> FilePath -> IO (Maybe Key) +keyValue :: FilePath -> Annex (Maybe Key) keyValue k = error "checksum keyValue unimplemented" -- TODO diff --git a/BackendFile.hs b/BackendFile.hs index a31cbfeb1e..9b82a0b202 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,6 +3,7 @@ module BackendFile (backend) where +import Control.Monad.State import System.IO import System.Cmd import Control.Exception @@ -21,28 +22,28 @@ backend = Backend { } -- direct mapping from filename to key -keyValue :: State -> FilePath -> IO (Maybe Key) -keyValue state file = return $ Just $ Key file +keyValue :: FilePath -> Annex (Maybe Key) +keyValue file = return $ Just $ Key file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. So storing or removing a key is - a no-op. TODO until support is added for git annex --push otherrepo, - then these could implement that.. -} -dummyStore :: State -> FilePath -> Key -> IO (Bool) -dummyStore state file key = return True -dummyRemove :: State -> Key -> IO Bool -dummyRemove state url = return False +dummyStore :: FilePath -> Key -> Annex (Bool) +dummyStore file key = return True +dummyRemove :: Key -> Annex Bool +dummyRemove url = return False {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} -copyKeyFile :: State -> Key -> FilePath -> IO (Bool) -copyKeyFile state key file = do - remotes <- remotesWithKey state key +copyKeyFile :: Key -> FilePath -> Annex (Bool) +copyKeyFile key file = do + remotes <- remotesWithKey key if (0 == length remotes) then error $ "no known remotes have: " ++ (keyFile key) ++ "\n" ++ "(Perhaps you need to git remote add a repository?)" - else trycopy remotes remotes + else liftIO $ trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ diff --git a/BackendUrl.hs b/BackendUrl.hs index 5b586497c4..43b0bc75a8 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,6 +3,7 @@ module BackendUrl (backend) where +import Control.Monad.State import System.Cmd import IO import Types @@ -16,19 +17,19 @@ backend = Backend { } -- cannot generate url from filename -keyValue :: State -> FilePath -> IO (Maybe Key) -keyValue repo file = return Nothing +keyValue :: FilePath -> Annex (Maybe Key) +keyValue file = return Nothing -- cannot change url contents -dummyStore :: State -> FilePath -> Key -> IO Bool -dummyStore repo file url = return False -dummyRemove :: State -> Key -> IO Bool -dummyRemove state url = return False +dummyStore :: FilePath -> Key -> Annex Bool +dummyStore file url = return False +dummyRemove :: Key -> Annex Bool +dummyRemove url = return False -downloadUrl :: State -> Key -> FilePath -> IO Bool -downloadUrl state url file = do - putStrLn $ "download: " ++ (show url) - result <- try $ rawSystem "curl" ["-#", "-o", file, (show url)] +downloadUrl :: Key -> FilePath -> Annex Bool +downloadUrl url file = do + liftIO $ putStrLn $ "download: " ++ (show url) + result <- liftIO $ try $ rawSystem "curl" ["-#", "-o", file, (show url)] case (result) of Left _ -> return False Right _ -> return True diff --git a/CmdLine.hs b/CmdLine.hs index 9da2b6493c..d23508aa27 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -6,7 +6,8 @@ module CmdLine ( argvToMode, - dispatch + dispatch, + Mode ) where import System.Console.GetOpt @@ -39,13 +40,13 @@ argvToMode argv = do (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) where header = "Usage: git-annex [mode] file" -dispatch :: State -> Mode -> FilePath -> IO () -dispatch state mode item = do +dispatch :: Mode -> FilePath -> Annex () +dispatch mode item = do case (mode) of - Add -> annexFile state item - Push -> annexPushRepo state item - Pull -> annexPullRepo state item - Want -> annexWantFile state item - Get -> annexGetFile state item - Drop -> annexDropFile state item - Unannex -> unannexFile state item + Add -> annexFile item + Push -> annexPushRepo item + Pull -> annexPullRepo item + Want -> annexWantFile item + Get -> annexGetFile item + Drop -> annexDropFile item + Unannex -> unannexFile item diff --git a/Remotes.hs b/Remotes.hs index ae709a3c23..3992914678 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -5,6 +5,7 @@ module Remotes ( remotesWithKey ) where +import Control.Monad.State (liftIO) import Types import GitRepo import LocationLog @@ -17,34 +18,43 @@ remotesList :: [GitRepo] -> String remotesList remotes = join " " $ map gitRepoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -remotesWithKey :: State -> Key -> IO [GitRepo] -remotesWithKey state key = do - uuids <- keyLocations (repo state) key - return $ reposByUUID state (remotesByCost state) uuids +remotesWithKey :: Key -> Annex [GitRepo] +remotesWithKey key = do + g <- gitAnnex + uuids <- liftIO $ keyLocations g key + remotes <- remotesByCost + reposByUUID remotes uuids {- Cost Ordered list of remotes. -} -remotesByCost :: State -> [GitRepo] -remotesByCost state = reposByCost state $ gitConfigRemotes (repo state) +remotesByCost :: Annex [GitRepo] +remotesByCost = do + g <- gitAnnex + reposByCost $ gitConfigRemotes g {- Orders a list of git repos by cost. -} -reposByCost :: State -> [GitRepo] -> [GitRepo] -reposByCost state l = - fst $ unzip $ sortBy (\(r1, c1) (r2, c2) -> compare c1 c2) $ costpairs l +reposByCost :: [GitRepo] -> Annex [GitRepo] +reposByCost l = do + costpairs <- mapM costpair l + return $ fst $ unzip $ sortBy bycost $ costpairs where - costpairs l = map (\r -> (r, repoCost state r)) l + costpair r = do + cost <- repoCost r + return (r, cost) + bycost (_, c1) (_, c2) = compare c1 c2 {- Calculates cost for a repo. - - The default cost is 100 for local repositories, and 200 for remote - repositories; it can also be configured by remote..annex-cost -} -repoCost :: State -> GitRepo -> Int -repoCost state r = - if ((length $ config state r) > 0) - then read $ config state r +repoCost :: GitRepo -> Annex Int +repoCost r = do + g <- gitAnnex + if ((length $ config g r) > 0) + then return $ read $ config g r else if (gitRepoIsLocal r) - then 100 - else 200 + then return 100 + else return 200 where - config state r = gitConfig (repo state) (configkey r) "" + config g r = gitConfig g (configkey r) "" configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" diff --git a/TODO b/TODO index a0f7c8b5fa..ea3f87c11b 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,9 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* implement retrieval for backendfile +* state monad -* query remotes for their annex.uuid settings +* query remotes for their annex.uuid settings and cache * --push/--pull/--want/--drop diff --git a/Types.hs b/Types.hs index 9b0bb00fd5..15c2ec89fd 100644 --- a/Types.hs +++ b/Types.hs @@ -1,20 +1,59 @@ {- git-annex core data types -} module Types ( - State(..), + Annex(..), + makeAnnexState, + runAnnexState, + gitAnnex, + gitAnnexChange, + backendsAnnex, + backendsAnnexChange, + + AnnexState(..), Key(..), Backend(..) ) where +import Control.Monad.State import Data.String.Utils import GitRepo -- git-annex's runtime state -data State = State { +data AnnexState = AnnexState { repo :: GitRepo, backends :: [Backend] } deriving (Show) +-- git-annex's monad +type Annex = StateT AnnexState IO + +-- constructor +makeAnnexState :: GitRepo -> AnnexState +makeAnnexState g = AnnexState { repo = g, backends = [] } + +-- performs an action in the Annex monad +runAnnexState state action = runStateT (action) state + +-- state accessors +gitAnnex :: Annex GitRepo +gitAnnex = do + state <- get + return (repo state) +gitAnnexChange :: GitRepo -> Annex () +gitAnnexChange r = do + state <- get + put state { repo = r } + return () +backendsAnnex :: Annex [Backend] +backendsAnnex = do + state <- get + return (backends state) +backendsAnnexChange :: [Backend] -> Annex () +backendsAnnexChange b = do + state <- get + put state { backends = b } + return () + -- annexed filenames are mapped into keys data Key = Key String deriving (Eq) @@ -27,13 +66,13 @@ data Backend = Backend { -- name of this backend name :: String, -- converts a filename to a key - getKey :: State -> FilePath -> IO (Maybe Key), + getKey :: FilePath -> Annex (Maybe Key), -- stores a file's contents to a key - storeFileKey :: State -> FilePath -> Key -> IO Bool, + storeFileKey :: FilePath -> Key -> Annex Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: State -> Key -> FilePath -> IO Bool, + retrieveKeyFile :: Key -> FilePath -> Annex Bool, -- removes a key - removeKey :: State -> Key -> IO Bool + removeKey :: Key -> Annex Bool } instance Show Backend where diff --git a/UUID.hs b/UUID.hs index b4c4c0cc0b..5c9f9179ea 100644 --- a/UUID.hs +++ b/UUID.hs @@ -13,6 +13,7 @@ module UUID ( reposByUUID ) where +import Control.Monad.State import Maybe import List import System.Cmd.Utils @@ -26,9 +27,8 @@ configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} -genUUID :: IO UUID -genUUID = do - pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h +genUUID :: Annex UUID +genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h {- Looks up a repo's UUID. May return "" if none is known. - @@ -36,28 +36,38 @@ genUUID = do - remote..annex-uuid - - -} -getUUID :: State -> GitRepo -> UUID -getUUID s r = - if ("" /= getUUID' r) - then getUUID' r - else cached s r +getUUID :: GitRepo -> Annex UUID +getUUID r = do + if ("" /= configured r) + then return $ configured r + else cached r where - cached s r = gitConfig (repo s) (configkey r) "" + configured r = gitConfig r "annex.uuid" "" + cached r = do + g <- gitAnnex + return $ gitConfig g (configkey r) "" configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-uuid" -getUUID' r = gitConfig r "annex.uuid" "" {- Make sure that the repo has an annex.uuid setting. -} -prepUUID :: GitRepo -> IO GitRepo -prepUUID repo = - if ("" == getUUID' repo) +prepUUID :: Annex () +prepUUID = do + g <- gitAnnex + u <- getUUID g + if ("" == u) then do uuid <- genUUID - gitRun repo ["config", configkey, uuid] - -- return new repo with updated config - gitConfigRead repo - else return repo + liftIO $ gitRun g ["config", configkey, uuid] + -- re-read git config and update the repo's state + u' <- liftIO $ gitConfigRead g + gitAnnexChange u' + return () + else return () {- Filters a list of repos to ones that have listed UUIDs. -} -reposByUUID :: State -> [GitRepo] -> [UUID] -> [GitRepo] -reposByUUID state repos uuids = - filter (\r -> isJust $ elemIndex (getUUID state r) uuids) repos +reposByUUID :: [GitRepo] -> [UUID] -> Annex [GitRepo] +reposByUUID repos uuids = do + filterM match repos + where + match r = do + u <- getUUID r + return $ isJust $ elemIndex u uuids diff --git a/git-annex.hs b/git-annex.hs index 7785e4f2d7..935be2f1ef 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,36 +1,43 @@ {- git-annex main program - -} +import Control.Monad.State import System.IO import System.Environment import Control.Exception import CmdLine +import Types import Annex main = do args <- getArgs - (mode, files) <- argvToMode args - + (mode, params) <- argvToMode args state <- startAnnex + tryRun state mode 0 0 params - tryRun 0 0 $ map (\f -> dispatch state mode f) files - -{- Tries to run a series of actions, not stopping if some error out, - - and propigating an overall error status at the end. -} -tryRun errnum oknum [] = do +{- Processes each param in the list by dispatching the handler function + - for the user-selection operation mode. Catches exceptions, not stopping + - if some error out, and propigates an overall error status at the end. + - + - This runs in the IO monad, not in the Annex monad. It seems that + - exceptions can only be caught in the IO monad, not in a stacked monad; + - or more likely I missed an easy way to do it. So, I have to laboriously + - thread AnnexState through this function. + -} +tryRun :: AnnexState -> Mode -> Int -> Int -> [String] -> IO () +tryRun state mode errnum oknum [] = do if (errnum > 0) then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" else return () -tryRun errnum oknum (a:as) = do - result <- try (a)::IO (Either SomeException ()) +tryRun state mode errnum oknum (f:fs) = do + result <- try (runAnnexState state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) case (result) of Left err -> do showErr err - tryRun (errnum + 1) oknum as - Right _ -> tryRun errnum (oknum + 1) as + tryRun state mode (errnum + 1) oknum fs + Right (_,state') -> tryRun state' mode errnum (oknum + 1) fs {- Exception pretty-printing. -} -showErr :: SomeException -> IO () showErr e = do hPutStrLn stderr $ "git-annex: " ++ (show e) return () From 89654751daacd9336c114bfc1c88c952dcc4ffe9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 21:35:10 -0400 Subject: [PATCH 0107/2835] bugfix --- Annex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index f3c8f533a4..f55a62c4d6 100644 --- a/Annex.hs +++ b/Annex.hs @@ -38,9 +38,9 @@ startAnnex = do where prep g = do -- setup git and read its config; update state - liftIO $ gitSetup g g' <- liftIO $ gitConfigRead g gitAnnexChange g' + liftIO $ gitSetup g' backendsAnnexChange $ parseBackendList $ gitConfig g' "annex.backends" "" prepUUID From 912d10e78b725b4d3d4105a0ffe5696c21fc0e10 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 22:59:43 -0400 Subject: [PATCH 0108/2835] implemented remotes config caching --- BackendFile.hs | 30 +++++++++++++----------------- GitRepo.hs | 50 +++++++++++++++++++++++++++++++++++--------------- Remotes.hs | 28 ++++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 34 deletions(-) diff --git a/BackendFile.hs b/BackendFile.hs index 9b82a0b202..d16f3611b1 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -43,16 +43,20 @@ copyKeyFile key file = do if (0 == length remotes) then error $ "no known remotes have: " ++ (keyFile key) ++ "\n" ++ "(Perhaps you need to git remote add a repository?)" - else liftIO $ trycopy remotes remotes + else trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ (remotesList full) trycopy full (r:rs) = do - result <- try (copyFromRemote r key file)::IO (Either SomeException ()) + -- annexLocation needs the git config to have been + -- read for a remote, so do that now, + -- if it hasn't been already + r' <- remoteEnsureGitConfigRead r + result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) case (result) of Left err -> do - hPutStrLn stderr (show err) + liftIO $ hPutStrLn stderr (show err) trycopy full rs Right succ -> return True @@ -61,19 +65,11 @@ copyFromRemote :: GitRepo -> Key -> FilePath -> IO () copyFromRemote r key file = do putStrLn $ "copy from " ++ (gitRepoDescribe r ) ++ " " ++ file - -- annexLocation needs the git config read for the remote first. - -- FIXME: Having this here means git-config is run repeatedly when - -- copying a series of files; need to use state monad to avoid - -- this. - r' <- gitConfigRead r - - _ <- if (gitRepoIsLocal r') - then getlocal r' - else getremote r' + if (gitRepoIsLocal r) + then getlocal + else getremote return () where - getlocal r = do - rawSystem "cp" ["-a", location r, file] - getremote r = do - error "get via network not yet implemented!" - location r = annexLocation r backend key + getlocal = rawSystem "cp" ["-a", location, file] + getremote = error "get via network not yet implemented!" + location = annexLocation r backend key diff --git a/GitRepo.hs b/GitRepo.hs index e1f086b693..d22218219c 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -12,15 +12,17 @@ module GitRepo ( gitRepoFromUrl, gitRepoIsLocal, gitRepoIsRemote, - gitConfigRemotes, gitRepoDescribe, gitWorkTree, gitDir, gitRelative, gitConfig, + gitConfigMap, gitConfigRead, gitRun, gitAttributes, + gitRepoRemotes, + gitRepoRemotesAdd, gitRepoRemoteName ) where @@ -46,12 +48,14 @@ data GitRepo = LocalGitRepo { top :: FilePath, config :: Map String String, + remotes :: [GitRepo], -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String } | RemoteGitRepo { url :: String, top :: FilePath, config :: Map String String, + remotes :: [GitRepo], remoteName :: Maybe String } deriving (Show, Read, Eq) @@ -61,6 +65,7 @@ gitRepoFromPath dir = LocalGitRepo { top = dir, config = Map.empty, + remotes = [], remoteName = Nothing } @@ -71,6 +76,7 @@ gitRepoFromUrl url = url = url, top = path url, config = Map.empty, + remotes = [], remoteName = Nothing } where path url = uriPath $ fromJust $ parseURI url @@ -83,6 +89,15 @@ gitRepoDescribe repo = then top repo else url repo +{- Returns the list of a repo's remotes. -} +gitRepoRemotes :: GitRepo -> [GitRepo] +gitRepoRemotes r = remotes r + +{- Constructs and returns an updated version of a repo with + - different remotes list. -} +gitRepoRemotesAdd :: GitRepo -> [GitRepo] -> GitRepo +gitRepoRemotesAdd repo rs = repo { remotes = rs } + {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} gitRepoRemoteName r = @@ -169,10 +184,24 @@ gitConfigRead repo = assertlocal repo $ do been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory bracket_ (changeWorkingDirectory (top repo)) - (\_ -> changeWorkingDirectory cwd) $ do + (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] $ \h -> do val <- hGetContentsStrict h - return repo { config = gitConfigParse val } + let r = repo { config = gitConfigParse val } + return r { remotes = gitConfigRemotes r } + +{- Calculates a list of a repo's configured remotes, by parsing its config. -} +gitConfigRemotes :: GitRepo -> [GitRepo] +gitConfigRemotes repo = map construct remotes + where + remotes = toList $ filter $ config repo + filter = filterWithKey (\k _ -> isremote k) + isremote k = (startswith "remote." k) && (endswith ".url" k) + remotename k = (split "." k) !! 1 + construct (k,v) = (gen v) { remoteName = Just $ remotename k } + gen v = if (isURI v) + then gitRepoFromUrl v + else gitRepoFromPath v {- Parses git config --list output into a config map. -} gitConfigParse :: String -> Map.Map String String @@ -189,18 +218,9 @@ gitConfig :: GitRepo -> String -> String -> String gitConfig repo key defaultValue = Map.findWithDefault defaultValue key (config repo) -{- Returns a list of a repo's configured remotes. -} -gitConfigRemotes :: GitRepo -> [GitRepo] -gitConfigRemotes repo = map construct remotes - where - remotes = toList $ filter $ config repo - filter = filterWithKey (\k _ -> isremote k) - isremote k = (startswith "remote." k) && (endswith ".url" k) - remotename k = (split "." k) !! 1 - construct (k,v) = (gen v) { remoteName = Just $ remotename k } - gen v = if (isURI v) - then gitRepoFromUrl v - else gitRepoFromPath v +{- Access to raw config Map -} +gitConfigMap :: GitRepo -> Map String String +gitConfigMap repo = config repo {- Finds the current git repository, which may be in a parent directory. -} gitRepoFromCwd :: IO GitRepo diff --git a/Remotes.hs b/Remotes.hs index 3992914678..13b87982c9 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -2,10 +2,12 @@ module Remotes ( remotesList, - remotesWithKey + remotesWithKey, + remoteEnsureGitConfigRead ) where import Control.Monad.State (liftIO) +import qualified Data.Map as Map import Types import GitRepo import LocationLog @@ -29,7 +31,7 @@ remotesWithKey key = do remotesByCost :: Annex [GitRepo] remotesByCost = do g <- gitAnnex - reposByCost $ gitConfigRemotes g + reposByCost $ gitRepoRemotes g {- Orders a list of git repos by cost. -} reposByCost :: [GitRepo] -> Annex [GitRepo] @@ -58,3 +60,25 @@ repoCost r = do where config g r = gitConfig g (configkey r) "" configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" + +{- The git configs for the git repo's remotes is not read on startup + - because reading it may be expensive. This function ensures that it is + - read for a specified remote, and updates state. It returns the + - updated git repo also. -} +remoteEnsureGitConfigRead :: GitRepo -> Annex GitRepo +remoteEnsureGitConfigRead r = do + if (Map.null $ gitConfigMap r) + then do + r' <- liftIO $ gitConfigRead r + g <- gitAnnex + let l = gitRepoRemotes g + let g' = gitRepoRemotesAdd g $ exchange l r' + gitAnnexChange g' + return r' + else return r + where + exchange [] new = [] + exchange (old:ls) new = + if ((gitRepoRemoteName old) == (gitRepoRemoteName new)) + then new:(exchange ls new) + else old:(exchange ls new) From 64b5167b0f9620bd96cd57b58f0e40be741e5420 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 23:03:01 -0400 Subject: [PATCH 0109/2835] update --- Types.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Types.hs b/Types.hs index 15c2ec89fd..ce377dda11 100644 --- a/Types.hs +++ b/Types.hs @@ -1,7 +1,8 @@ {- git-annex core data types -} module Types ( - Annex(..), + Annex, + AnnexState, makeAnnexState, runAnnexState, gitAnnex, @@ -9,7 +10,6 @@ module Types ( backendsAnnex, backendsAnnexChange, - AnnexState(..), Key(..), Backend(..) ) where @@ -34,7 +34,7 @@ makeAnnexState g = AnnexState { repo = g, backends = [] } -- performs an action in the Annex monad runAnnexState state action = runStateT (action) state --- state accessors +-- Annex monad state accessors gitAnnex :: Annex GitRepo gitAnnex = do state <- get From 8ab54401b609f49a603f3ed69bb8493a53f28db8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 23:18:58 -0400 Subject: [PATCH 0110/2835] update --- BackendFile.hs | 5 +---- Remotes.hs | 12 +++++++++--- TODO | 2 -- UUID.hs | 10 +++++++++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/BackendFile.hs b/BackendFile.hs index d16f3611b1..e821ac22b2 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -40,10 +40,7 @@ dummyRemove url = return False copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do remotes <- remotesWithKey key - if (0 == length remotes) - then error $ "no known remotes have: " ++ (keyFile key) ++ "\n" ++ - "(Perhaps you need to git remote add a repository?)" - else trycopy remotes remotes + trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ diff --git a/Remotes.hs b/Remotes.hs index 13b87982c9..f3af81f23d 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -8,10 +8,11 @@ module Remotes ( import Control.Monad.State (liftIO) import qualified Data.Map as Map +import Data.String.Utils import Types import GitRepo import LocationLog -import Data.String.Utils +import Locations import UUID import List @@ -24,8 +25,13 @@ remotesWithKey :: Key -> Annex [GitRepo] remotesWithKey key = do g <- gitAnnex uuids <- liftIO $ keyLocations g key - remotes <- remotesByCost - reposByUUID remotes uuids + allremotes <- remotesByCost + remotes <- reposByUUID allremotes uuids + if (0 == length remotes) + then error $ "no configured git remotes have: " ++ (keyFile key) ++ "\n" ++ + "It has been seen before in these repositories:\n" ++ + prettyPrintUUIDs uuids + else return remotes {- Cost Ordered list of remotes. -} remotesByCost :: Annex [GitRepo] diff --git a/TODO b/TODO index ea3f87c11b..40017c816a 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,6 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* state monad - * query remotes for their annex.uuid settings and cache * --push/--pull/--want/--drop diff --git a/UUID.hs b/UUID.hs index 5c9f9179ea..af6003bfb7 100644 --- a/UUID.hs +++ b/UUID.hs @@ -10,7 +10,8 @@ module UUID ( getUUID, prepUUID, genUUID, - reposByUUID + reposByUUID, + prettyPrintUUIDs ) where import Control.Monad.State @@ -71,3 +72,10 @@ reposByUUID repos uuids = do match r = do u <- getUUID r return $ isJust $ elemIndex u uuids + +{- Pretty-prints a list of UUIDs + - TODO: use lookup file to really show pretty names. -} +prettyPrintUUIDs :: [UUID] -> String +prettyPrintUUIDs uuids = + unwords $ map (\u -> "\tUUID "++u++"\n") uuids + From e47dca162a0ca0144172c9a61a47d1e0b5ad04b7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 23:31:08 -0400 Subject: [PATCH 0111/2835] update --- Locations.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Locations.hs b/Locations.hs index d6d7d42480..68a9581924 100644 --- a/Locations.hs +++ b/Locations.hs @@ -47,6 +47,7 @@ annexLocationRelative r backend key = keyFile :: Key -> FilePath keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key -{- Reverses keyFile -} +{- Reverses keyFile, converting a filename fragment (ie, the basename of + - the symlink target) into a key. -} fileKey :: FilePath -> Key fileKey file = Key $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file From eda80e44c5fb399fa4e5625388d6e0f993b0f779 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 02:12:41 -0400 Subject: [PATCH 0112/2835] add module that only exports abstract types --- AbstractTypes.hs | 17 +++++++++++++++++ Annex.hs | 2 +- CmdLine.hs | 2 +- LocationLog.hs | 2 +- Remotes.hs | 2 +- UUID.hs | 2 +- git-annex.hs | 2 +- 7 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 AbstractTypes.hs diff --git a/AbstractTypes.hs b/AbstractTypes.hs new file mode 100644 index 0000000000..510a37f0cc --- /dev/null +++ b/AbstractTypes.hs @@ -0,0 +1,17 @@ +{- git-annex data types, abstract only -} + +module AbstractTypes ( + Annex, + AnnexState, + makeAnnexState, + runAnnexState, + gitAnnex, + gitAnnexChange, + backendsAnnex, + backendsAnnexChange, + + Key, + Backend +) where + +import Types diff --git a/Annex.hs b/Annex.hs index f55a62c4d6..68bf0136a9 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,7 +24,7 @@ import Backend import BackendList import UUID import LocationLog -import Types +import AbstractTypes {- Create and returns an Annex state object. - Examines and prepares the git repo. diff --git a/CmdLine.hs b/CmdLine.hs index d23508aa27..bb908a2e40 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -11,7 +11,7 @@ module CmdLine ( ) where import System.Console.GetOpt -import Types +import AbstractTypes import Annex data Mode = Add | Push | Pull | Want | Get | Drop | Unannex diff --git a/LocationLog.hs b/LocationLog.hs index 28ac46b90c..a6d998e0af 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -32,7 +32,7 @@ import Data.Char import GitRepo import Utility import UUID -import Types +import AbstractTypes import Locations data LogLine = LogLine { diff --git a/Remotes.hs b/Remotes.hs index f3af81f23d..711cd6c83a 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -9,7 +9,7 @@ module Remotes ( import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils -import Types +import AbstractTypes import GitRepo import LocationLog import Locations diff --git a/UUID.hs b/UUID.hs index af6003bfb7..f334afdc9d 100644 --- a/UUID.hs +++ b/UUID.hs @@ -20,7 +20,7 @@ import List import System.Cmd.Utils import System.IO import GitRepo -import Types +import AbstractTypes type UUID = String diff --git a/git-annex.hs b/git-annex.hs index 935be2f1ef..be5168755f 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,7 +6,7 @@ import System.IO import System.Environment import Control.Exception import CmdLine -import Types +import AbstractTypes import Annex main = do From 48643b68b3ff05399b72f44b8b02ff34d6de046c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 02:36:41 -0400 Subject: [PATCH 0113/2835] convert GitRepo to qualified import --- Annex.hs | 78 ++++++++++++------------ Backend.hs | 4 +- BackendFile.hs | 8 +-- CmdLine.hs | 14 ++--- GitRepo.hs | 158 ++++++++++++++++++++++++------------------------- LocationLog.hs | 10 ++-- Locations.hs | 14 ++--- Remotes.hs | 34 +++++------ Types.hs | 24 ++------ UUID.hs | 16 ++--- git-annex.hs | 2 +- 11 files changed, 173 insertions(+), 189 deletions(-) diff --git a/Annex.hs b/Annex.hs index 68bf0136a9..54f9b9dff8 100644 --- a/Annex.hs +++ b/Annex.hs @@ -2,14 +2,14 @@ -} module Annex ( - startAnnex, - annexFile, - unannexFile, - annexGetFile, - annexWantFile, - annexDropFile, - annexPushRepo, - annexPullRepo + start, + annexCmd, + unannexCmd, + getCmd, + wantCmd, + dropCmd, + pushCmd, + pullCmd ) where import Control.Monad.State (liftIO) @@ -17,7 +17,7 @@ import System.Posix.Files import System.Directory import Data.String.Utils import List -import GitRepo +import qualified GitRepo as Git import Utility import Locations import Backend @@ -29,20 +29,20 @@ import AbstractTypes {- Create and returns an Annex state object. - Examines and prepares the git repo. -} -startAnnex :: IO AnnexState -startAnnex = do - g <- gitRepoFromCwd +start :: IO AnnexState +start = do + g <- Git.repoFromCwd let s = makeAnnexState g (_,s') <- runAnnexState s (prep g) return s' where prep g = do -- setup git and read its config; update state - g' <- liftIO $ gitConfigRead g + g' <- liftIO $ Git.configRead g gitAnnexChange g' liftIO $ gitSetup g' backendsAnnexChange $ parseBackendList $ - gitConfig g' "annex.backends" "" + Git.configGet g' "annex.backends" "" prepUUID inBackend file yes no = do @@ -54,8 +54,8 @@ notinBackend file yes no = inBackend file no yes {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} -annexFile :: FilePath -> Annex () -annexFile file = inBackend file err $ do +annexCmd :: FilePath -> Annex () +annexCmd file = inBackend file err $ do liftIO $ checkLegal file stored <- storeFile file g <- gitAnnex @@ -77,8 +77,8 @@ annexFile file = inBackend file err $ do createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink ((linkTarget file) ++ reldest) file - gitRun g ["add", file] - gitRun g ["commit", "-m", + Git.run g ["add", file] + Git.run g ["commit", "-m", ("git-annex annexed " ++ file), file] linkTarget file = -- relies on file being relative to the top of the @@ -90,9 +90,9 @@ annexFile file = inBackend file err $ do subdirs = (length $ split "/" file) - 1 -{- Inverse of annexFile. -} -unannexFile :: FilePath -> Annex () -unannexFile file = notinBackend file err $ \(key, backend) -> do +{- Inverse of annexCmd. -} +unannexCmd :: FilePath -> Annex () +unannexCmd file = notinBackend file err $ \(key, backend) -> do dropFile backend key logStatus key ValueMissing g <- gitAnnex @@ -102,8 +102,8 @@ unannexFile file = notinBackend file err $ \(key, backend) -> do err = error $ "not annexed " ++ file moveout g src = do removeFile file - gitRun g ["rm", file] - gitRun g ["commit", "-m", + Git.run g ["rm", file] + Git.run g ["commit", "-m", ("git-annex unannexed " ++ file), file] -- git rm deletes empty directories; -- put them back @@ -112,8 +112,8 @@ unannexFile file = notinBackend file err $ \(key, backend) -> do return () {- Gets an annexed file from one of the backends. -} -annexGetFile :: FilePath -> Annex () -annexGetFile file = notinBackend file err $ \(key, backend) -> do +getCmd :: FilePath -> Annex () +getCmd file = notinBackend file err $ \(key, backend) -> do inannex <- inAnnex backend key if (inannex) then return () @@ -131,23 +131,23 @@ annexGetFile file = notinBackend file err $ \(key, backend) -> do err = error $ "not annexed " ++ file {- Indicates a file is wanted. -} -annexWantFile :: FilePath -> Annex () -annexWantFile file = do error "not implemented" -- TODO +wantCmd :: FilePath -> Annex () +wantCmd file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} -annexDropFile :: FilePath -> Annex () -annexDropFile file = do error "not implemented" -- TODO +dropCmd :: FilePath -> Annex () +dropCmd file = do error "not implemented" -- TODO {- Pushes all files to a remote repository. -} -annexPushRepo :: String -> Annex () -annexPushRepo reponame = do error "not implemented" -- TODO +pushCmd :: String -> Annex () +pushCmd reponame = do error "not implemented" -- TODO {- Pulls all files from a remote repository. -} -annexPullRepo :: String -> Annex () -annexPullRepo reponame = do error "not implemented" -- TODO +pullCmd :: String -> Annex () +pullCmd reponame = do error "not implemented" -- TODO {- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: GitRepo -> IO () +gitSetup :: Git.Repo -> IO () gitSetup repo = do -- configure git to use union merge driver on state files exists <- doesFileExist attributes @@ -164,10 +164,10 @@ gitSetup repo = do else return () where attrLine = stateLoc ++ "/*.log merge=union" - attributes = gitAttributes repo + attributes = Git.attributes repo commit = do - gitRun repo ["add", attributes] - gitRun repo ["commit", "-m", "git-annex setup", + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", attributes] {- Updates the LocationLog when a key's presence changes. -} @@ -179,8 +179,8 @@ logStatus key status = do liftIO $ commit g f where commit g f = do - gitRun g ["add", f] - gitRun g ["commit", "-m", "git-annex log update", f] + Git.run g ["add", f] + Git.run g ["commit", "-m", "git-annex log update", f] {- Checks if a given key is currently present in the annexLocation -} inAnnex :: Backend -> Key -> Annex Bool diff --git a/Backend.hs b/Backend.hs index 775c4a02f2..1bd4efc1e8 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,7 +28,7 @@ import Data.String.Utils import System.Posix.Files import BackendList import Locations -import GitRepo +import qualified GitRepo as Git import Utility import Types @@ -36,7 +36,7 @@ import Types storeFile :: FilePath -> Annex (Maybe (Key, Backend)) storeFile file = do g <- gitAnnex - let relfile = gitRelative g file + let relfile = Git.relative g file b <- backendsAnnex storeFile' b file relfile storeFile' [] _ _ = return Nothing diff --git a/BackendFile.hs b/BackendFile.hs index e821ac22b2..6c1dc06233 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -11,7 +11,7 @@ import Types import LocationLog import Locations import Remotes -import GitRepo +import qualified GitRepo as Git backend = Backend { name = "file", @@ -58,11 +58,11 @@ copyKeyFile key file = do Right succ -> return True {- Tries to copy a file from a remote, exception on error. -} -copyFromRemote :: GitRepo -> Key -> FilePath -> IO () +copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () copyFromRemote r key file = do - putStrLn $ "copy from " ++ (gitRepoDescribe r ) ++ " " ++ file + putStrLn $ "copy from " ++ (Git.repoDescribe r ) ++ " " ++ file - if (gitRepoIsLocal r) + if (Git.repoIsLocal r) then getlocal else getremote return () diff --git a/CmdLine.hs b/CmdLine.hs index bb908a2e40..479be7e8b5 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -43,10 +43,10 @@ argvToMode argv = do dispatch :: Mode -> FilePath -> Annex () dispatch mode item = do case (mode) of - Add -> annexFile item - Push -> annexPushRepo item - Pull -> annexPullRepo item - Want -> annexWantFile item - Get -> annexGetFile item - Drop -> annexDropFile item - Unannex -> unannexFile item + Add -> annexCmd item + Push -> pushCmd item + Pull -> pullCmd item + Want -> wantCmd item + Get -> getCmd item + Drop -> dropCmd item + Unannex -> unannexCmd item diff --git a/GitRepo.hs b/GitRepo.hs index d22218219c..f3bb5427ad 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -3,27 +3,27 @@ - This is written to be completely independant of git-annex and should be - suitable for other uses. - - - -} + -} module GitRepo ( - GitRepo, - gitRepoFromCwd, - gitRepoFromPath, - gitRepoFromUrl, - gitRepoIsLocal, - gitRepoIsRemote, - gitRepoDescribe, - gitWorkTree, - gitDir, - gitRelative, - gitConfig, - gitConfigMap, - gitConfigRead, - gitRun, - gitAttributes, - gitRepoRemotes, - gitRepoRemotesAdd, - gitRepoRemoteName + Repo, + repoFromCwd, + repoFromPath, + repoFromUrl, + repoIsLocal, + repoIsRemote, + repoDescribe, + workTree, + dir, + relative, + configGet, + configMap, + configRead, + run, + attributes, + remotes, + remotesAdd, + repoRemoteName ) where import Directory @@ -44,35 +44,35 @@ import Utility {- A git repository can be on local disk or remote. Not to be confused - with a git repo's configured remotes, some of which may be on local - disk. -} -data GitRepo = - LocalGitRepo { +data Repo = + LocalRepo { top :: FilePath, config :: Map String String, - remotes :: [GitRepo], + remotes :: [Repo], -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String - } | RemoteGitRepo { + } | RemoteRepo { url :: String, top :: FilePath, config :: Map String String, - remotes :: [GitRepo], + remotes :: [Repo], remoteName :: Maybe String } deriving (Show, Read, Eq) -{- Local GitRepo constructor. -} -gitRepoFromPath :: FilePath -> GitRepo -gitRepoFromPath dir = - LocalGitRepo { +{- Local Repo constructor. -} +repoFromPath :: FilePath -> Repo +repoFromPath dir = + LocalRepo { top = dir, config = Map.empty, remotes = [], remoteName = Nothing } -{- Remote GitRepo constructor. Throws exception on invalid url. -} -gitRepoFromUrl :: String -> GitRepo -gitRepoFromUrl url = - RemoteGitRepo { +{- Remote Repo constructor. Throws exception on invalid url. -} +repoFromUrl :: String -> Repo +repoFromUrl url = + RemoteRepo { url = url, top = path url, config = Map.empty, @@ -82,72 +82,68 @@ gitRepoFromUrl url = where path url = uriPath $ fromJust $ parseURI url {- User-visible description of a git repo. -} -gitRepoDescribe repo = +repoDescribe repo = if (isJust $ remoteName repo) then fromJust $ remoteName repo - else if (gitRepoIsLocal repo) + else if (repoIsLocal repo) then top repo else url repo -{- Returns the list of a repo's remotes. -} -gitRepoRemotes :: GitRepo -> [GitRepo] -gitRepoRemotes r = remotes r - {- Constructs and returns an updated version of a repo with - different remotes list. -} -gitRepoRemotesAdd :: GitRepo -> [GitRepo] -> GitRepo -gitRepoRemotesAdd repo rs = repo { remotes = rs } +remotesAdd :: Repo -> [Repo] -> Repo +remotesAdd repo rs = repo { remotes = rs } {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} -gitRepoRemoteName r = +repoRemoteName r = if (isJust $ remoteName r) then fromJust $ remoteName r else "" {- Some code needs to vary between remote and local repos, or bare and - non-bare, these functions help with that. -} -gitRepoIsLocal repo = case (repo) of - LocalGitRepo {} -> True - RemoteGitRepo {} -> False -gitRepoIsRemote repo = not $ gitRepoIsLocal repo +repoIsLocal repo = case (repo) of + LocalRepo {} -> True + RemoteRepo {} -> False +repoIsRemote repo = not $ repoIsLocal repo assertlocal repo action = - if (gitRepoIsLocal repo) + if (repoIsLocal repo) then action - else error $ "acting on remote git repo " ++ (gitRepoDescribe repo) ++ + else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ " not supported" -bare :: GitRepo -> Bool +bare :: Repo -> Bool bare repo = if (member b (config repo)) then ("true" == fromJust (Map.lookup b (config repo))) - else error $ "it is not known if git repo " ++ (gitRepoDescribe repo) ++ + else error $ "it is not known if git repo " ++ (repoDescribe repo) ++ " is a bare repository; config not read" where b = "core.bare" {- Path to a repository's gitattributes file. -} -gitAttributes :: GitRepo -> String -gitAttributes repo = assertlocal repo $ do +attributes :: Repo -> String +attributes repo = assertlocal repo $ do if (bare repo) then (top repo) ++ "/info/.gitattributes" else (top repo) ++ "/.gitattributes" {- Path to a repository's .git directory, relative to its topdir. -} -gitDir :: GitRepo -> String -gitDir repo = assertlocal repo $ +dir :: Repo -> String +dir repo = assertlocal repo $ if (bare repo) then "" else ".git" {- Path to a repository's --work-tree. -} -gitWorkTree :: GitRepo -> FilePath -gitWorkTree repo = top repo +workTree :: Repo -> FilePath +workTree repo = top repo {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} -gitRelative :: GitRepo -> String -> String -gitRelative repo file = drop (length absrepo) absfile +relative :: Repo -> String -> String +relative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file @@ -159,27 +155,27 @@ gitRelative repo file = drop (length absrepo) absfile Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo {- Constructs a git command line operating on the specified repo. -} -gitCommandLine :: GitRepo -> [String] -> [String] +gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(top repo)++"/"++(gitDir repo), "--work-tree="++(top repo)] ++ params + ["--git-dir="++(top repo)++"/"++(dir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} -gitRun :: GitRepo -> [String] -> IO () -gitRun repo params = assertlocal repo $ do +run :: Repo -> [String] -> IO () +run repo params = assertlocal repo $ do r <- rawSystem "git" (gitCommandLine repo params) return () {- Runs a git subcommand and returns its output. -} -gitPipeRead :: GitRepo -> [String] -> IO String +gitPipeRead :: Repo -> [String] -> IO String gitPipeRead repo params = assertlocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do ret <- hGetContentsStrict h return ret {- Runs git config and populates a repo with its config. -} -gitConfigRead :: GitRepo -> IO GitRepo -gitConfigRead repo = assertlocal repo $ do +configRead :: Repo -> IO Repo +configRead repo = assertlocal repo $ do {- Cannot use gitPipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory @@ -187,12 +183,12 @@ gitConfigRead repo = assertlocal repo $ do (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] $ \h -> do val <- hGetContentsStrict h - let r = repo { config = gitConfigParse val } - return r { remotes = gitConfigRemotes r } + let r = repo { config = configParse val } + return r { remotes = configRemotes r } {- Calculates a list of a repo's configured remotes, by parsing its config. -} -gitConfigRemotes :: GitRepo -> [GitRepo] -gitConfigRemotes repo = map construct remotes +configRemotes :: Repo -> [Repo] +configRemotes repo = map construct remotes where remotes = toList $ filter $ config repo filter = filterWithKey (\k _ -> isremote k) @@ -200,12 +196,12 @@ gitConfigRemotes repo = map construct remotes remotename k = (split "." k) !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } gen v = if (isURI v) - then gitRepoFromUrl v - else gitRepoFromPath v + then repoFromUrl v + else repoFromPath v {- Parses git config --list output into a config map. -} -gitConfigParse :: String -> Map.Map String String -gitConfigParse s = Map.fromList $ map pair $ lines s +configParse :: String -> Map.Map String String +configParse s = Map.fromList $ map pair $ lines s where pair l = (key l, val l) key l = (keyval l) !! 0 @@ -214,21 +210,21 @@ gitConfigParse s = Map.fromList $ map pair $ lines s sep = "=" {- Returns a single git config setting, or a default value if not set. -} -gitConfig :: GitRepo -> String -> String -> String -gitConfig repo key defaultValue = +configGet :: Repo -> String -> String -> String +configGet repo key defaultValue = Map.findWithDefault defaultValue key (config repo) {- Access to raw config Map -} -gitConfigMap :: GitRepo -> Map String String -gitConfigMap repo = config repo +configMap :: Repo -> Map String String +configMap repo = config repo {- Finds the current git repository, which may be in a parent directory. -} -gitRepoFromCwd :: IO GitRepo -gitRepoFromCwd = do +repoFromCwd :: IO Repo +repoFromCwd = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> return $ gitRepoFromPath dir + (Just dir) -> return $ repoFromPath dir Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) @@ -241,11 +237,11 @@ seekUp dir want = do d -> seekUp d want isRepoTop dir = do - r <- isGitRepo dir + r <- isRepo dir b <- isBareRepo dir return (r || b) where - isGitRepo dir = gitSignature dir ".git" ".git/config" + isRepo dir = gitSignature dir ".git" ".git/config" isBareRepo dir = gitSignature dir "objects" "config" gitSignature dir subdir file = do s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) diff --git a/LocationLog.hs b/LocationLog.hs index a6d998e0af..7953b345f8 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -29,7 +29,7 @@ import qualified Data.Map as Map import System.IO import System.Directory import Data.Char -import GitRepo +import qualified GitRepo as Git import Utility import UUID import AbstractTypes @@ -81,7 +81,7 @@ instance Read LogLine where {- Log a change in the presence of a key's value in a repository, - and return the log filename. -} -logChange :: GitRepo -> Key -> UUID -> LogStatus -> IO FilePath +logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath logChange repo key uuid status = do log <- logNow status uuid ls <- readLog logfile @@ -127,13 +127,13 @@ logNow status uuid = do return $ LogLine now status uuid {- Returns the filename of the log file for a given key. -} -logFile :: GitRepo -> Key -> String +logFile :: Git.Repo -> Key -> String logFile repo key = - (gitStateDir repo) ++ (gitRelative repo (keyFile key)) ++ ".log" + (gitStateDir repo) ++ (Git.relative repo (keyFile key)) ++ ".log" {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} -keyLocations :: GitRepo -> Key -> IO [UUID] +keyLocations :: Git.Repo -> Key -> IO [UUID] keyLocations thisrepo key = do lines <- readLog $ logFile thisrepo key return $ map uuid (filterPresent lines) diff --git a/Locations.hs b/Locations.hs index 68a9581924..5d701681c1 100644 --- a/Locations.hs +++ b/Locations.hs @@ -12,27 +12,27 @@ module Locations ( import Data.String.Utils import Types -import GitRepo +import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex - directory, in the git repository's working tree. -} stateLoc = ".git-annex" -gitStateDir :: GitRepo -> FilePath -gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" +gitStateDir :: Git.Repo -> FilePath +gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc ++ "/" {- An annexed file's content is stored in - /path/to/repo/.git/annex// - - (That allows deriving the key and backend by looking at the symlink to it.) -} -annexLocation :: GitRepo -> Backend -> Key -> FilePath +annexLocation :: Git.Repo -> Backend -> Key -> FilePath annexLocation r backend key = - (gitWorkTree r) ++ "/" ++ (annexLocationRelative r backend key) + (Git.workTree r) ++ "/" ++ (annexLocationRelative r backend key) {- Annexed file's location relative to the gitWorkTree -} -annexLocationRelative :: GitRepo -> Backend -> Key -> FilePath +annexLocationRelative :: Git.Repo -> Backend -> Key -> FilePath annexLocationRelative r backend key = - gitDir r ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) + Git.dir r ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) {- Converts a key into a filename fragment. - diff --git a/Remotes.hs b/Remotes.hs index 711cd6c83a..39404bf191 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -10,18 +10,18 @@ import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils import AbstractTypes -import GitRepo +import qualified GitRepo as Git import LocationLog import Locations import UUID import List {- Human visible list of remotes. -} -remotesList :: [GitRepo] -> String -remotesList remotes = join " " $ map gitRepoDescribe remotes +remotesList :: [Git.Repo] -> String +remotesList remotes = join " " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -remotesWithKey :: Key -> Annex [GitRepo] +remotesWithKey :: Key -> Annex [Git.Repo] remotesWithKey key = do g <- gitAnnex uuids <- liftIO $ keyLocations g key @@ -34,13 +34,13 @@ remotesWithKey key = do else return remotes {- Cost Ordered list of remotes. -} -remotesByCost :: Annex [GitRepo] +remotesByCost :: Annex [Git.Repo] remotesByCost = do g <- gitAnnex - reposByCost $ gitRepoRemotes g + reposByCost $ Git.remotes g {- Orders a list of git repos by cost. -} -reposByCost :: [GitRepo] -> Annex [GitRepo] +reposByCost :: [Git.Repo] -> Annex [Git.Repo] reposByCost l = do costpairs <- mapM costpair l return $ fst $ unzip $ sortBy bycost $ costpairs @@ -55,36 +55,36 @@ reposByCost l = do - The default cost is 100 for local repositories, and 200 for remote - repositories; it can also be configured by remote..annex-cost -} -repoCost :: GitRepo -> Annex Int +repoCost :: Git.Repo -> Annex Int repoCost r = do g <- gitAnnex if ((length $ config g r) > 0) then return $ read $ config g r - else if (gitRepoIsLocal r) + else if (Git.repoIsLocal r) then return 100 else return 200 where - config g r = gitConfig g (configkey r) "" - configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" + config g r = Git.configGet g (configkey r) "" + configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function ensures that it is - read for a specified remote, and updates state. It returns the - updated git repo also. -} -remoteEnsureGitConfigRead :: GitRepo -> Annex GitRepo +remoteEnsureGitConfigRead :: Git.Repo -> Annex Git.Repo remoteEnsureGitConfigRead r = do - if (Map.null $ gitConfigMap r) + if (Map.null $ Git.configMap r) then do - r' <- liftIO $ gitConfigRead r + r' <- liftIO $ Git.configRead r g <- gitAnnex - let l = gitRepoRemotes g - let g' = gitRepoRemotesAdd g $ exchange l r' + let l = Git.remotes g + let g' = Git.remotesAdd g $ exchange l r' gitAnnexChange g' return r' else return r where exchange [] new = [] exchange (old:ls) new = - if ((gitRepoRemoteName old) == (gitRepoRemoteName new)) + if ((Git.repoRemoteName old) == (Git.repoRemoteName new)) then new:(exchange ls new) else old:(exchange ls new) diff --git a/Types.hs b/Types.hs index ce377dda11..c9d33affdd 100644 --- a/Types.hs +++ b/Types.hs @@ -1,26 +1,14 @@ {- git-annex core data types -} -module Types ( - Annex, - AnnexState, - makeAnnexState, - runAnnexState, - gitAnnex, - gitAnnexChange, - backendsAnnex, - backendsAnnexChange, - - Key(..), - Backend(..) -) where +module Types where import Control.Monad.State import Data.String.Utils -import GitRepo +import qualified GitRepo as Git -- git-annex's runtime state data AnnexState = AnnexState { - repo :: GitRepo, + repo :: Git.Repo, backends :: [Backend] } deriving (Show) @@ -28,18 +16,18 @@ data AnnexState = AnnexState { type Annex = StateT AnnexState IO -- constructor -makeAnnexState :: GitRepo -> AnnexState +makeAnnexState :: Git.Repo -> AnnexState makeAnnexState g = AnnexState { repo = g, backends = [] } -- performs an action in the Annex monad runAnnexState state action = runStateT (action) state -- Annex monad state accessors -gitAnnex :: Annex GitRepo +gitAnnex :: Annex Git.Repo gitAnnex = do state <- get return (repo state) -gitAnnexChange :: GitRepo -> Annex () +gitAnnexChange :: Git.Repo -> Annex () gitAnnexChange r = do state <- get put state { repo = r } diff --git a/UUID.hs b/UUID.hs index f334afdc9d..9c8b23a96a 100644 --- a/UUID.hs +++ b/UUID.hs @@ -19,7 +19,7 @@ import Maybe import List import System.Cmd.Utils import System.IO -import GitRepo +import qualified GitRepo as Git import AbstractTypes type UUID = String @@ -37,17 +37,17 @@ genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h - remote..annex-uuid - - -} -getUUID :: GitRepo -> Annex UUID +getUUID :: Git.Repo -> Annex UUID getUUID r = do if ("" /= configured r) then return $ configured r else cached r where - configured r = gitConfig r "annex.uuid" "" + configured r = Git.configGet r "annex.uuid" "" cached r = do g <- gitAnnex - return $ gitConfig g (configkey r) "" - configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-uuid" + return $ Git.configGet g (configkey r) "" + configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -57,15 +57,15 @@ prepUUID = do if ("" == u) then do uuid <- genUUID - liftIO $ gitRun g ["config", configkey, uuid] + liftIO $ Git.run g ["config", configkey, uuid] -- re-read git config and update the repo's state - u' <- liftIO $ gitConfigRead g + u' <- liftIO $ Git.configRead g gitAnnexChange u' return () else return () {- Filters a list of repos to ones that have listed UUIDs. -} -reposByUUID :: [GitRepo] -> [UUID] -> Annex [GitRepo] +reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] reposByUUID repos uuids = do filterM match repos where diff --git a/git-annex.hs b/git-annex.hs index be5168755f..2cf1c5305d 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -12,7 +12,7 @@ import Annex main = do args <- getArgs (mode, params) <- argvToMode args - state <- startAnnex + state <- start tryRun state mode 0 0 params {- Processes each param in the list by dispatching the handler function From 4c1d8b9689043c18214b1da7d5c145fb0859443d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 02:41:54 -0400 Subject: [PATCH 0114/2835] more namespace cleanup --- Annex.hs | 10 +++++----- BackendFile.hs | 8 ++++---- Remotes.hs | 18 +++++++++--------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Annex.hs b/Annex.hs index 54f9b9dff8..c26baabef5 100644 --- a/Annex.hs +++ b/Annex.hs @@ -20,7 +20,7 @@ import List import qualified GitRepo as Git import Utility import Locations -import Backend +import qualified Backend import BackendList import UUID import LocationLog @@ -46,7 +46,7 @@ start = do prepUUID inBackend file yes no = do - r <- liftIO $ lookupFile file + r <- liftIO $ Backend.lookupFile file case (r) of Just v -> yes v Nothing -> no @@ -57,7 +57,7 @@ notinBackend file yes no = inBackend file no yes annexCmd :: FilePath -> Annex () annexCmd file = inBackend file err $ do liftIO $ checkLegal file - stored <- storeFile file + stored <- Backend.storeFile file g <- gitAnnex case (stored) of Nothing -> error $ "no backend could store: " ++ file @@ -93,7 +93,7 @@ annexCmd file = inBackend file err $ do {- Inverse of annexCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - dropFile backend key + Backend.dropFile backend key logStatus key ValueMissing g <- gitAnnex let src = annexLocation g backend key @@ -121,7 +121,7 @@ getCmd file = notinBackend file err $ \(key, backend) -> do g <- gitAnnex let dest = annexLocation g backend key liftIO $ createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile backend key dest + success <- Backend.retrieveFile backend key dest if (success) then do logStatus key ValuePresent diff --git a/BackendFile.hs b/BackendFile.hs index 6c1dc06233..a0396f51da 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -10,7 +10,7 @@ import Control.Exception import Types import LocationLog import Locations -import Remotes +import qualified Remotes import qualified GitRepo as Git backend = Backend { @@ -39,17 +39,17 @@ dummyRemove url = return False - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do - remotes <- remotesWithKey key + remotes <- Remotes.withKey key trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ - (remotesList full) + (Remotes.list full) trycopy full (r:rs) = do -- annexLocation needs the git config to have been -- read for a remote, so do that now, -- if it hasn't been already - r' <- remoteEnsureGitConfigRead r + r' <- Remotes.ensureGitConfigRead r result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) case (result) of Left err -> do diff --git a/Remotes.hs b/Remotes.hs index 39404bf191..918ae2290c 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -1,9 +1,9 @@ {- git-annex remote repositories -} module Remotes ( - remotesList, - remotesWithKey, - remoteEnsureGitConfigRead + list, + withKey, + ensureGitConfigRead ) where import Control.Monad.State (liftIO) @@ -17,12 +17,12 @@ import UUID import List {- Human visible list of remotes. -} -remotesList :: [Git.Repo] -> String -remotesList remotes = join " " $ map Git.repoDescribe remotes +list :: [Git.Repo] -> String +list remotes = join " " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -remotesWithKey :: Key -> Annex [Git.Repo] -remotesWithKey key = do +withKey :: Key -> Annex [Git.Repo] +withKey key = do g <- gitAnnex uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost @@ -71,8 +71,8 @@ repoCost r = do - because reading it may be expensive. This function ensures that it is - read for a specified remote, and updates state. It returns the - updated git repo also. -} -remoteEnsureGitConfigRead :: Git.Repo -> Annex Git.Repo -remoteEnsureGitConfigRead r = do +ensureGitConfigRead :: Git.Repo -> Annex Git.Repo +ensureGitConfigRead r = do if (Map.null $ Git.configMap r) then do r' <- liftIO $ Git.configRead r From 0b55bd05de7b83a474ea58e9d45676934667f4bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 02:52:17 -0400 Subject: [PATCH 0115/2835] more namespace cleanup --- AbstractTypes.hs | 32 +++++++++++++++++++++++++++++++- Backend.hs | 3 ++- BackendChecksum.hs | 2 +- BackendFile.hs | 2 +- BackendList.hs | 2 +- Types.hs => BackendTypes.hs | 37 +++++++------------------------------ BackendUrl.hs | 2 +- Locations.hs | 8 +++++--- 8 files changed, 49 insertions(+), 39 deletions(-) rename Types.hs => BackendTypes.hs (55%) diff --git a/AbstractTypes.hs b/AbstractTypes.hs index 510a37f0cc..935d1de2f5 100644 --- a/AbstractTypes.hs +++ b/AbstractTypes.hs @@ -14,4 +14,34 @@ module AbstractTypes ( Backend ) where -import Types +import Control.Monad.State +import qualified GitRepo as Git +import BackendTypes + +-- constructor +makeAnnexState :: Git.Repo -> AnnexState +makeAnnexState g = AnnexState { repo = g, backends = [] } + +-- performs an action in the Annex monad +runAnnexState state action = runStateT (action) state + +-- Annex monad state accessors +gitAnnex :: Annex Git.Repo +gitAnnex = do + state <- get + return (repo state) +gitAnnexChange :: Git.Repo -> Annex () +gitAnnexChange r = do + state <- get + put state { repo = r } + return () +backendsAnnex :: Annex [Backend] +backendsAnnex = do + state <- get + return (backends state) +backendsAnnexChange :: [Backend] -> Annex () +backendsAnnexChange b = do + state <- get + put state { backends = b } + return () + diff --git a/Backend.hs b/Backend.hs index 1bd4efc1e8..251e436c75 100644 --- a/Backend.hs +++ b/Backend.hs @@ -30,7 +30,8 @@ import BackendList import Locations import qualified GitRepo as Git import Utility -import Types +import AbstractTypes +import BackendTypes {- Attempts to store a file in one of the backends. -} storeFile :: FilePath -> Annex (Maybe (Key, Backend)) diff --git a/BackendChecksum.hs b/BackendChecksum.hs index c6e68ffed4..50ef2ae6f6 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -5,7 +5,7 @@ module BackendChecksum (backend) where import qualified BackendFile import Data.Digest.Pure.SHA -import Types +import BackendTypes -- based on BackendFile just with a different key type backend = BackendFile.backend { diff --git a/BackendFile.hs b/BackendFile.hs index a0396f51da..284daca88e 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -7,7 +7,7 @@ import Control.Monad.State import System.IO import System.Cmd import Control.Exception -import Types +import BackendTypes import LocationLog import Locations import qualified Remotes diff --git a/BackendList.hs b/BackendList.hs index 104444dc20..91a2fa7fc3 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -7,7 +7,7 @@ module BackendList ( lookupBackendName ) where -import Types +import BackendTypes -- When adding a new backend, import it here and add it to the list. import qualified BackendFile diff --git a/Types.hs b/BackendTypes.hs similarity index 55% rename from Types.hs rename to BackendTypes.hs index c9d33affdd..2ef65f4691 100644 --- a/Types.hs +++ b/BackendTypes.hs @@ -1,12 +1,16 @@ -{- git-annex core data types -} +{- git-annex backend data types + - + - Mostly only backend implementations should need to import this. + -} -module Types where +module BackendTypes where import Control.Monad.State import Data.String.Utils import qualified GitRepo as Git --- git-annex's runtime state +-- git-annex's runtime state type doesn't really belong here, +-- but it uses Backend, so has to be here to avoid a depends loop. data AnnexState = AnnexState { repo :: Git.Repo, backends :: [Backend] @@ -15,33 +19,6 @@ data AnnexState = AnnexState { -- git-annex's monad type Annex = StateT AnnexState IO --- constructor -makeAnnexState :: Git.Repo -> AnnexState -makeAnnexState g = AnnexState { repo = g, backends = [] } - --- performs an action in the Annex monad -runAnnexState state action = runStateT (action) state - --- Annex monad state accessors -gitAnnex :: Annex Git.Repo -gitAnnex = do - state <- get - return (repo state) -gitAnnexChange :: Git.Repo -> Annex () -gitAnnexChange r = do - state <- get - put state { repo = r } - return () -backendsAnnex :: Annex [Backend] -backendsAnnex = do - state <- get - return (backends state) -backendsAnnexChange :: [Backend] -> Annex () -backendsAnnexChange b = do - state <- get - put state { backends = b } - return () - -- annexed filenames are mapped into keys data Key = Key String deriving (Eq) diff --git a/BackendUrl.hs b/BackendUrl.hs index 43b0bc75a8..fc0a8ae586 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -6,7 +6,7 @@ module BackendUrl (backend) where import Control.Monad.State import System.Cmd import IO -import Types +import BackendTypes backend = Backend { name = "url", diff --git a/Locations.hs b/Locations.hs index 5d701681c1..8c1915b021 100644 --- a/Locations.hs +++ b/Locations.hs @@ -11,7 +11,8 @@ module Locations ( ) where import Data.String.Utils -import Types +import AbstractTypes +import qualified BackendTypes as Backend import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex @@ -32,7 +33,7 @@ annexLocation r backend key = {- Annexed file's location relative to the gitWorkTree -} annexLocationRelative :: Git.Repo -> Backend -> Key -> FilePath annexLocationRelative r backend key = - Git.dir r ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) + Git.dir r ++ "/annex/" ++ (Backend.name backend) ++ "/" ++ (keyFile key) {- Converts a key into a filename fragment. - @@ -50,4 +51,5 @@ keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Key -fileKey file = Key $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file +fileKey file = Backend.Key $ + replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file From 6f3572e47f57bbe5cc76b58c8bcdc9c6c455dce0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:18:11 -0400 Subject: [PATCH 0116/2835] more reorg, spiffed up state monad --- AbstractTypes.hs | 47 ----------- Annex.hs | 215 ++++++++--------------------------------------- Backend.hs | 7 +- CmdLine.hs | 4 +- Commands.hs | 189 +++++++++++++++++++++++++++++++++++++++++ LocationLog.hs | 2 +- Locations.hs | 2 +- Remotes.hs | 13 +-- Types.hs | 10 +++ UUID.hs | 11 +-- git-annex.hs | 8 +- 11 files changed, 259 insertions(+), 249 deletions(-) delete mode 100644 AbstractTypes.hs create mode 100644 Commands.hs create mode 100644 Types.hs diff --git a/AbstractTypes.hs b/AbstractTypes.hs deleted file mode 100644 index 935d1de2f5..0000000000 --- a/AbstractTypes.hs +++ /dev/null @@ -1,47 +0,0 @@ -{- git-annex data types, abstract only -} - -module AbstractTypes ( - Annex, - AnnexState, - makeAnnexState, - runAnnexState, - gitAnnex, - gitAnnexChange, - backendsAnnex, - backendsAnnexChange, - - Key, - Backend -) where - -import Control.Monad.State -import qualified GitRepo as Git -import BackendTypes - --- constructor -makeAnnexState :: Git.Repo -> AnnexState -makeAnnexState g = AnnexState { repo = g, backends = [] } - --- performs an action in the Annex monad -runAnnexState state action = runStateT (action) state - --- Annex monad state accessors -gitAnnex :: Annex Git.Repo -gitAnnex = do - state <- get - return (repo state) -gitAnnexChange :: Git.Repo -> Annex () -gitAnnexChange r = do - state <- get - put state { repo = r } - return () -backendsAnnex :: Annex [Backend] -backendsAnnex = do - state <- get - return (backends state) -backendsAnnexChange :: [Backend] -> Annex () -backendsAnnexChange b = do - state <- get - put state { backends = b } - return () - diff --git a/Annex.hs b/Annex.hs index c26baabef5..fcd19ba033 100644 --- a/Annex.hs +++ b/Annex.hs @@ -1,189 +1,42 @@ -{- git-annex toplevel code - -} +{- git-annex monad -} module Annex ( - start, - annexCmd, - unannexCmd, - getCmd, - wantCmd, - dropCmd, - pushCmd, - pullCmd + new, + run, + gitRepo, + gitRepoChange, + backends, + backendsChange, ) where -import Control.Monad.State (liftIO) -import System.Posix.Files -import System.Directory -import Data.String.Utils -import List +import Control.Monad.State import qualified GitRepo as Git -import Utility -import Locations -import qualified Backend -import BackendList -import UUID -import LocationLog -import AbstractTypes +import Types +import qualified BackendTypes as Backend -{- Create and returns an Annex state object. - - Examines and prepares the git repo. - -} -start :: IO AnnexState -start = do - g <- Git.repoFromCwd - let s = makeAnnexState g - (_,s') <- runAnnexState s (prep g) - return s' - where - prep g = do - -- setup git and read its config; update state - g' <- liftIO $ Git.configRead g - gitAnnexChange g' - liftIO $ gitSetup g' - backendsAnnexChange $ parseBackendList $ - Git.configGet g' "annex.backends" "" - prepUUID +-- constructor +new :: Git.Repo -> AnnexState +new g = Backend.AnnexState { Backend.repo = g, Backend.backends = [] } -inBackend file yes no = do - r <- liftIO $ Backend.lookupFile file - case (r) of - Just v -> yes v - Nothing -> no -notinBackend file yes no = inBackend file no yes +-- performs an action in the Annex monad +run state action = runStateT (action) state -{- Annexes a file, storing it in a backend, and then moving it into - - the annex directory and setting up the symlink pointing to its content. -} -annexCmd :: FilePath -> Annex () -annexCmd file = inBackend file err $ do - liftIO $ checkLegal file - stored <- Backend.storeFile file - g <- gitAnnex - case (stored) of - Nothing -> error $ "no backend could store: " ++ file - Just (key, backend) -> do - logStatus key ValuePresent - liftIO $ setup g key backend - where - err = error $ "already annexed " ++ file - checkLegal file = do - s <- getSymbolicLinkStatus file - if ((isSymbolicLink s) || (not $ isRegularFile s)) - then error $ "not a regular file: " ++ file - else return () - setup g key backend = do - let dest = annexLocation g backend key - let reldest = annexLocationRelative g backend key - createDirectoryIfMissing True (parentDir dest) - renameFile file dest - createSymbolicLink ((linkTarget file) ++ reldest) file - Git.run g ["add", file] - Git.run g ["commit", "-m", - ("git-annex annexed " ++ file), file] - linkTarget file = - -- relies on file being relative to the top of the - -- git repo; just replace each subdirectory with ".." - if (subdirs > 0) - then (join "/" $ take subdirs $ repeat "..") ++ "/" - else "" - where - subdirs = (length $ split "/" file) - 1 - - -{- Inverse of annexCmd. -} -unannexCmd :: FilePath -> Annex () -unannexCmd file = notinBackend file err $ \(key, backend) -> do - Backend.dropFile backend key - logStatus key ValueMissing - g <- gitAnnex - let src = annexLocation g backend key - liftIO $ moveout g src - where - err = error $ "not annexed " ++ file - moveout g src = do - removeFile file - Git.run g ["rm", file] - Git.run g ["commit", "-m", - ("git-annex unannexed " ++ file), file] - -- git rm deletes empty directories; - -- put them back - createDirectoryIfMissing True (parentDir file) - renameFile src file - return () - -{- Gets an annexed file from one of the backends. -} -getCmd :: FilePath -> Annex () -getCmd file = notinBackend file err $ \(key, backend) -> do - inannex <- inAnnex backend key - if (inannex) - then return () - else do - g <- gitAnnex - let dest = annexLocation g backend key - liftIO $ createDirectoryIfMissing True (parentDir dest) - success <- Backend.retrieveFile backend key dest - if (success) - then do - logStatus key ValuePresent - return () - else error $ "failed to get " ++ file - where - err = error $ "not annexed " ++ file - -{- Indicates a file is wanted. -} -wantCmd :: FilePath -> Annex () -wantCmd file = do error "not implemented" -- TODO - -{- Indicates a file is not wanted. -} -dropCmd :: FilePath -> Annex () -dropCmd file = do error "not implemented" -- TODO - -{- Pushes all files to a remote repository. -} -pushCmd :: String -> Annex () -pushCmd reponame = do error "not implemented" -- TODO - -{- Pulls all files from a remote repository. -} -pullCmd :: String -> Annex () -pullCmd reponame = do error "not implemented" -- TODO - -{- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: Git.Repo -> IO () -gitSetup repo = do - -- configure git to use union merge driver on state files - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - commit - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () - where - attrLine = stateLoc ++ "/*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] - -{- Updates the LocationLog when a key's presence changes. -} -logStatus :: Key -> LogStatus -> Annex () -logStatus key status = do - g <- gitAnnex - u <- getUUID g - f <- liftIO $ logChange g key u status - liftIO $ commit g f - where - commit g f = do - Git.run g ["add", f] - Git.run g ["commit", "-m", "git-annex log update", f] - -{- Checks if a given key is currently present in the annexLocation -} -inAnnex :: Backend -> Key -> Annex Bool -inAnnex backend key = do - g <- gitAnnex - liftIO $ doesFileExist $ annexLocation g backend key +-- Annex monad state accessors +gitRepo :: Annex Git.Repo +gitRepo = do + state <- get + return (Backend.repo state) +gitRepoChange :: Git.Repo -> Annex () +gitRepoChange r = do + state <- get + put state { Backend.repo = r } + return () +backends :: Annex [Backend] +backends = do + state <- get + return (Backend.backends state) +backendsChange :: [Backend] -> Annex () +backendsChange b = do + state <- get + put state { Backend.backends = b } + return () diff --git a/Backend.hs b/Backend.hs index 251e436c75..2829fef9d2 100644 --- a/Backend.hs +++ b/Backend.hs @@ -29,16 +29,17 @@ import System.Posix.Files import BackendList import Locations import qualified GitRepo as Git +import qualified Annex import Utility -import AbstractTypes +import Types import BackendTypes {- Attempts to store a file in one of the backends. -} storeFile :: FilePath -> Annex (Maybe (Key, Backend)) storeFile file = do - g <- gitAnnex + g <- Annex.gitRepo let relfile = Git.relative g file - b <- backendsAnnex + b <- Annex.backends storeFile' b file relfile storeFile' [] _ _ = return Nothing storeFile' (b:bs) file relfile = do diff --git a/CmdLine.hs b/CmdLine.hs index 479be7e8b5..9737e0eb01 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -11,8 +11,8 @@ module CmdLine ( ) where import System.Console.GetOpt -import AbstractTypes -import Annex +import Types +import Commands data Mode = Add | Push | Pull | Want | Get | Drop | Unannex deriving Show diff --git a/Commands.hs b/Commands.hs new file mode 100644 index 0000000000..98e65b126a --- /dev/null +++ b/Commands.hs @@ -0,0 +1,189 @@ +{- git-annex subcommands -} + +module Commands ( + start, + annexCmd, + unannexCmd, + getCmd, + wantCmd, + dropCmd, + pushCmd, + pullCmd +) where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory +import Data.String.Utils +import List +import qualified GitRepo as Git +import qualified Annex +import Utility +import Locations +import qualified Backend +import BackendList +import UUID +import LocationLog +import Types + +{- Create and returns an Annex state object. + - Examines and prepares the git repo. + -} +start :: IO AnnexState +start = do + g <- Git.repoFromCwd + let s = Annex.new g + (_,s') <- Annex.run s (prep g) + return s' + where + prep g = do + -- setup git and read its config; update state + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' + liftIO $ gitSetup g' + Annex.backendsChange $ parseBackendList $ + Git.configGet g' "annex.backends" "" + prepUUID + +inBackend file yes no = do + r <- liftIO $ Backend.lookupFile file + case (r) of + Just v -> yes v + Nothing -> no +notinBackend file yes no = inBackend file no yes + +{- Annexes a file, storing it in a backend, and then moving it into + - the annex directory and setting up the symlink pointing to its content. -} +annexCmd :: FilePath -> Annex () +annexCmd file = inBackend file err $ do + liftIO $ checkLegal file + stored <- Backend.storeFile file + g <- Annex.gitRepo + case (stored) of + Nothing -> error $ "no backend could store: " ++ file + Just (key, backend) -> do + logStatus key ValuePresent + liftIO $ setup g key backend + where + err = error $ "already annexed " ++ file + checkLegal file = do + s <- getSymbolicLinkStatus file + if ((isSymbolicLink s) || (not $ isRegularFile s)) + then error $ "not a regular file: " ++ file + else return () + setup g key backend = do + let dest = annexLocation g backend key + let reldest = annexLocationRelative g backend key + createDirectoryIfMissing True (parentDir dest) + renameFile file dest + createSymbolicLink ((linkTarget file) ++ reldest) file + Git.run g ["add", file] + Git.run g ["commit", "-m", + ("git-annex annexed " ++ file), file] + linkTarget file = + -- relies on file being relative to the top of the + -- git repo; just replace each subdirectory with ".." + if (subdirs > 0) + then (join "/" $ take subdirs $ repeat "..") ++ "/" + else "" + where + subdirs = (length $ split "/" file) - 1 + + +{- Inverse of annexCmd. -} +unannexCmd :: FilePath -> Annex () +unannexCmd file = notinBackend file err $ \(key, backend) -> do + Backend.dropFile backend key + logStatus key ValueMissing + g <- Annex.gitRepo + let src = annexLocation g backend key + liftIO $ moveout g src + where + err = error $ "not annexed " ++ file + moveout g src = do + removeFile file + Git.run g ["rm", file] + Git.run g ["commit", "-m", + ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) + renameFile src file + return () + +{- Gets an annexed file from one of the backends. -} +getCmd :: FilePath -> Annex () +getCmd file = notinBackend file err $ \(key, backend) -> do + inannex <- inAnnex backend key + if (inannex) + then return () + else do + g <- Annex.gitRepo + let dest = annexLocation g backend key + liftIO $ createDirectoryIfMissing True (parentDir dest) + success <- Backend.retrieveFile backend key dest + if (success) + then do + logStatus key ValuePresent + return () + else error $ "failed to get " ++ file + where + err = error $ "not annexed " ++ file + +{- Indicates a file is wanted. -} +wantCmd :: FilePath -> Annex () +wantCmd file = do error "not implemented" -- TODO + +{- Indicates a file is not wanted. -} +dropCmd :: FilePath -> Annex () +dropCmd file = do error "not implemented" -- TODO + +{- Pushes all files to a remote repository. -} +pushCmd :: String -> Annex () +pushCmd reponame = do error "not implemented" -- TODO + +{- Pulls all files from a remote repository. -} +pullCmd :: String -> Annex () +pullCmd reponame = do error "not implemented" -- TODO + +{- Sets up a git repo for git-annex. May be called repeatedly. -} +gitSetup :: Git.Repo -> IO () +gitSetup repo = do + -- configure git to use union merge driver on state files + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then do + appendFile attributes $ attrLine ++ "\n" + commit + else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] + +{- Updates the LocationLog when a key's presence changes. -} +logStatus :: Key -> LogStatus -> Annex () +logStatus key status = do + g <- Annex.gitRepo + u <- getUUID g + f <- liftIO $ logChange g key u status + liftIO $ commit g f + where + commit g f = do + Git.run g ["add", f] + Git.run g ["commit", "-m", "git-annex log update", f] + +{- Checks if a given key is currently present in the annexLocation -} +inAnnex :: Backend -> Key -> Annex Bool +inAnnex backend key = do + g <- Annex.gitRepo + liftIO $ doesFileExist $ annexLocation g backend key diff --git a/LocationLog.hs b/LocationLog.hs index 7953b345f8..ba91787049 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -32,7 +32,7 @@ import Data.Char import qualified GitRepo as Git import Utility import UUID -import AbstractTypes +import Types import Locations data LogLine = LogLine { diff --git a/Locations.hs b/Locations.hs index 8c1915b021..7b8beb14f0 100644 --- a/Locations.hs +++ b/Locations.hs @@ -11,7 +11,7 @@ module Locations ( ) where import Data.String.Utils -import AbstractTypes +import Types import qualified BackendTypes as Backend import qualified GitRepo as Git diff --git a/Remotes.hs b/Remotes.hs index 918ae2290c..1802ff28eb 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -9,8 +9,9 @@ module Remotes ( import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils -import AbstractTypes +import Types import qualified GitRepo as Git +import qualified Annex import LocationLog import Locations import UUID @@ -23,7 +24,7 @@ list remotes = join " " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} withKey :: Key -> Annex [Git.Repo] withKey key = do - g <- gitAnnex + g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost remotes <- reposByUUID allremotes uuids @@ -36,7 +37,7 @@ withKey key = do {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] remotesByCost = do - g <- gitAnnex + g <- Annex.gitRepo reposByCost $ Git.remotes g {- Orders a list of git repos by cost. -} @@ -57,7 +58,7 @@ reposByCost l = do -} repoCost :: Git.Repo -> Annex Int repoCost r = do - g <- gitAnnex + g <- Annex.gitRepo if ((length $ config g r) > 0) then return $ read $ config g r else if (Git.repoIsLocal r) @@ -76,10 +77,10 @@ ensureGitConfigRead r = do if (Map.null $ Git.configMap r) then do r' <- liftIO $ Git.configRead r - g <- gitAnnex + g <- Annex.gitRepo let l = Git.remotes g let g' = Git.remotesAdd g $ exchange l r' - gitAnnexChange g' + Annex.gitRepoChange g' return r' else return r where diff --git a/Types.hs b/Types.hs new file mode 100644 index 0000000000..4262ed567d --- /dev/null +++ b/Types.hs @@ -0,0 +1,10 @@ +{- git-annex abstract data types -} + +module Types ( + Annex, + AnnexState, + Key, + Backend +) where + +import BackendTypes diff --git a/UUID.hs b/UUID.hs index 9c8b23a96a..1c31a343fa 100644 --- a/UUID.hs +++ b/UUID.hs @@ -20,7 +20,8 @@ import List import System.Cmd.Utils import System.IO import qualified GitRepo as Git -import AbstractTypes +import Types +import qualified Annex type UUID = String @@ -45,22 +46,22 @@ getUUID r = do where configured r = Git.configGet r "annex.uuid" "" cached r = do - g <- gitAnnex + g <- Annex.gitRepo return $ Git.configGet g (configkey r) "" configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () prepUUID = do - g <- gitAnnex + g <- Annex.gitRepo u <- getUUID g if ("" == u) then do uuid <- genUUID liftIO $ Git.run g ["config", configkey, uuid] -- re-read git config and update the repo's state - u' <- liftIO $ Git.configRead g - gitAnnexChange u' + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' return () else return () diff --git a/git-annex.hs b/git-annex.hs index 2cf1c5305d..ce3b2ac428 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,8 +6,9 @@ import System.IO import System.Environment import Control.Exception import CmdLine -import AbstractTypes -import Annex +import Types +import Commands +import qualified Annex main = do args <- getArgs @@ -30,7 +31,8 @@ tryRun state mode errnum oknum [] = do then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" else return () tryRun state mode errnum oknum (f:fs) = do - result <- try (runAnnexState state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) + result <- try + (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) case (result) of Left err -> do showErr err From f407f23a54d9152a382ee8e48629f40e1a72a26f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:40:26 -0400 Subject: [PATCH 0117/2835] more refactor --- Commands.hs | 58 +++++------------------------------ Core.hs | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++ git-annex.hs | 41 +++---------------------- 3 files changed, 98 insertions(+), 88 deletions(-) create mode 100644 Core.hs diff --git a/Commands.hs b/Commands.hs index 98e65b126a..be61c7c64e 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,7 +1,6 @@ {- git-annex subcommands -} module Commands ( - start, annexCmd, unannexCmd, getCmd, @@ -26,32 +25,6 @@ import UUID import LocationLog import Types -{- Create and returns an Annex state object. - - Examines and prepares the git repo. - -} -start :: IO AnnexState -start = do - g <- Git.repoFromCwd - let s = Annex.new g - (_,s') <- Annex.run s (prep g) - return s' - where - prep g = do - -- setup git and read its config; update state - g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' - liftIO $ gitSetup g' - Annex.backendsChange $ parseBackendList $ - Git.configGet g' "annex.backends" "" - prepUUID - -inBackend file yes no = do - r <- liftIO $ Backend.lookupFile file - case (r) of - Just v -> yes v - Nothing -> no -notinBackend file yes no = inBackend file no yes - {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexCmd :: FilePath -> Annex () @@ -146,30 +119,6 @@ pushCmd reponame = do error "not implemented" -- TODO pullCmd :: String -> Annex () pullCmd reponame = do error "not implemented" -- TODO -{- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: Git.Repo -> IO () -gitSetup repo = do - -- configure git to use union merge driver on state files - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - commit - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () - where - attrLine = stateLoc ++ "/*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] - {- Updates the LocationLog when a key's presence changes. -} logStatus :: Key -> LogStatus -> Annex () logStatus key status = do @@ -182,6 +131,13 @@ logStatus key status = do Git.run g ["add", f] Git.run g ["commit", "-m", "git-annex log update", f] +inBackend file yes no = do + r <- liftIO $ Backend.lookupFile file + case (r) of + Just v -> yes v + Nothing -> no +notinBackend file yes no = inBackend file no yes + {- Checks if a given key is currently present in the annexLocation -} inAnnex :: Backend -> Key -> Annex Bool inAnnex backend key = do diff --git a/Core.hs b/Core.hs new file mode 100644 index 0000000000..e3d2c64039 --- /dev/null +++ b/Core.hs @@ -0,0 +1,87 @@ +{- git-annex core functions -} + +module Core where + +import System.IO +import System.Directory +import Control.Monad.State (liftIO) +import Control.Exception +import CmdLine +import Types +import BackendList +import Locations +import UUID +import qualified GitRepo as Git +import qualified Annex + +{- Create and returns an Annex state object. + - Examines and prepares the git repo. + -} +start :: IO AnnexState +start = do + g <- Git.repoFromCwd + let s = Annex.new g + (_,s') <- Annex.run s (prep g) + return s' + where + prep g = do + -- setup git and read its config; update state + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' + liftIO $ gitSetup g' + Annex.backendsChange $ parseBackendList $ + Git.configGet g' "annex.backends" "" + prepUUID + +{- Processes each param in the list by dispatching the handler function + - for the user-selection operation mode. Catches exceptions, not stopping + - if some error out, and propigates an overall error status at the end. + - + - This runs in the IO monad, not in the Annex monad. It seems that + - exceptions can only be caught in the IO monad, not in a stacked monad; + - or more likely I missed an easy way to do it. So, I have to laboriously + - thread AnnexState through this function. + -} +tryRun :: AnnexState -> Mode -> [String] -> IO () +tryRun state mode params = tryRun' state mode 0 0 params +tryRun' state mode errnum oknum [] = do + if (errnum > 0) + then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" + else return () +tryRun' state mode errnum oknum (f:fs) = do + result <- try + (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) + case (result) of + Left err -> do + showErr err + tryRun' state mode (errnum + 1) oknum fs + Right (_,state') -> tryRun' state' mode errnum (oknum + 1) fs + +{- Exception pretty-printing. -} +showErr e = do + hPutStrLn stderr $ "git-annex: " ++ (show e) + return () + +{- Sets up a git repo for git-annex. May be called repeatedly. -} +gitSetup :: Git.Repo -> IO () +gitSetup repo = do + -- configure git to use union merge driver on state files + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then do + appendFile attributes $ attrLine ++ "\n" + commit + else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] diff --git a/git-annex.hs b/git-annex.hs index ce3b2ac428..b326b2b199 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,45 +1,12 @@ -{- git-annex main program - - -} +{- git-annex main program -} -import Control.Monad.State -import System.IO import System.Environment -import Control.Exception -import CmdLine -import Types -import Commands import qualified Annex +import Core +import CmdLine main = do args <- getArgs (mode, params) <- argvToMode args state <- start - tryRun state mode 0 0 params - -{- Processes each param in the list by dispatching the handler function - - for the user-selection operation mode. Catches exceptions, not stopping - - if some error out, and propigates an overall error status at the end. - - - - This runs in the IO monad, not in the Annex monad. It seems that - - exceptions can only be caught in the IO monad, not in a stacked monad; - - or more likely I missed an easy way to do it. So, I have to laboriously - - thread AnnexState through this function. - -} -tryRun :: AnnexState -> Mode -> Int -> Int -> [String] -> IO () -tryRun state mode errnum oknum [] = do - if (errnum > 0) - then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" - else return () -tryRun state mode errnum oknum (f:fs) = do - result <- try - (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) - case (result) of - Left err -> do - showErr err - tryRun state mode (errnum + 1) oknum fs - Right (_,state') -> tryRun state' mode errnum (oknum + 1) fs - -{- Exception pretty-printing. -} -showErr e = do - hPutStrLn stderr $ "git-annex: " ++ (show e) - return () + tryRun state mode params From 50630840ee6802fef9db136505975db40a81920a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:46:34 -0400 Subject: [PATCH 0118/2835] build in subdir --- .gitignore | 5 +---- Makefile | 5 +++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7dd8869b16..2b3e3aef1f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,2 @@ -*.o -*.hi -*.ho -*.a +build/* git-annex diff --git a/Makefile b/Makefile index 8b7c9d3a0e..876407de09 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ git-annex: - ghc --make git-annex + mkdir -p build + ghc -odir build -hidir build --make git-annex clean: - rm -f git-annex *.o *.hi *.ho *.a + rm -rf build git-annex .PHONY: git-annex From 7117702fddf521ed4f3675a91cd87119207eba02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:47:06 -0400 Subject: [PATCH 0119/2835] foo --- .gitattributes | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index b98b07d7d2..0000000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -.git-annex/*.log merge=union From 0f12bd16d829432f7b1c2efbba386262ed36fc27 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:50:28 -0400 Subject: [PATCH 0120/2835] subdir --- BackendChecksum.hs => Backend/Checksum.hs | 6 +++--- BackendFile.hs => Backend/File.hs | 2 +- BackendUrl.hs => Backend/Url.hs | 2 +- BackendList.hs | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) rename BackendChecksum.hs => Backend/Checksum.hs (75%) rename BackendFile.hs => Backend/File.hs (98%) rename BackendUrl.hs => Backend/Url.hs (95%) diff --git a/BackendChecksum.hs b/Backend/Checksum.hs similarity index 75% rename from BackendChecksum.hs rename to Backend/Checksum.hs index 50ef2ae6f6..bfc789e40e 100644 --- a/BackendChecksum.hs +++ b/Backend/Checksum.hs @@ -1,14 +1,14 @@ {- git-annex "checksum" backend - -} -module BackendChecksum (backend) where +module Backend.Checksum (backend) where -import qualified BackendFile +import qualified Backend.File import Data.Digest.Pure.SHA import BackendTypes -- based on BackendFile just with a different key type -backend = BackendFile.backend { +backend = Backend.File.backend { name = "checksum", getKey = keyValue } diff --git a/BackendFile.hs b/Backend/File.hs similarity index 98% rename from BackendFile.hs rename to Backend/File.hs index 284daca88e..107ef38517 100644 --- a/BackendFile.hs +++ b/Backend/File.hs @@ -1,7 +1,7 @@ {- git-annex "file" backend - -} -module BackendFile (backend) where +module Backend.File (backend) where import Control.Monad.State import System.IO diff --git a/BackendUrl.hs b/Backend/Url.hs similarity index 95% rename from BackendUrl.hs rename to Backend/Url.hs index fc0a8ae586..e4ba58e6d7 100644 --- a/BackendUrl.hs +++ b/Backend/Url.hs @@ -1,7 +1,7 @@ {- git-annex "url" backend - -} -module BackendUrl (backend) where +module Backend.Url (backend) where import Control.Monad.State import System.Cmd diff --git a/BackendList.hs b/BackendList.hs index 91a2fa7fc3..e9f926ce2a 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -10,13 +10,13 @@ module BackendList ( import BackendTypes -- When adding a new backend, import it here and add it to the list. -import qualified BackendFile -import qualified BackendChecksum -import qualified BackendUrl +import qualified Backend.File +import qualified Backend.Checksum +import qualified Backend.Url supportedBackends = - [ BackendFile.backend - , BackendChecksum.backend - , BackendUrl.backend + [ Backend.File.backend + , Backend.Checksum.backend + , Backend.Url.backend ] {- Parses a string with a list of backend names into From 282d9853682f457cc6dc8b095b230bd892f0a5f3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 12:36:40 -0400 Subject: [PATCH 0121/2835] default command --- CmdLine.hs | 8 ++++---- Commands.hs | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 9737e0eb01..98971a733b 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -14,7 +14,7 @@ import System.Console.GetOpt import Types import Commands -data Mode = Add | Push | Pull | Want | Get | Drop | Unannex +data Mode = Default | Add | Push | Pull | Want | Get | Drop | Unannex deriving Show options :: [OptDescr Mode] @@ -30,8 +30,7 @@ options = argvToMode argv = do case getOpt Permute options argv of - -- default mode is Add - ([],files,[]) -> return (Add, files) + ([],files,[]) -> return (Default, files) -- one mode is normal case (m:[],files,[]) -> return (m, files) -- multiple modes is an error @@ -43,7 +42,8 @@ argvToMode argv = do dispatch :: Mode -> FilePath -> Annex () dispatch mode item = do case (mode) of - Add -> annexCmd item + Default -> defaultCmd item + Add -> addCmd item Push -> pushCmd item Pull -> pullCmd item Want -> wantCmd item diff --git a/Commands.hs b/Commands.hs index be61c7c64e..b4f57d6fe4 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,7 +1,8 @@ {- git-annex subcommands -} module Commands ( - annexCmd, + defaultCmd, + addCmd, unannexCmd, getCmd, wantCmd, @@ -25,10 +26,19 @@ import UUID import LocationLog import Types +{- Default mode is to annex a file if it is not already, and otherwise + - get its content. -} +defaultCmd :: FilePath -> Annex () +defaultCmd file = do + r <- liftIO $ Backend.lookupFile file + case (r) of + Just v -> getCmd file + Nothing -> addCmd file + {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} -annexCmd :: FilePath -> Annex () -annexCmd file = inBackend file err $ do +addCmd :: FilePath -> Annex () +addCmd file = inBackend file err $ do liftIO $ checkLegal file stored <- Backend.storeFile file g <- Annex.gitRepo @@ -63,7 +73,7 @@ annexCmd file = inBackend file err $ do subdirs = (length $ split "/" file) - 1 -{- Inverse of annexCmd. -} +{- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do Backend.dropFile backend key From 8df3e2aa0227e426ade1d92f430e02e31bb97ad9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 13:11:42 -0400 Subject: [PATCH 0122/2835] query remotes for uuids (not cached yet) --- Backend/File.hs | 21 ++++++++++++-------- Remotes.hs | 52 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 107ef38517..78e1f55635 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -49,13 +49,16 @@ copyKeyFile key file = do -- annexLocation needs the git config to have been -- read for a remote, so do that now, -- if it hasn't been already - r' <- Remotes.ensureGitConfigRead r - result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) - case (result) of - Left err -> do - liftIO $ hPutStrLn stderr (show err) - trycopy full rs - Right succ -> return True + result <- Remotes.tryGitConfigRead r + case (result) of + Nothing -> trycopy full rs + Just r' -> do + result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) + case (result) of + Left err -> do + liftIO $ hPutStrLn stderr (show err) + trycopy full rs + Right succ -> return True {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () @@ -67,6 +70,8 @@ copyFromRemote r key file = do else getremote return () where - getlocal = rawSystem "cp" ["-a", location, file] + getlocal = do + rawSystem "cp" ["-a", location, file] + putStrLn "cp done" getremote = error "get via network not yet implemented!" location = annexLocation r backend key diff --git a/Remotes.hs b/Remotes.hs index 1802ff28eb..ecb0d96e38 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -3,19 +3,21 @@ module Remotes ( list, withKey, - ensureGitConfigRead + tryGitConfigRead ) where import Control.Monad.State (liftIO) +import IO import qualified Data.Map as Map import Data.String.Utils +import List +import Maybe import Types import qualified GitRepo as Git import qualified Annex import LocationLog import Locations import UUID -import List {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -27,12 +29,25 @@ withKey key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost + -- this only uses cached data, so may not find new remotes remotes <- reposByUUID allremotes uuids if (0 == length remotes) - then error $ "no configured git remotes have: " ++ (keyFile key) ++ "\n" ++ + then tryharder allremotes uuids + else return remotes + where + tryharder allremotes uuids = do + -- more expensive; check each remote's config + mayberemotes <- mapM tryGitConfigRead allremotes + let allremotes' = catMaybes mayberemotes + remotes' <- reposByUUID allremotes' uuids + if (0 == length remotes') + then err uuids + else return remotes' + err uuids = + error $ "no available git remotes have: " ++ + (keyFile key) ++ "\n" ++ "It has been seen before in these repositories:\n" ++ prettyPrintUUIDs uuids - else return remotes {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] @@ -69,20 +84,25 @@ repoCost r = do configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" {- The git configs for the git repo's remotes is not read on startup - - because reading it may be expensive. This function ensures that it is - - read for a specified remote, and updates state. It returns the - - updated git repo also. -} -ensureGitConfigRead :: Git.Repo -> Annex Git.Repo -ensureGitConfigRead r = do + - because reading it may be expensive. This function tries to read the + - config for a specified remote, and updates state. If successful, it + - returns the updated git repo. -} +tryGitConfigRead :: Git.Repo -> Annex (Maybe Git.Repo) +tryGitConfigRead r = do if (Map.null $ Git.configMap r) then do - r' <- liftIO $ Git.configRead r - g <- Annex.gitRepo - let l = Git.remotes g - let g' = Git.remotesAdd g $ exchange l r' - Annex.gitRepoChange g' - return r' - else return r + liftIO $ putStrLn $ "read config for " ++ (show r) + result <- liftIO $ try (Git.configRead r) + case (result) of + Left err -> return Nothing + Right r' -> do + g <- Annex.gitRepo + let l = Git.remotes g + let g' = Git.remotesAdd g $ + exchange l r' + Annex.gitRepoChange g' + return $ Just r' + else return $ Just r where exchange [] new = [] exchange (old:ls) new = From 7c975eab07d842e3d91626871027f803f34c6372 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 13:17:43 -0400 Subject: [PATCH 0123/2835] check rawSystem exit codes --- Backend/File.hs | 8 +++++--- Backend/Url.hs | 10 +++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 78e1f55635..2ac12487e1 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -6,6 +6,7 @@ module Backend.File (backend) where import Control.Monad.State import System.IO import System.Cmd +import System.Exit import Control.Exception import BackendTypes import LocationLog @@ -68,10 +69,11 @@ copyFromRemote r key file = do if (Git.repoIsLocal r) then getlocal else getremote - return () where getlocal = do - rawSystem "cp" ["-a", location, file] - putStrLn "cp done" + res <-rawSystem "cp" ["-a", location, file] + if (res == ExitSuccess) + then return () + else error "cp failed" getremote = error "get via network not yet implemented!" location = annexLocation r backend key diff --git a/Backend/Url.hs b/Backend/Url.hs index e4ba58e6d7..9831c337b8 100644 --- a/Backend/Url.hs +++ b/Backend/Url.hs @@ -5,7 +5,7 @@ module Backend.Url (backend) where import Control.Monad.State import System.Cmd -import IO +import System.Exit import BackendTypes backend = Backend { @@ -29,7 +29,7 @@ dummyRemove url = return False downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl url file = do liftIO $ putStrLn $ "download: " ++ (show url) - result <- liftIO $ try $ rawSystem "curl" ["-#", "-o", file, (show url)] - case (result) of - Left _ -> return False - Right _ -> return True + result <- liftIO $ rawSystem "curl" ["-#", "-o", file, (show url)] + if (result == ExitSuccess) + then return True + else return False From f9557d7c5e2aa7ef19a5d589594154a21c7f2caa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 13:49:45 -0400 Subject: [PATCH 0124/2835] uuid cache done --- Remotes.hs | 1 - TODO | 2 -- UUID.hs | 42 +++++++++++++++++++++++++++++------------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index ecb0d96e38..4f4e5a26cb 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -91,7 +91,6 @@ tryGitConfigRead :: Git.Repo -> Annex (Maybe Git.Repo) tryGitConfigRead r = do if (Map.null $ Git.configMap r) then do - liftIO $ putStrLn $ "read config for " ++ (show r) result <- liftIO $ try (Git.configRead r) case (result) of Left err -> return Nothing diff --git a/TODO b/TODO index 40017c816a..54411185a8 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,6 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* query remotes for their annex.uuid settings and cache - * --push/--pull/--want/--drop * how to handle git mv file? diff --git a/UUID.hs b/UUID.hs index 1c31a343fa..c770045270 100644 --- a/UUID.hs +++ b/UUID.hs @@ -40,15 +40,25 @@ genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h - -} getUUID :: Git.Repo -> Annex UUID getUUID r = do - if ("" /= configured r) - then return $ configured r - else cached r + g <- Annex.gitRepo + let uuid = cached r g + if (uuid /= "") + then return $ uuid + else do + let uuid = uncached r + if (uuid /= "") + then do + updatecache r g uuid + return uuid + else return "" where - configured r = Git.configGet r "annex.uuid" "" - cached r = do - g <- Annex.gitRepo - return $ Git.configGet g (configkey r) "" - configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" + uncached r = Git.configGet r "annex.uuid" "" + cached r g = Git.configGet g (cachekey r) "" + updatecache r g uuid = do + if (g /= r) + then setConfig (cachekey r) uuid + else return () + cachekey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -58,13 +68,19 @@ prepUUID = do if ("" == u) then do uuid <- genUUID - liftIO $ Git.run g ["config", configkey, uuid] - -- re-read git config and update the repo's state - g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' - return () + setConfig configkey uuid else return () +{- Changes a git config setting in both internal state and .git/config -} +setConfig :: String -> String -> Annex () +setConfig key value = do + g <- Annex.gitRepo + liftIO $ Git.run g ["config", key, value] + -- re-read git config and update the repo's state + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' + return () + {- Filters a list of repos to ones that have listed UUIDs. -} reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] reposByUUID repos uuids = do From a200761e66f01a271c90ce67482105befca6ef09 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:14:19 -0400 Subject: [PATCH 0125/2835] implemented basic --drop --- Backend.hs | 34 +++++++++++++++++----------------- Backend/File.hs | 6 ++++-- Backend/Url.hs | 4 +++- Commands.hs | 24 ++++++++++++++++++++---- Remotes.hs | 7 ++++--- TODO | 2 +- 6 files changed, 49 insertions(+), 28 deletions(-) diff --git a/Backend.hs b/Backend.hs index 2829fef9d2..7a8a41a4ba 100644 --- a/Backend.hs +++ b/Backend.hs @@ -14,9 +14,9 @@ - -} module Backend ( - storeFile, - dropFile, - retrieveFile, + storeFileKey, + removeKey, + retrieveKeyFile, lookupFile ) where @@ -32,37 +32,37 @@ import qualified GitRepo as Git import qualified Annex import Utility import Types -import BackendTypes +import qualified BackendTypes as B {- Attempts to store a file in one of the backends. -} -storeFile :: FilePath -> Annex (Maybe (Key, Backend)) -storeFile file = do +storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) +storeFileKey file = do g <- Annex.gitRepo let relfile = Git.relative g file b <- Annex.backends - storeFile' b file relfile -storeFile' [] _ _ = return Nothing -storeFile' (b:bs) file relfile = do - try <- (getKey b) relfile + storeFileKey' b file relfile +storeFileKey' [] _ _ = return Nothing +storeFileKey' (b:bs) file relfile = do + try <- (B.getKey b) relfile case (try) of Nothing -> nextbackend Just key -> do - stored <- (storeFileKey b) file key + stored <- (B.storeFileKey b) file key if (not stored) then nextbackend else do return $ Just (key, b) where - nextbackend = storeFile' bs file relfile + nextbackend = storeFileKey' bs file relfile {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveFile :: Backend -> Key -> FilePath -> Annex Bool -retrieveFile backend key dest = (retrieveKeyFile backend) key dest +retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool +retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest -{- Drops a key from a backend. -} -dropFile :: Backend -> Key -> Annex Bool -dropFile backend key = (removeKey backend) key +{- Removes a key from a backend. -} +removeKey :: Backend -> Key -> Annex Bool +removeKey backend key = (B.removeKey backend) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/Backend/File.hs b/Backend/File.hs index 2ac12487e1..311fe820b2 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -28,13 +28,15 @@ keyValue file = return $ Just $ Key file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - - and other accessible repos. So storing or removing a key is + - and other accessible repos. So storing a key is - a no-op. TODO until support is added for git annex --push otherrepo, - then these could implement that.. -} dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True + +{- Allow keys to be removed. -} dummyRemove :: Key -> Annex Bool -dummyRemove url = return False +dummyRemove url = return True {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} diff --git a/Backend/Url.hs b/Backend/Url.hs index 9831c337b8..3d971864ab 100644 --- a/Backend/Url.hs +++ b/Backend/Url.hs @@ -23,8 +23,10 @@ keyValue file = return Nothing -- cannot change url contents dummyStore :: FilePath -> Key -> Annex Bool dummyStore file url = return False + +-- allow keys to be removed dummyRemove :: Key -> Annex Bool -dummyRemove url = return False +dummyRemove url = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl url file = do diff --git a/Commands.hs b/Commands.hs index b4f57d6fe4..65f6f6efd6 100644 --- a/Commands.hs +++ b/Commands.hs @@ -40,7 +40,7 @@ defaultCmd file = do addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file - stored <- Backend.storeFile file + stored <- Backend.storeFileKey file g <- Annex.gitRepo case (stored) of Nothing -> error $ "no backend could store: " ++ file @@ -76,7 +76,7 @@ addCmd file = inBackend file err $ do {- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - Backend.dropFile backend key + Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo let src = annexLocation g backend key @@ -104,7 +104,7 @@ getCmd file = notinBackend file err $ \(key, backend) -> do g <- Annex.gitRepo let dest = annexLocation g backend key liftIO $ createDirectoryIfMissing True (parentDir dest) - success <- Backend.retrieveFile backend key dest + success <- Backend.retrieveKeyFile backend key dest if (success) then do logStatus key ValuePresent @@ -119,7 +119,23 @@ wantCmd file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} dropCmd :: FilePath -> Annex () -dropCmd file = do error "not implemented" -- TODO +dropCmd file = notinBackend file err $ \(key, backend) -> do + -- TODO only remove if enough copies are present elsewhere + success <- Backend.removeKey backend key + if (success) + then do + logStatus key ValueMissing + inannex <- inAnnex backend key + if (inannex) + then do + g <- Annex.gitRepo + let loc = annexLocation g backend key + liftIO $ removeFile loc + return () + else return () + else error $ "backend refused to drop " ++ file + where + err = error $ "not annexed " ++ file {- Pushes all files to a remote repository. -} pushCmd :: String -> Annex () diff --git a/Remotes.hs b/Remotes.hs index 4f4e5a26cb..f20d51ab35 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -45,9 +45,10 @@ withKey key = do else return remotes' err uuids = error $ "no available git remotes have: " ++ - (keyFile key) ++ "\n" ++ - "It has been seen before in these repositories:\n" ++ - prettyPrintUUIDs uuids + (keyFile key) ++ (uuidlist uuids) + uuidlist [] = "" + uuidlist uuids = "\nIt has been seen before in these repositories:\n" ++ + prettyPrintUUIDs uuids {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] diff --git a/TODO b/TODO index 54411185a8..c4ce74e198 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,7 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* --push/--pull/--want/--drop +* --push/--pull/--want * how to handle git mv file? From 65e4f9cc73f4800fd4dcb5503f7a428539e1e959 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:22:05 -0400 Subject: [PATCH 0126/2835] update cached uuids if it's noticed they changed --- UUID.hs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/UUID.hs b/UUID.hs index c770045270..9348c7b437 100644 --- a/UUID.hs +++ b/UUID.hs @@ -41,22 +41,21 @@ genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h getUUID :: Git.Repo -> Annex UUID getUUID r = do g <- Annex.gitRepo - let uuid = cached r g - if (uuid /= "") - then return $ uuid - else do - let uuid = uncached r - if (uuid /= "") - then do - updatecache r g uuid - return uuid - else return "" + + let c = cached r g + let u = uncached r + + if (c /= u && u /= "") + then do + updatecache g r u + return u + else return c where uncached r = Git.configGet r "annex.uuid" "" cached r g = Git.configGet g (cachekey r) "" - updatecache r g uuid = do - if (g /= r) - then setConfig (cachekey r) uuid + updatecache g r u = do + if (g /= r) + then setConfig (cachekey r) u else return () cachekey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" From 90cdc61c7c8d08590e054018c54c542c463be7e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:38:29 -0400 Subject: [PATCH 0127/2835] refactor --- CmdLine.hs | 52 -------------------------------------------- Commands.hs | 58 +++++++++++++++++++++++++++++++++++++------------- Core.hs | 37 ++++++-------------------------- git-annex.hs | 34 ++++++++++++++++++++++++++++- git-annex.mdwn | 6 ++++-- 5 files changed, 86 insertions(+), 101 deletions(-) delete mode 100644 CmdLine.hs diff --git a/CmdLine.hs b/CmdLine.hs deleted file mode 100644 index 98971a733b..0000000000 --- a/CmdLine.hs +++ /dev/null @@ -1,52 +0,0 @@ -{- git-annex command line - - - - TODO: This is very rough and stupid; I would like to use - - System.Console.CmdArgs.Implicit but it is not yet packaged in Debian. - -} - -module CmdLine ( - argvToMode, - dispatch, - Mode -) where - -import System.Console.GetOpt -import Types -import Commands - -data Mode = Default | Add | Push | Pull | Want | Get | Drop | Unannex - deriving Show - -options :: [OptDescr Mode] -options = - [ Option ['a'] ["add"] (NoArg Add) "add files to annex" - , Option ['p'] ["push"] (NoArg Push) "push annex to repos" - , Option ['P'] ["pull"] (NoArg Pull) "pull annex from repos" - , Option ['w'] ["want"] (NoArg Want) "request file contents" - , Option ['g'] ["get"] (NoArg Get) "transfer file contents" - , Option ['d'] ["drop"] (NoArg Drop) "indicate file contents not needed" - , Option ['u'] ["unannex"] (NoArg Unannex) "undo --add" - ] - -argvToMode argv = do - case getOpt Permute options argv of - ([],files,[]) -> return (Default, files) - -- one mode is normal case - (m:[],files,[]) -> return (m, files) - -- multiple modes is an error - (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) - -- error case - (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) - where header = "Usage: git-annex [mode] file" - -dispatch :: Mode -> FilePath -> Annex () -dispatch mode item = do - case (mode) of - Default -> defaultCmd item - Add -> addCmd item - Push -> pushCmd item - Pull -> pullCmd item - Want -> wantCmd item - Get -> getCmd item - Drop -> dropCmd item - Unannex -> unannexCmd item diff --git a/Commands.hs b/Commands.hs index 65f6f6efd6..b631664d61 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,16 +1,12 @@ -{- git-annex subcommands -} +{- git-annex command line -} module Commands ( - defaultCmd, - addCmd, - unannexCmd, - getCmd, - wantCmd, - dropCmd, - pushCmd, - pullCmd + argvToMode, + dispatch, + Mode ) where +import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory @@ -25,6 +21,44 @@ import BackendList import UUID import LocationLog import Types +import Core + +data Mode = Default | Add | Push | Pull | Want | Get | Drop | Unannex + deriving Show + +options :: [OptDescr Mode] +options = + [ Option ['a'] ["add"] (NoArg Add) "add files to annex" + , Option ['p'] ["push"] (NoArg Push) "push annex to repos" + , Option ['P'] ["pull"] (NoArg Pull) "pull annex from repos" + , Option ['w'] ["want"] (NoArg Want) "request file contents" + , Option ['g'] ["get"] (NoArg Get) "transfer file contents" + , Option ['d'] ["drop"] (NoArg Drop) "indicate file contents not needed" + , Option ['u'] ["unannex"] (NoArg Unannex) "undo --add" + ] + +argvToMode argv = do + case getOpt Permute options argv of + ([],files,[]) -> return (Default, files) + -- one mode is normal case + (m:[],files,[]) -> return (m, files) + -- multiple modes is an error + (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) + -- error case + (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) + where header = "Usage: git-annex [mode] file" + +dispatch :: Mode -> FilePath -> Annex () +dispatch mode item = do + case (mode) of + Default -> defaultCmd item + Add -> addCmd item + Push -> pushCmd item + Pull -> pullCmd item + Want -> wantCmd item + Get -> getCmd item + Drop -> dropCmd item + Unannex -> unannexCmd item {- Default mode is to annex a file if it is not already, and otherwise - get its content. -} @@ -163,9 +197,3 @@ inBackend file yes no = do Just v -> yes v Nothing -> no notinBackend file yes no = inBackend file no yes - -{- Checks if a given key is currently present in the annexLocation -} -inAnnex :: Backend -> Key -> Annex Bool -inAnnex backend key = do - g <- Annex.gitRepo - liftIO $ doesFileExist $ annexLocation g backend key diff --git a/Core.hs b/Core.hs index e3d2c64039..1eb9da687d 100644 --- a/Core.hs +++ b/Core.hs @@ -5,8 +5,6 @@ module Core where import System.IO import System.Directory import Control.Monad.State (liftIO) -import Control.Exception -import CmdLine import Types import BackendList import Locations @@ -33,35 +31,6 @@ start = do Git.configGet g' "annex.backends" "" prepUUID -{- Processes each param in the list by dispatching the handler function - - for the user-selection operation mode. Catches exceptions, not stopping - - if some error out, and propigates an overall error status at the end. - - - - This runs in the IO monad, not in the Annex monad. It seems that - - exceptions can only be caught in the IO monad, not in a stacked monad; - - or more likely I missed an easy way to do it. So, I have to laboriously - - thread AnnexState through this function. - -} -tryRun :: AnnexState -> Mode -> [String] -> IO () -tryRun state mode params = tryRun' state mode 0 0 params -tryRun' state mode errnum oknum [] = do - if (errnum > 0) - then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" - else return () -tryRun' state mode errnum oknum (f:fs) = do - result <- try - (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) - case (result) of - Left err -> do - showErr err - tryRun' state mode (errnum + 1) oknum fs - Right (_,state') -> tryRun' state' mode errnum (oknum + 1) fs - -{- Exception pretty-printing. -} -showErr e = do - hPutStrLn stderr $ "git-annex: " ++ (show e) - return () - {- Sets up a git repo for git-annex. May be called repeatedly. -} gitSetup :: Git.Repo -> IO () gitSetup repo = do @@ -85,3 +54,9 @@ gitSetup repo = do Git.run repo ["add", attributes] Git.run repo ["commit", "-m", "git-annex setup", attributes] + +{- Checks if a given key is currently present in the annexLocation -} +inAnnex :: Backend -> Key -> Annex Bool +inAnnex backend key = do + g <- Annex.gitRepo + liftIO $ doesFileExist $ annexLocation g backend key diff --git a/git-annex.hs b/git-annex.hs index b326b2b199..a038107e9e 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,12 +1,44 @@ {- git-annex main program -} +import Control.Exception +import System.IO import System.Environment import qualified Annex +import Types import Core -import CmdLine +import Commands main = do args <- getArgs (mode, params) <- argvToMode args state <- start tryRun state mode params + +{- Processes each param in the list by dispatching the handler function + - for the user-selection operation mode. Catches exceptions, not stopping + - if some error out, and propigates an overall error status at the end. + - + - This runs in the IO monad, not in the Annex monad. It seems that + - exceptions can only be caught in the IO monad, not in a stacked monad; + - or more likely I missed an easy way to do it. So, I have to laboriously + - thread AnnexState through this function. + -} +tryRun :: AnnexState -> Mode -> [String] -> IO () +tryRun state mode params = tryRun' state mode 0 0 params +tryRun' state mode errnum oknum (f:fs) = do + result <- try + (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) + case (result) of + Left err -> do + showErr err + tryRun' state mode (errnum + 1) oknum fs + Right (_,state') -> tryRun' state' mode errnum (oknum + 1) fs +tryRun' state mode errnum oknum [] = do + if (errnum > 0) + then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" + else return () + +{- Exception pretty-printing. -} +showErr e = do + hPutStrLn stderr $ "git-annex: " ++ (show e) + return () diff --git a/git-annex.mdwn b/git-annex.mdwn index 6852ed0080..a7a0539077 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -48,9 +48,11 @@ git-annex can be configured to try to keep N copies of a file's content available across all repositories. By default, N is 1 (configured by annex.numcopies). -`git annex --drop` attempts to communicate with all other configured +`git annex --drop` attempts to check all other configured repositories, to check that N copies of the file exist. If enough -repositories cannot be contacted, it will retain the file content. +repositories cannot be verified to have it, it will retain the file content +to avoid data loss. + You can later use `git annex --drop --retry` to retry pending drops. Or you can use `git annex --drop --force $file` to force dropping of file content. From a0c013605699a4b1509ba1b04b4522ac43f033c6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:49:19 -0400 Subject: [PATCH 0128/2835] cooler command-line handling --- Commands.hs | 44 +++++++++++++++----------------------------- git-annex.hs | 18 +++++++++--------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/Commands.hs b/Commands.hs index b631664d61..05ea658801 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,9 +1,7 @@ {- git-annex command line -} module Commands ( - argvToMode, - dispatch, - Mode + argvToActions ) where import System.Console.GetOpt @@ -23,43 +21,31 @@ import LocationLog import Types import Core -data Mode = Default | Add | Push | Pull | Want | Get | Drop | Unannex - deriving Show - -options :: [OptDescr Mode] +options :: [OptDescr (String -> Annex ())] options = - [ Option ['a'] ["add"] (NoArg Add) "add files to annex" - , Option ['p'] ["push"] (NoArg Push) "push annex to repos" - , Option ['P'] ["pull"] (NoArg Pull) "pull annex from repos" - , Option ['w'] ["want"] (NoArg Want) "request file contents" - , Option ['g'] ["get"] (NoArg Get) "transfer file contents" - , Option ['d'] ["drop"] (NoArg Drop) "indicate file contents not needed" - , Option ['u'] ["unannex"] (NoArg Unannex) "undo --add" + [ Option ['a'] ["add"] (NoArg addCmd) "add files to annex" + , Option ['p'] ["push"] (NoArg pushCmd) "push annex to repos" + , Option ['P'] ["pull"] (NoArg pullCmd) "pull annex from repos" + , Option ['w'] ["want"] (NoArg wantCmd) "request file contents" + , Option ['g'] ["get"] (NoArg getCmd) "transfer file contents" + , Option ['d'] ["drop"] (NoArg dropCmd) "indicate file contents not needed" + , Option ['u'] ["unannex"] (NoArg unannexCmd) "undo --add" ] -argvToMode argv = do +{- Parses command line and returns a list of actons to be run in the Annex + - monad. -} +argvToActions :: [String] -> IO [Annex ()] +argvToActions argv = do case getOpt Permute options argv of - ([],files,[]) -> return (Default, files) + ([],files,[]) -> return $ map defaultCmd files -- one mode is normal case - (m:[],files,[]) -> return (m, files) + (m:[],files,[]) -> return $ map m files -- multiple modes is an error (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) -- error case (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) where header = "Usage: git-annex [mode] file" -dispatch :: Mode -> FilePath -> Annex () -dispatch mode item = do - case (mode) of - Default -> defaultCmd item - Add -> addCmd item - Push -> pushCmd item - Pull -> pullCmd item - Want -> wantCmd item - Get -> getCmd item - Drop -> dropCmd item - Unannex -> unannexCmd item - {- Default mode is to annex a file if it is not already, and otherwise - get its content. -} defaultCmd :: FilePath -> Annex () diff --git a/git-annex.hs b/git-annex.hs index a038107e9e..77dd29fac7 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -10,9 +10,9 @@ import Commands main = do args <- getArgs - (mode, params) <- argvToMode args + actions <- argvToActions args state <- start - tryRun state mode params + tryRun state actions {- Processes each param in the list by dispatching the handler function - for the user-selection operation mode. Catches exceptions, not stopping @@ -23,17 +23,17 @@ main = do - or more likely I missed an easy way to do it. So, I have to laboriously - thread AnnexState through this function. -} -tryRun :: AnnexState -> Mode -> [String] -> IO () -tryRun state mode params = tryRun' state mode 0 0 params -tryRun' state mode errnum oknum (f:fs) = do +tryRun :: AnnexState -> [Annex ()] -> IO () +tryRun state actions = tryRun' state 0 0 actions +tryRun' state errnum oknum (a:as) = do result <- try - (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) + (Annex.run state a)::IO (Either SomeException ((), AnnexState)) case (result) of Left err -> do showErr err - tryRun' state mode (errnum + 1) oknum fs - Right (_,state') -> tryRun' state' mode errnum (oknum + 1) fs -tryRun' state mode errnum oknum [] = do + tryRun' state (errnum + 1) oknum as + Right (_,state') -> tryRun' state' errnum (oknum + 1) as +tryRun' state errnum oknum [] = do if (errnum > 0) then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" else return () From 1d628ff2380d1bec0c260bc19349c67360fa7a1f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:50:46 -0400 Subject: [PATCH 0129/2835] comment --- Commands.hs | 4 +--- git-annex.hs | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index 05ea658801..feb35d1fb7 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,8 +1,6 @@ {- git-annex command line -} -module Commands ( - argvToActions -) where +module Commands (argvToActions) where import System.Console.GetOpt import Control.Monad.State (liftIO) diff --git a/git-annex.hs b/git-annex.hs index 77dd29fac7..78e8750147 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -14,8 +14,7 @@ main = do state <- start tryRun state actions -{- Processes each param in the list by dispatching the handler function - - for the user-selection operation mode. Catches exceptions, not stopping +{- Runs a list of Annex actions. Catches exceptions, not stopping - if some error out, and propigates an overall error status at the end. - - This runs in the IO monad, not in the Annex monad. It seems that From 40df205881c6bfa180dd37d1a6e67afb3ce3593f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:54:56 -0400 Subject: [PATCH 0130/2835] indent --- Commands.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Commands.hs b/Commands.hs index feb35d1fb7..730663b0d6 100644 --- a/Commands.hs +++ b/Commands.hs @@ -21,12 +21,12 @@ import Core options :: [OptDescr (String -> Annex ())] options = - [ Option ['a'] ["add"] (NoArg addCmd) "add files to annex" - , Option ['p'] ["push"] (NoArg pushCmd) "push annex to repos" - , Option ['P'] ["pull"] (NoArg pullCmd) "pull annex from repos" - , Option ['w'] ["want"] (NoArg wantCmd) "request file contents" - , Option ['g'] ["get"] (NoArg getCmd) "transfer file contents" - , Option ['d'] ["drop"] (NoArg dropCmd) "indicate file contents not needed" + [ Option ['a'] ["add"] (NoArg addCmd) "add files to annex" + , Option ['p'] ["push"] (NoArg pushCmd) "push annex to repos" + , Option ['P'] ["pull"] (NoArg pullCmd) "pull annex from repos" + , Option ['w'] ["want"] (NoArg wantCmd) "request file contents" + , Option ['g'] ["get"] (NoArg getCmd) "transfer file contents" + , Option ['d'] ["drop"] (NoArg dropCmd) "indicate file contents not needed" , Option ['u'] ["unannex"] (NoArg unannexCmd) "undo --add" ] From 4b7e54eddb953ee3ec3ce1734e2748fc3e2f2f9f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 15:05:10 -0400 Subject: [PATCH 0131/2835] tweak docs --- git-annex.mdwn | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index a7a0539077..62e4301eb5 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -53,10 +53,6 @@ repositories, to check that N copies of the file exist. If enough repositories cannot be verified to have it, it will retain the file content to avoid data loss. -You can later use `git annex --drop --retry` to retry pending drops. -Or you can use `git annex --drop --force $file` to force dropping of -file content. - For example, consider three repositories: Server, Laptop, and USB. Both Server and USB have a copy of a file, and N=1. If on Laptop, you `git annex --get $file`, this will transfer it from either Server or USB (depending on which @@ -66,14 +62,15 @@ Suppose you want to free up space on laptop again, and you --drop the file there. If USB is connected, or Server can be contacted, git-annex can check that it still has a copy of the file, and the content is removed from Laptop. But if USB is currently disconnected, and Server also cannot be -contacted, it can't check that and will retain the file content. +contacted, it can't verify that it is safe to drop the file, and will +refuse to do so. With N=2, in order to drop the file content from Laptop, it would need access to both USB and Server. Note that different repositories can be configured with different values of N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1, and of they have the +copies falling to 1, when USB and Server have N=1, and if they have the only copies of a file. ## the .git-annex directory From 859731ee5b09072d112296a073cb152007d7785a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 15:31:44 -0400 Subject: [PATCH 0132/2835] add hasKey --- Backend/File.hs | 12 +++++++++--- Backend/Url.hs | 9 +++++---- BackendTypes.hs | 4 +++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 311fe820b2..893850a695 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -13,13 +13,16 @@ import LocationLog import Locations import qualified Remotes import qualified GitRepo as Git +import Utility +import Core backend = Backend { name = "file", getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, - removeKey = dummyRemove + removeKey = dummyRemove, + hasKey = checkKeyFile } -- direct mapping from filename to key @@ -29,8 +32,7 @@ keyValue file = return $ Just $ Key file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. So storing a key is - - a no-op. TODO until support is added for git annex --push otherrepo, - - then these could implement that.. -} + - a no-op. -} dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True @@ -38,6 +40,10 @@ dummyStore file key = return True dummyRemove :: Key -> Annex Bool dummyRemove url = return True +{- Just check if the .git/annex/ file for the key exists. -} +checkKeyFile :: Key -> Annex Bool +checkKeyFile k = inAnnex backend k + {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex (Bool) diff --git a/Backend/Url.hs b/Backend/Url.hs index 3d971864ab..325a7e217a 100644 --- a/Backend/Url.hs +++ b/Backend/Url.hs @@ -13,7 +13,8 @@ backend = Backend { getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = downloadUrl, - removeKey = dummyRemove + removeKey = dummyOk, + hasKey = dummyOk } -- cannot generate url from filename @@ -24,9 +25,9 @@ keyValue file = return Nothing dummyStore :: FilePath -> Key -> Annex Bool dummyStore file url = return False --- allow keys to be removed -dummyRemove :: Key -> Annex Bool -dummyRemove url = return True +-- allow keys to be removed; presumably they can always be downloaded again +dummyOk :: Key -> Annex Bool +dummyOk url = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl url file = do diff --git a/BackendTypes.hs b/BackendTypes.hs index 2ef65f4691..e480f725b8 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -37,7 +37,9 @@ data Backend = Backend { -- retrieves a key's contents to a file retrieveKeyFile :: Key -> FilePath -> Annex Bool, -- removes a key - removeKey :: Key -> Annex Bool + removeKey :: Key -> Annex Bool, + -- checks if a backend is storing the content of a key + hasKey :: Key -> Annex Bool } instance Show Backend where From d4ce0724527fa0155f737b5d3e94e190c27d29dc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 15:58:53 -0400 Subject: [PATCH 0133/2835] break depends cycle --- Backend.hs | 13 +++++++++++++ Backend/File.hs | 1 - Core.hs | 2 -- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Backend.hs b/Backend.hs index 7a8a41a4ba..01c7c6823f 100644 --- a/Backend.hs +++ b/Backend.hs @@ -33,6 +33,19 @@ import qualified Annex import Utility import Types import qualified BackendTypes as B +import BackendList + +{- List of backends in the order to try them when storing a new key. -} +backendList :: Annex [Backend] +backendList = do + l <- Annex.backends + if (0 < length l) + then return l + else do + g <- Annex.gitRepo + let l = parseBackendList $ Git.configGet g "annex.backends" "" + Annex.backendsChange l + return l {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) diff --git a/Backend/File.hs b/Backend/File.hs index 893850a695..92f5932ce5 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -14,7 +14,6 @@ import Locations import qualified Remotes import qualified GitRepo as Git import Utility -import Core backend = Backend { name = "file", diff --git a/Core.hs b/Core.hs index 1eb9da687d..431c9c9e63 100644 --- a/Core.hs +++ b/Core.hs @@ -27,8 +27,6 @@ start = do g' <- liftIO $ Git.configRead g Annex.gitRepoChange g' liftIO $ gitSetup g' - Annex.backendsChange $ parseBackendList $ - Git.configGet g' "annex.backends" "" prepUUID {- Sets up a git repo for git-annex. May be called repeatedly. -} From aa2f4bd81049e3bcaad6f5f1334864ce14887527 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 16:13:43 -0400 Subject: [PATCH 0134/2835] bug --- Annex.hs | 15 ++++++++--- Backend.hs | 2 +- Backend/File.hs | 1 + Core.hs | 67 ++++++++++++++++++++----------------------------- git-annex.hs | 7 ++++-- 5 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Annex.hs b/Annex.hs index fcd19ba033..9be86c9481 100644 --- a/Annex.hs +++ b/Annex.hs @@ -14,9 +14,18 @@ import qualified GitRepo as Git import Types import qualified BackendTypes as Backend --- constructor -new :: Git.Repo -> AnnexState -new g = Backend.AnnexState { Backend.repo = g, Backend.backends = [] } +{- Create and returns an Annex state object for the specified git repo. + -} +new :: Git.Repo -> IO AnnexState +new g = do + let s = Backend.AnnexState { Backend.repo = g, Backend.backends = [] } + (_,s') <- Annex.run s (prep g) + return s' + where + prep g = do + -- read git config and update state + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' -- performs an action in the Annex monad run state action = runStateT (action) state diff --git a/Backend.hs b/Backend.hs index 01c7c6823f..7a8178a8eb 100644 --- a/Backend.hs +++ b/Backend.hs @@ -52,7 +52,7 @@ storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) storeFileKey file = do g <- Annex.gitRepo let relfile = Git.relative g file - b <- Annex.backends + b <- backendList storeFileKey' b file relfile storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do diff --git a/Backend/File.hs b/Backend/File.hs index 92f5932ce5..893850a695 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -14,6 +14,7 @@ import Locations import qualified Remotes import qualified GitRepo as Git import Utility +import Core backend = Backend { name = "file", diff --git a/Core.hs b/Core.hs index 431c9c9e63..644bedd00f 100644 --- a/Core.hs +++ b/Core.hs @@ -6,52 +6,39 @@ import System.IO import System.Directory import Control.Monad.State (liftIO) import Types -import BackendList import Locations import UUID import qualified GitRepo as Git import qualified Annex - -{- Create and returns an Annex state object. - - Examines and prepares the git repo. - -} -start :: IO AnnexState -start = do - g <- Git.repoFromCwd - let s = Annex.new g - (_,s') <- Annex.run s (prep g) - return s' - where - prep g = do - -- setup git and read its config; update state - g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' - liftIO $ gitSetup g' - prepUUID - + {- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: Git.Repo -> IO () -gitSetup repo = do - -- configure git to use union merge driver on state files - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - commit - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () +gitSetup :: Annex () +gitSetup = do + g <- Annex.gitRepo + liftIO $ setupattributes g + prepUUID where - attrLine = stateLoc ++ "/*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] + -- configure git to use union merge driver on state files + setupattributes repo = do + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then do + appendFile attributes $ attrLine ++ "\n" + commit + else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] {- Checks if a given key is currently present in the annexLocation -} inAnnex :: Backend -> Key -> Annex Bool diff --git a/git-annex.hs b/git-annex.hs index 78e8750147..f9d9311eb2 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -7,12 +7,15 @@ import qualified Annex import Types import Core import Commands +import Annex +import qualified GitRepo as Git main = do args <- getArgs actions <- argvToActions args - state <- start - tryRun state actions + gitrepo <- Git.repoFromCwd + state <- new gitrepo + tryRun state (gitSetup:actions) {- Runs a list of Annex actions. Catches exceptions, not stopping - if some error out, and propigates an overall error status at the end. From 508a3b65ed675c9322940578614f088ea2c74e7f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 17:37:20 -0400 Subject: [PATCH 0135/2835] annex.numcopies works --- Backend.hs | 15 ++++++++++++++- Backend/File.hs | 13 +++++++++++++ Commands.hs | 41 ++++++++++++++++++++++++++++++++++++++++- Remotes.hs | 10 +--------- TODO | 3 +++ 5 files changed, 71 insertions(+), 11 deletions(-) diff --git a/Backend.hs b/Backend.hs index 7a8178a8eb..47e42b8229 100644 --- a/Backend.hs +++ b/Backend.hs @@ -15,8 +15,9 @@ module Backend ( storeFileKey, - removeKey, retrieveKeyFile, + removeKey, + hasKey, lookupFile ) where @@ -77,6 +78,18 @@ retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest removeKey :: Backend -> Key -> Annex Bool removeKey backend key = (B.removeKey backend) key +{- Checks if any backend has a key. -} +hasKey :: Key -> Annex Bool +hasKey key = do + b <- backendList + hasKey' b key +hasKey' [] key = return False +hasKey' (b:bs) key = do + has <- (B.hasKey b) key + if (has) + then return True + else hasKey' bs key + {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} lookupFile :: FilePath -> IO (Maybe (Key, Backend)) diff --git a/Backend/File.hs b/Backend/File.hs index 893850a695..def2f30910 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,6 +15,8 @@ import qualified Remotes import qualified GitRepo as Git import Utility import Core +import qualified Annex +import UUID backend = Backend { name = "file", @@ -49,6 +51,9 @@ checkKeyFile k = inAnnex backend k copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do remotes <- Remotes.withKey key + if (0 == length remotes) + then cantfind + else return () trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ @@ -68,6 +73,14 @@ copyKeyFile key file = do liftIO $ hPutStrLn stderr (show err) trycopy full rs Right succ -> return True + cantfind = do + g <- Annex.gitRepo + uuids <- liftIO $ keyLocations g key + error $ "no available git remotes have: " ++ + (keyFile key) ++ (uuidlist uuids) + uuidlist [] = "" + uuidlist uuids = "\nIt has been seen before in these repositories:\n" ++ + prettyPrintUUIDs uuids {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () diff --git a/Commands.hs b/Commands.hs index 730663b0d6..6128b76aad 100644 --- a/Commands.hs +++ b/Commands.hs @@ -8,6 +8,7 @@ import System.Posix.Files import System.Directory import Data.String.Utils import List +import IO import qualified GitRepo as Git import qualified Annex import Utility @@ -18,6 +19,7 @@ import UUID import LocationLog import Types import Core +import qualified Remotes options :: [OptDescr (String -> Annex ())] options = @@ -138,7 +140,7 @@ wantCmd file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} dropCmd :: FilePath -> Annex () dropCmd file = notinBackend file err $ \(key, backend) -> do - -- TODO only remove if enough copies are present elsewhere + requireEnoughCopies key success <- Backend.removeKey backend key if (success) then do @@ -181,3 +183,40 @@ inBackend file yes no = do Just v -> yes v Nothing -> no notinBackend file yes no = inBackend file no yes + +{- Checks remotes to verify that enough copies of a key exist to allow + - for a key to be safely removed (with no data loss), and fails with an + - error if not. -} +requireEnoughCopies :: Key -> Annex () +requireEnoughCopies key = do + g <- Annex.gitRepo + let numcopies = read $ Git.configGet g config "1" + remotes <- Remotes.withKey key + if (numcopies > length remotes) + then error $ "I only know about " ++ (show $ length remotes) ++ + " out of " ++ (show numcopies) ++ + " necessary copies of: " ++ (keyFile key) ++ + unsafe + else findcopies numcopies remotes [] + where + findcopies 0 _ _ = return () -- success, enough copies found + findcopies _ [] bad = die bad + findcopies n (r:rs) bad = do + result <- liftIO $ try $ haskey r + case (result) of + Right True -> findcopies (n-1) rs bad + Left _ -> findcopies n rs (r:bad) + haskey r = do + -- To check if a remote has a key, construct a new + -- Annex monad and query its backend. + a <- Annex.new r + (result, _) <- Annex.run a (Backend.hasKey key) + return result + die bad = + error $ "I failed to find enough other copies of: " ++ + (keyFile key) ++ "\n" ++ + "I was unable to access these remotes: " ++ + (Remotes.list bad) ++ unsafe + unsafe = "\n -- According to the " ++ config ++ + " setting, it is not safe to remove it!" + config = "annex.numcopies" diff --git a/Remotes.hs b/Remotes.hs index f20d51ab35..2fffcffa7e 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -40,15 +40,7 @@ withKey key = do mayberemotes <- mapM tryGitConfigRead allremotes let allremotes' = catMaybes mayberemotes remotes' <- reposByUUID allremotes' uuids - if (0 == length remotes') - then err uuids - else return remotes' - err uuids = - error $ "no available git remotes have: " ++ - (keyFile key) ++ (uuidlist uuids) - uuidlist [] = "" - uuidlist uuids = "\nIt has been seen before in these repositories:\n" ++ - prettyPrintUUIDs uuids + return remotes' {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] diff --git a/TODO b/TODO index c4ce74e198..70ace863ea 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,9 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing +* need to include backend name as part of the key, because currently + if two backends have overlapping key spaces, it can confuse things + * --push/--pull/--want * how to handle git mv file? From 467c4b2751921818f86561d85b0927254e48d956 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 17:57:04 -0400 Subject: [PATCH 0136/2835] better shutdown --- Commands.hs | 9 +++------ Core.hs | 57 ++++++++++++++++++++++++++++++---------------------- git-annex.hs | 14 ++++++------- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/Commands.hs b/Commands.hs index 6128b76aad..58d88aa3b4 100644 --- a/Commands.hs +++ b/Commands.hs @@ -171,11 +171,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - liftIO $ commit g f - where - commit g f = do - Git.run g ["add", f] - Git.run g ["commit", "-m", "git-annex log update", f] + liftIO $ Git.run g ["add", f] -- committed at shutdown inBackend file yes no = do r <- liftIO $ Backend.lookupFile file @@ -204,7 +200,8 @@ requireEnoughCopies key = do findcopies n (r:rs) bad = do result <- liftIO $ try $ haskey r case (result) of - Right True -> findcopies (n-1) rs bad + Right True -> do + findcopies (n-1) rs bad Left _ -> findcopies n rs (r:bad) haskey r = do -- To check if a remote has a key, construct a new diff --git a/Core.hs b/Core.hs index 644bedd00f..5182a6855a 100644 --- a/Core.hs +++ b/Core.hs @@ -11,34 +11,43 @@ import UUID import qualified GitRepo as Git import qualified Annex -{- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: Annex () -gitSetup = do +{- Sets up a git repo for git-annex. -} +setup :: Annex () +setup = do g <- Annex.gitRepo - liftIO $ setupattributes g + liftIO $ gitAttributes g prepUUID - where - -- configure git to use union merge driver on state files - setupattributes repo = do - exists <- doesFileExist attributes - if (not exists) + +{- When git-annex is done, it runs this. -} +shutdown :: Annex () +shutdown = do + g <- Annex.gitRepo + liftIO $ Git.run g ["commit", "-m", + "git-annex log update", ".git-annex"] + +{- configure git to use union merge driver on state files, if it is not + - already -} +gitAttributes :: Git.Repo -> IO () +gitAttributes repo = do + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) then do - writeFile attributes $ attrLine ++ "\n" + appendFile attributes $ attrLine ++ "\n" commit - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () - where - attrLine = stateLoc ++ "/*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] + else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] {- Checks if a given key is currently present in the annexLocation -} inAnnex :: Backend -> Key -> Annex Bool diff --git a/git-annex.hs b/git-annex.hs index f9d9311eb2..e147391957 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -15,7 +15,7 @@ main = do actions <- argvToActions args gitrepo <- Git.repoFromCwd state <- new gitrepo - tryRun state (gitSetup:actions) + tryRun state $ [setup] ++ actions ++ [shutdown] {- Runs a list of Annex actions. Catches exceptions, not stopping - if some error out, and propigates an overall error status at the end. @@ -26,18 +26,18 @@ main = do - thread AnnexState through this function. -} tryRun :: AnnexState -> [Annex ()] -> IO () -tryRun state actions = tryRun' state 0 0 actions -tryRun' state errnum oknum (a:as) = do +tryRun state actions = tryRun' state 0 actions +tryRun' state errnum (a:as) = do result <- try (Annex.run state a)::IO (Either SomeException ((), AnnexState)) case (result) of Left err -> do showErr err - tryRun' state (errnum + 1) oknum as - Right (_,state') -> tryRun' state' errnum (oknum + 1) as -tryRun' state errnum oknum [] = do + tryRun' state (errnum + 1) as + Right (_,state') -> tryRun' state' errnum as +tryRun' state errnum [] = do if (errnum > 0) - then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" + then error $ (show errnum) ++ " failed" else return () {- Exception pretty-printing. -} From c4959fee47f168857998dea6d11395158251158d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 18:48:21 -0400 Subject: [PATCH 0137/2835] bugfix --- Commands.hs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Commands.hs b/Commands.hs index 58d88aa3b4..ce8f00fd6f 100644 --- a/Commands.hs +++ b/Commands.hs @@ -200,9 +200,9 @@ requireEnoughCopies key = do findcopies n (r:rs) bad = do result <- liftIO $ try $ haskey r case (result) of - Right True -> do - findcopies (n-1) rs bad - Left _ -> findcopies n rs (r:bad) + Right True -> findcopies (n-1) rs bad + Right False -> findcopies n rs bad + Left _ -> findcopies n rs (r:bad) haskey r = do -- To check if a remote has a key, construct a new -- Annex monad and query its backend. @@ -211,9 +211,11 @@ requireEnoughCopies key = do return result die bad = error $ "I failed to find enough other copies of: " ++ - (keyFile key) ++ "\n" ++ - "I was unable to access these remotes: " ++ - (Remotes.list bad) ++ unsafe + (keyFile key) ++ + (if (0 /= length bad) then listbad bad else "") + ++ unsafe + listbad bad = "\nI was unable to access these remotes: " ++ + (Remotes.list bad) unsafe = "\n -- According to the " ++ config ++ " setting, it is not safe to remove it!" config = "annex.numcopies" From b8ba60428a0b4c077482560757e830e9ba02a823 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 19:36:11 -0400 Subject: [PATCH 0138/2835] changed key to include backend name --- Backend/File.hs | 6 +++--- BackendTypes.hs | 20 +++++++++++++++----- Commands.hs | 18 +++++++++--------- Core.hs | 6 +++--- Locations.hs | 19 ++++++++++--------- 5 files changed, 40 insertions(+), 29 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index def2f30910..6267b478a0 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -29,7 +29,7 @@ backend = Backend { -- direct mapping from filename to key keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return $ Just $ Key file +keyValue file = return $ Just $ Key ((name backend), file) {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, @@ -44,7 +44,7 @@ dummyRemove url = return True {- Just check if the .git/annex/ file for the key exists. -} checkKeyFile :: Key -> Annex Bool -checkKeyFile k = inAnnex backend k +checkKeyFile k = inAnnex k {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} @@ -97,4 +97,4 @@ copyFromRemote r key file = do then return () else error "cp failed" getremote = error "get via network not yet implemented!" - location = annexLocation r backend key + location = annexLocation r key diff --git a/BackendTypes.hs b/BackendTypes.hs index e480f725b8..e0f5f73738 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -5,7 +5,7 @@ module BackendTypes where -import Control.Monad.State +import Control.Monad.State (StateT) import Data.String.Utils import qualified GitRepo as Git @@ -19,12 +19,22 @@ data AnnexState = AnnexState { -- git-annex's monad type Annex = StateT AnnexState IO --- annexed filenames are mapped into keys -data Key = Key String deriving (Eq) +-- annexed filenames are mapped through a backend into keys +type KeyFrag = String +type BackendName = String +data Key = Key (BackendName, KeyFrag) deriving (Eq) --- show a key to convert it to a string +-- show a key to convert it to a string; the string includes the +-- name of the backend to avoid collisions between key strings instance Show Key where - show (Key v) = v + show (Key (b, k)) = b ++ ":" ++ k + +instance Read Key where + readsPrec _ s = [((Key (b,k)) ,"")] + where + l = split ":" s + b = l !! 0 + k = join ":" $ drop 1 l -- this structure represents a key/value backend data Backend = Backend { diff --git a/Commands.hs b/Commands.hs index ce8f00fd6f..7ff33ab020 100644 --- a/Commands.hs +++ b/Commands.hs @@ -66,7 +66,7 @@ addCmd file = inBackend file err $ do Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> do logStatus key ValuePresent - liftIO $ setup g key backend + liftIO $ setup g key where err = error $ "already annexed " ++ file checkLegal file = do @@ -74,9 +74,9 @@ addCmd file = inBackend file err $ do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - setup g key backend = do - let dest = annexLocation g backend key - let reldest = annexLocationRelative g backend key + setup g key = do + let dest = annexLocation g key + let reldest = annexLocationRelative g key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink ((linkTarget file) ++ reldest) file @@ -99,7 +99,7 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo - let src = annexLocation g backend key + let src = annexLocation g key liftIO $ moveout g src where err = error $ "not annexed " ++ file @@ -117,12 +117,12 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () getCmd file = notinBackend file err $ \(key, backend) -> do - inannex <- inAnnex backend key + inannex <- inAnnex key if (inannex) then return () else do g <- Annex.gitRepo - let dest = annexLocation g backend key + let dest = annexLocation g key liftIO $ createDirectoryIfMissing True (parentDir dest) success <- Backend.retrieveKeyFile backend key dest if (success) @@ -145,11 +145,11 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do if (success) then do logStatus key ValueMissing - inannex <- inAnnex backend key + inannex <- inAnnex key if (inannex) then do g <- Annex.gitRepo - let loc = annexLocation g backend key + let loc = annexLocation g key liftIO $ removeFile loc return () else return () diff --git a/Core.hs b/Core.hs index 5182a6855a..6f05394bb7 100644 --- a/Core.hs +++ b/Core.hs @@ -50,7 +50,7 @@ gitAttributes repo = do attributes] {- Checks if a given key is currently present in the annexLocation -} -inAnnex :: Backend -> Key -> Annex Bool -inAnnex backend key = do +inAnnex :: Key -> Annex Bool +inAnnex key = do g <- Annex.gitRepo - liftIO $ doesFileExist $ annexLocation g backend key + liftIO $ doesFileExist $ annexLocation g key diff --git a/Locations.hs b/Locations.hs index 7b8beb14f0..960a8938d8 100644 --- a/Locations.hs +++ b/Locations.hs @@ -22,18 +22,19 @@ gitStateDir :: Git.Repo -> FilePath gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc ++ "/" {- An annexed file's content is stored in - - /path/to/repo/.git/annex// + - /path/to/repo/.git/annex/, where is of the form + - - - - (That allows deriving the key and backend by looking at the symlink to it.) + - That allows deriving the key and backend by looking at the symlink to it. -} -annexLocation :: Git.Repo -> Backend -> Key -> FilePath -annexLocation r backend key = - (Git.workTree r) ++ "/" ++ (annexLocationRelative r backend key) +annexLocation :: Git.Repo -> Key -> FilePath +annexLocation r key = + (Git.workTree r) ++ "/" ++ (annexLocationRelative r key) {- Annexed file's location relative to the gitWorkTree -} -annexLocationRelative :: Git.Repo -> Backend -> Key -> FilePath -annexLocationRelative r backend key = - Git.dir r ++ "/annex/" ++ (Backend.name backend) ++ "/" ++ (keyFile key) +annexLocationRelative :: Git.Repo -> Key -> FilePath +annexLocationRelative r key = + Git.dir r ++ "/annex/" ++ (keyFile key) {- Converts a key into a filename fragment. - @@ -51,5 +52,5 @@ keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Key -fileKey file = Backend.Key $ +fileKey file = read $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file From 4c3ad80f320d3c4cccc3e41e4f2364155bae22a1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 20:05:04 -0400 Subject: [PATCH 0139/2835] bugfix --- Backend.hs | 20 +++++++------------- Backend/Url.hs | 11 +++++++---- BackendList.hs | 2 +- BackendTypes.hs | 8 ++++++++ TODO | 3 --- Types.hs | 4 +++- 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Backend.hs b/Backend.hs index 47e42b8229..f419831d28 100644 --- a/Backend.hs +++ b/Backend.hs @@ -78,17 +78,9 @@ retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest removeKey :: Backend -> Key -> Annex Bool removeKey backend key = (B.removeKey backend) key -{- Checks if any backend has a key. -} +{- Checks if a backend has its key. -} hasKey :: Key -> Annex Bool -hasKey key = do - b <- backendList - hasKey' b key -hasKey' [] key = return False -hasKey' (b:bs) key = do - has <- (B.hasKey b) key - if (has) - then return True - else hasKey' bs key +hasKey key = (B.hasKey (lookupBackendName $ backendName key)) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} @@ -101,6 +93,8 @@ lookupFile file = do where lookup = do l <- readSymbolicLink file - return $ Just (k l, b l) - k l = fileKey $ takeFileName $ l - b l = lookupBackendName $ takeFileName $ parentDir $ l + return $ Just $ pair $ takeFileName l + pair file = (k, b) + where + k = fileKey file + b = lookupBackendName $ backendName k diff --git a/Backend/Url.hs b/Backend/Url.hs index 325a7e217a..e237672084 100644 --- a/Backend/Url.hs +++ b/Backend/Url.hs @@ -3,7 +3,8 @@ module Backend.Url (backend) where -import Control.Monad.State +import Control.Monad.State (liftIO) +import Data.String.Utils import System.Cmd import System.Exit import BackendTypes @@ -30,9 +31,11 @@ dummyOk :: Key -> Annex Bool dummyOk url = return True downloadUrl :: Key -> FilePath -> Annex Bool -downloadUrl url file = do - liftIO $ putStrLn $ "download: " ++ (show url) - result <- liftIO $ rawSystem "curl" ["-#", "-o", file, (show url)] +downloadUrl key file = do + liftIO $ putStrLn $ "download: " ++ url + result <- liftIO $ rawSystem "curl" ["-#", "-o", file, url] if (result == ExitSuccess) then return True else return False + where + url = join ":" $ drop 1 $ split ":" $ show key diff --git a/BackendList.hs b/BackendList.hs index e9f926ce2a..b66110905a 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -28,7 +28,7 @@ parseBackendList s = then supportedBackends else map (lookupBackendName) $ words s -{- Looks up a supported backed by name. -} +{- Looks up a supported backend by name. -} lookupBackendName :: String -> Backend lookupBackendName s = if ((length matches) /= 1) diff --git a/BackendTypes.hs b/BackendTypes.hs index e0f5f73738..41ff7e506f 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -36,6 +36,14 @@ instance Read Key where b = l !! 0 k = join ":" $ drop 1 l +-- pulls the backend name out +backendName :: Key -> BackendName +backendName (Key (b,k)) = b + +-- pulls the key fragment out +keyFrag :: Key -> KeyFrag +keyFrag (Key (b,k)) = k + -- this structure represents a key/value backend data Backend = Backend { -- name of this backend diff --git a/TODO b/TODO index 70ace863ea..c4ce74e198 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* need to include backend name as part of the key, because currently - if two backends have overlapping key spaces, it can confuse things - * --push/--pull/--want * how to handle git mv file? diff --git a/Types.hs b/Types.hs index 4262ed567d..a0f120db0b 100644 --- a/Types.hs +++ b/Types.hs @@ -3,8 +3,10 @@ module Types ( Annex, AnnexState, + Backend, Key, - Backend + backendName, + keyFrag ) where import BackendTypes From 29039fdf97f541a1077c9af65ccbe09dd2ae2b28 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 21:10:59 -0400 Subject: [PATCH 0140/2835] add flags, and change to subcommand style --- Annex.hs | 18 +++++++++++++++++- BackendTypes.hs | 7 ++++++- Commands.hs | 50 ++++++++++++++++++++++++++----------------------- Core.hs | 5 +++-- TODO | 2 ++ Types.hs | 3 ++- git-annex.hs | 4 ++-- 7 files changed, 59 insertions(+), 30 deletions(-) diff --git a/Annex.hs b/Annex.hs index 9be86c9481..9e76b9b042 100644 --- a/Annex.hs +++ b/Annex.hs @@ -7,6 +7,9 @@ module Annex ( gitRepoChange, backends, backendsChange, + flagIsSet, + flagsChange, + Flag(..) ) where import Control.Monad.State @@ -18,7 +21,11 @@ import qualified BackendTypes as Backend -} new :: Git.Repo -> IO AnnexState new g = do - let s = Backend.AnnexState { Backend.repo = g, Backend.backends = [] } + let s = Backend.AnnexState { + Backend.repo = g, + Backend.backends = [], + Backend.flags = [] + } (_,s') <- Annex.run s (prep g) return s' where @@ -49,3 +56,12 @@ backendsChange b = do state <- get put state { Backend.backends = b } return () +flagIsSet :: Flag -> Annex Bool +flagIsSet flag = do + state <- get + return $ elem flag $ Backend.flags state +flagsChange :: [Flag] -> Annex () +flagsChange b = do + state <- get + put state { Backend.flags = b } + return () diff --git a/BackendTypes.hs b/BackendTypes.hs index 41ff7e506f..1b67ef584d 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -9,11 +9,16 @@ import Control.Monad.State (StateT) import Data.String.Utils import qualified GitRepo as Git +-- command-line flags +data Flag = Force + deriving (Eq, Read, Show) + -- git-annex's runtime state type doesn't really belong here, -- but it uses Backend, so has to be here to avoid a depends loop. data AnnexState = AnnexState { repo :: Git.Repo, - backends :: [Backend] + backends :: [Backend], + flags :: [Flag] } deriving (Show) -- git-annex's monad diff --git a/Commands.hs b/Commands.hs index 7ff33ab020..a16470fe38 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,6 +1,6 @@ {- git-annex command line -} -module Commands (argvToActions) where +module Commands (parseCmd) where import System.Console.GetOpt import Control.Monad.State (liftIO) @@ -21,30 +21,34 @@ import Types import Core import qualified Remotes -options :: [OptDescr (String -> Annex ())] -options = - [ Option ['a'] ["add"] (NoArg addCmd) "add files to annex" - , Option ['p'] ["push"] (NoArg pushCmd) "push annex to repos" - , Option ['P'] ["pull"] (NoArg pullCmd) "pull annex from repos" - , Option ['w'] ["want"] (NoArg wantCmd) "request file contents" - , Option ['g'] ["get"] (NoArg getCmd) "transfer file contents" - , Option ['d'] ["drop"] (NoArg dropCmd) "indicate file contents not needed" - , Option ['u'] ["unannex"] (NoArg unannexCmd) "undo --add" - ] - {- Parses command line and returns a list of actons to be run in the Annex - monad. -} -argvToActions :: [String] -> IO [Annex ()] -argvToActions argv = do - case getOpt Permute options argv of - ([],files,[]) -> return $ map defaultCmd files - -- one mode is normal case - (m:[],files,[]) -> return $ map m files - -- multiple modes is an error - (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) - -- error case - (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) - where header = "Usage: git-annex [mode] file" +parseCmd :: [String] -> IO ([Flag], [Annex ()]) +parseCmd argv = do + (flags, nonopts) <- getopt + case (length nonopts) of + 0 -> error header + _ -> do + let c = lookupCmd (nonopts !! 0) + if (0 == length c) + then return $ (flags, map defaultCmd nonopts) + else do + return $ (flags, map (snd $ c !! 0) $ drop 1 nonopts) + where + getopt = case getOpt Permute options argv of + (flags, nonopts, []) -> return (flags, nonopts) + (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) + lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds + cmds = [ ("add", addCmd) + , ("push", pushCmd) + , ("pull", pullCmd) + , ("want", wantCmd) + , ("drop", dropCmd) + , ("unannex", unannexCmd) + ] + header = "Usage: git-annex [" ++ + (join "|" $ map fst cmds) ++ "] file ..." + options = [ Option ['f'] ["force"] (NoArg Force) "" ] {- Default mode is to annex a file if it is not already, and otherwise - get its content. -} diff --git a/Core.hs b/Core.hs index 6f05394bb7..765b1e6a7e 100644 --- a/Core.hs +++ b/Core.hs @@ -12,8 +12,9 @@ import qualified GitRepo as Git import qualified Annex {- Sets up a git repo for git-annex. -} -setup :: Annex () -setup = do +startup :: [Flag] -> Annex () +startup flags = do + Annex.flagsChange flags g <- Annex.gitRepo liftIO $ gitAttributes g prepUUID diff --git a/TODO b/TODO index c4ce74e198..b800097a0e 100644 --- a/TODO +++ b/TODO @@ -3,6 +3,8 @@ * --push/--pull/--want +* recurse on directories + * how to handle git mv file? * finish BackendChecksum diff --git a/Types.hs b/Types.hs index a0f120db0b..6bf26d36e6 100644 --- a/Types.hs +++ b/Types.hs @@ -6,7 +6,8 @@ module Types ( Backend, Key, backendName, - keyFrag + keyFrag, + Flag(..), ) where import BackendTypes diff --git a/git-annex.hs b/git-annex.hs index e147391957..cd67242afa 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -12,10 +12,10 @@ import qualified GitRepo as Git main = do args <- getArgs - actions <- argvToActions args + (flags, actions) <- parseCmd args gitrepo <- Git.repoFromCwd state <- new gitrepo - tryRun state $ [setup] ++ actions ++ [shutdown] + tryRun state $ [startup flags] ++ actions ++ [shutdown] {- Runs a list of Annex actions. Catches exceptions, not stopping - if some error out, and propigates an overall error status at the end. From 1ab3e54ca8e56f8d7b8fd6ad4ceda888e19205f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 21:12:54 -0400 Subject: [PATCH 0141/2835] docs --- git-annex.mdwn | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index 62e4301eb5..70bd66e954 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -13,7 +13,7 @@ associated with annexed files but that benefit from full revision control. Enough broad picture, here's how it actually looks: -* `git annex --add $file` moves the file into `.git/annex/`, and replaces +* `git annex add $file` moves the file into `.git/annex/`, and replaces it with a symlink pointing at the annexed file, and then calls `git add` to version the *symlink*. (If the file has already been annexed, it does nothing.) @@ -24,23 +24,23 @@ Enough broad picture, here's how it actually looks: * If you use normal git push/pull commands, the annexed file contents won't be sent, but the symlinks will be. So different clones of a repository can have different sets of annexed files available. -* `git annex --push $repository` pushes *all* annexed files to the specified +* `git annex push $repository` pushes *all* annexed files to the specified repository. -* `git annex --pull $repository` pulls *all* annexed files from the specified +* `git annex pull $repository` pulls *all* annexed files from the specified repository. -* `git annex --want $file` indicates that you want access to a file's +* `git annex want $file` indicates that you want access to a file's content, without immediatly transferring it. -* `git annex --get $file` is used to transfer a specified file, and/or - files previously indicated with --want. If a configured repository has it, +* `git annex get $file` is used to transfer a specified file, and/or + files previously indicated with `git annex want`. If a configured repository has it, or it is available from other key/value storage, it will be immediatly downloaded. -* `git annex --drop $file` indicates that you no longer want the file's +* `git annex drop $file` indicates that you no longer want the file's content to be available in this repository. -* `git annex --unannex $file` undoes a `git annex --add`. But use `--drop` - if you're just done with a file; only use `--unannex` if you +* `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` + if you're just done with a file; only use `unannex` if you accidentially added a file. -* `git annex $file` is a shorthand for either --add or --get. If the file - is already known, it does --get, otherwise it does --add. +* `git annex $file` is a shorthand. If the file + is already known, it does `git annex get`, otherwise it does `git annex add`. ## copies @@ -48,17 +48,17 @@ git-annex can be configured to try to keep N copies of a file's content available across all repositories. By default, N is 1 (configured by annex.numcopies). -`git annex --drop` attempts to check all other configured +`git annex drop` attempts to check all other configured repositories, to check that N copies of the file exist. If enough repositories cannot be verified to have it, it will retain the file content to avoid data loss. For example, consider three repositories: Server, Laptop, and USB. Both Server -and USB have a copy of a file, and N=1. If on Laptop, you `git annex --get +and USB have a copy of a file, and N=1. If on Laptop, you `git annex get $file`, this will transfer it from either Server or USB (depending on which is available), and there are now 3 copies of the file. -Suppose you want to free up space on laptop again, and you --drop the file +Suppose you want to free up space on laptop again, and you `git annex drop` the file there. If USB is connected, or Server can be contacted, git-annex can check that it still has a copy of the file, and the content is removed from Laptop. But if USB is currently disconnected, and Server also cannot be @@ -108,11 +108,11 @@ to store different files' contents in a given repository. git-annex keeps track of on which repository it last saw a file's content. This can be useful when using it for archiving with offline storage. When -you indicate you --want a file, git-annex will tell you which repositories +you indicate you want a file, git-annex will tell you which repositories have the file's content. Location tracking information is stored in `.git-annex/$key.log`. -Repositories record their UUID and the date when they --get or --drop +Repositories record their UUID and the date when they get or drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) @@ -150,7 +150,7 @@ If the symlink to annexed content is relative, moving it to a subdir will break it. But it it's absolute, moving the git repo (or mounting its drive elsewhere) will break it. Either: -* Use relative links and need `git annex --mv` to move (or post-commit +* Use relative links and need `git annex mv` to move (or post-commit hook that caches moves and updates links). * Use absolute links and need `git annex fixlinks` when location changes; note that would also mean that git would see the symlink targets changed From c977b6b1f3833ed1ead9212d956d8f83a4ca9028 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 22:09:03 -0400 Subject: [PATCH 0142/2835] forcing --- Commands.hs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Commands.hs b/Commands.hs index a16470fe38..6c519c294a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -144,7 +144,10 @@ wantCmd file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} dropCmd :: FilePath -> Annex () dropCmd file = notinBackend file err $ \(key, backend) -> do - requireEnoughCopies key + force <- Annex.flagIsSet Force + if (not force) + then requireEnoughCopies key + else return () success <- Backend.removeKey backend key if (success) then do @@ -220,6 +223,9 @@ requireEnoughCopies key = do ++ unsafe listbad bad = "\nI was unable to access these remotes: " ++ (Remotes.list bad) - unsafe = "\n -- According to the " ++ config ++ - " setting, it is not safe to remove it!" + unsafe = "\n" ++ + " -- According to the " ++ config ++ + " setting, it is not safe to remove it!\n" ++ + " (Use --force to override.)" + config = "annex.numcopies" From bbbe9858fe2e83767661282f7ab8ed3470ec6568 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 23:52:45 -0400 Subject: [PATCH 0143/2835] avoid empty commits --- Annex.hs | 11 +++++++---- BackendTypes.hs | 2 +- Commands.hs | 3 ++- Core.hs | 9 ++++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Annex.hs b/Annex.hs index 9e76b9b042..08607cafa8 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,7 +8,7 @@ module Annex ( backends, backendsChange, flagIsSet, - flagsChange, + flagChange, Flag(..) ) where @@ -60,8 +60,11 @@ flagIsSet :: Flag -> Annex Bool flagIsSet flag = do state <- get return $ elem flag $ Backend.flags state -flagsChange :: [Flag] -> Annex () -flagsChange b = do +flagChange :: Flag -> Bool -> Annex () +flagChange flag set = do state <- get - put state { Backend.flags = b } + let f = filter (/= flag) $ Backend.flags state + if (set) + then put state { Backend.flags = (flag:f) } + else put state { Backend.flags = f } return () diff --git a/BackendTypes.hs b/BackendTypes.hs index 1b67ef584d..13ffde7f89 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -10,7 +10,7 @@ import Data.String.Utils import qualified GitRepo as Git -- command-line flags -data Flag = Force +data Flag = Force | NeedCommit deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, diff --git a/Commands.hs b/Commands.hs index 6c519c294a..5b5bc269bc 100644 --- a/Commands.hs +++ b/Commands.hs @@ -178,7 +178,8 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - liftIO $ Git.run g ["add", f] -- committed at shutdown + liftIO $ Git.run g ["add", f] + Annex.flagChange NeedCommit True inBackend file yes no = do r <- liftIO $ Backend.lookupFile file diff --git a/Core.hs b/Core.hs index 765b1e6a7e..8f1c9cc802 100644 --- a/Core.hs +++ b/Core.hs @@ -14,7 +14,7 @@ import qualified Annex {- Sets up a git repo for git-annex. -} startup :: [Flag] -> Annex () startup flags = do - Annex.flagsChange flags + mapM (\f -> Annex.flagChange f True) flags g <- Annex.gitRepo liftIO $ gitAttributes g prepUUID @@ -23,8 +23,11 @@ startup flags = do shutdown :: Annex () shutdown = do g <- Annex.gitRepo - liftIO $ Git.run g ["commit", "-m", - "git-annex log update", ".git-annex"] + needcommit <- Annex.flagIsSet NeedCommit + if (needcommit) + then liftIO $ Git.run g ["commit", "-m", + "git-annex log update", ".git-annex"] + else return () {- configure git to use union merge driver on state files, if it is not - already -} From e7ffa5b594deb9d89d555b24f8ed7842951905af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 23:57:22 -0400 Subject: [PATCH 0144/2835] update --- Commands.hs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index 5b5bc269bc..5021ba260a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -21,8 +21,8 @@ import Types import Core import qualified Remotes -{- Parses command line and returns a list of actons to be run in the Annex - - monad. -} +{- Parses command line and returns a list of flags and a list of + - actions to be run in the Annex monad. -} parseCmd :: [String] -> IO ([Flag], [Annex ()]) parseCmd argv = do (flags, nonopts) <- getopt @@ -40,15 +40,16 @@ parseCmd argv = do (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds cmds = [ ("add", addCmd) + , ("get", getCmd) + , ("drop", dropCmd) + , ("want", wantCmd) , ("push", pushCmd) , ("pull", pullCmd) - , ("want", wantCmd) - , ("drop", dropCmd) , ("unannex", unannexCmd) ] header = "Usage: git-annex [" ++ (join "|" $ map fst cmds) ++ "] file ..." - options = [ Option ['f'] ["force"] (NoArg Force) "" ] + options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] {- Default mode is to annex a file if it is not already, and otherwise - get its content. -} From d7b170c9a2f9a7f52b6ef88243e249a04685764a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 14:26:16 -0400 Subject: [PATCH 0145/2835] cleanup --- Commands.hs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Commands.hs b/Commands.hs index 5021ba260a..aed3a19d32 100644 --- a/Commands.hs +++ b/Commands.hs @@ -25,16 +25,17 @@ import qualified Remotes - actions to be run in the Annex monad. -} parseCmd :: [String] -> IO ([Flag], [Annex ()]) parseCmd argv = do - (flags, nonopts) <- getopt - case (length nonopts) of + (flags, files) <- getopt + case (length files) of 0 -> error header _ -> do - let c = lookupCmd (nonopts !! 0) + let c = lookupCmd (files !! 0) if (0 == length c) - then return $ (flags, map defaultCmd nonopts) - else do - return $ (flags, map (snd $ c !! 0) $ drop 1 nonopts) + then ret flags defaultCmd files + else ret flags (snd $ c !! 0) $ drop 1 files where + ret flags cmd files = return (flags, makeactions cmd files) + makeactions cmd files = map cmd files getopt = case getOpt Permute options argv of (flags, nonopts, []) -> return (flags, nonopts) (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) From 80104eab9a28b9a94fb36653b7cd95b734e16e4d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 14:31:06 -0400 Subject: [PATCH 0146/2835] bugfix --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 8f1c9cc802..7e719888a0 100644 --- a/Core.hs +++ b/Core.hs @@ -26,7 +26,7 @@ shutdown = do needcommit <- Annex.flagIsSet NeedCommit if (needcommit) then liftIO $ Git.run g ["commit", "-m", - "git-annex log update", ".git-annex"] + "git-annex log update", gitStateDir g] else return () {- configure git to use union merge driver on state files, if it is not From e577656fea6f66ef64547374e962adb7fd4ce80a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 16:09:30 -0400 Subject: [PATCH 0147/2835] relative link fix --- Commands.hs | 25 ++++++++++++------------- Locations.hs | 5 ++--- Utility.hs | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/Commands.hs b/Commands.hs index aed3a19d32..a403a5a48a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -6,6 +6,7 @@ import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory +import System.Path import Data.String.Utils import List import IO @@ -66,13 +67,14 @@ defaultCmd file = do addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file - stored <- Backend.storeFileKey file g <- Annex.gitRepo + link <- liftIO $ calcGitLink file g + stored <- Backend.storeFileKey file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> do logStatus key ValuePresent - liftIO $ setup g key + liftIO $ setup g key link where err = error $ "already annexed " ++ file checkLegal file = do @@ -80,24 +82,21 @@ addCmd file = inBackend file err $ do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - setup g key = do + calcGitLink file g = do + cwd <- getCurrentDirectory + let absfile = case (absNormPath cwd file) of + Just f -> f + Nothing -> error $ "unable to normalize " ++ file + return $ relPathDirToDir (parentDir absfile) (Git.workTree g) + setup g key link = do let dest = annexLocation g key let reldest = annexLocationRelative g key createDirectoryIfMissing True (parentDir dest) renameFile file dest - createSymbolicLink ((linkTarget file) ++ reldest) file + createSymbolicLink (link ++ reldest) file Git.run g ["add", file] Git.run g ["commit", "-m", ("git-annex annexed " ++ file), file] - linkTarget file = - -- relies on file being relative to the top of the - -- git repo; just replace each subdirectory with ".." - if (subdirs > 0) - then (join "/" $ take subdirs $ repeat "..") ++ "/" - else "" - where - subdirs = (length $ split "/" file) - 1 - {- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () diff --git a/Locations.hs b/Locations.hs index 960a8938d8..733e745537 100644 --- a/Locations.hs +++ b/Locations.hs @@ -31,10 +31,9 @@ annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = (Git.workTree r) ++ "/" ++ (annexLocationRelative r key) -{- Annexed file's location relative to the gitWorkTree -} +{- Annexed file's location relative to git's working tree. -} annexLocationRelative :: Git.Repo -> Key -> FilePath -annexLocationRelative r key = - Git.dir r ++ "/annex/" ++ (keyFile key) +annexLocationRelative r key = Git.dir r ++ "/annex/" ++ (keyFile key) {- Converts a key into a filename fragment. - diff --git a/Utility.hs b/Utility.hs index 349dd9355f..a8324815e5 100644 --- a/Utility.hs +++ b/Utility.hs @@ -4,12 +4,16 @@ module Utility ( withFileLocked, hGetContentsStrict, - parentDir + parentDir, + relPathCwdToDir, + relPathDirToDir, ) where import System.IO import System.Posix.IO import Data.String.Utils +import System.Path +import System.Directory {- Let's just say that Haskell makes reading/writing a file with - file locking excessively difficult. -} @@ -39,3 +43,45 @@ parentDir dir = where dirs = filter (\x -> length x > 0) $ split "/" dir absolute = if ((dir !! 0) == '/') then "/" else "" + +{- Constructs a relative path from the CWD to a directory. + - + - For example, assuming CWD is /tmp/foo/bar: + - relPathCwdToDir "/tmp/foo" == "../" + - relPathCwdToDir "/tmp/foo/bar" == "" + - relPathCwdToDir "/tmp/foo/bar" == "" + -} +relPathCwdToDir :: FilePath -> IO FilePath +relPathCwdToDir dir = do + cwd <- getCurrentDirectory + let absdir = abs cwd dir + return $ relPathDirToDir cwd absdir + where + -- absolute, normalized form of the directory + abs cwd dir = + case (absNormPath cwd dir) of + Just d -> d + Nothing -> error $ "unable to normalize " ++ dir + +{- Constructs a relative path from one directory to another. + - + - Both directories must be absolute, and normalized (eg with absNormpath). + - + - The path will end with "/", unless it is empty. + - -} +relPathDirToDir :: FilePath -> FilePath -> FilePath +relPathDirToDir from to = + if (0 < length path) + then if (endswith "/" path) + then path + else path ++ "/" + else "" + where + pfrom = split "/" from + pto = split "/" to + common = map fst $ filter same $ zip pfrom pto + same (c,d) = c == d + uncommon = drop numcommon pto + dotdots = take ((length pfrom) - numcommon) $ repeat ".." + numcommon = length $ common + path = join "/" $ dotdots ++ uncommon From 0e8cb63aabaa4a80769792fb07b0db2594efd6b0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 16:25:52 -0400 Subject: [PATCH 0148/2835] update --- TODO | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/TODO b/TODO index b800097a0e..ddd0761658 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,4 @@ -* bug when annexing files while in a subdir of a git repo -* bug when specifying absolute path to files when annexing +* bug: cannot "git annex ../foo" (GitRepo.relative is buggy) * --push/--pull/--want From 395625d0a7c00457f63925beb31078f3eb3d9f79 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 16:42:36 -0400 Subject: [PATCH 0149/2835] rename file -> WORM --- Backend/Checksum.hs | 1 - Backend/File.hs | 22 ++++++++++------------ Backend/Worm.hs | 16 ++++++++++++++++ BackendList.hs | 4 ++-- git-annex.mdwn | 4 ++-- 5 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 Backend/Worm.hs diff --git a/Backend/Checksum.hs b/Backend/Checksum.hs index bfc789e40e..de98fbf446 100644 --- a/Backend/Checksum.hs +++ b/Backend/Checksum.hs @@ -7,7 +7,6 @@ import qualified Backend.File import Data.Digest.Pure.SHA import BackendTypes --- based on BackendFile just with a different key type backend = Backend.File.backend { name = "checksum", getKey = keyValue diff --git a/Backend/File.hs b/Backend/File.hs index 6267b478a0..eba4b88f83 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -1,5 +1,12 @@ -{- git-annex "file" backend - - -} +{- git-annex pseudo-backend + - + - This backend does not really do any independant data storage, + - it relies on the file contents in .git/annex/ in this repo, + - and other accessible repos. + - + - This is an abstract backend; getKey has to be implemented to complete + - it. + -} module Backend.File (backend) where @@ -19,22 +26,13 @@ import qualified Annex import UUID backend = Backend { - name = "file", - getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, removeKey = dummyRemove, hasKey = checkKeyFile } --- direct mapping from filename to key -keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return $ Just $ Key ((name backend), file) - -{- This backend does not really do any independant data storage, - - it relies on the file contents in .git/annex/ in this repo, - - and other accessible repos. So storing a key is - - a no-op. -} +{- Storing a key is a no-op. -} dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True diff --git a/Backend/Worm.hs b/Backend/Worm.hs new file mode 100644 index 0000000000..26fffab521 --- /dev/null +++ b/Backend/Worm.hs @@ -0,0 +1,16 @@ +{- git-annex "WORM" backend -- Write Once, Read Many + - -} + +module Backend.Worm (backend) where + +import qualified Backend.File +import BackendTypes + +backend = Backend.File.backend { + name = "WORM", + getKey = keyValue +} + +-- direct mapping from filename to key +keyValue :: FilePath -> Annex (Maybe Key) +keyValue file = return $ Just $ Key ((name backend), file) diff --git a/BackendList.hs b/BackendList.hs index b66110905a..93c0464f1b 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -10,11 +10,11 @@ module BackendList ( import BackendTypes -- When adding a new backend, import it here and add it to the list. -import qualified Backend.File +import qualified Backend.Worm import qualified Backend.Checksum import qualified Backend.Url supportedBackends = - [ Backend.File.backend + [ Backend.Worm.backend , Backend.Checksum.backend , Backend.Url.backend ] diff --git a/git-annex.mdwn b/git-annex.mdwn index 70bd66e954..fba9648dba 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -93,8 +93,8 @@ stable for a given file content, name, and size. Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. -* `file` -- This backend stores the file's content in - `.git/annex/`, and assumes that any file with the same basename +* `WORM` ("Write Once, Read Many") This backend stores the file's content + in `.git/annex/`, and assumes that any file with the same basename has the same content. So with this backend, files can be moved around, but should never be added to or changed. This is the default, and the least expensive backend. From 1f585912e2097234ecad599a072610000e7744f0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 16:52:47 -0400 Subject: [PATCH 0150/2835] use basename as key --- Backend/Worm.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Backend/Worm.hs b/Backend/Worm.hs index 26fffab521..ba79428efa 100644 --- a/Backend/Worm.hs +++ b/Backend/Worm.hs @@ -5,12 +5,14 @@ module Backend.Worm (backend) where import qualified Backend.File import BackendTypes +import Utility +import System.FilePath backend = Backend.File.backend { name = "WORM", getKey = keyValue } --- direct mapping from filename to key +-- direct mapping from basename of filename to key keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return $ Just $ Key ((name backend), file) +keyValue file = return $ Just $ Key ((name backend), (takeFileName file)) From 8e742bd89e6bd3d83c44847c0455043809c64c89 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 18:24:31 -0400 Subject: [PATCH 0151/2835] use some library functions --- Utility.hs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Utility.hs b/Utility.hs index a8324815e5..105c533ece 100644 --- a/Utility.hs +++ b/Utility.hs @@ -13,6 +13,7 @@ import System.IO import System.Posix.IO import Data.String.Utils import System.Path +import System.FilePath import System.Directory {- Let's just say that Haskell makes reading/writing a file with @@ -38,11 +39,13 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s parentDir :: String -> String parentDir dir = if length dirs > 0 - then absolute ++ (join "/" $ take ((length dirs) - 1) dirs) + then slash ++ (join s $ take ((length dirs) - 1) dirs) else "" where - dirs = filter (\x -> length x > 0) $ split "/" dir - absolute = if ((dir !! 0) == '/') then "/" else "" + dirs = filter (\x -> length x > 0) $ + split s dir + slash = if (isAbsolute dir) then "" else s + s = [pathSeparator] {- Constructs a relative path from the CWD to a directory. - @@ -68,20 +71,19 @@ relPathCwdToDir dir = do - Both directories must be absolute, and normalized (eg with absNormpath). - - The path will end with "/", unless it is empty. - - -} + -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = if (0 < length path) - then if (endswith "/" path) - then path - else path ++ "/" + then addTrailingPathSeparator path else "" where - pfrom = split "/" from - pto = split "/" to + s = [pathSeparator] + pfrom = split s from + pto = split s to common = map fst $ filter same $ zip pfrom pto same (c,d) = c == d uncommon = drop numcommon pto dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common - path = join "/" $ dotdots ++ uncommon + path = join s $ dotdots ++ uncommon From 44b8f7c95de84018044ce3669e62d40eac1b91a7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 18:57:05 -0400 Subject: [PATCH 0152/2835] better worm keys --- Backend/Worm.hs | 24 ++++++++++++++++++++++-- git-annex.mdwn | 8 ++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/Backend/Worm.hs b/Backend/Worm.hs index ba79428efa..89fe4bf572 100644 --- a/Backend/Worm.hs +++ b/Backend/Worm.hs @@ -3,16 +3,36 @@ module Backend.Worm (backend) where +import Control.Monad.State import qualified Backend.File import BackendTypes import Utility import System.FilePath +import System.Posix.Files +import Data.Digest.Pure.SHA -- slow, but we only checksum filenames +import qualified Data.ByteString.Lazy.Char8 as B backend = Backend.File.backend { name = "WORM", getKey = keyValue } --- direct mapping from basename of filename to key +-- A SHA1 of the basename of the filename, plus the file size and +-- modification time, is used as the unique part of the key. That +-- allows multiple files with the same names to have different keys, +-- while also allowing a file to be moved around while retaining the +-- same key. +-- +-- The basename of the filename is also included in the key, so it's clear +-- what the original filename was when a user sees the value. keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return $ Just $ Key ((name backend), (takeFileName file)) +keyValue file = do + stat <- liftIO $ getFileStatus file + return $ Just $ Key ((name backend), key stat) + where + key stat = (checksum $ uniqueid stat) ++ sep ++ base + checksum s = show $ sha1 $ B.pack s + uniqueid stat = (show $ fileSize stat) ++ sep ++ + (show $ modificationTime stat) + base = takeFileName file + sep = ":" diff --git a/git-annex.mdwn b/git-annex.mdwn index fba9648dba..2079b5b466 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -94,10 +94,10 @@ Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. * `WORM` ("Write Once, Read Many") This backend stores the file's content - in `.git/annex/`, and assumes that any file with the same basename - has the same content. So with this backend, files can be moved around, - but should never be added to or changed. This is the default, and - the least expensive backend. + in `.git/annex/`, and assumes that any file with the same basename, + size, and modification time has the same content. So with this backend, + files can be moved around, but should never be added to or changed. + This is the default, and the least expensive backend. * `sha1sum` -- This backend stores the file's content in `.git/annex/`, with a name based on its sha1 checksum. This backend allows modifications of files to be tracked. Its need to generate checksums From 0989dd2694e4be1bc851d0a50903ceaaa988907a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 18:58:26 -0400 Subject: [PATCH 0153/2835] Revert "use some library functions" This reverts commit 8e742bd89e6bd3d83c44847c0455043809c64c89. meh? --- Utility.hs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Utility.hs b/Utility.hs index 105c533ece..a8324815e5 100644 --- a/Utility.hs +++ b/Utility.hs @@ -13,7 +13,6 @@ import System.IO import System.Posix.IO import Data.String.Utils import System.Path -import System.FilePath import System.Directory {- Let's just say that Haskell makes reading/writing a file with @@ -39,13 +38,11 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s parentDir :: String -> String parentDir dir = if length dirs > 0 - then slash ++ (join s $ take ((length dirs) - 1) dirs) + then absolute ++ (join "/" $ take ((length dirs) - 1) dirs) else "" where - dirs = filter (\x -> length x > 0) $ - split s dir - slash = if (isAbsolute dir) then "" else s - s = [pathSeparator] + dirs = filter (\x -> length x > 0) $ split "/" dir + absolute = if ((dir !! 0) == '/') then "/" else "" {- Constructs a relative path from the CWD to a directory. - @@ -71,19 +68,20 @@ relPathCwdToDir dir = do - Both directories must be absolute, and normalized (eg with absNormpath). - - The path will end with "/", unless it is empty. - -} + - -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = if (0 < length path) - then addTrailingPathSeparator path + then if (endswith "/" path) + then path + else path ++ "/" else "" where - s = [pathSeparator] - pfrom = split s from - pto = split s to + pfrom = split "/" from + pto = split "/" to common = map fst $ filter same $ zip pfrom pto same (c,d) = c == d uncommon = drop numcommon pto dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common - path = join s $ dotdots ++ uncommon + path = join "/" $ dotdots ++ uncommon From 23f95ac6df5f25613ac2904c23821f3ca3054246 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 19:01:20 -0400 Subject: [PATCH 0154/2835] use some library functions retry with a bugfix --- Utility.hs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Utility.hs b/Utility.hs index a8324815e5..e04b44e6ff 100644 --- a/Utility.hs +++ b/Utility.hs @@ -13,6 +13,7 @@ import System.IO import System.Posix.IO import Data.String.Utils import System.Path +import System.FilePath import System.Directory {- Let's just say that Haskell makes reading/writing a file with @@ -38,11 +39,13 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s parentDir :: String -> String parentDir dir = if length dirs > 0 - then absolute ++ (join "/" $ take ((length dirs) - 1) dirs) + then slash ++ (join s $ take ((length dirs) - 1) dirs) else "" where - dirs = filter (\x -> length x > 0) $ split "/" dir - absolute = if ((dir !! 0) == '/') then "/" else "" + dirs = filter (\x -> length x > 0) $ + split s dir + slash = if (not $ isAbsolute dir) then "" else s + s = [pathSeparator] {- Constructs a relative path from the CWD to a directory. - @@ -68,20 +71,19 @@ relPathCwdToDir dir = do - Both directories must be absolute, and normalized (eg with absNormpath). - - The path will end with "/", unless it is empty. - - -} + -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = if (0 < length path) - then if (endswith "/" path) - then path - else path ++ "/" + then addTrailingPathSeparator path else "" where - pfrom = split "/" from - pto = split "/" to + s = [pathSeparator] + pfrom = split s from + pto = split s to common = map fst $ filter same $ zip pfrom pto same (c,d) = c == d uncommon = drop numcommon pto dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common - path = join "/" $ dotdots ++ uncommon + path = join s $ dotdots ++ uncommon From 946a7f3f2128704c7b4eeea265a1375c1b60c622 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 19:32:33 -0400 Subject: [PATCH 0155/2835] update --- git-annex.mdwn | 59 ++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index 2079b5b466..21649bfd16 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -1,15 +1,15 @@ git-annex allows managing files with git, without checking the file -contents into git. This is useful when dealing with files larger than git -can currently easily handle, whether due to limitations in memory, -checksumming time, or disk space (only one copy need be stored of an -annexed file). +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. -Even without file content tracking, being able to manage file metadata with -git, move files around and delete files with versioned directory trees, and use -branches and distributed clone, are all very handy reasons to use git. And -annexed files can co-exist in the same git repository with regularly versioned -files, which is convenient for maintaining code, Makefiles, etc that are -associated with annexed files but that benefit from full revision control. +Even without file content tracking, being able to manage files with git, +move files around and delete files with versioned directory trees, and use +branches and distributed clones, are all very handy reasons to use git. And +annexed files can co-exist in the same git repository with regularly +versioned files, which is convenient for maintaining documents, Makefiles, +etc that are associated with annexed files but that benefit from full +revision control. Enough broad picture, here's how it actually looks: @@ -17,13 +17,13 @@ Enough broad picture, here's how it actually looks: it with a symlink pointing at the annexed file, and then calls `git add` to version the *symlink*. (If the file has already been annexed, it does nothing.) +* If you use normal git push/pull commands, the annexed file content + won't be transferred, but the symlinks will be. So different clones of a + repository can have different sets of annexed files available. * You can move the symlink around, copy it, delete it, etc, and commit changes as desired using git. Reading the symlink will always get you the annexed file content, or the link may be broken if the content is not currently available. -* If you use normal git push/pull commands, the annexed file contents - won't be sent, but the symlinks will be. So different clones of a repository - can have different sets of annexed files available. * `git annex push $repository` pushes *all* annexed files to the specified repository. * `git annex pull $repository` pulls *all* annexed files from the specified @@ -31,9 +31,9 @@ Enough broad picture, here's how it actually looks: * `git annex want $file` indicates that you want access to a file's content, without immediatly transferring it. * `git annex get $file` is used to transfer a specified file, and/or - files previously indicated with `git annex want`. If a configured repository has it, - or it is available from other key/value storage, it will be immediatly - downloaded. + files previously indicated with `git annex want`. If a configured + repository has it, or it is available from other key/value storage, + it will be immediatly downloaded. * `git annex drop $file` indicates that you no longer want the file's content to be available in this repository. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` @@ -48,17 +48,16 @@ git-annex can be configured to try to keep N copies of a file's content available across all repositories. By default, N is 1 (configured by annex.numcopies). -`git annex drop` attempts to check all other configured -repositories, to check that N copies of the file exist. If enough -repositories cannot be verified to have it, it will retain the file content -to avoid data loss. +`git annex drop` attempts to check with other git remotes, to check that N +copies of the file exist. If enough repositories cannot be verified to have +it, it will retain the file content to avoid data loss. For example, consider three repositories: Server, Laptop, and USB. Both Server and USB have a copy of a file, and N=1. If on Laptop, you `git annex get $file`, this will transfer it from either Server or USB (depending on which is available), and there are now 3 copies of the file. -Suppose you want to free up space on laptop again, and you `git annex drop` the file +Suppose you want to free up space on Laptop again, and you `git annex drop` the file there. If USB is connected, or Server can be contacted, git-annex can check that it still has a copy of the file, and the content is removed from Laptop. But if USB is currently disconnected, and Server also cannot be @@ -70,17 +69,11 @@ to both USB and Server. Note that different repositories can be configured with different values of N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1, and if they have the -only copies of a file. - -## the .git-annex directory - -The `.git-annex` directory at the top of the repository is used to store -git-annex information that should be propigated between repositories. +copies falling to 1, when USB and Server have N=1. ## key/value storage -git-annex uses a key/value abstraction layer to allow files contents to be +git-annex uses a key/value abstraction layer to allow file contents to be stored in different ways. In theory, any key/value storage system could be used to store the file contents, and git-annex would then retrieve them as needed and put them in `.git/annex/`. @@ -94,15 +87,15 @@ Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. * `WORM` ("Write Once, Read Many") This backend stores the file's content - in `.git/annex/`, and assumes that any file with the same basename, + only in `.git/annex/`, and assumes that any file with the same basename, size, and modification time has the same content. So with this backend, files can be moved around, but should never be added to or changed. This is the default, and the least expensive backend. -* `sha1sum` -- This backend stores the file's content in +* `SHA1` -- This backend stores the file's content in `.git/annex/`, with a name based on its sha1 checksum. This backend allows modifications of files to be tracked. Its need to generate checksums can make it slow for large files. -* `url` -- This backend downloads the file's content from an external URL. +* `URL` -- This backend downloads the file's content from an external URL. ## location tracking @@ -132,7 +125,7 @@ example: * `annex.numcopies` -- number of copies of files to keep (default: 1) * `annex.backends` -- space-separated list of names of the key/value backends to use. The first listed is used to store - new files. (default: file, checksum, url) + new files. (default: "WORM SHA1 URL") * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 100 for local repositories, and 200 for remote From e67887d98b61aeabffc9d1a231421bb00848dd13 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 19:32:56 -0400 Subject: [PATCH 0156/2835] lift to IO --- Backend/File.hs | 13 +++++++++---- UUID.hs | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index eba4b88f83..b2c5c90ebb 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -26,12 +26,16 @@ import qualified Annex import UUID backend = Backend { + name = mustProvide, + getKey = mustProvide, storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, removeKey = dummyRemove, hasKey = checkKeyFile } +mustProvide = error "must provide this field" + {- Storing a key is a no-op. -} dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True @@ -74,11 +78,12 @@ copyKeyFile key file = do cantfind = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key + ppuuids <- prettyPrintUUIDs uuids error $ "no available git remotes have: " ++ - (keyFile key) ++ (uuidlist uuids) - uuidlist [] = "" - uuidlist uuids = "\nIt has been seen before in these repositories:\n" ++ - prettyPrintUUIDs uuids + (keyFile key) ++ + if (0 < length uuids) + then "\nIt has been seen before in these repositories:\n" ++ ppuuids + else "" {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () diff --git a/UUID.hs b/UUID.hs index 9348c7b437..3653eeec42 100644 --- a/UUID.hs +++ b/UUID.hs @@ -91,7 +91,7 @@ reposByUUID repos uuids = do {- Pretty-prints a list of UUIDs - TODO: use lookup file to really show pretty names. -} -prettyPrintUUIDs :: [UUID] -> String +prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = - unwords $ map (\u -> "\tUUID "++u++"\n") uuids + return $ unwords $ map (\u -> "\tUUID "++u++"\n") uuids From 5de102d5b90fb621bdb1bd81cf5f562a9a2549e4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 19:33:10 -0400 Subject: [PATCH 0157/2835] rename backends more --- Backend/{Checksum.hs => SHA1.hs} | 8 ++++---- Backend/{Url.hs => URL.hs} | 6 +++--- Backend/{Worm.hs => WORM.hs} | 2 +- BackendList.hs | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) rename Backend/{Checksum.hs => SHA1.hs} (58%) rename Backend/{Url.hs => URL.hs} (92%) rename Backend/{Worm.hs => WORM.hs} (97%) diff --git a/Backend/Checksum.hs b/Backend/SHA1.hs similarity index 58% rename from Backend/Checksum.hs rename to Backend/SHA1.hs index de98fbf446..8c7c99bbd7 100644 --- a/Backend/Checksum.hs +++ b/Backend/SHA1.hs @@ -1,17 +1,17 @@ -{- git-annex "checksum" backend +{- git-annex "SHA1" backend - -} -module Backend.Checksum (backend) where +module Backend.SHA1 (backend) where import qualified Backend.File import Data.Digest.Pure.SHA import BackendTypes backend = Backend.File.backend { - name = "checksum", + name = "SHA1", getKey = keyValue } -- checksum the file to get its key keyValue :: FilePath -> Annex (Maybe Key) -keyValue k = error "checksum keyValue unimplemented" -- TODO +keyValue k = error "SHA1 keyValue unimplemented" -- TODO diff --git a/Backend/Url.hs b/Backend/URL.hs similarity index 92% rename from Backend/Url.hs rename to Backend/URL.hs index e237672084..4e87ca4c26 100644 --- a/Backend/Url.hs +++ b/Backend/URL.hs @@ -1,7 +1,7 @@ -{- git-annex "url" backend +{- git-annex "URL" backend - -} -module Backend.Url (backend) where +module Backend.URL (backend) where import Control.Monad.State (liftIO) import Data.String.Utils @@ -10,7 +10,7 @@ import System.Exit import BackendTypes backend = Backend { - name = "url", + name = "URL", getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = downloadUrl, diff --git a/Backend/Worm.hs b/Backend/WORM.hs similarity index 97% rename from Backend/Worm.hs rename to Backend/WORM.hs index 89fe4bf572..9a1e17ec5f 100644 --- a/Backend/Worm.hs +++ b/Backend/WORM.hs @@ -1,7 +1,7 @@ {- git-annex "WORM" backend -- Write Once, Read Many - -} -module Backend.Worm (backend) where +module Backend.WORM (backend) where import Control.Monad.State import qualified Backend.File diff --git a/BackendList.hs b/BackendList.hs index 93c0464f1b..42e237204f 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -10,13 +10,13 @@ module BackendList ( import BackendTypes -- When adding a new backend, import it here and add it to the list. -import qualified Backend.Worm -import qualified Backend.Checksum -import qualified Backend.Url +import qualified Backend.WORM +import qualified Backend.SHA1 +import qualified Backend.URL supportedBackends = - [ Backend.Worm.backend - , Backend.Checksum.backend - , Backend.Url.backend + [ Backend.WORM.backend + , Backend.SHA1.backend + , Backend.URL.backend ] {- Parses a string with a list of backend names into From 46ac19a51d8994aa0ac978fef3359729ed91c6ba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 20:20:16 -0400 Subject: [PATCH 0158/2835] implemented uuid.log --- UUID.hs | 50 +++++++++++++++++++++++++++++++++++++++++++------- git-annex.mdwn | 2 +- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/UUID.hs b/UUID.hs index 3653eeec42..8cdee43de6 100644 --- a/UUID.hs +++ b/UUID.hs @@ -11,7 +11,8 @@ module UUID ( prepUUID, genUUID, reposByUUID, - prettyPrintUUIDs + prettyPrintUUIDs, + describeUUID ) where import Control.Monad.State @@ -19,8 +20,10 @@ import Maybe import List import System.Cmd.Utils import System.IO +import qualified Data.Map as M import qualified GitRepo as Git import Types +import Locations import qualified Annex type UUID = String @@ -29,7 +32,7 @@ configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} -genUUID :: Annex UUID +genUUID :: IO UUID genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h {- Looks up a repo's UUID. May return "" if none is known. @@ -66,7 +69,7 @@ prepUUID = do u <- getUUID g if ("" == u) then do - uuid <- genUUID + uuid <- liftIO $ genUUID setConfig configkey uuid else return () @@ -89,9 +92,42 @@ reposByUUID repos uuids = do u <- getUUID r return $ isJust $ elemIndex u uuids -{- Pretty-prints a list of UUIDs - - TODO: use lookup file to really show pretty names. -} +{- Pretty-prints a list of UUIDs -} prettyPrintUUIDs :: [UUID] -> Annex String -prettyPrintUUIDs uuids = - return $ unwords $ map (\u -> "\tUUID "++u++"\n") uuids +prettyPrintUUIDs uuids = do + m <- uuidMap + return $ unwords $ map (\u -> " "++(prettify m u)++"\n") uuids + where + prettify m u = + if (0 < (length $ findlog m u)) + then u ++ " -- " ++ (findlog m u) + else u + findlog m u = M.findWithDefault "" u m +{- Records a description for a uuid in the uuidLog. -} +describeUUID :: UUID -> String -> Annex () +describeUUID uuid desc = do + m <- uuidMap + let m' = M.insert uuid desc m + log <- uuidLog + liftIO $ writeFile log $ serialize m' + where + serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m + +{- Read and parse the uuidLog into a Map -} +uuidMap :: Annex (M.Map UUID String) +uuidMap = do + log <- uuidLog + s <- liftIO $ catch (readFile log) (\error -> return "") + return $ M.fromList $ map (\l -> pair l) $ lines s + where + pair l = + if (1 < (length $ words l)) + then ((words l) !! 0, unwords $ drop 1 $ words l) + else ("", "") + +{- Filename of uuid.log. -} +uuidLog :: Annex String +uuidLog = do + g <- Annex.gitRepo + return $ (gitStateDir g) ++ "uuid.log" diff --git a/git-annex.mdwn b/git-annex.mdwn index 21649bfd16..1261a196fe 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -109,7 +109,7 @@ Repositories record their UUID and the date when they get or drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) -The optional file `.git-annex/uuid.map` can be created to add a description +The optional file `.git-annex/uuid.log` can be created to add a description to a UUID. If git-annex needs a file from a repository and it cannot find the repository amoung the remotes, it will use the description from this file when asking for the repository to be made available. The file format From 645bc94d3d9e5f08bda74a99e0584768b32da81c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 13:22:48 -0400 Subject: [PATCH 0159/2835] quiet commit of logs --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 7e719888a0..006973fc9d 100644 --- a/Core.hs +++ b/Core.hs @@ -25,7 +25,7 @@ shutdown = do g <- Annex.gitRepo needcommit <- Annex.flagIsSet NeedCommit if (needcommit) - then liftIO $ Git.run g ["commit", "-m", + then liftIO $ Git.run g ["commit", "-q", "-m", "git-annex log update", gitStateDir g] else return () From 1260adbd7700ab9e35f61f4ad94b9cc0536f243e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 13:38:59 -0400 Subject: [PATCH 0160/2835] basic recursion done; skipping git stuff still todo --- Commands.hs | 17 +++++++++-------- TODO | 2 ++ Utility.hs | 14 ++++++++++++++ git-annex.mdwn | 5 +++++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Commands.hs b/Commands.hs index a403a5a48a..f28b3e72b9 100644 --- a/Commands.hs +++ b/Commands.hs @@ -26,20 +26,21 @@ import qualified Remotes - actions to be run in the Annex monad. -} parseCmd :: [String] -> IO ([Flag], [Annex ()]) parseCmd argv = do - (flags, files) <- getopt - case (length files) of + (flags, params) <- getopt + case (length params) of 0 -> error header _ -> do - let c = lookupCmd (files !! 0) - if (0 == length c) - then ret flags defaultCmd files - else ret flags (snd $ c !! 0) $ drop 1 files + let (cmd, locs) = takeCmd params $ lookupCmd (params !! 0) + files <- mapM recurseFiles locs + return (flags, map cmd $ foldl (++) [] files) where - ret flags cmd files = return (flags, makeactions cmd files) - makeactions cmd files = map cmd files getopt = case getOpt Permute options argv of (flags, nonopts, []) -> return (flags, nonopts) (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) + takeCmd files cmds = + if (0 == length cmds) + then (defaultCmd, files) + else ((snd $ cmds !! 0), drop 1 files) lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds cmds = [ ("add", addCmd) , ("get", getCmd) diff --git a/TODO b/TODO index ddd0761658..8cb77fe9fb 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,8 @@ * --push/--pull/--want +* isn't pull the same as get? + * recurse on directories * how to handle git mv file? diff --git a/Utility.hs b/Utility.hs index e04b44e6ff..8005fd17cc 100644 --- a/Utility.hs +++ b/Utility.hs @@ -7,12 +7,14 @@ module Utility ( parentDir, relPathCwdToDir, relPathDirToDir, + recurseFiles, ) where import System.IO import System.Posix.IO import Data.String.Utils import System.Path +import System.IO.HVFS import System.FilePath import System.Directory @@ -87,3 +89,15 @@ relPathDirToDir from to = dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common path = join s $ dotdots ++ uncommon + +{- Recursively returns all files and symlinks (to anything) in the specified + - path. If the path is a file, returns only it. Does not follow symlinks to + - directories. -} +recurseFiles :: FilePath -> IO [FilePath] +recurseFiles path = do + find <- recurseDirStat SystemFS path + return $ filesOnly find + where + filesOnly l = map (\(f,s) -> f) $ filter isFile l + isFile (f, HVFSStatEncap s) = + vIsRegularFile s || vIsSymbolicLink s diff --git a/git-annex.mdwn b/git-annex.mdwn index 1261a196fe..1922a1b63c 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -42,6 +42,11 @@ Enough broad picture, here's how it actually looks: * `git annex $file` is a shorthand. If the file is already known, it does `git annex get`, otherwise it does `git annex add`. +Oh yeah, "$file" in the above can be any number of files, or directories. +git-annex automatically recurses into directories, but skips files that are +checked into git (as well as skipping `.git` itself), so "git annex ." works +fine. + ## copies git-annex can be configured to try to keep N copies of a file's content From 5f73fd5b661ecdeae164cc3d5a6c4d0b6113eba7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 13:59:48 -0400 Subject: [PATCH 0161/2835] dropped defaultCmd With recusrion, it doesn't really make sense. --- Commands.hs | 23 ++++++----------------- Core.hs | 2 ++ git-annex.mdwn | 2 -- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/Commands.hs b/Commands.hs index f28b3e72b9..3d85b12b93 100644 --- a/Commands.hs +++ b/Commands.hs @@ -29,18 +29,16 @@ parseCmd argv = do (flags, params) <- getopt case (length params) of 0 -> error header - _ -> do - let (cmd, locs) = takeCmd params $ lookupCmd (params !! 0) - files <- mapM recurseFiles locs - return (flags, map cmd $ foldl (++) [] files) + _ -> case (lookupCmd (params !! 0)) of + [] -> error header + [(_,cmd)] -> do + let locs = drop 1 params + files <- mapM recurseFiles locs + return (flags, map cmd $ foldl (++) [] files) where getopt = case getOpt Permute options argv of (flags, nonopts, []) -> return (flags, nonopts) (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) - takeCmd files cmds = - if (0 == length cmds) - then (defaultCmd, files) - else ((snd $ cmds !! 0), drop 1 files) lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds cmds = [ ("add", addCmd) , ("get", getCmd) @@ -54,15 +52,6 @@ parseCmd argv = do (join "|" $ map fst cmds) ++ "] file ..." options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] -{- Default mode is to annex a file if it is not already, and otherwise - - get its content. -} -defaultCmd :: FilePath -> Annex () -defaultCmd file = do - r <- liftIO $ Backend.lookupFile file - case (r) of - Just v -> getCmd file - Nothing -> addCmd file - {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () diff --git a/Core.hs b/Core.hs index 006973fc9d..19d1737c3a 100644 --- a/Core.hs +++ b/Core.hs @@ -58,3 +58,5 @@ inAnnex :: Key -> Annex Bool inAnnex key = do g <- Annex.gitRepo liftIO $ doesFileExist $ annexLocation g key + +{- -} diff --git a/git-annex.mdwn b/git-annex.mdwn index 1922a1b63c..2796f48fb1 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -39,8 +39,6 @@ Enough broad picture, here's how it actually looks: * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` if you're just done with a file; only use `unannex` if you accidentially added a file. -* `git annex $file` is a shorthand. If the file - is already known, it does `git annex get`, otherwise it does `git annex add`. Oh yeah, "$file" in the above can be any number of files, or directories. git-annex automatically recurses into directories, but skips files that are From 5a32804115a73d3c6fb2de17a1f9a6c628beba5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 14:20:43 -0400 Subject: [PATCH 0162/2835] add inGit/notInGit --- GitRepo.hs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index f3bb5427ad..5981a6ca11 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -20,10 +20,13 @@ module GitRepo ( configMap, configRead, run, + pipeRead, attributes, remotes, remotesAdd, - repoRemoteName + repoRemoteName, + inGit, + notInGit ) where import Directory @@ -167,16 +170,30 @@ run repo params = assertlocal repo $ do return () {- Runs a git subcommand and returns its output. -} -gitPipeRead :: Repo -> [String] -> IO String -gitPipeRead repo params = assertlocal repo $ do +pipeRead :: Repo -> [String] -> IO String +pipeRead repo params = assertlocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do ret <- hGetContentsStrict h return ret +{- Passed a location, recursively scans for all files that + - are checked into git at that location. -} +inGit :: Repo -> FilePath -> IO [FilePath] +inGit repo location = do + s <- pipeRead repo ["ls-files", "--cached", "--exclude-standard"] + return $ lines s + +{- Passed a location, recursively scans for all files that are not checked + - into git, and not gitignored. -} +notInGit :: Repo -> FilePath -> IO [FilePath] +notInGit repo location = do + s <- pipeRead repo ["ls-files", "--others", "--exclude-standard"] + return $ lines s + {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo configRead repo = assertlocal repo $ do - {- Cannot use gitPipeRead because it relies on the config having + {- Cannot use pipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory bracket_ (changeWorkingDirectory (top repo)) From bfa581a218719c46dbc19a212a005b0cf2e145c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 14:58:14 -0400 Subject: [PATCH 0163/2835] bugfix --- GitRepo.hs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 5981a6ca11..76150b3095 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -25,8 +25,8 @@ module GitRepo ( remotes, remotesAdd, repoRemoteName, - inGit, - notInGit + inRepo, + notInRepo ) where import Directory @@ -178,16 +178,16 @@ pipeRead repo params = assertlocal repo $ do {- Passed a location, recursively scans for all files that - are checked into git at that location. -} -inGit :: Repo -> FilePath -> IO [FilePath] -inGit repo location = do - s <- pipeRead repo ["ls-files", "--cached", "--exclude-standard"] +inRepo :: Repo -> FilePath -> IO [FilePath] +inRepo repo location = do + s <- pipeRead repo ["ls-files", "--cached", "--exclude-standard", location] return $ lines s {- Passed a location, recursively scans for all files that are not checked - into git, and not gitignored. -} -notInGit :: Repo -> FilePath -> IO [FilePath] -notInGit repo location = do - s <- pipeRead repo ["ls-files", "--others", "--exclude-standard"] +notInRepo :: Repo -> FilePath -> IO [FilePath] +notInRepo repo location = do + s <- pipeRead repo ["ls-files", "--others", "--exclude-standard", location] return $ lines s {- Runs git config and populates a repo with its config. -} From e80160380a16fbeb38f21f4683917b49a9221a91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 14:58:35 -0400 Subject: [PATCH 0164/2835] now finds files in git or not depending on what command wants --- Commands.hs | 56 ++++++++++++++++++++++++++++++++++++---------------- Utility.hs | 15 +------------- git-annex.hs | 2 +- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/Commands.hs b/Commands.hs index 3d85b12b93..a2535001e7 100644 --- a/Commands.hs +++ b/Commands.hs @@ -21,35 +21,57 @@ import LocationLog import Types import Core import qualified Remotes +import qualified BackendTypes + +data CmdWants = FilesInGit | FilesNotInGit | RepoName +data Command = Command { + cmdname :: String, + cmdaction :: (String -> Annex ()), + cmdwants :: CmdWants +} + +cmds :: [Command] +cmds = [ (Command "add" addCmd FilesNotInGit) + , (Command "get" getCmd FilesInGit) + , (Command "drop" dropCmd FilesInGit) + , (Command "want" wantCmd FilesInGit) + , (Command "push" pushCmd RepoName) + , (Command "pull" pullCmd RepoName) + , (Command "unannex" unannexCmd FilesInGit) + ] + +{- Finds the type of parameters a command wants, from among the passed + - parameter list. -} +findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] +findWanted FilesNotInGit params repo = do + files <- mapM (Git.notInRepo repo) params + return $ foldl (++) [] files +findWanted FilesInGit params repo = do + files <- mapM (Git.inRepo repo) params + return $ foldl (++) [] files +findWanted RepoName params _ = do + return $ params {- Parses command line and returns a list of flags and a list of - actions to be run in the Annex monad. -} -parseCmd :: [String] -> IO ([Flag], [Annex ()]) -parseCmd argv = do +parseCmd :: [String] -> AnnexState -> IO ([Flag], [Annex ()]) +parseCmd argv state = do (flags, params) <- getopt case (length params) of 0 -> error header _ -> case (lookupCmd (params !! 0)) of [] -> error header - [(_,cmd)] -> do - let locs = drop 1 params - files <- mapM recurseFiles locs - return (flags, map cmd $ foldl (++) [] files) + [Command _ action want] -> do + f <- findWanted want (drop 1 params) + (BackendTypes.repo state) + return (flags, map action f) where getopt = case getOpt Permute options argv of - (flags, nonopts, []) -> return (flags, nonopts) + (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) - lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds - cmds = [ ("add", addCmd) - , ("get", getCmd) - , ("drop", dropCmd) - , ("want", wantCmd) - , ("push", pushCmd) - , ("pull", pullCmd) - , ("unannex", unannexCmd) - ] + lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds header = "Usage: git-annex [" ++ - (join "|" $ map fst cmds) ++ "] file ..." + (join "|" $ map cmdname cmds) ++ "] file ..." options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] {- Annexes a file, storing it in a backend, and then moving it into diff --git a/Utility.hs b/Utility.hs index 8005fd17cc..e4278ff3f6 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,8 +6,7 @@ module Utility ( hGetContentsStrict, parentDir, relPathCwdToDir, - relPathDirToDir, - recurseFiles, + relPathDirToDir ) where import System.IO @@ -89,15 +88,3 @@ relPathDirToDir from to = dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common path = join s $ dotdots ++ uncommon - -{- Recursively returns all files and symlinks (to anything) in the specified - - path. If the path is a file, returns only it. Does not follow symlinks to - - directories. -} -recurseFiles :: FilePath -> IO [FilePath] -recurseFiles path = do - find <- recurseDirStat SystemFS path - return $ filesOnly find - where - filesOnly l = map (\(f,s) -> f) $ filter isFile l - isFile (f, HVFSStatEncap s) = - vIsRegularFile s || vIsSymbolicLink s diff --git a/git-annex.hs b/git-annex.hs index cd67242afa..01416f6dd9 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -12,9 +12,9 @@ import qualified GitRepo as Git main = do args <- getArgs - (flags, actions) <- parseCmd args gitrepo <- Git.repoFromCwd state <- new gitrepo + (flags, actions) <- parseCmd args state tryRun state $ [startup flags] ++ actions ++ [shutdown] {- Runs a list of Annex actions. Catches exceptions, not stopping From eed4a7fcdfbae821485d120055c8aec4824ecb3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 15:10:07 -0400 Subject: [PATCH 0165/2835] tweak --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index a2535001e7..11f808c211 100644 --- a/Commands.hs +++ b/Commands.hs @@ -71,7 +71,7 @@ parseCmd argv state = do (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds header = "Usage: git-annex [" ++ - (join "|" $ map cmdname cmds) ++ "] file ..." + (join "|" $ map cmdname cmds) ++ "] ..." options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] {- Annexes a file, storing it in a backend, and then moving it into From 684011175cc75bb6a667e65ba0ec6cabd1f0897a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 15:22:47 -0400 Subject: [PATCH 0166/2835] update --- git-annex.mdwn | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index 2796f48fb1..bb216f0383 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -40,16 +40,15 @@ Enough broad picture, here's how it actually looks: if you're just done with a file; only use `unannex` if you accidentially added a file. -Oh yeah, "$file" in the above can be any number of files, or directories. -git-annex automatically recurses into directories, but skips files that are -checked into git (as well as skipping `.git` itself), so "git annex ." works -fine. +Oh yeah, "$file" in the above can be any number of files, or directories, +same as you'd pass to "git add" or "git rm". +So "git annex add ." or "git annex get dir/" work fine. ## copies git-annex can be configured to try to keep N copies of a file's content -available across all repositories. By default, N is 1 (configured by -annex.numcopies). +available across all repositories. By default, N is 1; it is configured by +annex.numcopies. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have @@ -105,7 +104,11 @@ to store different files' contents in a given repository. git-annex keeps track of on which repository it last saw a file's content. This can be useful when using it for archiving with offline storage. When you indicate you want a file, git-annex will tell you which repositories -have the file's content. +have the file's content. For example: + + # git annex get myfile + git-annex: unable to get: myfile + To get that file, need access to one of these remotes: usbdrive Location tracking information is stored in `.git-annex/$key.log`. Repositories record their UUID and the date when they get or drop @@ -113,7 +116,7 @@ a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) The optional file `.git-annex/uuid.log` can be created to add a description -to a UUID. If git-annex needs a file from a repository and it cannot find +to a UUID. If git-annex needs a file from some repository, and it cannot find the repository amoung the remotes, it will use the description from this file when asking for the repository to be made available. The file format is a UUID, a space, and the rest of the line is its description. For From a31dc74806f165e01f56dbc3322e738a921cc6e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 15:23:03 -0400 Subject: [PATCH 0167/2835] update --- TODO | 7 ++----- git-annex.mdwn => doc/git-annex.mdwn | 0 2 files changed, 2 insertions(+), 5 deletions(-) rename git-annex.mdwn => doc/git-annex.mdwn (100%) diff --git a/TODO b/TODO index 8cb77fe9fb..8fc17fca99 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,8 @@ -* bug: cannot "git annex ../foo" (GitRepo.relative is buggy) +* bug: cannot "git annex ../foo" (GitRepo.relative is buggy and + git-ls-files also refuses w/o --full-name, which would need other changes) * --push/--pull/--want -* isn't pull the same as get? - -* recurse on directories - * how to handle git mv file? * finish BackendChecksum diff --git a/git-annex.mdwn b/doc/git-annex.mdwn similarity index 100% rename from git-annex.mdwn rename to doc/git-annex.mdwn From 81d628a8cd6f20c2ef336271ae03376dc75b6920 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 15:58:42 -0400 Subject: [PATCH 0168/2835] updatte --- doc/git-annex.mdwn | 102 +++++++++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index bb216f0383..ad45c08426 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -11,34 +11,48 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. +My motivation for git-annex was the growing number of external drives I +use. Some are used to archive data, others hold backups, and yet others +come with me when I'm away from home to carry data that doesn't fit on my +netbook. Maintaining all that was a nightmare, lots of ad-hoc moving files +around, rsyncing files (unison is too slow), and deleting multiple copies +of files from multiple places. I realized what what I needed was revision +control where each drive was a repository, and where copying the files +around, and deciding which copies were safe to delete was automated. +I posted about this to the VCS-home mailing list and got a great suggestion +to make it support arbitrary key-value stores. A week of coding later, +and git-annex is born. + Enough broad picture, here's how it actually looks: * `git annex add $file` moves the file into `.git/annex/`, and replaces it with a symlink pointing at the annexed file, and then calls `git add` to version the *symlink*. (If the file has already been annexed, it does - nothing.) -* If you use normal git push/pull commands, the annexed file content - won't be transferred, but the symlinks will be. So different clones of a - repository can have different sets of annexed files available. -* You can move the symlink around, copy it, delete it, etc, and commit changes + nothing.) + + If you then use normal git push/pull commands, the annexed file content + won't be transferred between repositories, but the symlinks will be. + So different clones of a repository can have different sets of annexed + files available. + + You can move the symlink around, copy it, delete it, etc, and commit changes as desired using git. Reading the symlink will always get you the annexed file content, or the link may be broken if the content is not currently available. +* `git annex get $file` is used to transfer a specified file from the + backend storage to the current repository. +* `git annex drop $file` indicates that you no longer want the file's + content to be available in this repository. * `git annex push $repository` pushes *all* annexed files to the specified repository. * `git annex pull $repository` pulls *all* annexed files from the specified repository. -* `git annex want $file` indicates that you want access to a file's - content, without immediatly transferring it. -* `git annex get $file` is used to transfer a specified file, and/or - files previously indicated with `git annex want`. If a configured - repository has it, or it is available from other key/value storage, - it will be immediatly downloaded. -* `git annex drop $file` indicates that you no longer want the file's - content to be available in this repository. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` if you're just done with a file; only use `unannex` if you accidentially added a file. +* `git annex describe "some description"` allows associating some description + (such as "USB archive drive 1") with a repository. This can help with + finding it later, see "Location Tracking" below. Oh yeah, "$file" in the above can be any number of files, or directories, same as you'd pass to "git add" or "git rm". @@ -73,10 +87,10 @@ Note that different repositories can be configured with different values of N. So just because Laptop has N=2, this does not prevent the number of copies falling to 1, when USB and Server have N=1. -## key/value storage +## key-value storage -git-annex uses a key/value abstraction layer to allow file contents to be -stored in different ways. In theory, any key/value storage system could be +git-annex uses a key-value abstraction layer to allow file contents to be +stored in different ways. In theory, any key-value storage system could be used to store the file contents, and git-annex would then retrieve them as needed and put them in `.git/annex/`. @@ -101,36 +115,40 @@ to store different files' contents in a given repository. ## location tracking -git-annex keeps track of on which repository it last saw a file's content. -This can be useful when using it for archiving with offline storage. When -you indicate you want a file, git-annex will tell you which repositories -have the file's content. For example: - - # git annex get myfile - git-annex: unable to get: myfile - To get that file, need access to one of these remotes: usbdrive - -Location tracking information is stored in `.git-annex/$key.log`. +git-annex keeps track of in which repositories it last saw a file's content. +This location tracking information is stored in `.git-annex/$key.log`. Repositories record their UUID and the date when they get or drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) -The optional file `.git-annex/uuid.log` can be created to add a description -to a UUID. If git-annex needs a file from some repository, and it cannot find -the repository amoung the remotes, it will use the description from this -file when asking for the repository to be made available. The file format -is a UUID, a space, and the rest of the line is its description. For -example: +This location tracking information is useful if you have multiple +repositories, and not all are always accessible. For example, perhaps one +is on a home file server, and you are away from home. Then git-annex can +tell you what git remote it needs access to in order to get a file: - UUID d3d2474c-d5c3-11df-80a9-002170d25c55 USB drive in red enclosure - UUID 60cf39c8-d5c6-11df-aa8b-93fda39008d6 my colocated server + # git annex get myfile + git-annex: unable to get file with key: WORM:8b01f6d371178722367393eb26043482e1820306:myfile + To get that file, need access to one of these remotes: home + +Another way the location tracking comes in handy is if you put repositories +on removable USB drives, that might be archived away offline in a safe +place. In this sort of case, you probably don't have a git remotes +configured for every USB drive. So git-annex may have to resort to talking +about repository UUIDs. If you have previously used "git annex describe" +in those repositories, it will include their description to help you with +finding them: + + git-annex: no available git remotes have file with key: WORM:8b01f6d371178722367393eb26043482e1820306:myfile + It has been seen before in these repositories: + c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 + e1938fee-d95b-11df-96cc-002170d25c55 ## configuration * `annex.uuid` -- a unique UUID for this repository * `annex.numcopies` -- number of copies of files to keep (default: 1) * `annex.backends` -- space-separated list of names of - the key/value backends to use. The first listed is used to store + the key-value backends to use. The first listed is used to store new files. (default: "WORM SHA1 URL") * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. @@ -165,13 +183,13 @@ Need a way to tell how much free space is available on the disk containing a given repository. The repository may be remote, so ssh may need to be used. -Similarly, need a way to tell the size of a file before downloading it from -remote, to check local disk space. +Similarly, need a way to tell the size of a file before copying it from +a remote, to check local disk space. -### auto-drop files on rm +### auto-drop on rm -When git-rm removed a file, it should get dropped too. Of course, it may -not be dropped right away, depending on number of copies available. +When git-rm removed a file, its key should get dropped too. Of course, it +may not be dropped right away, depending on number of copies available. ### branching @@ -180,3 +198,7 @@ and the user switched between them, git-annex will see different logs in the different branches, and so may miss info about what remotes have which files (though it can re-learn). An alternative would be to store the log data directly in the git repo as `pristine-tar` does. + +## contact + +Joey Hess From 6d13ae10cf1d295b64855984f5a526f8209f3341 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 16:15:31 -0400 Subject: [PATCH 0169/2835] git annex describe --- Backend.hs | 6 +++--- Backend/File.hs | 4 ++-- BackendTypes.hs | 2 +- Commands.hs | 24 +++++++++++++++++------- UUID.hs | 10 +++++++--- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/Backend.hs b/Backend.hs index f419831d28..636557d7d4 100644 --- a/Backend.hs +++ b/Backend.hs @@ -1,7 +1,7 @@ -{- git-annex key/value storage backends +{- git-annex key-value storage backends - - - git-annex uses a key/value abstraction layer to allow files contents to be - - stored in different ways. In theory, any key/value storage system could be + - git-annex uses a key-value abstraction layer to allow files contents to be + - stored in different ways. In theory, any key-value storage system could be - used to store the file contents, and git-annex would then retrieve them - as needed and put them in `.git/annex/`. - diff --git a/Backend/File.hs b/Backend/File.hs index b2c5c90ebb..c443b4f7ad 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -58,7 +58,7 @@ copyKeyFile key file = do else return () trycopy remotes remotes where - trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ + trycopy full [] = error $ "unable to get file with key: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ (Remotes.list full) trycopy full (r:rs) = do @@ -79,7 +79,7 @@ copyKeyFile key file = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key ppuuids <- prettyPrintUUIDs uuids - error $ "no available git remotes have: " ++ + error $ "no available git remotes have file with key: " ++ (keyFile key) ++ if (0 < length uuids) then "\nIt has been seen before in these repositories:\n" ++ ppuuids diff --git a/BackendTypes.hs b/BackendTypes.hs index 13ffde7f89..41bc77858e 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -49,7 +49,7 @@ backendName (Key (b,k)) = b keyFrag :: Key -> KeyFrag keyFrag (Key (b,k)) = k --- this structure represents a key/value backend +-- this structure represents a key-value backend data Backend = Backend { -- name of this backend name :: String, diff --git a/Commands.hs b/Commands.hs index 11f808c211..1f91280112 100644 --- a/Commands.hs +++ b/Commands.hs @@ -23,7 +23,7 @@ import Core import qualified Remotes import qualified BackendTypes -data CmdWants = FilesInGit | FilesNotInGit | RepoName +data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), @@ -34,10 +34,10 @@ cmds :: [Command] cmds = [ (Command "add" addCmd FilesNotInGit) , (Command "get" getCmd FilesInGit) , (Command "drop" dropCmd FilesInGit) - , (Command "want" wantCmd FilesInGit) , (Command "push" pushCmd RepoName) , (Command "pull" pullCmd RepoName) , (Command "unannex" unannexCmd FilesInGit) + , (Command "describe" describeCmd SingleString) ] {- Finds the type of parameters a command wants, from among the passed @@ -49,6 +49,8 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files +findWanted SingleString params _ = do + return $ [unwords params] findWanted RepoName params _ = do return $ params @@ -150,11 +152,8 @@ getCmd file = notinBackend file err $ \(key, backend) -> do where err = error $ "not annexed " ++ file -{- Indicates a file is wanted. -} -wantCmd :: FilePath -> Annex () -wantCmd file = do error "not implemented" -- TODO - -{- Indicates a file is not wanted. -} +{- Indicates a file's content is not wanted anymore, and should be removed + - if it's safe to do so. -} dropCmd :: FilePath -> Annex () dropCmd file = notinBackend file err $ \(key, backend) -> do force <- Annex.flagIsSet Force @@ -185,6 +184,17 @@ pushCmd reponame = do error "not implemented" -- TODO pullCmd :: String -> Annex () pullCmd reponame = do error "not implemented" -- TODO +{- Stores description for the repository. -} +describeCmd :: String -> Annex () +describeCmd description = do + g <- Annex.gitRepo + u <- getUUID g + describeUUID u description + log <- uuidLog + liftIO $ Git.run g ["add", log] + Annex.flagChange NeedCommit True + liftIO $ putStrLn "description set" + {- Updates the LocationLog when a key's presence changes. -} logStatus :: Key -> LogStatus -> Annex () logStatus key status = do diff --git a/UUID.hs b/UUID.hs index 8cdee43de6..3e6991d485 100644 --- a/UUID.hs +++ b/UUID.hs @@ -12,7 +12,8 @@ module UUID ( genUUID, reposByUUID, prettyPrintUUIDs, - describeUUID + describeUUID, + uuidLog ) where import Control.Monad.State @@ -25,6 +26,7 @@ import qualified GitRepo as Git import Types import Locations import qualified Annex +import Utility type UUID = String @@ -110,7 +112,7 @@ describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m log <- uuidLog - liftIO $ writeFile log $ serialize m' + liftIO $ withFileLocked log WriteMode (\h -> hPutStr h $ serialize m') where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m @@ -118,7 +120,9 @@ describeUUID uuid desc = do uuidMap :: Annex (M.Map UUID String) uuidMap = do log <- uuidLog - s <- liftIO $ catch (readFile log) (\error -> return "") + s <- liftIO $ catch + (withFileLocked log ReadMode $ \h -> hGetContentsStrict h) + (\error -> return "") return $ M.fromList $ map (\l -> pair l) $ lines s where pair l = From 909f619c07699fe6c76d40bb4649e07737a0b9ae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 16:20:49 -0400 Subject: [PATCH 0170/2835] tweaks --- Annex.hs | 1 + Backend.hs | 1 + Backend/File.hs | 1 + Backend/SHA1.hs | 3 ++- Backend/URL.hs | 1 + Backend/WORM.hs | 7 ++++--- BackendTypes.hs | 1 + Commands.hs | 1 + Core.hs | 1 + GitRepo.hs | 1 + LocationLog.hs | 1 + Locations.hs | 1 + Remotes.hs | 1 + UUID.hs | 3 +++ doc/git-annex.mdwn | 10 +++++----- git-annex.hs | 4 ++-- 16 files changed, 27 insertions(+), 11 deletions(-) diff --git a/Annex.hs b/Annex.hs index 08607cafa8..68c0cb88e2 100644 --- a/Annex.hs +++ b/Annex.hs @@ -13,6 +13,7 @@ module Annex ( ) where import Control.Monad.State + import qualified GitRepo as Git import Types import qualified BackendTypes as Backend diff --git a/Backend.hs b/Backend.hs index 636557d7d4..874191924d 100644 --- a/Backend.hs +++ b/Backend.hs @@ -27,6 +27,7 @@ import System.Directory import System.FilePath import Data.String.Utils import System.Posix.Files + import BackendList import Locations import qualified GitRepo as Git diff --git a/Backend/File.hs b/Backend/File.hs index c443b4f7ad..f5237f7210 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,6 +15,7 @@ import System.IO import System.Cmd import System.Exit import Control.Exception + import BackendTypes import LocationLog import Locations diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 8c7c99bbd7..c01e01a723 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -3,8 +3,9 @@ module Backend.SHA1 (backend) where -import qualified Backend.File import Data.Digest.Pure.SHA + +import qualified Backend.File import BackendTypes backend = Backend.File.backend { diff --git a/Backend/URL.hs b/Backend/URL.hs index 4e87ca4c26..9e64e04996 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -7,6 +7,7 @@ import Control.Monad.State (liftIO) import Data.String.Utils import System.Cmd import System.Exit + import BackendTypes backend = Backend { diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 9a1e17ec5f..420f336e90 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -4,14 +4,15 @@ module Backend.WORM (backend) where import Control.Monad.State -import qualified Backend.File -import BackendTypes -import Utility import System.FilePath import System.Posix.Files import Data.Digest.Pure.SHA -- slow, but we only checksum filenames import qualified Data.ByteString.Lazy.Char8 as B +import qualified Backend.File +import BackendTypes +import Utility + backend = Backend.File.backend { name = "WORM", getKey = keyValue diff --git a/BackendTypes.hs b/BackendTypes.hs index 41bc77858e..49bd1bceb8 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -7,6 +7,7 @@ module BackendTypes where import Control.Monad.State (StateT) import Data.String.Utils + import qualified GitRepo as Git -- command-line flags diff --git a/Commands.hs b/Commands.hs index 1f91280112..c477a81fde 100644 --- a/Commands.hs +++ b/Commands.hs @@ -10,6 +10,7 @@ import System.Path import Data.String.Utils import List import IO + import qualified GitRepo as Git import qualified Annex import Utility diff --git a/Core.hs b/Core.hs index 19d1737c3a..fcbce4163a 100644 --- a/Core.hs +++ b/Core.hs @@ -5,6 +5,7 @@ module Core where import System.IO import System.Directory import Control.Monad.State (liftIO) + import Types import Locations import UUID diff --git a/GitRepo.hs b/GitRepo.hs index 76150b3095..32383197b5 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -42,6 +42,7 @@ import Data.String.Utils import Data.Map as Map hiding (map, split) import Network.URI import Maybe + import Utility {- A git repository can be on local disk or remote. Not to be confused diff --git a/LocationLog.hs b/LocationLog.hs index ba91787049..c0d6170b2e 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -29,6 +29,7 @@ import qualified Data.Map as Map import System.IO import System.Directory import Data.Char + import qualified GitRepo as Git import Utility import UUID diff --git a/Locations.hs b/Locations.hs index 733e745537..4978500624 100644 --- a/Locations.hs +++ b/Locations.hs @@ -11,6 +11,7 @@ module Locations ( ) where import Data.String.Utils + import Types import qualified BackendTypes as Backend import qualified GitRepo as Git diff --git a/Remotes.hs b/Remotes.hs index 2fffcffa7e..3774f993ce 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -12,6 +12,7 @@ import qualified Data.Map as Map import Data.String.Utils import List import Maybe + import Types import qualified GitRepo as Git import qualified Annex diff --git a/UUID.hs b/UUID.hs index 3e6991d485..6bd483a18c 100644 --- a/UUID.hs +++ b/UUID.hs @@ -21,7 +21,9 @@ import Maybe import List import System.Cmd.Utils import System.IO +import System.Directory import qualified Data.Map as M + import qualified GitRepo as Git import Types import Locations @@ -112,6 +114,7 @@ describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m log <- uuidLog + liftIO $ createDirectoryIfMissing True (parentDir log) liftIO $ withFileLocked log WriteMode (\h -> hPutStr h $ serialize m') where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index ad45c08426..e552dc770a 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -16,12 +16,12 @@ use. Some are used to archive data, others hold backups, and yet others come with me when I'm away from home to carry data that doesn't fit on my netbook. Maintaining all that was a nightmare, lots of ad-hoc moving files around, rsyncing files (unison is too slow), and deleting multiple copies -of files from multiple places. I realized what what I needed was revision -control where each drive was a repository, and where copying the files -around, and deciding which copies were safe to delete was automated. +of files from multiple places. I realized what what I needed was a form of +revision control where each drive was a repository, and where copying the +files around, and deciding which copies were safe to delete was automated. I posted about this to the VCS-home mailing list and got a great suggestion -to make it support arbitrary key-value stores. A week of coding later, -and git-annex is born. +to make it support arbitrary key-value stores, for more generality and +flexability. A week of coding later, and git-annex is born. Enough broad picture, here's how it actually looks: diff --git a/git-annex.hs b/git-annex.hs index 01416f6dd9..f4f0cfcdfd 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -3,17 +3,17 @@ import Control.Exception import System.IO import System.Environment + import qualified Annex import Types import Core import Commands -import Annex import qualified GitRepo as Git main = do args <- getArgs gitrepo <- Git.repoFromCwd - state <- new gitrepo + state <- Annex.new gitrepo (flags, actions) <- parseCmd args state tryRun state $ [startup flags] ++ actions ++ [shutdown] From 117e97ea30f6e414a99f413d1e2050da84edd9df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 16:41:24 -0400 Subject: [PATCH 0171/2835] debianization --- INSTALL | 5 +++++ Makefile | 4 ++++ debian/changelog | 5 +++++ debian/compat | 1 + debian/control | 26 ++++++++++++++++++++++++++ debian/docs | 1 + debian/rules | 7 +++++++ 7 files changed, 49 insertions(+) create mode 100644 INSTALL create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/docs create mode 100755 debian/rules diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000000..5c149dc453 --- /dev/null +++ b/INSTALL @@ -0,0 +1,5 @@ +To build and use git-annex, you will need: + +* ghc +* These haskell libraries: MissingH +* a "uuid" command diff --git a/Makefile b/Makefile index 876407de09..d1fcbbeeeb 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,10 @@ git-annex: mkdir -p build ghc -odir build -hidir build --make git-annex +install: + install -d $(DESTDIR)/usr/bin + install git-annex $(DESTDIR)/usr/bin + clean: rm -rf build git-annex diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000000..998754777e --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +git-annex (0.01) UNRELEASED; urgency=low + + * First release + + -- Joey Hess Thu, 09 Sep 2010 08:24:58 -0400 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000000..7f8f011eb7 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +7 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000000..fa4fd61159 --- /dev/null +++ b/debian/control @@ -0,0 +1,26 @@ +Source: git-annex +Section: utils +Priority: optional +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev +Maintainer: Joey Hess +Standards-Version: 3.9.1 +Vcs-Git: git://git.kitenet.net/git-annex +Homepage: http://kitenet.net/~joey/code/git-annex/ + +Package: git-annex +Architecture: any +Section: utils +Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid +Description: manage files with git, without checking their contents into git + git-annex allows managing files with git, without checking the file + contents into git. While that may seem paradoxical, it is useful when + dealing with files larger than git can currently easily handle, whether due + to limitations in memory, checksumming time, or disk space. + . + Even without file content tracking, being able to manage files with git, + move files around and delete files with versioned directory trees, and use + branches and distributed clones, are all very handy reasons to use git. And + annexed files can co-exist in the same git repository with regularly + versioned files, which is convenient for maintaining documents, Makefiles, + etc that are associated with annexed files but that benefit from full + revision control. diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000000..9de86edc7e --- /dev/null +++ b/debian/docs @@ -0,0 +1 @@ +doc/*.mdwn diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000000..e0a209a72c --- /dev/null +++ b/debian/rules @@ -0,0 +1,7 @@ +#!/usr/bin/make -f +%: + dh $@ + +# Not intended for use by anyone except the author. +announcedir: + @echo ${HOME}/src/joeywiki/code/git-annex/news From d1a455bdb4bd96c4ecf590b56f0517538c9d8eb0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 16:52:20 -0400 Subject: [PATCH 0172/2835] need SHA too --- INSTALL | 2 +- debian/control | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL b/INSTALL index 5c149dc453..11f573488c 100644 --- a/INSTALL +++ b/INSTALL @@ -1,5 +1,5 @@ To build and use git-annex, you will need: * ghc -* These haskell libraries: MissingH +* These haskell libraries: MissingH SHA * a "uuid" command diff --git a/debian/control b/debian/control index fa4fd61159..e58f55af9e 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-dev Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From 4da793b51441e65c48bbf680d8650c57a4c9874d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 17:08:08 -0400 Subject: [PATCH 0173/2835] up --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 8fc17fca99..ed150cff02 100644 --- a/TODO +++ b/TODO @@ -5,4 +5,7 @@ * how to handle git mv file? +* Support for remote git repositories (ssh:// specifically can be made to + work, although the other end probably needs to have git-annex installed..) + * finish BackendChecksum From 91c9cd2b8eb9934eebf9a20adde7794a103d144a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 17:15:58 -0400 Subject: [PATCH 0174/2835] todo --- INSTALL | 2 +- TODO | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index 11f573488c..9594448e70 100644 --- a/INSTALL +++ b/INSTALL @@ -2,4 +2,4 @@ To build and use git-annex, you will need: * ghc * These haskell libraries: MissingH SHA -* a "uuid" command +* uuid diff --git a/TODO b/TODO index ed150cff02..5bef280e21 100644 --- a/TODO +++ b/TODO @@ -8,4 +8,8 @@ * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) +* Find a way to copy a file with a progress bar, while still preserving + stat. Easiest way might be to use pv and fix up the permissions etc + after? + * finish BackendChecksum From b3e5590fb2995d73d5e69a3954fcb11d9fe98d28 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 17:44:59 -0400 Subject: [PATCH 0175/2835] update --- Remotes.hs | 6 ++++-- TODO | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 3774f993ce..f21f5a6ba4 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -6,8 +6,8 @@ module Remotes ( tryGitConfigRead ) where +import Control.Exception import Control.Monad.State (liftIO) -import IO import qualified Data.Map as Map import Data.String.Utils import List @@ -85,7 +85,9 @@ tryGitConfigRead :: Git.Repo -> Annex (Maybe Git.Repo) tryGitConfigRead r = do if (Map.null $ Git.configMap r) then do - result <- liftIO $ try (Git.configRead r) + -- configRead can fail due to IO error or + -- for other reasons; catch all possible exceptions + result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) case (result) of Left err -> return Nothing Right r' -> do diff --git a/TODO b/TODO index 5bef280e21..cd94f03bcb 100644 --- a/TODO +++ b/TODO @@ -8,6 +8,9 @@ * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) +* Copy files atomically, don't leaf a partial key on interrupt. + (Fix for URL download too..) + * Find a way to copy a file with a progress bar, while still preserving stat. Easiest way might be to use pv and fix up the permissions etc after? From be5b1defeb2f3b5499fd3c002fcdba5b5e9d15f5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 19:43:32 -0400 Subject: [PATCH 0176/2835] add --no-commit option --- BackendTypes.hs | 2 +- Commands.hs | 37 ++++++++++++++++++++++++++----------- Core.hs | 3 ++- TODO | 3 +++ 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/BackendTypes.hs b/BackendTypes.hs index 49bd1bceb8..06ecfb8fe5 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -11,7 +11,7 @@ import Data.String.Utils import qualified GitRepo as Git -- command-line flags -data Flag = Force | NeedCommit +data Flag = Force | NoCommit | NeedCommit deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, diff --git a/Commands.hs b/Commands.hs index c477a81fde..63ca6b5e4e 100644 --- a/Commands.hs +++ b/Commands.hs @@ -32,7 +32,8 @@ data Command = Command { } cmds :: [Command] -cmds = [ (Command "add" addCmd FilesNotInGit) +cmds = [ + (Command "add" addCmd FilesNotInGit) , (Command "get" getCmd FilesInGit) , (Command "drop" dropCmd FilesInGit) , (Command "push" pushCmd RepoName) @@ -41,6 +42,11 @@ cmds = [ (Command "add" addCmd FilesNotInGit) , (Command "describe" describeCmd SingleString) ] +options = [ + Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" + , Option ['N'] ["no-commit"] (NoArg NoCommit) "do not stage or commit changes" + ] + {- Finds the type of parameters a command wants, from among the passed - parameter list. -} findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] @@ -75,7 +81,6 @@ parseCmd argv state = do lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." - options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} @@ -89,7 +94,7 @@ addCmd file = inBackend file err $ do Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> do logStatus key ValuePresent - liftIO $ setup g key link + setup g key link where err = error $ "already annexed " ++ file checkLegal file = do @@ -106,12 +111,16 @@ addCmd file = inBackend file err $ do setup g key link = do let dest = annexLocation g key let reldest = annexLocationRelative g key - createDirectoryIfMissing True (parentDir dest) - renameFile file dest - createSymbolicLink (link ++ reldest) file - Git.run g ["add", file] - Git.run g ["commit", "-m", - ("git-annex annexed " ++ file), file] + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ renameFile file dest + liftIO $ createSymbolicLink (link ++ reldest) file + nocommit <- Annex.flagIsSet NoCommit + if (not nocommit) + then do + liftIO $ Git.run g ["add", file] + liftIO $ Git.run g ["commit", "-m", + ("git-annex annexed " ++ file), file] + else return () {- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () @@ -192,7 +201,10 @@ describeCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - liftIO $ Git.run g ["add", log] + nocommit <- Annex.flagIsSet NoCommit + if (not nocommit) + then liftIO $ Git.run g ["add", log] + else return () Annex.flagChange NeedCommit True liftIO $ putStrLn "description set" @@ -202,7 +214,10 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - liftIO $ Git.run g ["add", f] + nocommit <- Annex.flagIsSet NoCommit + if (not nocommit) + then liftIO $ Git.run g ["add", f] + else return () Annex.flagChange NeedCommit True inBackend file yes no = do diff --git a/Core.hs b/Core.hs index fcbce4163a..6e48068f96 100644 --- a/Core.hs +++ b/Core.hs @@ -24,8 +24,9 @@ startup flags = do shutdown :: Annex () shutdown = do g <- Annex.gitRepo + nocommit <- Annex.flagIsSet NoCommit needcommit <- Annex.flagIsSet NeedCommit - if (needcommit) + if (needcommit && not nocommit) then liftIO $ Git.run g ["commit", "-q", "-m", "git-annex log update", gitStateDir g] else return () diff --git a/TODO b/TODO index cd94f03bcb..fedcce6dd9 100644 --- a/TODO +++ b/TODO @@ -5,6 +5,9 @@ * how to handle git mv file? +* how to handle git rm file? (should try to drop keys that have no + referring file, if it seems safe..) + * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) From c69e747d383d308d0cf65d88dc1c3be139d056a9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 19:57:56 -0400 Subject: [PATCH 0177/2835] refactor --- Commands.hs | 20 +++----------------- Core.hs | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Commands.hs b/Commands.hs index 63ca6b5e4e..f33be5393c 100644 --- a/Commands.hs +++ b/Commands.hs @@ -114,13 +114,7 @@ addCmd file = inBackend file err $ do liftIO $ createDirectoryIfMissing True (parentDir dest) liftIO $ renameFile file dest liftIO $ createSymbolicLink (link ++ reldest) file - nocommit <- Annex.flagIsSet NoCommit - if (not nocommit) - then do - liftIO $ Git.run g ["add", file] - liftIO $ Git.run g ["commit", "-m", - ("git-annex annexed " ++ file), file] - else return () + gitAdd file $ Just $ "git-annex annexed " ++ file {- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () @@ -201,11 +195,7 @@ describeCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - nocommit <- Annex.flagIsSet NoCommit - if (not nocommit) - then liftIO $ Git.run g ["add", log] - else return () - Annex.flagChange NeedCommit True + gitAdd log Nothing -- all logs are committed at end liftIO $ putStrLn "description set" {- Updates the LocationLog when a key's presence changes. -} @@ -214,11 +204,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - nocommit <- Annex.flagIsSet NoCommit - if (not nocommit) - then liftIO $ Git.run g ["add", f] - else return () - Annex.flagChange NeedCommit True + gitAdd f Nothing -- all logs are committed at end inBackend file yes no = do r <- liftIO $ Backend.lookupFile file diff --git a/Core.hs b/Core.hs index 6e48068f96..5f5cba2957 100644 --- a/Core.hs +++ b/Core.hs @@ -2,6 +2,7 @@ module Core where +import Maybe import System.IO import System.Directory import Control.Monad.State (liftIO) @@ -61,4 +62,22 @@ inAnnex key = do g <- Annex.gitRepo liftIO $ doesFileExist $ annexLocation g key -{- -} +{- Adds, optionally also commits a file to git. + - + - All changes to the git repository should go through this function. + - + - This is careful to not rely on the index. It may have staged changes, + - so only use operations that avoid committing such changes. + -} +gitAdd :: FilePath -> Maybe String -> Annex () +gitAdd file commitmessage = do + nocommit <- Annex.flagIsSet NoCommit + if (nocommit) + then Annex.flagChange NeedCommit True + else do + g <- Annex.gitRepo + liftIO $ Git.run g ["add", file] + if (isJust commitmessage) + then liftIO $ Git.run g ["commit", "-m", + (fromJust commitmessage), file] + else return () From 96347a25a26d01ae4814e9eeb44e7c82a68fb560 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 20:03:41 -0400 Subject: [PATCH 0178/2835] show full usage --- Commands.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Commands.hs b/Commands.hs index f33be5393c..b9f31a56cd 100644 --- a/Commands.hs +++ b/Commands.hs @@ -67,9 +67,9 @@ parseCmd :: [String] -> AnnexState -> IO ([Flag], [Annex ()]) parseCmd argv state = do (flags, params) <- getopt case (length params) of - 0 -> error header + 0 -> error usage _ -> case (lookupCmd (params !! 0)) of - [] -> error header + [] -> error usage [Command _ action want] -> do f <- findWanted want (drop 1 params) (BackendTypes.repo state) @@ -77,10 +77,11 @@ parseCmd argv state = do where getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) - (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) + (_, _, errs) -> ioError (userError (concat errs ++ usage)) lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." + usage = usageInfo header options {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} From b02a3b3f5b264ca12fcbf225db3c3ddd341ac51a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 21:03:25 -0400 Subject: [PATCH 0179/2835] add fix subcommand --- Commands.hs | 37 +++++++++++++++++++++++++------------ Core.hs | 13 +++++++++++++ doc/git-annex.mdwn | 2 ++ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/Commands.hs b/Commands.hs index b9f31a56cd..8afe66b91b 100644 --- a/Commands.hs +++ b/Commands.hs @@ -40,6 +40,7 @@ cmds = [ , (Command "pull" pullCmd RepoName) , (Command "unannex" unannexCmd FilesInGit) , (Command "describe" describeCmd SingleString) + , (Command "fix" fixCmd FilesInGit) ] options = [ @@ -89,13 +90,12 @@ addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file g <- Annex.gitRepo - link <- liftIO $ calcGitLink file g stored <- Backend.storeFileKey file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> do logStatus key ValuePresent - setup g key link + setup g key where err = error $ "already annexed " ++ file checkLegal file = do @@ -103,21 +103,15 @@ addCmd file = inBackend file err $ do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - calcGitLink file g = do - cwd <- getCurrentDirectory - let absfile = case (absNormPath cwd file) of - Just f -> f - Nothing -> error $ "unable to normalize " ++ file - return $ relPathDirToDir (parentDir absfile) (Git.workTree g) - setup g key link = do + setup g key = do let dest = annexLocation g key - let reldest = annexLocationRelative g key liftIO $ createDirectoryIfMissing True (parentDir dest) liftIO $ renameFile file dest - liftIO $ createSymbolicLink (link ++ reldest) file + link <- calcGitLink file key + liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex annexed " ++ file -{- Inverse of addCmd. -} +{- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do Backend.removeKey backend key @@ -181,6 +175,25 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do where err = error $ "not annexed " ++ file +{- Fixes the symlink to an annexed file. -} +fixCmd :: String -> Annex () +fixCmd file = notinBackend file err $ \(key, backend) -> do + link <- calcGitLink file key + checkLegal file + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + gitAdd file $ Just $ "git-annex fix " ++ file + where + checkLegal file = do + s <- liftIO $ getSymbolicLinkStatus file + force <- Annex.flagIsSet Force + if (not (isSymbolicLink s) && not force) + then error $ "not a symbolic link : " ++ file ++ + " (use --force to override this sanity check)" + else return () + err = error $ "not annexed " ++ file + {- Pushes all files to a remote repository. -} pushCmd :: String -> Annex () pushCmd reponame = do error "not implemented" -- TODO diff --git a/Core.hs b/Core.hs index 5f5cba2957..021595f8b6 100644 --- a/Core.hs +++ b/Core.hs @@ -6,12 +6,14 @@ import Maybe import System.IO import System.Directory import Control.Monad.State (liftIO) +import System.Path import Types import Locations import UUID import qualified GitRepo as Git import qualified Annex +import Utility {- Sets up a git repo for git-annex. -} startup :: [Flag] -> Annex () @@ -81,3 +83,14 @@ gitAdd file commitmessage = do then liftIO $ Git.run g ["commit", "-m", (fromJust commitmessage), file] else return () + +{- Calculates the relative path to use to link a file to a key. -} +calcGitLink :: FilePath -> Key -> Annex FilePath +calcGitLink file key = do + g <- Annex.gitRepo + cwd <- liftIO $ getCurrentDirectory + let absfile = case (absNormPath cwd file) of + Just f -> f + Nothing -> error $ "unable to normalize " ++ file + return $ (relPathDirToDir (parentDir absfile) (Git.workTree g)) ++ + annexLocationRelative g key diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e552dc770a..e65ad5b020 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -47,6 +47,8 @@ Enough broad picture, here's how it actually looks: repository. * `git annex pull $repository` pulls *all* annexed files from the specified repository. +* `git annex file $file` adjusts the symlink for the file to point to its + content again. Use this if you've moved the file around. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` if you're just done with a file; only use `unannex` if you accidentially added a file. From 0c0ae028386aaf17aed1771eee6731c62b72e839 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 21:15:23 -0400 Subject: [PATCH 0180/2835] add fix subcommand --- Commands.hs | 20 +++++++++++++++----- TODO | 6 ++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Commands.hs b/Commands.hs index 8afe66b91b..9a3f925244 100644 --- a/Commands.hs +++ b/Commands.hs @@ -24,7 +24,8 @@ import Core import qualified Remotes import qualified BackendTypes -data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString +data CmdWants = FilesInGit | FilesNotInGit | FilesInOrNotInGit | + RepoName | SingleString data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), @@ -40,7 +41,7 @@ cmds = [ , (Command "pull" pullCmd RepoName) , (Command "unannex" unannexCmd FilesInGit) , (Command "describe" describeCmd SingleString) - , (Command "fix" fixCmd FilesInGit) + , (Command "fix" fixCmd FilesInOrNotInGit) ] options = [ @@ -57,6 +58,10 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files +findWanted FilesInOrNotInGit params repo = do + a <- findWanted FilesInGit params repo + b <- findWanted FilesNotInGit params repo + return $ union a b findWanted SingleString params _ = do return $ [unwords params] findWanted RepoName params _ = do @@ -178,20 +183,25 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do {- Fixes the symlink to an annexed file. -} fixCmd :: String -> Annex () fixCmd file = notinBackend file err $ \(key, backend) -> do + liftIO $ putStrLn $ "fix " ++ file link <- calcGitLink file key - checkLegal file + checkLegal file link liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex fix " ++ file where - checkLegal file = do + checkLegal file link = do s <- liftIO $ getSymbolicLinkStatus file force <- Annex.flagIsSet Force if (not (isSymbolicLink s) && not force) then error $ "not a symbolic link : " ++ file ++ " (use --force to override this sanity check)" - else return () + else do + l <- liftIO $ readSymbolicLink file + if (link == l) + then error $ "symbolic link already ok for: " ++ file + else return () err = error $ "not annexed " ++ file {- Pushes all files to a remote repository. -} diff --git a/TODO b/TODO index fedcce6dd9..038ed0d4a4 100644 --- a/TODO +++ b/TODO @@ -3,15 +3,17 @@ * --push/--pull/--want -* how to handle git mv file? +* how to handle git mv file? -> git annex fix -> run automatically? * how to handle git rm file? (should try to drop keys that have no referring file, if it seems safe..) +* add a git annex fsck that finds keys that have no referring file + * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) -* Copy files atomically, don't leaf a partial key on interrupt. +* Copy files atomically, don't leave a partial key on interrupt. (Fix for URL download too..) * Find a way to copy a file with a progress bar, while still preserving From 38825f48645c0220b6b2e0368c2ebdbb625f6703 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 21:18:21 -0400 Subject: [PATCH 0181/2835] remove useless checks the file will always be a symlink at this point --- Commands.hs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Commands.hs b/Commands.hs index 9a3f925244..4346a35e22 100644 --- a/Commands.hs +++ b/Commands.hs @@ -192,16 +192,10 @@ fixCmd file = notinBackend file err $ \(key, backend) -> do gitAdd file $ Just $ "git-annex fix " ++ file where checkLegal file link = do - s <- liftIO $ getSymbolicLinkStatus file - force <- Annex.flagIsSet Force - if (not (isSymbolicLink s) && not force) - then error $ "not a symbolic link : " ++ file ++ - " (use --force to override this sanity check)" - else do - l <- liftIO $ readSymbolicLink file - if (link == l) - then error $ "symbolic link already ok for: " ++ file - else return () + l <- liftIO $ readSymbolicLink file + if (link == l) + then error $ "symbolic link already ok for: " ++ file + else return () err = error $ "not annexed " ++ file {- Pushes all files to a remote repository. -} From 19daf3fca40d99dd305a75e10dcaa8fbc734598b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 21:50:33 -0400 Subject: [PATCH 0182/2835] oops, should commit descriptions! --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index 4346a35e22..af6b5aad78 100644 --- a/Commands.hs +++ b/Commands.hs @@ -213,7 +213,7 @@ describeCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - gitAdd log Nothing -- all logs are committed at end + gitAdd log $ Just $ "description for UUID " ++ (show u) liftIO $ putStrLn "description set" {- Updates the LocationLog when a key's presence changes. -} From da453ba70149444672b8cd64e36fe34604edce73 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 22:36:35 -0400 Subject: [PATCH 0183/2835] bugfix: don't add files under .git-annex That could happen if git annex add -N were used repeatedly.. --- Commands.hs | 5 ++++- Core.hs | 2 +- Locations.hs | 4 ++-- doc/git-annex.mdwn | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index af6b5aad78..1364a1b35e 100644 --- a/Commands.hs +++ b/Commands.hs @@ -54,7 +54,10 @@ options = [ findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] findWanted FilesNotInGit params repo = do files <- mapM (Git.notInRepo repo) params - return $ foldl (++) [] files + return $ filter notstate $ foldl (++) [] files + where + -- never include files in the state directory + notstate f = f /= take (length stateLoc) f findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files diff --git a/Core.hs b/Core.hs index 021595f8b6..70e6e66804 100644 --- a/Core.hs +++ b/Core.hs @@ -51,7 +51,7 @@ gitAttributes repo = do commit else return () where - attrLine = stateLoc ++ "/*.log merge=union" + attrLine = stateLoc ++ "*.log merge=union" attributes = Git.attributes repo commit = do Git.run repo ["add", attributes] diff --git a/Locations.hs b/Locations.hs index 4978500624..76516224cf 100644 --- a/Locations.hs +++ b/Locations.hs @@ -18,9 +18,9 @@ import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex - directory, in the git repository's working tree. -} -stateLoc = ".git-annex" +stateLoc = ".git-annex/" gitStateDir :: Git.Repo -> FilePath -gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc ++ "/" +gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc {- An annexed file's content is stored in - /path/to/repo/.git/annex/, where is of the form diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e65ad5b020..50fd28e82a 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -51,7 +51,8 @@ Enough broad picture, here's how it actually looks: content again. Use this if you've moved the file around. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` if you're just done with a file; only use `unannex` if you - accidentially added a file. + accidentially added a file. (You can also run this on all your annexed + files come the Singularity. ;-) * `git annex describe "some description"` allows associating some description (such as "USB archive drive 1") with a repository. This can help with finding it later, see "Location Tracking" below. From 96451ac392d42973f508da08d7c1197c83c659a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 22:43:38 -0400 Subject: [PATCH 0184/2835] nocommit does not make sense in unannex mode --- Commands.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index 1364a1b35e..be0a8e37ff 100644 --- a/Commands.hs +++ b/Commands.hs @@ -122,11 +122,15 @@ addCmd file = inBackend file err $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - Backend.removeKey backend key - logStatus key ValueMissing - g <- Annex.gitRepo - let src = annexLocation g key - liftIO $ moveout g src + nocommit <- Annex.flagIsSet NoCommit + if (nocommit) + then error "--nocommit cannot be used in unannex mode" + else do + Backend.removeKey backend key + logStatus key ValueMissing + g <- Annex.gitRepo + let src = annexLocation g key + liftIO $ moveout g src where err = error $ "not annexed " ++ file moveout g src = do From c0b16e0a7306bacce96564d80911bbdb5a246847 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 22:49:09 -0400 Subject: [PATCH 0185/2835] actually, unannex w/o commit can work just have to git rm --- Commands.hs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index be0a8e37ff..67b8fe8ad3 100644 --- a/Commands.hs +++ b/Commands.hs @@ -122,26 +122,25 @@ addCmd file = inBackend file err $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - nocommit <- Annex.flagIsSet NoCommit - if (nocommit) - then error "--nocommit cannot be used in unannex mode" - else do - Backend.removeKey backend key - logStatus key ValueMissing - g <- Annex.gitRepo - let src = annexLocation g key - liftIO $ moveout g src + Backend.removeKey backend key + logStatus key ValueMissing + g <- Annex.gitRepo + let src = annexLocation g key + moveout g src where err = error $ "not annexed " ++ file moveout g src = do - removeFile file - Git.run g ["rm", file] - Git.run g ["commit", "-m", - ("git-annex unannexed " ++ file), file] + nocommit <- Annex.flagIsSet NoCommit + liftIO removeFile file + liftIO Git.run g ["rm", file] + if (not nocommit) + then liftIO Git.run g ["commit", "-m", + ("git-annex unannexed " ++ file), file] + else return () -- git rm deletes empty directories; -- put them back - createDirectoryIfMissing True (parentDir file) - renameFile src file + liftIO createDirectoryIfMissing True (parentDir file) + liftIO renameFile src file return () {- Gets an annexed file from one of the backends. -} From 20b447782a974408594692c7aa6ce8bc26f87858 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 22:59:19 -0400 Subject: [PATCH 0186/2835] typo --- Commands.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index 67b8fe8ad3..dfb3eef436 100644 --- a/Commands.hs +++ b/Commands.hs @@ -131,16 +131,16 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do err = error $ "not annexed " ++ file moveout g src = do nocommit <- Annex.flagIsSet NoCommit - liftIO removeFile file - liftIO Git.run g ["rm", file] + liftIO $ removeFile file + liftIO $ Git.run g ["rm", file] if (not nocommit) - then liftIO Git.run g ["commit", "-m", + then liftIO $ Git.run g ["commit", "-m", ("git-annex unannexed " ++ file), file] else return () -- git rm deletes empty directories; -- put them back - liftIO createDirectoryIfMissing True (parentDir file) - liftIO renameFile src file + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ renameFile src file return () {- Gets an annexed file from one of the backends. -} From aaee8e231f111b9b4a2ead95eaaeb3d635cc1699 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 23:36:45 -0400 Subject: [PATCH 0187/2835] bugfix --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index dfb3eef436..258490996d 100644 --- a/Commands.hs +++ b/Commands.hs @@ -57,7 +57,7 @@ findWanted FilesNotInGit params repo = do return $ filter notstate $ foldl (++) [] files where -- never include files in the state directory - notstate f = f /= take (length stateLoc) f + notstate f = stateLoc /= take (length stateLoc) f findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files From c57b1a697c5de4e20ef10c2c4a39a77c20fde85b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 00:10:04 -0400 Subject: [PATCH 0188/2835] add visible size and time to worm keys --- Backend/WORM.hs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 420f336e90..7e86d4d243 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -23,7 +23,10 @@ backend = Backend.File.backend { -- allows multiple files with the same names to have different keys, -- while also allowing a file to be moved around while retaining the -- same key. --- +-- +-- The file size and modification time are also included in the key, +-- unhashed. This could be used as a sanity check. +-- -- The basename of the filename is also included in the key, so it's clear -- what the original filename was when a user sees the value. keyValue :: FilePath -> Annex (Maybe Key) @@ -31,9 +34,10 @@ keyValue file = do stat <- liftIO $ getFileStatus file return $ Just $ Key ((name backend), key stat) where - key stat = (checksum $ uniqueid stat) ++ sep ++ base + key stat = (checksum $ uniqueid stat) ++ sep ++ + uniqueid stat ++ sep ++ base checksum s = show $ sha1 $ B.pack s - uniqueid stat = (show $ fileSize stat) ++ sep ++ - (show $ modificationTime stat) + uniqueid stat = (show $ modificationTime stat) ++ sep ++ + (show $ fileSize stat) base = takeFileName file sep = ":" From a0b040524a595c16ddb2dbead205ca8ccb6890aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 00:33:05 -0400 Subject: [PATCH 0189/2835] remove checksum from WORM with size and mtime in the key, it's redundant --- Backend/WORM.hs | 18 +++++------------- Commands.hs | 9 ++++----- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 7e86d4d243..463b0ac8e2 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -6,7 +6,6 @@ module Backend.WORM (backend) where import Control.Monad.State import System.FilePath import System.Posix.Files -import Data.Digest.Pure.SHA -- slow, but we only checksum filenames import qualified Data.ByteString.Lazy.Char8 as B import qualified Backend.File @@ -18,25 +17,18 @@ backend = Backend.File.backend { getKey = keyValue } --- A SHA1 of the basename of the filename, plus the file size and --- modification time, is used as the unique part of the key. That --- allows multiple files with the same names to have different keys, +-- The key is formed from the file size, modification time, and the +-- basename of the filename. +-- +-- That allows multiple files with the same names to have different keys, -- while also allowing a file to be moved around while retaining the -- same key. --- --- The file size and modification time are also included in the key, --- unhashed. This could be used as a sanity check. --- --- The basename of the filename is also included in the key, so it's clear --- what the original filename was when a user sees the value. keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do stat <- liftIO $ getFileStatus file return $ Just $ Key ((name backend), key stat) where - key stat = (checksum $ uniqueid stat) ++ sep ++ - uniqueid stat ++ sep ++ base - checksum s = show $ sha1 $ B.pack s + key stat = uniqueid stat ++ sep ++ base uniqueid stat = (show $ modificationTime stat) ++ sep ++ (show $ fileSize stat) base = takeFileName file diff --git a/Commands.hs b/Commands.hs index 258490996d..5931bf0a9a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -54,10 +54,7 @@ options = [ findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] findWanted FilesNotInGit params repo = do files <- mapM (Git.notInRepo repo) params - return $ filter notstate $ foldl (++) [] files - where - -- never include files in the state directory - notstate f = stateLoc /= take (length stateLoc) f + return $ foldl (++) [] files findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files @@ -82,8 +79,10 @@ parseCmd argv state = do [Command _ action want] -> do f <- findWanted want (drop 1 params) (BackendTypes.repo state) - return (flags, map action f) + return (flags, map action $ filter notstate f) where + -- never include files from the state directory + notstate f = stateLoc /= take (length stateLoc) f getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) From 6bfa534aa4d7552c4ccfdb9523b55da19fac8883 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 10:47:46 -0400 Subject: [PATCH 0190/2835] git annex drop -- do not try to drop if key is not in backend --- Commands.hs | 23 ++++++++++++++--------- TODO | 6 +++++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Commands.hs b/Commands.hs index 5931bf0a9a..b446dbfac6 100644 --- a/Commands.hs +++ b/Commands.hs @@ -165,13 +165,20 @@ getCmd file = notinBackend file err $ \(key, backend) -> do - if it's safe to do so. -} dropCmd :: FilePath -> Annex () dropCmd file = notinBackend file err $ \(key, backend) -> do - force <- Annex.flagIsSet Force - if (not force) - then requireEnoughCopies key - else return () - success <- Backend.removeKey backend key - if (success) - then do + inbackend <- Backend.hasKey key + if (not inbackend) + then return () -- no-op + else do + force <- Annex.flagIsSet Force + if (not force) + then requireEnoughCopies key + else return () + success <- Backend.removeKey backend key + if (success) + then cleanup key + else error $ "backend refused to drop " ++ file + where + cleanup key = do logStatus key ValueMissing inannex <- inAnnex key if (inannex) @@ -181,8 +188,6 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do liftIO $ removeFile loc return () else return () - else error $ "backend refused to drop " ++ file - where err = error $ "not annexed " ++ file {- Fixes the symlink to an annexed file. -} diff --git a/TODO b/TODO index 038ed0d4a4..807df32d84 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,11 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) -* --push/--pull/--want +* bug: git annex add file is silent if file was a symlink and got replaced + with a file. The you then git command -a, you'll check in the fil contents.. + +* --push/--pull should take a reponame and files, and push those files + to that repo; dropping them from the current repo * how to handle git mv file? -> git annex fix -> run automatically? From b471822cfe4476995f539c6e7e7da7f7bf2b5e02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 11:47:36 -0400 Subject: [PATCH 0191/2835] move supportedBackends list into annex monad This was necessary so the File backend could import Backend w/o a cycle. Moved code that checks whether enough backends have a file into File backend. --- Annex.hs | 26 +++++++++++++++------ Backend.hs | 35 ++++++++++++++++++++--------- Backend/File.hs | 60 ++++++++++++++++++++++++++++++++++++++++++++----- BackendList.hs | 25 ++------------------- BackendTypes.hs | 1 + Commands.hs | 50 +---------------------------------------- git-annex.hs | 3 ++- 7 files changed, 105 insertions(+), 95 deletions(-) diff --git a/Annex.hs b/Annex.hs index 68c0cb88e2..e76ccd1dcb 100644 --- a/Annex.hs +++ b/Annex.hs @@ -7,6 +7,8 @@ module Annex ( gitRepoChange, backends, backendsChange, + supportedBackends, + supportedBackendsChange, flagIsSet, flagChange, Flag(..) @@ -20,20 +22,21 @@ import qualified BackendTypes as Backend {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> IO AnnexState -new g = do +new :: Git.Repo -> [Backend] -> IO AnnexState +new gitrepo allbackends = do let s = Backend.AnnexState { - Backend.repo = g, + Backend.repo = gitrepo, Backend.backends = [], + Backend.supportedBackends = allbackends, Backend.flags = [] } - (_,s') <- Annex.run s (prep g) + (_,s') <- Annex.run s (prep gitrepo) return s' where - prep g = do + prep gitrepo = do -- read git config and update state - g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' + gitrepo' <- liftIO $ Git.configRead gitrepo + Annex.gitRepoChange gitrepo' -- performs an action in the Annex monad run state action = runStateT (action) state @@ -57,6 +60,15 @@ backendsChange b = do state <- get put state { Backend.backends = b } return () +supportedBackends :: Annex [Backend] +supportedBackends = do + state <- get + return (Backend.supportedBackends state) +supportedBackendsChange :: [Backend] -> Annex () +supportedBackendsChange b = do + state <- get + put state { Backend.supportedBackends = b } + return () flagIsSet :: Flag -> Annex Bool flagIsSet flag = do state <- get diff --git a/Backend.hs b/Backend.hs index 874191924d..dfaa559702 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,14 +28,12 @@ import System.FilePath import Data.String.Utils import System.Posix.Files -import BackendList import Locations import qualified GitRepo as Git import qualified Annex import Utility import Types import qualified BackendTypes as B -import BackendList {- List of backends in the order to try them when storing a new key. -} backendList :: Annex [Backend] @@ -44,10 +42,24 @@ backendList = do if (0 < length l) then return l else do + all <- Annex.supportedBackends g <- Annex.gitRepo - let l = parseBackendList $ Git.configGet g "annex.backends" "" + let l = parseBackendList all $ Git.configGet g "annex.backends" "" Annex.backendsChange l return l + where + parseBackendList all s = + if (length s == 0) + then all + else map (lookupBackendName all) $ words s + +{- Looks up a backend in the list of supportedBackends -} +lookupBackendName :: [Backend] -> String -> Backend +lookupBackendName all s = + if ((length matches) /= 1) + then error $ "unknown backend " ++ s + else matches !! 0 + where matches = filter (\b -> s == B.name b) all {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) @@ -81,21 +93,24 @@ removeKey backend key = (B.removeKey backend) key {- Checks if a backend has its key. -} hasKey :: Key -> Annex Bool -hasKey key = (B.hasKey (lookupBackendName $ backendName key)) key +hasKey key = do + all <- Annex.supportedBackends + (B.hasKey (lookupBackendName all $ backendName key)) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} -lookupFile :: FilePath -> IO (Maybe (Key, Backend)) +lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) lookupFile file = do - result <- try (lookup)::IO (Either SomeException (Maybe (Key, Backend))) + all <- Annex.supportedBackends + result <- liftIO $ (try (lookup all)::IO (Either SomeException (Maybe (Key, Backend)))) case (result) of Left err -> return Nothing Right succ -> return succ where - lookup = do + lookup all = do l <- readSymbolicLink file - return $ Just $ pair $ takeFileName l - pair file = (k, b) + return $ Just $ pair all $ takeFileName l + pair all file = (k, b) where k = fileKey file - b = lookupBackendName $ backendName k + b = lookupBackendName all $ backendName k diff --git a/Backend/File.hs b/Backend/File.hs index f5237f7210..591ff3db41 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -25,13 +25,14 @@ import Utility import Core import qualified Annex import UUID +import qualified Backend backend = Backend { name = mustProvide, getKey = mustProvide, storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, - removeKey = dummyRemove, + removeKey = checkRemoveKey, hasKey = checkKeyFile } @@ -41,10 +42,6 @@ mustProvide = error "must provide this field" dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True -{- Allow keys to be removed. -} -dummyRemove :: Key -> Annex Bool -dummyRemove url = return True - {- Just check if the .git/annex/ file for the key exists. -} checkKeyFile :: Key -> Annex Bool checkKeyFile k = inAnnex k @@ -102,3 +99,56 @@ copyFromRemote r key file = do else error "cp failed" getremote = error "get via network not yet implemented!" location = annexLocation r key + +{- Checks remotes to verify that enough copies of a key exist to allow + - for a key to be safely removed (with no data loss), and fails with an + - error if not. -} +checkRemoveKey :: Key -> Annex (Bool) +checkRemoveKey key = do + force <- Annex.flagIsSet Force + if (force) + then return True + else do + g <- Annex.gitRepo + let numcopies = read $ Git.configGet g config "1" + remotes <- Remotes.withKey key + if (numcopies > length remotes) + then retNotEnoughCopiesKnown remotes numcopies + else findcopies numcopies remotes [] + where + failMsg w = do + liftIO $ hPutStrLn stderr $ "git-annex: " ++ w + return False -- failure, not enough copies found + findcopies 0 _ _ = return True -- success, enough copies found + findcopies _ [] bad = notEnoughCopiesSeen bad + findcopies n (r:rs) bad = do + all <- Annex.supportedBackends + result <- liftIO $ ((try $ remoteHasKey r all)::IO (Either SomeException Bool)) + case (result) of + Right True -> findcopies (n-1) rs bad + Right False -> findcopies n rs bad + Left _ -> findcopies n rs (r:bad) + remoteHasKey r all = do + -- To check if a remote has a key, construct a new + -- Annex monad and query its backend. + a <- Annex.new r all + (result, _) <- Annex.run a (Backend.hasKey key) + return result + notEnoughCopiesSeen bad = failMsg $ + "I failed to find enough other copies of: " ++ + (keyFile key) ++ + (if (0 /= length bad) then listbad bad else "") + ++ unsafe + listbad bad = "\nI was unable to access these remotes: " ++ + (Remotes.list bad) + retNotEnoughCopiesKnown remotes numcopies = failMsg $ + "I only know about " ++ (show $ length remotes) ++ + " out of " ++ (show numcopies) ++ + " necessary copies of: " ++ (keyFile key) ++ + unsafe + unsafe = "\n" ++ + " -- According to the " ++ config ++ + " setting, it is not safe to remove it!\n" ++ + " (Use --force to override.)" + + config = "annex.numcopies" diff --git a/BackendList.hs b/BackendList.hs index 42e237204f..920f8fc0a6 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -1,11 +1,7 @@ {- git-annex backend list - -} -module BackendList ( - supportedBackends, - parseBackendList, - lookupBackendName -) where +module BackendList (allBackends) where import BackendTypes @@ -13,25 +9,8 @@ import BackendTypes import qualified Backend.WORM import qualified Backend.SHA1 import qualified Backend.URL -supportedBackends = +allBackends = [ Backend.WORM.backend , Backend.SHA1.backend , Backend.URL.backend ] - -{- Parses a string with a list of backend names into - - a list of Backend objects. If the list is empty, - - defaults to supportedBackends. -} -parseBackendList :: String -> [Backend] -parseBackendList s = - if (length s == 0) - then supportedBackends - else map (lookupBackendName) $ words s - -{- Looks up a supported backend by name. -} -lookupBackendName :: String -> Backend -lookupBackendName s = - if ((length matches) /= 1) - then error $ "unknown backend " ++ s - else matches !! 0 - where matches = filter (\b -> s == name b) supportedBackends diff --git a/BackendTypes.hs b/BackendTypes.hs index 06ecfb8fe5..e372099b2d 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -19,6 +19,7 @@ data Flag = Force | NoCommit | NeedCommit data AnnexState = AnnexState { repo :: Git.Repo, backends :: [Backend], + supportedBackends :: [Backend], flags :: [Flag] } deriving (Show) diff --git a/Commands.hs b/Commands.hs index b446dbfac6..62376e4dde 100644 --- a/Commands.hs +++ b/Commands.hs @@ -16,7 +16,6 @@ import qualified Annex import Utility import Locations import qualified Backend -import BackendList import UUID import LocationLog import Types @@ -169,10 +168,6 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do if (not inbackend) then return () -- no-op else do - force <- Annex.flagIsSet Force - if (not force) - then requireEnoughCopies key - else return () success <- Backend.removeKey backend key if (success) then cleanup key @@ -235,51 +230,8 @@ logStatus key status = do gitAdd f Nothing -- all logs are committed at end inBackend file yes no = do - r <- liftIO $ Backend.lookupFile file + r <- Backend.lookupFile file case (r) of Just v -> yes v Nothing -> no notinBackend file yes no = inBackend file no yes - -{- Checks remotes to verify that enough copies of a key exist to allow - - for a key to be safely removed (with no data loss), and fails with an - - error if not. -} -requireEnoughCopies :: Key -> Annex () -requireEnoughCopies key = do - g <- Annex.gitRepo - let numcopies = read $ Git.configGet g config "1" - remotes <- Remotes.withKey key - if (numcopies > length remotes) - then error $ "I only know about " ++ (show $ length remotes) ++ - " out of " ++ (show numcopies) ++ - " necessary copies of: " ++ (keyFile key) ++ - unsafe - else findcopies numcopies remotes [] - where - findcopies 0 _ _ = return () -- success, enough copies found - findcopies _ [] bad = die bad - findcopies n (r:rs) bad = do - result <- liftIO $ try $ haskey r - case (result) of - Right True -> findcopies (n-1) rs bad - Right False -> findcopies n rs bad - Left _ -> findcopies n rs (r:bad) - haskey r = do - -- To check if a remote has a key, construct a new - -- Annex monad and query its backend. - a <- Annex.new r - (result, _) <- Annex.run a (Backend.hasKey key) - return result - die bad = - error $ "I failed to find enough other copies of: " ++ - (keyFile key) ++ - (if (0 /= length bad) then listbad bad else "") - ++ unsafe - listbad bad = "\nI was unable to access these remotes: " ++ - (Remotes.list bad) - unsafe = "\n" ++ - " -- According to the " ++ config ++ - " setting, it is not safe to remove it!\n" ++ - " (Use --force to override.)" - - config = "annex.numcopies" diff --git a/git-annex.hs b/git-annex.hs index f4f0cfcdfd..947868f238 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -9,11 +9,12 @@ import Types import Core import Commands import qualified GitRepo as Git +import BackendList main = do args <- getArgs gitrepo <- Git.repoFromCwd - state <- Annex.new gitrepo + state <- Annex.new gitrepo allBackends (flags, actions) <- parseCmd args state tryRun state $ [startup flags] ++ actions ++ [shutdown] From cb1a0a387f93e882ced50709f938bd0a28cd14be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 11:51:53 -0400 Subject: [PATCH 0192/2835] update --- doc/git-annex.mdwn | 63 ++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 50fd28e82a..4d2872aa3d 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -61,35 +61,6 @@ Oh yeah, "$file" in the above can be any number of files, or directories, same as you'd pass to "git add" or "git rm". So "git annex add ." or "git annex get dir/" work fine. -## copies - -git-annex can be configured to try to keep N copies of a file's content -available across all repositories. By default, N is 1; it is configured by -annex.numcopies. - -`git annex drop` attempts to check with other git remotes, to check that N -copies of the file exist. If enough repositories cannot be verified to have -it, it will retain the file content to avoid data loss. - -For example, consider three repositories: Server, Laptop, and USB. Both Server -and USB have a copy of a file, and N=1. If on Laptop, you `git annex get -$file`, this will transfer it from either Server or USB (depending on which -is available), and there are now 3 copies of the file. - -Suppose you want to free up space on Laptop again, and you `git annex drop` the file -there. If USB is connected, or Server can be contacted, git-annex can check -that it still has a copy of the file, and the content is removed from -Laptop. But if USB is currently disconnected, and Server also cannot be -contacted, it can't verify that it is safe to drop the file, and will -refuse to do so. - -With N=2, in order to drop the file content from Laptop, it would need access -to both USB and Server. - -Note that different repositories can be configured with different values of -N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1. - ## key-value storage git-annex uses a key-value abstraction layer to allow file contents to be @@ -116,6 +87,37 @@ to store different files' contents in a given repository. can make it slow for large files. * `URL` -- This backend downloads the file's content from an external URL. +## copies + +The WORM and SHA1 key-value backends store data inside your git repository. +It's important that data not get lost by an ill-though `git annex drop` +command. So, then using those backends, git-annex can be configured to try +to keep N copies of a file's content available across all repositories. By +default, N is 1; it is configured by annex.numcopies. + +`git annex drop` attempts to check with other git remotes, to check that N +copies of the file exist. If enough repositories cannot be verified to have +it, it will retain the file content to avoid data loss. + +For example, consider three repositories: Server, Laptop, and USB. Both Server +and USB have a copy of a file, and N=1. If on Laptop, you `git annex get +$file`, this will transfer it from either Server or USB (depending on which +is available), and there are now 3 copies of the file. + +Suppose you want to free up space on Laptop again, and you `git annex drop` the file +there. If USB is connected, or Server can be contacted, git-annex can check +that it still has a copy of the file, and the content is removed from +Laptop. But if USB is currently disconnected, and Server also cannot be +contacted, it can't verify that it is safe to drop the file, and will +refuse to do so. + +With N=2, in order to drop the file content from Laptop, it would need access +to both USB and Server. + +Note that different repositories can be configured with different values of +N. So just because Laptop has N=2, this does not prevent the number of +copies falling to 1, when USB and Server have N=1. + ## location tracking git-annex keeps track of in which repositories it last saw a file's content. @@ -149,7 +151,8 @@ finding them: ## configuration * `annex.uuid` -- a unique UUID for this repository -* `annex.numcopies` -- number of copies of files to keep (default: 1) +* `annex.numcopies` -- number of copies of files to keep across all + repositories (default: 1) * `annex.backends` -- space-separated list of names of the key-value backends to use. The first listed is used to store new files. (default: "WORM SHA1 URL") From ae4d20d157eb288046dddf4555bfc9f2660ed675 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 11:57:39 -0400 Subject: [PATCH 0193/2835] bugfix --- Core.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core.hs b/Core.hs index 70e6e66804..1832b35138 100644 --- a/Core.hs +++ b/Core.hs @@ -75,14 +75,14 @@ gitAdd :: FilePath -> Maybe String -> Annex () gitAdd file commitmessage = do nocommit <- Annex.flagIsSet NoCommit if (nocommit) - then Annex.flagChange NeedCommit True + then return () else do g <- Annex.gitRepo liftIO $ Git.run g ["add", file] if (isJust commitmessage) then liftIO $ Git.run g ["commit", "-m", (fromJust commitmessage), file] - else return () + else Annex.flagChange NeedCommit True {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath From 76ba2d003072da67bd9b0fb5b84bf7a268a956ee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 12:08:59 -0400 Subject: [PATCH 0194/2835] reorg --- Commands.hs | 9 +-------- Core.hs | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Commands.hs b/Commands.hs index 62376e4dde..2ff8d0d7b3 100644 --- a/Commands.hs +++ b/Commands.hs @@ -221,14 +221,7 @@ describeCmd description = do gitAdd log $ Just $ "description for UUID " ++ (show u) liftIO $ putStrLn "description set" -{- Updates the LocationLog when a key's presence changes. -} -logStatus :: Key -> LogStatus -> Annex () -logStatus key status = do - g <- Annex.gitRepo - u <- getUUID g - f <- liftIO $ logChange g key u status - gitAdd f Nothing -- all logs are committed at end - +-- helpers inBackend file yes no = do r <- Backend.lookupFile file case (r) of diff --git a/Core.hs b/Core.hs index 1832b35138..70ec2eca01 100644 --- a/Core.hs +++ b/Core.hs @@ -10,6 +10,7 @@ import System.Path import Types import Locations +import LocationLog import UUID import qualified GitRepo as Git import qualified Annex @@ -94,3 +95,12 @@ calcGitLink file key = do Nothing -> error $ "unable to normalize " ++ file return $ (relPathDirToDir (parentDir absfile) (Git.workTree g)) ++ annexLocationRelative g key + +{- Updates the LocationLog when a key's presence changes. -} +logStatus :: Key -> LogStatus -> Annex () +logStatus key status = do + g <- Annex.gitRepo + u <- getUUID g + f <- liftIO $ logChange g key u status + gitAdd f Nothing -- all logs are committed at end + From 98676928c8dec5183951006450cc26cf5fb6a985 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 12:09:52 -0400 Subject: [PATCH 0195/2835] prune --- Annex.hs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Annex.hs b/Annex.hs index e76ccd1dcb..5fd500d949 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,7 +8,6 @@ module Annex ( backends, backendsChange, supportedBackends, - supportedBackendsChange, flagIsSet, flagChange, Flag(..) @@ -64,11 +63,6 @@ supportedBackends :: Annex [Backend] supportedBackends = do state <- get return (Backend.supportedBackends state) -supportedBackendsChange :: [Backend] -> Annex () -supportedBackendsChange b = do - state <- get - put state { Backend.supportedBackends = b } - return () flagIsSet :: Flag -> Annex Bool flagIsSet flag = do state <- get From 8f6e5da18f80ea5dfc124afed0ff2671a3909d56 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 12:12:23 -0400 Subject: [PATCH 0196/2835] verbosity --- Commands.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Commands.hs b/Commands.hs index 2ff8d0d7b3..e9b5ddcbdf 100644 --- a/Commands.hs +++ b/Commands.hs @@ -95,6 +95,7 @@ parseCmd argv state = do addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file + liftIO $ putStrLn $ "add " ++ file g <- Annex.gitRepo stored <- Backend.storeFileKey file case (stored) of @@ -120,6 +121,7 @@ addCmd file = inBackend file err $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do + liftIO $ putStrLn $ "unannex " ++ file Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo @@ -168,6 +170,7 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do if (not inbackend) then return () -- no-op else do + liftIO $ putStrLn $ "drop " ++ file success <- Backend.removeKey backend key if (success) then cleanup key From 6d4fc0ca7eb220298e42d368ead57622e80929a3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 13:13:49 -0400 Subject: [PATCH 0197/2835] command output cleanup --- Backend/File.hs | 59 +++++++++++++++++++++++++------------------------ Commands.hs | 33 ++++++++++++++++----------- Core.hs | 26 ++++++++++++++++++++-- UUID.hs | 2 +- 4 files changed, 75 insertions(+), 45 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 591ff3db41..f7796532be 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -53,12 +53,13 @@ copyKeyFile key file = do remotes <- Remotes.withKey key if (0 == length remotes) then cantfind - else return () - trycopy remotes remotes + else trycopy remotes remotes where - trycopy full [] = error $ "unable to get file with key: " ++ (keyFile key) ++ "\n" ++ - "To get that file, need access to one of these remotes: " ++ - (Remotes.list full) + trycopy full [] = do + showNote $ + "need access to one of these remotes: " ++ + (Remotes.list full) + return False trycopy full (r:rs) = do -- annexLocation needs the git config to have been -- read for a remote, so do that now, @@ -67,6 +68,7 @@ copyKeyFile key file = do case (result) of Nothing -> trycopy full rs Just r' -> do + showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) case (result) of Left err -> do @@ -77,17 +79,15 @@ copyKeyFile key file = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key ppuuids <- prettyPrintUUIDs uuids - error $ "no available git remotes have file with key: " ++ - (keyFile key) ++ - if (0 < length uuids) - then "\nIt has been seen before in these repositories:\n" ++ ppuuids - else "" + showNote $ "No available git remotes have the file." + if (0 < length uuids) + then showLongNote $ "It has been seen before in these repositories:\n" ++ ppuuids + else return () + return False {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () copyFromRemote r key file = do - putStrLn $ "copy from " ++ (Git.repoDescribe r ) ++ " " ++ file - if (Git.repoIsLocal r) then getlocal else getremote @@ -116,9 +116,6 @@ checkRemoveKey key = do then retNotEnoughCopiesKnown remotes numcopies else findcopies numcopies remotes [] where - failMsg w = do - liftIO $ hPutStrLn stderr $ "git-annex: " ++ w - return False -- failure, not enough copies found findcopies 0 _ _ = return True -- success, enough copies found findcopies _ [] bad = notEnoughCopiesSeen bad findcopies n (r:rs) bad = do @@ -134,21 +131,25 @@ checkRemoveKey key = do a <- Annex.new r all (result, _) <- Annex.run a (Backend.hasKey key) return result - notEnoughCopiesSeen bad = failMsg $ - "I failed to find enough other copies of: " ++ - (keyFile key) ++ - (if (0 /= length bad) then listbad bad else "") - ++ unsafe - listbad bad = "\nI was unable to access these remotes: " ++ - (Remotes.list bad) - retNotEnoughCopiesKnown remotes numcopies = failMsg $ + notEnoughCopiesSeen bad = do + showNote "failed to find enough other copies of the file" + if (0 /= length bad) then listbad bad else return () + unsafe + return False + listbad bad = + showLongNote $ + "I was unable to access these remotes: " ++ + (Remotes.list bad) + retNotEnoughCopiesKnown remotes numcopies = do + showNote $ "I only know about " ++ (show $ length remotes) ++ " out of " ++ (show numcopies) ++ - " necessary copies of: " ++ (keyFile key) ++ - unsafe - unsafe = "\n" ++ - " -- According to the " ++ config ++ - " setting, it is not safe to remove it!\n" ++ - " (Use --force to override.)" + " necessary copies of the file" + unsafe + return False + unsafe = do + showLongNote $ "According to the " ++ config ++ + " setting, it is not safe to remove it!" + showLongNote "(Use --force to override.)" config = "annex.numcopies" diff --git a/Commands.hs b/Commands.hs index e9b5ddcbdf..8591dbf6ac 100644 --- a/Commands.hs +++ b/Commands.hs @@ -95,11 +95,11 @@ parseCmd argv state = do addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file - liftIO $ putStrLn $ "add " ++ file + showStart "add" file g <- Annex.gitRepo stored <- Backend.storeFileKey file case (stored) of - Nothing -> error $ "no backend could store: " ++ file + Nothing -> showEndFail "no backend could store" file Just (key, backend) -> do logStatus key ValuePresent setup g key @@ -117,11 +117,13 @@ addCmd file = inBackend file err $ do link <- calcGitLink file key liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex annexed " ++ file + showEndOk {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - liftIO $ putStrLn $ "unannex " ++ file + showStart "unannex" file + Annex.flagChange Force True -- force backend to always remove Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo @@ -132,16 +134,17 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do moveout g src = do nocommit <- Annex.flagIsSet NoCommit liftIO $ removeFile file - liftIO $ Git.run g ["rm", file] + liftIO $ Git.run g ["rm", "--quiet", file] if (not nocommit) - then liftIO $ Git.run g ["commit", "-m", - ("git-annex unannexed " ++ file), file] + then liftIO $ Git.run g ["commit", "--quiet", + "-m", ("git-annex unannexed " ++ file), + file] else return () -- git rm deletes empty directories; -- put them back liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ renameFile src file - return () + showEndOk {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () @@ -150,6 +153,7 @@ getCmd file = notinBackend file err $ \(key, backend) -> do if (inannex) then return () else do + showStart "get" file g <- Annex.gitRepo let dest = annexLocation g key liftIO $ createDirectoryIfMissing True (parentDir dest) @@ -157,8 +161,8 @@ getCmd file = notinBackend file err $ \(key, backend) -> do if (success) then do logStatus key ValuePresent - return () - else error $ "failed to get " ++ file + showEndOk + else showEndFail "get" file where err = error $ "not annexed " ++ file @@ -170,11 +174,13 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do if (not inbackend) then return () -- no-op else do - liftIO $ putStrLn $ "drop " ++ file + showStart "drop" file success <- Backend.removeKey backend key if (success) - then cleanup key - else error $ "backend refused to drop " ++ file + then do + cleanup key + showEndOk + else showEndFail "backend refused to drop" file where cleanup key = do logStatus key ValueMissing @@ -191,13 +197,14 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do {- Fixes the symlink to an annexed file. -} fixCmd :: String -> Annex () fixCmd file = notinBackend file err $ \(key, backend) -> do - liftIO $ putStrLn $ "fix " ++ file link <- calcGitLink file key checkLegal file link + showStart "fix" file liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex fix " ++ file + showEndOk where checkLegal file link = do l <- liftIO $ readSymbolicLink file diff --git a/Core.hs b/Core.hs index 70ec2eca01..27411b2e64 100644 --- a/Core.hs +++ b/Core.hs @@ -7,6 +7,7 @@ import System.IO import System.Directory import Control.Monad.State (liftIO) import System.Path +import Data.String.Utils import Types import Locations @@ -81,8 +82,8 @@ gitAdd file commitmessage = do g <- Annex.gitRepo liftIO $ Git.run g ["add", file] if (isJust commitmessage) - then liftIO $ Git.run g ["commit", "-m", - (fromJust commitmessage), file] + then liftIO $ Git.run g ["commit", "--quiet", + "-m", (fromJust commitmessage), file] else Annex.flagChange NeedCommit True {- Calculates the relative path to use to link a file to a key. -} @@ -104,3 +105,24 @@ logStatus key status = do f <- liftIO $ logChange g key u status gitAdd f Nothing -- all logs are committed at end +{- Output logging -} +showStart :: String -> String -> Annex () +showStart command file = do + liftIO $ putStr $ command ++ " " ++ file + liftIO $ hFlush stdout +showNote :: String -> Annex () +showNote s = do + liftIO $ putStr $ " (" ++ s ++ ")" + liftIO $ hFlush stdout +showLongNote :: String -> Annex () +showLongNote s = do + liftIO $ putStr $ "\n" ++ (indent s) + where + indent s = join "\n" $ map (\l -> " " ++ l) $ lines s +showEndOk :: Annex () +showEndOk = do + liftIO $ putStrLn " ok" +showEndFail :: String -> String -> Annex () +showEndFail command file = do + liftIO $ putStrLn "" + error $ command ++ " " ++ file ++ " failed" diff --git a/UUID.hs b/UUID.hs index 6bd483a18c..b665c27e93 100644 --- a/UUID.hs +++ b/UUID.hs @@ -100,7 +100,7 @@ reposByUUID repos uuids = do prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = do m <- uuidMap - return $ unwords $ map (\u -> " "++(prettify m u)++"\n") uuids + return $ unwords $ map (\u -> "\t"++(prettify m u)++"\n") uuids where prettify m u = if (0 < (length $ findlog m u)) From 8398b9ab4a654f3f6ec570b70229a8a0030e8ab6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 13:17:34 -0400 Subject: [PATCH 0198/2835] cleanup output --- Backend/URL.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index 9e64e04996..7535207661 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -9,6 +9,7 @@ import System.Cmd import System.Exit import BackendTypes +import Core backend = Backend { name = "URL", @@ -33,7 +34,8 @@ dummyOk url = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do - liftIO $ putStrLn $ "download: " ++ url + showNote "downloading" + liftIO $ putStrLn "" -- make way for curl progress bar result <- liftIO $ rawSystem "curl" ["-#", "-o", file, url] if (result == ExitSuccess) then return True From a020b0c25c4e7c2e14d685eac8c4d3aa0e1fef8a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 16:39:30 -0400 Subject: [PATCH 0199/2835] atomic file retrieval from backends --- Commands.hs | 9 ++++++--- Core.hs | 9 +++++++++ Locations.hs | 8 +++++++- TODO | 6 ------ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Commands.hs b/Commands.hs index 8591dbf6ac..05af0ab2db 100644 --- a/Commands.hs +++ b/Commands.hs @@ -156,13 +156,16 @@ getCmd file = notinBackend file err $ \(key, backend) -> do showStart "get" file g <- Annex.gitRepo let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - success <- Backend.retrieveKeyFile backend key dest + let tmp = (annexTmpLocation g) ++ (keyFile key) + liftIO $ createDirectoryIfMissing True (parentDir tmp) + success <- Backend.retrieveKeyFile backend key tmp if (success) then do + liftIO $ renameFile tmp dest logStatus key ValuePresent showEndOk - else showEndFail "get" file + else do + showEndFail "get" file where err = error $ "not annexed " ++ file diff --git a/Core.hs b/Core.hs index 27411b2e64..302e304e49 100644 --- a/Core.hs +++ b/Core.hs @@ -29,6 +29,8 @@ startup flags = do shutdown :: Annex () shutdown = do g <- Annex.gitRepo + + -- handle pending commits nocommit <- Annex.flagIsSet NoCommit needcommit <- Annex.flagIsSet NeedCommit if (needcommit && not nocommit) @@ -36,6 +38,13 @@ shutdown = do "git-annex log update", gitStateDir g] else return () + -- clean up any files left in the temp directory + let tmp = annexTmpLocation g + exists <- liftIO $ doesDirectoryExist tmp + if (exists) + then liftIO $ removeDirectoryRecursive $ tmp + else return () + {- configure git to use union merge driver on state files, if it is not - already -} gitAttributes :: Git.Repo -> IO () diff --git a/Locations.hs b/Locations.hs index 76516224cf..2b0adb7ba8 100644 --- a/Locations.hs +++ b/Locations.hs @@ -7,7 +7,8 @@ module Locations ( keyFile, fileKey, annexLocation, - annexLocationRelative + annexLocationRelative, + annexTmpLocation ) where import Data.String.Utils @@ -36,6 +37,11 @@ annexLocation r key = annexLocationRelative :: Git.Repo -> Key -> FilePath annexLocationRelative r key = Git.dir r ++ "/annex/" ++ (keyFile key) +{- .git-annex/tmp is used for temp files + -} +annexTmpLocation :: Git.Repo -> FilePath +annexTmpLocation r = (Git.workTree r) ++ "/" ++ Git.dir r ++ "/annex/tmp/" + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid diff --git a/TODO b/TODO index 807df32d84..410c694c28 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) -* bug: git annex add file is silent if file was a symlink and got replaced - with a file. The you then git command -a, you'll check in the fil contents.. - * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo @@ -17,9 +14,6 @@ * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) -* Copy files atomically, don't leave a partial key on interrupt. - (Fix for URL download too..) - * Find a way to copy a file with a progress bar, while still preserving stat. Easiest way might be to use pv and fix up the permissions etc after? From a4dc920f6b2c31cbdd2c727f1ba7550216303991 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 16:44:06 -0400 Subject: [PATCH 0200/2835] remove some old todos --- Commands.hs | 10 ---------- LocationLog.hs | 1 - doc/git-annex.mdwn | 4 ---- 3 files changed, 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index 05af0ab2db..48186928ae 100644 --- a/Commands.hs +++ b/Commands.hs @@ -36,8 +36,6 @@ cmds = [ (Command "add" addCmd FilesNotInGit) , (Command "get" getCmd FilesInGit) , (Command "drop" dropCmd FilesInGit) - , (Command "push" pushCmd RepoName) - , (Command "pull" pullCmd RepoName) , (Command "unannex" unannexCmd FilesInGit) , (Command "describe" describeCmd SingleString) , (Command "fix" fixCmd FilesInOrNotInGit) @@ -216,14 +214,6 @@ fixCmd file = notinBackend file err $ \(key, backend) -> do else return () err = error $ "not annexed " ++ file -{- Pushes all files to a remote repository. -} -pushCmd :: String -> Annex () -pushCmd reponame = do error "not implemented" -- TODO - -{- Pulls all files from a remote repository. -} -pullCmd :: String -> Annex () -pullCmd reponame = do error "not implemented" -- TODO - {- Stores description for the repository. -} describeCmd :: String -> Annex () describeCmd description = do diff --git a/LocationLog.hs b/LocationLog.hs index c0d6170b2e..4a5fe449cd 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -112,7 +112,6 @@ appendLog file line = do createDirectoryIfMissing True (parentDir file) withFileLocked file AppendMode $ \h -> hPutStrLn h $ show line - -- TODO git add log {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4d2872aa3d..66d9897d02 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -43,10 +43,6 @@ Enough broad picture, here's how it actually looks: backend storage to the current repository. * `git annex drop $file` indicates that you no longer want the file's content to be available in this repository. -* `git annex push $repository` pushes *all* annexed files to the specified - repository. -* `git annex pull $repository` pulls *all* annexed files from the specified - repository. * `git annex file $file` adjusts the symlink for the file to point to its content again. Use this if you've moved the file around. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` From 632a4e2c6de54aec47a5553d68edd4921231d3c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 17:10:20 -0400 Subject: [PATCH 0201/2835] rename describe to init and show usage --- Commands.hs | 93 +++++++++++++++++++++++++++------------------- doc/git-annex.mdwn | 22 ++++++----- 2 files changed, 66 insertions(+), 49 deletions(-) diff --git a/Commands.hs b/Commands.hs index 48186928ae..6d68fc4e75 100644 --- a/Commands.hs +++ b/Commands.hs @@ -23,22 +23,28 @@ import Core import qualified Remotes import qualified BackendTypes -data CmdWants = FilesInGit | FilesNotInGit | FilesInOrNotInGit | - RepoName | SingleString +data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), - cmdwants :: CmdWants + cmdwants :: CmdWants, + cmddesc :: String } cmds :: [Command] cmds = [ - (Command "add" addCmd FilesNotInGit) - , (Command "get" getCmd FilesInGit) - , (Command "drop" dropCmd FilesInGit) - , (Command "unannex" unannexCmd FilesInGit) - , (Command "describe" describeCmd SingleString) - , (Command "fix" fixCmd FilesInOrNotInGit) + (Command "add" addCmd FilesNotInGit + "add files to annex") + , (Command "get" getCmd FilesInGit + "make content of annexed files available") + , (Command "drop" dropCmd FilesInGit + "indicate content of files not currently wanted") + , (Command "unannex" unannexCmd FilesInGit + "undo accidential add command") + , (Command "init" initCmd SingleString + "initialize git-annex with repository description") + , (Command "fix" fixCmd FilesInGit + "fix up files' symlinks to point to annexed content") ] options = [ @@ -46,6 +52,17 @@ options = [ , Option ['N'] ["no-commit"] (NoArg NoCommit) "do not stage or commit changes" ] +header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." + +usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs + where + cmddescs = unlines $ map (\c -> indent $ showcmd c) cmds + showcmd c = + (cmdname c) ++ + (take (10 - (length (cmdname c))) $ repeat ' ') ++ + (cmddesc c) + indent l = " " ++ l + {- Finds the type of parameters a command wants, from among the passed - parameter list. -} findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] @@ -55,10 +72,6 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files -findWanted FilesInOrNotInGit params repo = do - a <- findWanted FilesInGit params repo - b <- findWanted FilesNotInGit params repo - return $ union a b findWanted SingleString params _ = do return $ [unwords params] findWanted RepoName params _ = do @@ -73,7 +86,7 @@ parseCmd argv state = do 0 -> error usage _ -> case (lookupCmd (params !! 0)) of [] -> error usage - [Command _ action want] -> do + [Command _ action want _] -> do f <- findWanted want (drop 1 params) (BackendTypes.repo state) return (flags, map action $ filter notstate f) @@ -84,9 +97,6 @@ parseCmd argv state = do (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds - header = "Usage: git-annex [" ++ - (join "|" $ map cmdname cmds) ++ "] ..." - usage = usageInfo header options {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} @@ -197,32 +207,37 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do {- Fixes the symlink to an annexed file. -} fixCmd :: String -> Annex () -fixCmd file = notinBackend file err $ \(key, backend) -> do +fixCmd file = notinBackend file skip $ \(key, backend) -> do link <- calcGitLink file key - checkLegal file link - showStart "fix" file - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ removeFile file - liftIO $ createSymbolicLink link file - gitAdd file $ Just $ "git-annex fix " ++ file - showEndOk + l <- liftIO $ readSymbolicLink file + if (link == l) + then skip + else do + showStart "fix" file + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + gitAdd file $ Just $ "git-annex fix " ++ file + showEndOk where - checkLegal file link = do - l <- liftIO $ readSymbolicLink file - if (link == l) - then error $ "symbolic link already ok for: " ++ file - else return () - err = error $ "not annexed " ++ file + -- quietly skip non-annexed files, so this can be used + -- as a commit hook + skip = return () {- Stores description for the repository. -} -describeCmd :: String -> Annex () -describeCmd description = do - g <- Annex.gitRepo - u <- getUUID g - describeUUID u description - log <- uuidLog - gitAdd log $ Just $ "description for UUID " ++ (show u) - liftIO $ putStrLn "description set" +initCmd :: String -> Annex () +initCmd description = do + if (0 == length description) + then error $ + "please specify a description of this repository\n" ++ + usage + else do + g <- Annex.gitRepo + u <- getUUID g + describeUUID u description + log <- uuidLog + gitAdd log $ Just $ "description for UUID " ++ (show u) + liftIO $ putStrLn "description set" -- helpers inBackend file yes no = do diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 66d9897d02..4647eb0587 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -49,7 +49,7 @@ Enough broad picture, here's how it actually looks: if you're just done with a file; only use `unannex` if you accidentially added a file. (You can also run this on all your annexed files come the Singularity. ;-) -* `git annex describe "some description"` allows associating some description +* `git annex init "some description"` allows associating some description (such as "USB archive drive 1") with a repository. This can help with finding it later, see "Location Tracking" below. @@ -128,21 +128,23 @@ is on a home file server, and you are away from home. Then git-annex can tell you what git remote it needs access to in order to get a file: # git annex get myfile - git-annex: unable to get file with key: WORM:8b01f6d371178722367393eb26043482e1820306:myfile - To get that file, need access to one of these remotes: home + get myfile (need access to one of these remotes: home) + git-annex: get myfile failed Another way the location tracking comes in handy is if you put repositories on removable USB drives, that might be archived away offline in a safe place. In this sort of case, you probably don't have a git remotes configured for every USB drive. So git-annex may have to resort to talking -about repository UUIDs. If you have previously used "git annex describe" -in those repositories, it will include their description to help you with -finding them: +about repository UUIDs. If you have previously used "git annex init" +to attach descriptions to those repositories, it will include their +descriptions to help you with finding them: - git-annex: no available git remotes have file with key: WORM:8b01f6d371178722367393eb26043482e1820306:myfile - It has been seen before in these repositories: - c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 - e1938fee-d95b-11df-96cc-002170d25c55 + # git annex get myfile + get myfile (No available git remotes have the file.) + It has been seen before in these repositories: + c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 + e1938fee-d95b-11df-96cc-002170d25c55 + git-annex: get myfile failed ## configuration From 97b20e7ffeae1e6b678a77c72871af8b03d62cc6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 17:13:37 -0400 Subject: [PATCH 0202/2835] update --- INSTALL | 2 ++ 1 file changed, 2 insertions(+) diff --git a/INSTALL b/INSTALL index 9594448e70..a7fc7f6f31 100644 --- a/INSTALL +++ b/INSTALL @@ -3,3 +3,5 @@ To build and use git-annex, you will need: * ghc * These haskell libraries: MissingH SHA * uuid + +Then just run make; make install From e602238cd8cd309cea812ae85044f7d9f79be530 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 18:27:37 -0400 Subject: [PATCH 0203/2835] don't complain if a file is not annexed --- Commands.hs | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/Commands.hs b/Commands.hs index 6d68fc4e75..028c458cae 100644 --- a/Commands.hs +++ b/Commands.hs @@ -101,7 +101,7 @@ parseCmd argv state = do {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () -addCmd file = inBackend file err $ do +addCmd file = inBackend file $ do liftIO $ checkLegal file showStart "add" file g <- Annex.gitRepo @@ -112,7 +112,6 @@ addCmd file = inBackend file err $ do logStatus key ValuePresent setup g key where - err = error $ "already annexed " ++ file checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -129,7 +128,7 @@ addCmd file = inBackend file err $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () -unannexCmd file = notinBackend file err $ \(key, backend) -> do +unannexCmd file = notinBackend file $ \(key, backend) -> do showStart "unannex" file Annex.flagChange Force True -- force backend to always remove Backend.removeKey backend key @@ -138,7 +137,6 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do let src = annexLocation g key moveout g src where - err = error $ "not annexed " ++ file moveout g src = do nocommit <- Annex.flagIsSet NoCommit liftIO $ removeFile file @@ -156,7 +154,7 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () -getCmd file = notinBackend file err $ \(key, backend) -> do +getCmd file = notinBackend file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return () @@ -174,13 +172,11 @@ getCmd file = notinBackend file err $ \(key, backend) -> do showEndOk else do showEndFail "get" file - where - err = error $ "not annexed " ++ file {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} dropCmd :: FilePath -> Annex () -dropCmd file = notinBackend file err $ \(key, backend) -> do +dropCmd file = notinBackend file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return () -- no-op @@ -203,15 +199,14 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do liftIO $ removeFile loc return () else return () - err = error $ "not annexed " ++ file {- Fixes the symlink to an annexed file. -} fixCmd :: String -> Annex () -fixCmd file = notinBackend file skip $ \(key, backend) -> do +fixCmd file = notinBackend file $ \(key, backend) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) - then skip + then return () else do showStart "fix" file liftIO $ createDirectoryIfMissing True (parentDir file) @@ -219,10 +214,6 @@ fixCmd file = notinBackend file skip $ \(key, backend) -> do liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex fix " ++ file showEndOk - where - -- quietly skip non-annexed files, so this can be used - -- as a commit hook - skip = return () {- Stores description for the repository. -} initCmd :: String -> Annex () @@ -240,9 +231,13 @@ initCmd description = do liftIO $ putStrLn "description set" -- helpers -inBackend file yes no = do +inBackend file a = do r <- Backend.lookupFile file case (r) of - Just v -> yes v - Nothing -> no -notinBackend file yes no = inBackend file no yes + Just v -> return () + Nothing -> a +notinBackend file a = do + r <- Backend.lookupFile file + case (r) of + Just v -> a v + Nothing -> return () From bb6707020d08f7509c21c1229088bb6017438caf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 18:52:01 -0400 Subject: [PATCH 0204/2835] update --- doc/git-annex.mdwn | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4647eb0587..d15ca4a9f2 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -146,6 +146,19 @@ descriptions to help you with finding them: e1938fee-d95b-11df-96cc-002170d25c55 git-annex: get myfile failed +## symlink farming commit hook + +git-annex does use a lot of symlinks. Specicially, relative symlinks, +that are checked into git. To allow you to move those around without +annoyance, git-annex can run as a post-commit hook. This way, you can `git mv` +a symlink to an annexed file, and as soon as you commit, it will be fixed +up. + +`git annex init` tries to set up a post-commit hook that is itself a symlink +back to git-annex. If you want to have your own shell script in the post-commit +hook, just make it call `git annex` with no parameters. git-annex will detect +when it's run from a git hook and do the necessary fixups. + ## configuration * `annex.uuid` -- a unique UUID for this repository @@ -165,22 +178,6 @@ descriptions to help you with finding them: ## issues -### symlinks - -If the symlink to annexed content is relative, moving it to a subdir will -break it. But it it's absolute, moving the git repo (or mounting its drive -elsewhere) will break it. Either: - -* Use relative links and need `git annex mv` to move (or post-commit - hook that caches moves and updates links). -* Use absolute links and need `git annex fixlinks` when location changes; - note that would also mean that git would see the symlink targets changed - and want to commit the change. And, other clones of the repo would - diverge and there would be conflicts on the symlink text. Ugh. - -Hard links are not an option, because git would then happily commit the -file content. Amoung other reasons.. - ### free space determination Need a way to tell how much free space is available on the disk containing From 335c06171ac9a45a76b3b92d647615142bcc6ba0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 18:52:09 -0400 Subject: [PATCH 0205/2835] commit logs at end; faster --- Backend/File.hs | 4 ++-- Commands.hs | 6 +++--- Core.hs | 20 +++++++++----------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index f7796532be..9b81bef9a1 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -116,6 +116,8 @@ checkRemoveKey key = do then retNotEnoughCopiesKnown remotes numcopies else findcopies numcopies remotes [] where + config = "annex.numcopies" + findcopies 0 _ _ = return True -- success, enough copies found findcopies _ [] bad = notEnoughCopiesSeen bad findcopies n (r:rs) bad = do @@ -151,5 +153,3 @@ checkRemoveKey key = do showLongNote $ "According to the " ++ config ++ " setting, it is not safe to remove it!" showLongNote "(Use --force to override.)" - - config = "annex.numcopies" diff --git a/Commands.hs b/Commands.hs index 028c458cae..9a79e9d0c5 100644 --- a/Commands.hs +++ b/Commands.hs @@ -123,7 +123,7 @@ addCmd file = inBackend file $ do liftIO $ renameFile file dest link <- calcGitLink file key liftIO $ createSymbolicLink link file - gitAdd file $ Just $ "git-annex annexed " ++ file + gitAdd file $ "git-annex annexed " ++ file showEndOk {- Undo addCmd. -} @@ -212,7 +212,7 @@ fixCmd file = notinBackend file $ \(key, backend) -> do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file - gitAdd file $ Just $ "git-annex fix " ++ file + gitAdd file $ "git-annex fix " ++ file showEndOk {- Stores description for the repository. -} @@ -227,7 +227,7 @@ initCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - gitAdd log $ Just $ "description for UUID " ++ (show u) + gitAdd log $ "description for UUID " ++ (show u) liftIO $ putStrLn "description set" -- helpers diff --git a/Core.hs b/Core.hs index 302e304e49..5f63002c1f 100644 --- a/Core.hs +++ b/Core.hs @@ -34,8 +34,10 @@ shutdown = do nocommit <- Annex.flagIsSet NoCommit needcommit <- Annex.flagIsSet NeedCommit if (needcommit && not nocommit) - then liftIO $ Git.run g ["commit", "-q", "-m", - "git-annex log update", gitStateDir g] + then do + liftIO $ Git.run g ["add", gitStateDir g] + liftIO $ Git.run g ["commit", "-q", "-m", + "git-annex log update", gitStateDir g] else return () -- clean up any files left in the temp directory @@ -75,14 +77,12 @@ inAnnex key = do g <- Annex.gitRepo liftIO $ doesFileExist $ annexLocation g key -{- Adds, optionally also commits a file to git. - - - - All changes to the git repository should go through this function. +{- Adds and commits a file to git. - - This is careful to not rely on the index. It may have staged changes, - so only use operations that avoid committing such changes. -} -gitAdd :: FilePath -> Maybe String -> Annex () +gitAdd :: FilePath -> String -> Annex () gitAdd file commitmessage = do nocommit <- Annex.flagIsSet NoCommit if (nocommit) @@ -90,10 +90,8 @@ gitAdd file commitmessage = do else do g <- Annex.gitRepo liftIO $ Git.run g ["add", file] - if (isJust commitmessage) - then liftIO $ Git.run g ["commit", "--quiet", - "-m", (fromJust commitmessage), file] - else Annex.flagChange NeedCommit True + liftIO $ Git.run g ["commit", "--quiet", + "-m", commitmessage, file] {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath @@ -112,7 +110,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - gitAdd f Nothing -- all logs are committed at end + Annex.flagChange NeedCommit True -- commit all logs at end {- Output logging -} showStart :: String -> String -> Annex () From 8f634a5e16a7fae58c5bb24628e19c319905606e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 20:35:33 -0400 Subject: [PATCH 0206/2835] cleanup --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index 9a79e9d0c5..de3089a566 100644 --- a/Commands.hs +++ b/Commands.hs @@ -187,7 +187,7 @@ dropCmd file = notinBackend file $ \(key, backend) -> do then do cleanup key showEndOk - else showEndFail "backend refused to drop" file + else showEndFail "drop" file where cleanup key = do logStatus key ValueMissing From 939a6f860e1a2eea58e46a05861076e1b174cbd2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 23:53:01 -0400 Subject: [PATCH 0207/2835] thoughts --- doc/git-annex.mdwn | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d15ca4a9f2..4c85a03b69 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -197,8 +197,39 @@ may not be dropped right away, depending on number of copies available. The use of `.git-annex` to store logs means that if a repo has branches and the user switched between them, git-annex will see different logs in the different branches, and so may miss info about what remotes have which -files (though it can re-learn). An alternative would be to -store the log data directly in the git repo as `pristine-tar` does. +files (though it can re-learn). + +An alternative would be to store the log data directly in the git repo +as `pristine-tar` does. Problem with that approach is that git won't merge +conflicting changes to log files if they are not in the currently checked +out branch. + +It would be possible to use a branch with a tree like this, to avoid +conflicts: + +key/uuid/time/status + +As long as new files are only added, and old timestamped files deleted, +there would be no conflicts. + +A related problem though is the size of the tree objects git needs to +commit. Having the logs in a separate branch doesn't help with that. +As more keys are added, the tree object size will increase, and git will +take longer and longer to commit, and use more space. One way to deal with +this is simply by splitting the logs amoung subdirectories. Git then can +reuse trees for most directories. (Check: Does it still have to build +dup trees in memory?) + +Another approach would be to have git-annex *delete* old logs. Keep logs +for the currently available files, or something like that. If other log +info is needed, look back through history to find the first occurance of a +log. Maybe even look at other branches -- so if the logs were on master, +a new empty branch could be made and git-annex would still know where to +get keys in that branch. + +Would have to be careful about conflicts when deleting and bringing back +files with the same name. And would need to avoid expensive searching thru +all history to try to find an old log file. ## contact From 4b1086cc7d1dd9cb4eba78210976a731a683948d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 01:52:06 -0400 Subject: [PATCH 0208/2835] experimentally, removing all actual git committing Idea is the user will commit when ready, just stage everything. --- BackendTypes.hs | 3 +-- Commands.hs | 14 ++++---------- Core.hs | 29 ++--------------------------- LocationLog.hs | 6 ++---- 4 files changed, 9 insertions(+), 43 deletions(-) diff --git a/BackendTypes.hs b/BackendTypes.hs index e372099b2d..548ef17a2e 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -11,8 +11,7 @@ import Data.String.Utils import qualified GitRepo as Git -- command-line flags -data Flag = Force | NoCommit | NeedCommit - deriving (Eq, Read, Show) +data Flag = Force deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, -- but it uses Backend, so has to be here to avoid a depends loop. diff --git a/Commands.hs b/Commands.hs index de3089a566..718e991c9a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -49,7 +49,6 @@ cmds = [ options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" - , Option ['N'] ["no-commit"] (NoArg NoCommit) "do not stage or commit changes" ] header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." @@ -123,7 +122,7 @@ addCmd file = inBackend file $ do liftIO $ renameFile file dest link <- calcGitLink file key liftIO $ createSymbolicLink link file - gitAdd file $ "git-annex annexed " ++ file + liftIO $ Git.run g ["add", file] showEndOk {- Undo addCmd. -} @@ -138,14 +137,8 @@ unannexCmd file = notinBackend file $ \(key, backend) -> do moveout g src where moveout g src = do - nocommit <- Annex.flagIsSet NoCommit liftIO $ removeFile file liftIO $ Git.run g ["rm", "--quiet", file] - if (not nocommit) - then liftIO $ Git.run g ["commit", "--quiet", - "-m", ("git-annex unannexed " ++ file), - file] - else return () -- git rm deletes empty directories; -- put them back liftIO $ createDirectoryIfMissing True (parentDir file) @@ -212,7 +205,8 @@ fixCmd file = notinBackend file $ \(key, backend) -> do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file - gitAdd file $ "git-annex fix " ++ file + g <- Annex.gitRepo + liftIO $ Git.run g ["add", file] showEndOk {- Stores description for the repository. -} @@ -227,7 +221,7 @@ initCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - gitAdd log $ "description for UUID " ++ (show u) + liftIO $ Git.run g ["add", log] liftIO $ putStrLn "description set" -- helpers diff --git a/Core.hs b/Core.hs index 5f63002c1f..0af22ee73e 100644 --- a/Core.hs +++ b/Core.hs @@ -30,15 +30,7 @@ shutdown :: Annex () shutdown = do g <- Annex.gitRepo - -- handle pending commits - nocommit <- Annex.flagIsSet NoCommit - needcommit <- Annex.flagIsSet NeedCommit - if (needcommit && not nocommit) - then do - liftIO $ Git.run g ["add", gitStateDir g] - liftIO $ Git.run g ["commit", "-q", "-m", - "git-annex log update", gitStateDir g] - else return () + liftIO $ Git.run g ["add", gitStateDir g] -- clean up any files left in the temp directory let tmp = annexTmpLocation g @@ -77,22 +69,6 @@ inAnnex key = do g <- Annex.gitRepo liftIO $ doesFileExist $ annexLocation g key -{- Adds and commits a file to git. - - - - This is careful to not rely on the index. It may have staged changes, - - so only use operations that avoid committing such changes. - -} -gitAdd :: FilePath -> String -> Annex () -gitAdd file commitmessage = do - nocommit <- Annex.flagIsSet NoCommit - if (nocommit) - then return () - else do - g <- Annex.gitRepo - liftIO $ Git.run g ["add", file] - liftIO $ Git.run g ["commit", "--quiet", - "-m", commitmessage, file] - {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath calcGitLink file key = do @@ -109,8 +85,7 @@ logStatus :: Key -> LogStatus -> Annex () logStatus key status = do g <- Annex.gitRepo u <- getUUID g - f <- liftIO $ logChange g key u status - Annex.flagChange NeedCommit True -- commit all logs at end + liftIO $ logChange g key u status {- Output logging -} showStart :: String -> String -> Annex () diff --git a/LocationLog.hs b/LocationLog.hs index 4a5fe449cd..785b3330db 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -80,14 +80,12 @@ instance Read LogLine where undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] -{- Log a change in the presence of a key's value in a repository, - - and return the log filename. -} -logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath +{- Log a change in the presence of a key's value in a repository. -} +logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO () logChange repo key uuid status = do log <- logNow status uuid ls <- readLog logfile writeLog logfile (compactLog $ log:ls) - return logfile where logfile = logFile repo key From 0382d26cdbdc52c1e985cba9667a4d50d0653216 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 01:57:32 -0400 Subject: [PATCH 0209/2835] speling --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index 718e991c9a..bdeab5fc91 100644 --- a/Commands.hs +++ b/Commands.hs @@ -48,7 +48,7 @@ cmds = [ ] options = [ - Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" + Option ['f'] ["force"] (NoArg Force) "allow actions that may lose annexed data" ] header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." From f3dcc8489d7b7f9417f9752987a298976838ce47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 02:06:27 -0400 Subject: [PATCH 0210/2835] gratuitous rename --- Annex.hs | 30 ++++++++++++++--------------- Backend.hs | 14 +++++++------- Backend/File.hs | 2 +- Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Backend/WORM.hs | 2 +- BackendList.hs | 2 -- Commands.hs | 4 ++-- Locations.hs | 2 +- BackendTypes.hs => TypeInternals.hs | 6 +++--- Types.hs | 2 +- 11 files changed, 33 insertions(+), 35 deletions(-) rename BackendTypes.hs => TypeInternals.hs (93%) diff --git a/Annex.hs b/Annex.hs index 5fd500d949..b68e513553 100644 --- a/Annex.hs +++ b/Annex.hs @@ -17,17 +17,17 @@ import Control.Monad.State import qualified GitRepo as Git import Types -import qualified BackendTypes as Backend +import qualified TypeInternals as Internals {- Create and returns an Annex state object for the specified git repo. -} new :: Git.Repo -> [Backend] -> IO AnnexState new gitrepo allbackends = do - let s = Backend.AnnexState { - Backend.repo = gitrepo, - Backend.backends = [], - Backend.supportedBackends = allbackends, - Backend.flags = [] + let s = Internals.AnnexState { + Internals.repo = gitrepo, + Internals.backends = [], + Internals.supportedBackends = allbackends, + Internals.flags = [] } (_,s') <- Annex.run s (prep gitrepo) return s' @@ -44,34 +44,34 @@ run state action = runStateT (action) state gitRepo :: Annex Git.Repo gitRepo = do state <- get - return (Backend.repo state) + return (Internals.repo state) gitRepoChange :: Git.Repo -> Annex () gitRepoChange r = do state <- get - put state { Backend.repo = r } + put state { Internals.repo = r } return () backends :: Annex [Backend] backends = do state <- get - return (Backend.backends state) + return (Internals.backends state) backendsChange :: [Backend] -> Annex () backendsChange b = do state <- get - put state { Backend.backends = b } + put state { Internals.backends = b } return () supportedBackends :: Annex [Backend] supportedBackends = do state <- get - return (Backend.supportedBackends state) + return (Internals.supportedBackends state) flagIsSet :: Flag -> Annex Bool flagIsSet flag = do state <- get - return $ elem flag $ Backend.flags state + return $ elem flag $ Internals.flags state flagChange :: Flag -> Bool -> Annex () flagChange flag set = do state <- get - let f = filter (/= flag) $ Backend.flags state + let f = filter (/= flag) $ Internals.flags state if (set) - then put state { Backend.flags = (flag:f) } - else put state { Backend.flags = f } + then put state { Internals.flags = (flag:f) } + else put state { Internals.flags = f } return () diff --git a/Backend.hs b/Backend.hs index dfaa559702..a427234d7c 100644 --- a/Backend.hs +++ b/Backend.hs @@ -33,7 +33,7 @@ import qualified GitRepo as Git import qualified Annex import Utility import Types -import qualified BackendTypes as B +import qualified TypeInternals as Internals {- List of backends in the order to try them when storing a new key. -} backendList :: Annex [Backend] @@ -59,7 +59,7 @@ lookupBackendName all s = if ((length matches) /= 1) then error $ "unknown backend " ++ s else matches !! 0 - where matches = filter (\b -> s == B.name b) all + where matches = filter (\b -> s == Internals.name b) all {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) @@ -70,11 +70,11 @@ storeFileKey file = do storeFileKey' b file relfile storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do - try <- (B.getKey b) relfile + try <- (Internals.getKey b) relfile case (try) of Nothing -> nextbackend Just key -> do - stored <- (B.storeFileKey b) file key + stored <- (Internals.storeFileKey b) file key if (not stored) then nextbackend else do @@ -85,17 +85,17 @@ storeFileKey' (b:bs) file relfile = do {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool -retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest +retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} removeKey :: Backend -> Key -> Annex Bool -removeKey backend key = (B.removeKey backend) key +removeKey backend key = (Internals.removeKey backend) key {- Checks if a backend has its key. -} hasKey :: Key -> Annex Bool hasKey key = do all <- Annex.supportedBackends - (B.hasKey (lookupBackendName all $ backendName key)) key + (Internals.hasKey (lookupBackendName all $ backendName key)) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/Backend/File.hs b/Backend/File.hs index 9b81bef9a1..c97a354d0c 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -16,7 +16,7 @@ import System.Cmd import System.Exit import Control.Exception -import BackendTypes +import TypeInternals import LocationLog import Locations import qualified Remotes diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index c01e01a723..2143a6af59 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -6,7 +6,7 @@ module Backend.SHA1 (backend) where import Data.Digest.Pure.SHA import qualified Backend.File -import BackendTypes +import TypeInternals backend = Backend.File.backend { name = "SHA1", diff --git a/Backend/URL.hs b/Backend/URL.hs index 7535207661..5c1fd74c93 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -8,7 +8,7 @@ import Data.String.Utils import System.Cmd import System.Exit -import BackendTypes +import TypeInternals import Core backend = Backend { diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 463b0ac8e2..0588ddaf83 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -9,7 +9,7 @@ import System.Posix.Files import qualified Data.ByteString.Lazy.Char8 as B import qualified Backend.File -import BackendTypes +import TypeInternals import Utility backend = Backend.File.backend { diff --git a/BackendList.hs b/BackendList.hs index 920f8fc0a6..25f3ae5eac 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -3,8 +3,6 @@ module BackendList (allBackends) where -import BackendTypes - -- When adding a new backend, import it here and add it to the list. import qualified Backend.WORM import qualified Backend.SHA1 diff --git a/Commands.hs b/Commands.hs index bdeab5fc91..fab72160a2 100644 --- a/Commands.hs +++ b/Commands.hs @@ -21,7 +21,7 @@ import LocationLog import Types import Core import qualified Remotes -import qualified BackendTypes +import qualified TypeInternals data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString data Command = Command { @@ -87,7 +87,7 @@ parseCmd argv state = do [] -> error usage [Command _ action want _] -> do f <- findWanted want (drop 1 params) - (BackendTypes.repo state) + (TypeInternals.repo state) return (flags, map action $ filter notstate f) where -- never include files from the state directory diff --git a/Locations.hs b/Locations.hs index 2b0adb7ba8..18d416eb4a 100644 --- a/Locations.hs +++ b/Locations.hs @@ -14,7 +14,7 @@ module Locations ( import Data.String.Utils import Types -import qualified BackendTypes as Backend +import qualified TypeInternals as Internals import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex diff --git a/BackendTypes.hs b/TypeInternals.hs similarity index 93% rename from BackendTypes.hs rename to TypeInternals.hs index 548ef17a2e..e8f7cb9e74 100644 --- a/BackendTypes.hs +++ b/TypeInternals.hs @@ -1,9 +1,9 @@ -{- git-annex backend data types +{- git-annex internal data types - - - Mostly only backend implementations should need to import this. + - Most things should not need this, using Types and/or Annex instead. -} -module BackendTypes where +module TypeInternals where import Control.Monad.State (StateT) import Data.String.Utils diff --git a/Types.hs b/Types.hs index 6bf26d36e6..2284d92674 100644 --- a/Types.hs +++ b/Types.hs @@ -10,4 +10,4 @@ module Types ( Flag(..), ) where -import BackendTypes +import TypeInternals From 33432a32775d23dff5e5b6f499656b247e4ed604 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 21:08:27 -0400 Subject: [PATCH 0211/2835] bug --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 410c694c28..6fa34046db 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,9 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) +* bug: ctrl+c does not stop it from running another action; need to + not catch UserInterrupt exceptions. + * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo From 15986f01d1fd565da151dcb08697e21a94fc9037 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 21:36:26 -0400 Subject: [PATCH 0212/2835] bug --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 6fa34046db..eca1922b7a 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,9 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) +* bug: doesn't learn new remote's uuids if a known (but maybe not accessible) + uuids has a wanted file + * bug: ctrl+c does not stop it from running another action; need to not catch UserInterrupt exceptions. From c7664588f81fe27b3e88d49523ef3c483ac6481a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 01:19:56 -0400 Subject: [PATCH 0213/2835] use safesystem --- Backend/File.hs | 7 ++----- Backend/URL.hs | 11 +++++++---- GitRepo.hs | 2 +- TODO | 3 --- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index c97a354d0c..8969d7556e 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -13,6 +13,7 @@ module Backend.File (backend) where import Control.Monad.State import System.IO import System.Cmd +import System.Cmd.Utils import System.Exit import Control.Exception @@ -92,11 +93,7 @@ copyFromRemote r key file = do then getlocal else getremote where - getlocal = do - res <-rawSystem "cp" ["-a", location, file] - if (res == ExitSuccess) - then return () - else error "cp failed" + getlocal = safeSystem "cp" ["-a", location, file] getremote = error "get via network not yet implemented!" location = annexLocation r key diff --git a/Backend/URL.hs b/Backend/URL.hs index 5c1fd74c93..c9b6ab6df8 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -3,9 +3,11 @@ module Backend.URL (backend) where +import Control.Exception import Control.Monad.State (liftIO) import Data.String.Utils import System.Cmd +import System.Cmd.Utils import System.Exit import TypeInternals @@ -36,9 +38,10 @@ downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" liftIO $ putStrLn "" -- make way for curl progress bar - result <- liftIO $ rawSystem "curl" ["-#", "-o", file, url] - if (result == ExitSuccess) - then return True - else return False + result <- liftIO $ (try curl::IO (Either SomeException ())) + case result of + Left err -> return False + Right succ -> return True where + curl = safeSystem "curl" ["-#", "-o", file, url] url = join ":" $ drop 1 $ split ":" $ show key diff --git a/GitRepo.hs b/GitRepo.hs index 32383197b5..5b0e68cd62 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -167,7 +167,7 @@ gitCommandLine repo params = assertlocal repo $ {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () run repo params = assertlocal repo $ do - r <- rawSystem "git" (gitCommandLine repo params) + r <- safeSystem "git" (gitCommandLine repo params) return () {- Runs a git subcommand and returns its output. -} diff --git a/TODO b/TODO index eca1922b7a..e6fdcd0b29 100644 --- a/TODO +++ b/TODO @@ -4,9 +4,6 @@ * bug: doesn't learn new remote's uuids if a known (but maybe not accessible) uuids has a wanted file -* bug: ctrl+c does not stop it from running another action; need to - not catch UserInterrupt exceptions. - * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo From 7afac113443b8e93e19ad87d769a24c52706f551 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 01:45:45 -0400 Subject: [PATCH 0214/2835] add boolSystem --- Utility.hs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index e4278ff3f6..09b973002f 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,10 +6,15 @@ module Utility ( hGetContentsStrict, parentDir, relPathCwdToDir, - relPathDirToDir + relPathDirToDir, + boolSystem ) where import System.IO +import System.Cmd +import System.Exit +import System.Posix.Signals +import Data.Typeable import System.Posix.IO import Data.String.Utils import System.Path @@ -88,3 +93,18 @@ relPathDirToDir from to = dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common path = join s $ dotdots ++ uncommon + +{- Run a system command, and returns True or False + - if it succeeded or failed. + - + - An error is thrown if the command exits due to SIGINT, + - to propigate ctrl-c. + -} +boolSystem :: FilePath -> [String] -> IO Bool +boolSystem command params = do + r <- rawSystem command params + case r of + ExitSuccess -> return True + ExitFailure e -> if Just e == cast sigINT + then error $ command ++ "interrupted" + else return False From 470e0a2fbd1f554df677127212643d534c2f7857 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 01:46:07 -0400 Subject: [PATCH 0215/2835] use boolSystem --- Backend/File.hs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 8969d7556e..6b2e82726d 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -14,7 +14,6 @@ import Control.Monad.State import System.IO import System.Cmd import System.Cmd.Utils -import System.Exit import Control.Exception import TypeInternals @@ -70,12 +69,7 @@ copyKeyFile key file = do Nothing -> trycopy full rs Just r' -> do showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." - result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) - case (result) of - Left err -> do - liftIO $ hPutStrLn stderr (show err) - trycopy full rs - Right succ -> return True + liftIO $ copyFromRemote r' key file cantfind = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key @@ -86,15 +80,15 @@ copyKeyFile key file = do else return () return False -{- Tries to copy a file from a remote, exception on error. -} -copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () +{- Tries to copy a file from a remote. -} +copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool copyFromRemote r key file = do if (Git.repoIsLocal r) then getlocal else getremote where - getlocal = safeSystem "cp" ["-a", location, file] - getremote = error "get via network not yet implemented!" + getlocal = boolSystem "cp" ["-a", location, file] + getremote = return False -- TODO implement get from remote location = annexLocation r key {- Checks remotes to verify that enough copies of a key exist to allow From 2caf711827470976f935bb06bb3b6b87e1776299 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 01:46:20 -0400 Subject: [PATCH 0216/2835] stop trapping all exceptions Need to allow exceptions to be thrown for SIGPIPE propigation. Converted places that used error unncessarily to not. --- Commands.hs | 26 ++++++++++++-------------- git-annex.hs | 10 +++++----- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/Commands.hs b/Commands.hs index fab72160a2..115c3b3edb 100644 --- a/Commands.hs +++ b/Commands.hs @@ -101,21 +101,19 @@ parseCmd argv state = do - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () addCmd file = inBackend file $ do - liftIO $ checkLegal file - showStart "add" file - g <- Annex.gitRepo - stored <- Backend.storeFileKey file - case (stored) of - Nothing -> showEndFail "no backend could store" file - Just (key, backend) -> do - logStatus key ValuePresent - setup g key + s <- liftIO $ getSymbolicLinkStatus file + if ((isSymbolicLink s) || (not $ isRegularFile s)) + then return () + else do + showStart "add" file + g <- Annex.gitRepo + stored <- Backend.storeFileKey file + case (stored) of + Nothing -> showEndFail "no backend could store" file + Just (key, backend) -> do + logStatus key ValuePresent + setup g key where - checkLegal file = do - s <- getSymbolicLinkStatus file - if ((isSymbolicLink s) || (not $ isRegularFile s)) - then error $ "not a regular file: " ++ file - else return () setup g key = do let dest = annexLocation g key liftIO $ createDirectoryIfMissing True (parentDir dest) diff --git a/git-annex.hs b/git-annex.hs index 947868f238..71a21379dc 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,6 +1,6 @@ {- git-annex main program -} -import Control.Exception +import IO (try) import System.IO import System.Environment @@ -18,8 +18,9 @@ main = do (flags, actions) <- parseCmd args state tryRun state $ [startup flags] ++ actions ++ [shutdown] -{- Runs a list of Annex actions. Catches exceptions, not stopping - - if some error out, and propigates an overall error status at the end. +{- Runs a list of Annex actions. Catches IO errors and continues + - (but explicitly thrown errors terminate the whole command). + - Propigates an overall error status at the end. - - This runs in the IO monad, not in the Annex monad. It seems that - exceptions can only be caught in the IO monad, not in a stacked monad; @@ -29,8 +30,7 @@ main = do tryRun :: AnnexState -> [Annex ()] -> IO () tryRun state actions = tryRun' state 0 actions tryRun' state errnum (a:as) = do - result <- try - (Annex.run state a)::IO (Either SomeException ((), AnnexState)) + result <- try $ Annex.run state a case (result) of Left err -> do showErr err From 2ea589e117bcc36ee613454ffb52b8e52cc96bc9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 12:53:51 -0400 Subject: [PATCH 0217/2835] don't throw a fatal error --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 0af22ee73e..5bf1083938 100644 --- a/Core.hs +++ b/Core.hs @@ -107,4 +107,4 @@ showEndOk = do showEndFail :: String -> String -> Annex () showEndFail command file = do liftIO $ putStrLn "" - error $ command ++ " " ++ file ++ " failed" + liftIO $ hPutStrLn stderr $ command ++ " " ++ file ++ " failed" From d23fc22f0e17c95765f940f81f733f9580e19107 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 12:55:40 -0400 Subject: [PATCH 0218/2835] less verbose failures seem better here --- Commands.hs | 6 +++--- Core.hs | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Commands.hs b/Commands.hs index 115c3b3edb..e38f9c3725 100644 --- a/Commands.hs +++ b/Commands.hs @@ -109,7 +109,7 @@ addCmd file = inBackend file $ do g <- Annex.gitRepo stored <- Backend.storeFileKey file case (stored) of - Nothing -> showEndFail "no backend could store" file + Nothing -> showEndFail Just (key, backend) -> do logStatus key ValuePresent setup g key @@ -162,7 +162,7 @@ getCmd file = notinBackend file $ \(key, backend) -> do logStatus key ValuePresent showEndOk else do - showEndFail "get" file + showEndFail {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} @@ -178,7 +178,7 @@ dropCmd file = notinBackend file $ \(key, backend) -> do then do cleanup key showEndOk - else showEndFail "drop" file + else showEndFail where cleanup key = do logStatus key ValueMissing diff --git a/Core.hs b/Core.hs index 5bf1083938..3532c71d52 100644 --- a/Core.hs +++ b/Core.hs @@ -104,7 +104,6 @@ showLongNote s = do showEndOk :: Annex () showEndOk = do liftIO $ putStrLn " ok" -showEndFail :: String -> String -> Annex () -showEndFail command file = do - liftIO $ putStrLn "" - liftIO $ hPutStrLn stderr $ command ++ " " ++ file ++ " failed" +showEndFail :: Annex () +showEndFail = do + liftIO $ putStrLn " failed" From 3531ce5c54e380d15d54d838c90f4ebe311782af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 13:08:05 -0400 Subject: [PATCH 0219/2835] fix remote uuid learning bug --- Remotes.hs | 11 ++++++++--- TODO | 3 --- TypeInternals.hs | 6 ++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index f21f5a6ba4..828dc753fe 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -30,17 +30,22 @@ withKey key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost - -- this only uses cached data, so may not find new remotes + -- This only uses cached data, so may not include new remotes + -- or remotes whose uuid has changed (eg by a different drive being + -- mounted at their location). So unless it happens to find all + -- remotes, try harder, loading the remotes' configs. remotes <- reposByUUID allremotes uuids - if (0 == length remotes) + remotesread <- Annex.flagIsSet RemotesRead + if ((length allremotes /= length remotes) && not remotesread) then tryharder allremotes uuids else return remotes where tryharder allremotes uuids = do - -- more expensive; check each remote's config + -- more expensive; read each remote's config mayberemotes <- mapM tryGitConfigRead allremotes let allremotes' = catMaybes mayberemotes remotes' <- reposByUUID allremotes' uuids + Annex.flagChange RemotesRead True return remotes' {- Cost Ordered list of remotes. -} diff --git a/TODO b/TODO index e6fdcd0b29..410c694c28 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) -* bug: doesn't learn new remote's uuids if a known (but maybe not accessible) - uuids has a wanted file - * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo diff --git a/TypeInternals.hs b/TypeInternals.hs index e8f7cb9e74..4a9d2653e1 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -10,8 +10,10 @@ import Data.String.Utils import qualified GitRepo as Git --- command-line flags -data Flag = Force deriving (Eq, Read, Show) +data Flag = + Force | -- command-line flags + RemotesRead -- indicates that remote repo configs have been read + deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, -- but it uses Backend, so has to be here to avoid a depends loop. From ed3f6653b664d72e4b89c4dd0c56f4b7db7cbab9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 13:39:53 -0400 Subject: [PATCH 0220/2835] better drop error messages --- Backend/File.hs | 49 +++++++++++++++++++++++++++++-------------------- UUID.hs | 2 +- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 6b2e82726d..6944a8b62f 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,6 +15,8 @@ import System.IO import System.Cmd import System.Cmd.Utils import Control.Exception +import List +import Maybe import TypeInternals import LocationLog @@ -52,7 +54,10 @@ copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do remotes <- Remotes.withKey key if (0 == length remotes) - then cantfind + then do + showNote $ "No available git remotes have the file." + showLocations key + return False else trycopy remotes remotes where trycopy full [] = do @@ -70,15 +75,6 @@ copyKeyFile key file = do Just r' -> do showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." liftIO $ copyFromRemote r' key file - cantfind = do - g <- Annex.gitRepo - uuids <- liftIO $ keyLocations g key - ppuuids <- prettyPrintUUIDs uuids - showNote $ "No available git remotes have the file." - if (0 < length uuids) - then showLongNote $ "It has been seen before in these repositories:\n" ++ ppuuids - else return () - return False {- Tries to copy a file from a remote. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool @@ -90,6 +86,17 @@ copyFromRemote r key file = do getlocal = boolSystem "cp" ["-a", location, file] getremote = return False -- TODO implement get from remote location = annexLocation r key + +showLocations :: Key -> Annex () +showLocations key = do + g <- Annex.gitRepo + u <- getUUID g + uuids <- liftIO $ keyLocations g key + let uuidsf = filter (\v -> v /= u) uuids + ppuuids <- prettyPrintUUIDs uuidsf + if (0 < length uuidsf) + then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids + else showLongNote $ "No other repository is known to contain the file." {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an @@ -125,22 +132,24 @@ checkRemoveKey key = do (result, _) <- Annex.run a (Backend.hasKey key) return result notEnoughCopiesSeen bad = do - showNote "failed to find enough other copies of the file" - if (0 /= length bad) then listbad bad else return () unsafe + if (0 /= length bad) then listbad bad else return () + showLocations key + hint return False listbad bad = showLongNote $ "I was unable to access these remotes: " ++ (Remotes.list bad) retNotEnoughCopiesKnown remotes numcopies = do - showNote $ - "I only know about " ++ (show $ length remotes) ++ - " out of " ++ (show numcopies) ++ - " necessary copies of the file" unsafe + showLongNote $ + "Could only verify the existence of " ++ + (show $ length remotes) ++ + " out of " ++ (show numcopies) ++ + " necessary copies" + showLocations key + hint return False - unsafe = do - showLongNote $ "According to the " ++ config ++ - " setting, it is not safe to remove it!" - showLongNote "(Use --force to override.)" + unsafe = showNote "unsafe" + hint = showLongNote $ "(Use --force to override this check, or adjust annex.numcopies.)" diff --git a/UUID.hs b/UUID.hs index b665c27e93..47d305c4fb 100644 --- a/UUID.hs +++ b/UUID.hs @@ -51,7 +51,7 @@ getUUID r = do let c = cached r g let u = uncached r - + if (c /= u && u /= "") then do updatecache g r u From 4f8d28819da8e13bfa741dae8d84b0000e38e083 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:13:48 -0400 Subject: [PATCH 0221/2835] improved messages when a file is not available in remotes --- Backend/File.hs | 55 ++++++++++++++++++++++--------------------------- Remotes.hs | 13 ++++++------ 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 6944a8b62f..4ea25daa73 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -55,15 +55,15 @@ copyKeyFile key file = do remotes <- Remotes.withKey key if (0 == length remotes) then do - showNote $ "No available git remotes have the file." + showNote "not available" showLocations key return False else trycopy remotes remotes where trycopy full [] = do - showNote $ - "need access to one of these remotes: " ++ - (Remotes.list full) + showNote "not available" + showTriedRemotes full + showLocations key return False trycopy full (r:rs) = do -- annexLocation needs the git config to have been @@ -71,8 +71,8 @@ copyKeyFile key file = do -- if it hasn't been already result <- Remotes.tryGitConfigRead r case (result) of - Nothing -> trycopy full rs - Just r' -> do + Left err -> trycopy full rs + Right r' -> do showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." liftIO $ copyFromRemote r' key file @@ -86,7 +86,7 @@ copyFromRemote r key file = do getlocal = boolSystem "cp" ["-a", location, file] getremote = return False -- TODO implement get from remote location = annexLocation r key - + showLocations :: Key -> Annex () showLocations key = do g <- Annex.gitRepo @@ -97,6 +97,10 @@ showLocations key = do if (0 < length uuidsf) then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids else showLongNote $ "No other repository is known to contain the file." + +showTriedRemotes remotes = + showLongNote $ "I was unable to access these remotes: " ++ + (Remotes.list remotes) {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an @@ -108,46 +112,37 @@ checkRemoveKey key = do then return True else do g <- Annex.gitRepo - let numcopies = read $ Git.configGet g config "1" remotes <- Remotes.withKey key + let numcopies = read $ Git.configGet g config "1" if (numcopies > length remotes) - then retNotEnoughCopiesKnown remotes numcopies - else findcopies numcopies remotes [] + then notEnoughCopies numcopies (length remotes) [] + else findcopies numcopies 0 remotes [] where config = "annex.numcopies" - - findcopies 0 _ _ = return True -- success, enough copies found - findcopies _ [] bad = notEnoughCopiesSeen bad - findcopies n (r:rs) bad = do + findcopies need have [] bad = + if (have >= need) + then return True + else notEnoughCopies need have bad + findcopies need have (r:rs) bad = do all <- Annex.supportedBackends result <- liftIO $ ((try $ remoteHasKey r all)::IO (Either SomeException Bool)) case (result) of - Right True -> findcopies (n-1) rs bad - Right False -> findcopies n rs bad - Left _ -> findcopies n rs (r:bad) + Right True -> findcopies need (have+1) rs bad + Right False -> findcopies need have rs bad + Left _ -> findcopies need have rs (r:bad) remoteHasKey r all = do -- To check if a remote has a key, construct a new -- Annex monad and query its backend. a <- Annex.new r all (result, _) <- Annex.run a (Backend.hasKey key) return result - notEnoughCopiesSeen bad = do - unsafe - if (0 /= length bad) then listbad bad else return () - showLocations key - hint - return False - listbad bad = - showLongNote $ - "I was unable to access these remotes: " ++ - (Remotes.list bad) - retNotEnoughCopiesKnown remotes numcopies = do + notEnoughCopies need have bad = do unsafe showLongNote $ "Could only verify the existence of " ++ - (show $ length remotes) ++ - " out of " ++ (show numcopies) ++ + (show have) ++ " out of " ++ (show need) ++ " necessary copies" + if (0 /= length bad) then showTriedRemotes bad else return () showLocations key hint return False diff --git a/Remotes.hs b/Remotes.hs index 828dc753fe..a0894f418b 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -10,6 +10,7 @@ import Control.Exception import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils +import Data.Either.Utils import List import Maybe @@ -42,8 +43,8 @@ withKey key = do where tryharder allremotes uuids = do -- more expensive; read each remote's config - mayberemotes <- mapM tryGitConfigRead allremotes - let allremotes' = catMaybes mayberemotes + eitherremotes <- mapM tryGitConfigRead allremotes + let allremotes' = map fromEither eitherremotes remotes' <- reposByUUID allremotes' uuids Annex.flagChange RemotesRead True return remotes' @@ -86,7 +87,7 @@ repoCost r = do - because reading it may be expensive. This function tries to read the - config for a specified remote, and updates state. If successful, it - returns the updated git repo. -} -tryGitConfigRead :: Git.Repo -> Annex (Maybe Git.Repo) +tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r = do if (Map.null $ Git.configMap r) then do @@ -94,15 +95,15 @@ tryGitConfigRead r = do -- for other reasons; catch all possible exceptions result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) case (result) of - Left err -> return Nothing + Left err -> return $ Left r Right r' -> do g <- Annex.gitRepo let l = Git.remotes g let g' = Git.remotesAdd g $ exchange l r' Annex.gitRepoChange g' - return $ Just r' - else return $ Just r + return $ Right r' + else return $ Right r -- config already read where exchange [] new = [] exchange (old:ls) new = From 731cabbe3d083bcca6535fc9751b63cc16a83b83 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:15:53 -0400 Subject: [PATCH 0222/2835] newlines before failed message needed if a long message was shown --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 3532c71d52..8dc4bff6fc 100644 --- a/Core.hs +++ b/Core.hs @@ -106,4 +106,4 @@ showEndOk = do liftIO $ putStrLn " ok" showEndFail :: Annex () showEndFail = do - liftIO $ putStrLn " failed" + liftIO $ putStrLn "\nfailed" From f3c5a8543b2793f507b4a4801315d1f333e758cc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:17:41 -0400 Subject: [PATCH 0223/2835] update --- TODO | 2 +- debian/docs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index 410c694c28..a804597b88 100644 --- a/TODO +++ b/TODO @@ -4,7 +4,7 @@ * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo -* how to handle git mv file? -> git annex fix -> run automatically? +* how to handle git mv file? -> git annex fix -> run automatically on commit * how to handle git rm file? (should try to drop keys that have no referring file, if it seems safe..) diff --git a/debian/docs b/debian/docs index 9de86edc7e..d6fa57dd7b 100644 --- a/debian/docs +++ b/debian/docs @@ -1 +1,2 @@ doc/*.mdwn +TODO From e8267f1b9e99cce79209eb2f47fce02d52d60b56 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:37:19 -0400 Subject: [PATCH 0224/2835] add doc wiki --- Makefile | 12 +++++++ debian/control | 2 +- doc/Makefile | 15 +++++++++ doc/bugs.mdwn | 4 +++ doc/bugs/branching.mdwn | 36 ++++++++++++++++++++ doc/bugs/done.mdwn | 3 ++ doc/bugs/free_space_checking.mdwn | 8 +++++ doc/contact.mdwn | 4 +++ doc/download.mdwn | 5 +++ doc/git-annex.mdwn | 55 ------------------------------- doc/index.mdwn | 23 +++++++++++++ doc/news.mdwn | 5 +++ 12 files changed, 116 insertions(+), 56 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/bugs.mdwn create mode 100644 doc/bugs/branching.mdwn create mode 100644 doc/bugs/done.mdwn create mode 100644 doc/bugs/free_space_checking.mdwn create mode 100644 doc/contact.mdwn create mode 100644 doc/download.mdwn create mode 100644 doc/index.mdwn create mode 100644 doc/news.mdwn diff --git a/Makefile b/Makefile index d1fcbbeeeb..87c725efea 100644 --- a/Makefile +++ b/Makefile @@ -8,5 +8,17 @@ install: clean: rm -rf build git-annex + rm -rf doc/.ikiwiki html + +# Build static html docs suitable for being shipped in the software +# package. This depends on ikiwiki being installed to build the docs. +ifeq ($(shell which ikiwiki),) +IKIWIKI=echo "** ikiwiki not found" >&2 ; echo ikiwiki +else +IKIWIKI=ikiwiki +endif + +docs: + $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff .PHONY: git-annex diff --git a/debian/control b/debian/control index e58f55af9e..846844c32a 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-dev +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-dev, ikiwiki Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000000..f2c4d8e54d --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,15 @@ +# Build static html docs suitable for being shipped in the software +# package. This depends on ikiwiki being installed to build the docs. + +ifeq ($(shell which ikiwiki),) +IKIWIKI=echo "** ikiwiki not found" >&2 ; echo ikiwiki +else +IKIWIKI=ikiwiki +endif + +all: + $(IKIWIKI) `pwd` html -v --wikiname FooBar --plugin=goodstuff \ + --exclude=html --exclude=Makefile + +clean: + rm -rf .ikiwiki html diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn new file mode 100644 index 0000000000..dd2a9b4035 --- /dev/null +++ b/doc/bugs.mdwn @@ -0,0 +1,4 @@ +This is git-annex's bug list. Link bugs to [[bugs/done]] when done. + +[[!inline pages="./bugs/* and !./bugs/done and !link(done) +and !*/Discussion" actions=yes postform=yes show=0]] diff --git a/doc/bugs/branching.mdwn b/doc/bugs/branching.mdwn new file mode 100644 index 0000000000..21996ecc00 --- /dev/null +++ b/doc/bugs/branching.mdwn @@ -0,0 +1,36 @@ +The use of `.git-annex` to store logs means that if a repo has branches +and the user switched between them, git-annex will see different logs in +the different branches, and so may miss info about what remotes have which +files (though it can re-learn). + +An alternative would be to store the log data directly in the git repo +as `pristine-tar` does. Problem with that approach is that git won't merge +conflicting changes to log files if they are not in the currently checked +out branch. + +It would be possible to use a branch with a tree like this, to avoid +conflicts: + +key/uuid/time/status + +As long as new files are only added, and old timestamped files deleted, +there would be no conflicts. + +A related problem though is the size of the tree objects git needs to +commit. Having the logs in a separate branch doesn't help with that. +As more keys are added, the tree object size will increase, and git will +take longer and longer to commit, and use more space. One way to deal with +this is simply by splitting the logs amoung subdirectories. Git then can +reuse trees for most directories. (Check: Does it still have to build +dup trees in memory?) + +Another approach would be to have git-annex *delete* old logs. Keep logs +for the currently available files, or something like that. If other log +info is needed, look back through history to find the first occurance of a +log. Maybe even look at other branches -- so if the logs were on master, +a new empty branch could be made and git-annex would still know where to +get keys in that branch. + +Would have to be careful about conflicts when deleting and bringing back +files with the same name. And would need to avoid expensive searching thru +all history to try to find an old log file. diff --git a/doc/bugs/done.mdwn b/doc/bugs/done.mdwn new file mode 100644 index 0000000000..ad332e2a26 --- /dev/null +++ b/doc/bugs/done.mdwn @@ -0,0 +1,3 @@ +recently fixed [[bugs]] + +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10]] diff --git a/doc/bugs/free_space_checking.mdwn b/doc/bugs/free_space_checking.mdwn new file mode 100644 index 0000000000..34528a7b35 --- /dev/null +++ b/doc/bugs/free_space_checking.mdwn @@ -0,0 +1,8 @@ +Should check that there is enough free space before trying to copy a +file around. + +* Need a way to tell how much free space is available on the disk containing + a given repository. + +* And, need a way to tell the size of a file before copying it from + a remote, to check local disk space. diff --git a/doc/contact.mdwn b/doc/contact.mdwn new file mode 100644 index 0000000000..1238ca0403 --- /dev/null +++ b/doc/contact.mdwn @@ -0,0 +1,4 @@ +Joey Hess is the author of git-annex. + +The [VCS-home mailing list](http://lists.madduck.net/listinfo/vcs-home) +is a good place to discuss it. diff --git a/doc/download.mdwn b/doc/download.mdwn new file mode 100644 index 0000000000..2ceb73193a --- /dev/null +++ b/doc/download.mdwn @@ -0,0 +1,5 @@ +The main git repository for git-annex is `git://git.kitenet.net/git-annex` +[[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] + +There are no binary packages yet, but you can build Debian packages from +the source tree with `dpkg-buildpackage`. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4c85a03b69..eb5fa9ced9 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -176,61 +176,6 @@ when it's run from a git hook and do the necessary fixups. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. -## issues - -### free space determination - -Need a way to tell how much free space is available on the disk containing -a given repository. The repository may be remote, so ssh may need to be -used. - -Similarly, need a way to tell the size of a file before copying it from -a remote, to check local disk space. - -### auto-drop on rm - -When git-rm removed a file, its key should get dropped too. Of course, it -may not be dropped right away, depending on number of copies available. - -### branching - -The use of `.git-annex` to store logs means that if a repo has branches -and the user switched between them, git-annex will see different logs in -the different branches, and so may miss info about what remotes have which -files (though it can re-learn). - -An alternative would be to store the log data directly in the git repo -as `pristine-tar` does. Problem with that approach is that git won't merge -conflicting changes to log files if they are not in the currently checked -out branch. - -It would be possible to use a branch with a tree like this, to avoid -conflicts: - -key/uuid/time/status - -As long as new files are only added, and old timestamped files deleted, -there would be no conflicts. - -A related problem though is the size of the tree objects git needs to -commit. Having the logs in a separate branch doesn't help with that. -As more keys are added, the tree object size will increase, and git will -take longer and longer to commit, and use more space. One way to deal with -this is simply by splitting the logs amoung subdirectories. Git then can -reuse trees for most directories. (Check: Does it still have to build -dup trees in memory?) - -Another approach would be to have git-annex *delete* old logs. Keep logs -for the currently available files, or something like that. If other log -info is needed, look back through history to find the first occurance of a -log. Maybe even look at other branches -- so if the logs were on master, -a new empty branch could be made and git-annex would still know where to -get keys in that branch. - -Would have to be careful about conflicts when deleting and bringing back -files with the same name. And would need to avoid expensive searching thru -all history to try to find an old log file. - ## contact Joey Hess diff --git a/doc/index.mdwn b/doc/index.mdwn new file mode 100644 index 0000000000..8d2cb1ef57 --- /dev/null +++ b/doc/index.mdwn @@ -0,0 +1,23 @@ +git-annex allows managing files with git, without checking the file +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. + +Even without file content tracking, being able to manage files with git, +move files around and delete files with versioned directory trees, and use +branches and distributed clones, are all very handy reasons to use git. And +annexed files can co-exist in the same git repository with regularly +versioned files, which is convenient for maintaining documents, Makefiles, +etc that are associated with annexed files but that benefit from full +revision control. + +* [[man page|git-annex]] +* **[[download]]** +* [[news]] +* [[bugs]] +* [[contact]] + +---- + +git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and +hosted by [Branchable](http://branchable.com/). diff --git a/doc/news.mdwn b/doc/news.mdwn new file mode 100644 index 0000000000..d0ff1ca2c1 --- /dev/null +++ b/doc/news.mdwn @@ -0,0 +1,5 @@ +This is where announcements of new releases, features, and other news is +posted. git-annex users are recommended to subscribe to this page's RSS +feed. + +[[!inline pages="./news/* and !*/Discussion" rootpage="news" show="30"]] From 21128c88e71cb2050b78f996888d9e251a448807 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:39:40 -0400 Subject: [PATCH 0225/2835] tweak --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 87c725efea..a0911df04c 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +all: git-annex docs + git-annex: mkdir -p build ghc -odir build -hidir build --make git-annex @@ -10,15 +12,16 @@ clean: rm -rf build git-annex rm -rf doc/.ikiwiki html -# Build static html docs suitable for being shipped in the software -# package. This depends on ikiwiki being installed to build the docs. +# If ikiwiki is available, build static html docs suitable for being +# shipped in the software package. ifeq ($(shell which ikiwiki),) -IKIWIKI=echo "** ikiwiki not found" >&2 ; echo ikiwiki +IKIWIKI=echo "** ikiwiki not found, skipping building docs" >&2 else IKIWIKI=ikiwiki endif docs: - $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff + $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ + --no-usedirs .PHONY: git-annex From 6ef1c2d2daf37dc92b4c364ea34802b62688018b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 15:17:26 -0400 Subject: [PATCH 0226/2835] allow lines with leading tab, to be preformatted text --- mdwn2man | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 mdwn2man diff --git a/mdwn2man b/mdwn2man new file mode 100755 index 0000000000..c212539454 --- /dev/null +++ b/mdwn2man @@ -0,0 +1,43 @@ +#!/usr/bin/perl +# Warning: hack + +my $prog=shift; +my $section=shift; + +print ".TH $prog $section\n"; + +while (<>) { + s{(\\?)\[\[([^\s\|\]]+)(\|[^\s\]]+)?\]\]}{$1 ? "[[$2]]" : $2}eg; + s/\`//g; + s/^\s*\./\\&./g; + if (/^#\s/) { + s/^#\s/.SH /; + <>; # blank; + } + s/^ +//; + s/^\t/ /; + s/-/\\-/g; + s/^Warning:.*//g; + s/^$/.PP\n/; + s/^\*\s+(.*)/.IP "$1"/; + next if $_ eq ".PP\n" && $skippara; + if (/^.IP /) { + $inlist=1; + $spippara=0; + } + elsif (/.SH/) { + $skippara=0; + $inlist=0; + } + elsif (/^\./) { + $skippara=1; + } + else { + $skippara=0; + } + if ($inlist && $_ eq ".PP\n") { + $_=".IP\n"; + } + + print $_; +} From 7bc4435ffdc1760a7ac8638cdc1cfac78aebaabb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 15:59:40 -0400 Subject: [PATCH 0227/2835] update --- .gitignore | 3 + Commands.hs | 2 +- Makefile | 9 +- doc/Makefile | 15 -- doc/backends.mdwn | 21 ++ doc/bugs/symlink_farming_commit_hook.mdwn | 12 ++ doc/copies.mdwn | 30 +++ doc/git-annex.mdwn | 223 +++++++++------------- doc/index.mdwn | 9 +- doc/location_tracking.mdwn | 28 +++ 10 files changed, 202 insertions(+), 150 deletions(-) delete mode 100644 doc/Makefile create mode 100644 doc/backends.mdwn create mode 100644 doc/bugs/symlink_farming_commit_hook.mdwn create mode 100644 doc/copies.mdwn create mode 100644 doc/location_tracking.mdwn diff --git a/.gitignore b/.gitignore index 2b3e3aef1f..13deb526a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ build/* git-annex +git-annex.1 +doc/.ikiwiki +html diff --git a/Commands.hs b/Commands.hs index e38f9c3725..2addf714e8 100644 --- a/Commands.hs +++ b/Commands.hs @@ -51,7 +51,7 @@ options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may lose annexed data" ] -header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." +header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) ++ " [path ...]" usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where diff --git a/Makefile b/Makefile index a0911df04c..f18f076da9 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,6 @@ install: install -d $(DESTDIR)/usr/bin install git-annex $(DESTDIR)/usr/bin -clean: - rm -rf build git-annex - rm -rf doc/.ikiwiki html - # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. ifeq ($(shell which ikiwiki),) @@ -21,7 +17,12 @@ IKIWIKI=ikiwiki endif docs: + ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs +clean: + rm -rf build git-annex git-annex.1 + rm -rf doc/.ikiwiki html + .PHONY: git-annex diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index f2c4d8e54d..0000000000 --- a/doc/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# Build static html docs suitable for being shipped in the software -# package. This depends on ikiwiki being installed to build the docs. - -ifeq ($(shell which ikiwiki),) -IKIWIKI=echo "** ikiwiki not found" >&2 ; echo ikiwiki -else -IKIWIKI=ikiwiki -endif - -all: - $(IKIWIKI) `pwd` html -v --wikiname FooBar --plugin=goodstuff \ - --exclude=html --exclude=Makefile - -clean: - rm -rf .ikiwiki html diff --git a/doc/backends.mdwn b/doc/backends.mdwn new file mode 100644 index 0000000000..d3ccaec491 --- /dev/null +++ b/doc/backends.mdwn @@ -0,0 +1,21 @@ +git-annex uses a key-value abstraction layer to allow file contents to be +stored in different ways. In theory, any key-value storage system could be +used to store file contents. + +When a file is annexed, a key is generated from its content and/or metadata. +The file checked into git symlinks to the key. This key can later be used +to retrieve the file's content (its value). + +Multiple pluggable backends are supported, and more than one can be used +to store different files' contents in a given repository. + +* `WORM` ("Write Once, Read Many") This backend stores the file's content + only in `.git/annex/`, and assumes that any file with the same basename, + size, and modification time has the same content. So with this backend, + files can be moved around, but should never be added to or changed. + This is the default, and the least expensive backend. +* `SHA1` -- This backend stores the file's content in + `.git/annex/`, with a name based on its sha1 checksum. This backend allows + modifications of files to be tracked. Its need to generate checksums + can make it slower for large files. +* `URL` -- This backend downloads the file's content from an external URL. diff --git a/doc/bugs/symlink_farming_commit_hook.mdwn b/doc/bugs/symlink_farming_commit_hook.mdwn new file mode 100644 index 0000000000..af03beb70e --- /dev/null +++ b/doc/bugs/symlink_farming_commit_hook.mdwn @@ -0,0 +1,12 @@ +TODO: implement below + +git-annex does use a lot of symlinks. Specicially, relative symlinks, +that are checked into git. To allow you to move those around without +annoyance, git-annex can run as a post-commit hook. This way, you can `git mv` +a symlink to an annexed file, and as soon as you commit, it will be fixed +up. + +`git annex init` tries to set up a post-commit hook that is itself a symlink +back to git-annex. If you want to have your own shell script in the post-commit +hook, just make it call `git annex` with no parameters. git-annex will detect +when it's run from a git hook and do the necessary fixups. diff --git a/doc/copies.mdwn b/doc/copies.mdwn new file mode 100644 index 0000000000..ff66f4e8ae --- /dev/null +++ b/doc/copies.mdwn @@ -0,0 +1,30 @@ +The WORM and SHA1 key-value [[backends|backend]] store data inside +your git repository's `.git` directory, not in some external data store. + +It's important that data not get lost by an ill-considered `git annex drop` +command. So, then using those backends, git-annex can be configured to try +to keep N copies of a file's content available across all repositories. By +default, N is 1; it is configured by annex.numcopies. + +`git annex drop` attempts to check with other git remotes, to check that N +copies of the file exist. If enough repositories cannot be verified to have +it, it will retain the file content to avoid data loss. + +For example, consider three repositories: Server, Laptop, and USB. Both Server +and USB have a copy of a file, and N=1. If on Laptop, you `git annex get +$file`, this will transfer it from either Server or USB (depending on which +is available), and there are now 3 copies of the file. + +Suppose you want to free up space on Laptop again, and you `git annex drop` the file +there. If USB is connected, or Server can be contacted, git-annex can check +that it still has a copy of the file, and the content is removed from +Laptop. But if USB is currently disconnected, and Server also cannot be +contacted, it can't verify that it is safe to drop the file, and will +refuse to do so. + +With N=2, in order to drop the file content from Laptop, it would need access +to both USB and Server. + +Note that different repositories can be configured with different values of +N. So just because Laptop has N=2, this does not prevent the number of +copies falling to 1, when USB and Server have N=1. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index eb5fa9ced9..25cf6f776c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1,3 +1,13 @@ +# NAME + +git-annex - manage files with git, without checking their contents in + +# SYNOPSIS + +git annex subcommand [path ...] + +# DESCRIPTION + git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when dealing with files larger than git can currently easily handle, whether due @@ -11,157 +21,94 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. -My motivation for git-annex was the growing number of external drives I -use. Some are used to archive data, others hold backups, and yet others -come with me when I'm away from home to carry data that doesn't fit on my -netbook. Maintaining all that was a nightmare, lots of ad-hoc moving files -around, rsyncing files (unison is too slow), and deleting multiple copies -of files from multiple places. I realized what what I needed was a form of -revision control where each drive was a repository, and where copying the -files around, and deciding which copies were safe to delete was automated. -I posted about this to the VCS-home mailing list and got a great suggestion -to make it support arbitrary key-value stores, for more generality and -flexability. A week of coding later, and git-annex is born. +When a file is annexed, its content is moved into a key-value store, and +a symlink is made that points to the content. These symlinks are checked into +git and versioned like regular files. You can move them around, delete +them, and so on. Pushing to another git repository will make git-annex +there aware of the annexed file, and it can be used to retrieve its +content from the key-value store. -Enough broad picture, here's how it actually looks: +# EXAMPLES -* `git annex add $file` moves the file into `.git/annex/`, and replaces - it with a symlink pointing at the annexed file, and then calls `git add` - to version the *symlink*. (If the file has already been annexed, it does - nothing.) - - If you then use normal git push/pull commands, the annexed file content - won't be transferred between repositories, but the symlinks will be. - So different clones of a repository can have different sets of annexed - files available. - - You can move the symlink around, copy it, delete it, etc, and commit changes - as desired using git. Reading the symlink will always get you the annexed - file content, or the link may be broken if the content is not currently - available. -* `git annex get $file` is used to transfer a specified file from the - backend storage to the current repository. -* `git annex drop $file` indicates that you no longer want the file's - content to be available in this repository. -* `git annex file $file` adjusts the symlink for the file to point to its - content again. Use this if you've moved the file around. -* `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` - if you're just done with a file; only use `unannex` if you - accidentially added a file. (You can also run this on all your annexed - files come the Singularity. ;-) -* `git annex init "some description"` allows associating some description - (such as "USB archive drive 1") with a repository. This can help with - finding it later, see "Location Tracking" below. + # git annex get video/hackity_hack_and_kaxxt.mov + get video/_why_hackity_hack_and_kaxxt.mov (not available) + I was unable to access these remotes: server + Try making some of these repositories available: + 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + failed + # sudo mount /media/usb + # git remote add usbdrive /media/usb + # git annex get video/hackity_hack_and_kaxxt.mov + get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok + # git commit -a -m "got a video I want to rewatch on the plane" + + # git annex add iso + add iso/Debian_5.0.iso ok + # git commit -a -m "saving Debian CD for later" + + # git annex push usbdrive iso + error: push not yet implemented! + # git annex drop iso + drop iso/Debian_5.0.iso ok + # git commit -a -m "freed up space" -Oh yeah, "$file" in the above can be any number of files, or directories, -same as you'd pass to "git add" or "git rm". -So "git annex add ." or "git annex get dir/" work fine. +# SUBCOMMANDS -## key-value storage +Like many git commands, git-annex can be passed a path that +is either a file or a directory. In the latter case it acts on all relevant +files in the directory. -git-annex uses a key-value abstraction layer to allow file contents to be -stored in different ways. In theory, any key-value storage system could be -used to store the file contents, and git-annex would then retrieve them -as needed and put them in `.git/annex/`. +Many git-annex subcommands will stage changes for later `git commit` by you. -When a file is annexed, a key is generated from its content and/or metadata. -The file checked into git symlinks to the key. This key can later be used -to retrieve the file's content (its value). This key generation must be -stable for a given file content, name, and size. +* add [path ...] -Multiple pluggable backends are supported, and more than one can be used -to store different files' contents in a given repository. + Adds files in the path to the annex. Files that are already checked into + git, or that git has been configured to ignore will be silently skipped. -* `WORM` ("Write Once, Read Many") This backend stores the file's content - only in `.git/annex/`, and assumes that any file with the same basename, - size, and modification time has the same content. So with this backend, - files can be moved around, but should never be added to or changed. - This is the default, and the least expensive backend. -* `SHA1` -- This backend stores the file's content in - `.git/annex/`, with a name based on its sha1 checksum. This backend allows - modifications of files to be tracked. Its need to generate checksums - can make it slow for large files. -* `URL` -- This backend downloads the file's content from an external URL. +* get [path ...] -## copies + Makes the content of annexed files available in this repository. Depending + on the backend used, this will involve copying them from another repository, + or downloading them, or transferring them from some kind of key-value store. -The WORM and SHA1 key-value backends store data inside your git repository. -It's important that data not get lost by an ill-though `git annex drop` -command. So, then using those backends, git-annex can be configured to try -to keep N copies of a file's content available across all repositories. By -default, N is 1; it is configured by annex.numcopies. +* drop [path ...] -`git annex drop` attempts to check with other git remotes, to check that N -copies of the file exist. If enough repositories cannot be verified to have -it, it will retain the file content to avoid data loss. + Drops the content of annexed files from this repository. -For example, consider three repositories: Server, Laptop, and USB. Both Server -and USB have a copy of a file, and N=1. If on Laptop, you `git annex get -$file`, this will transfer it from either Server or USB (depending on which -is available), and there are now 3 copies of the file. + git-annex may refuse to drop a content if the backend does not think + it is safe to do so. -Suppose you want to free up space on Laptop again, and you `git annex drop` the file -there. If USB is connected, or Server can be contacted, git-annex can check -that it still has a copy of the file, and the content is removed from -Laptop. But if USB is currently disconnected, and Server also cannot be -contacted, it can't verify that it is safe to drop the file, and will -refuse to do so. +* unannex [path ...] -With N=2, in order to drop the file content from Laptop, it would need access -to both USB and Server. + Use this to undo an accidental add command. This is not the command you + should use if you intentionally annexed a file and don't want its contents + any more. In that case you should use `git annex drop` instead, and you + can also `git rm` the file. -Note that different repositories can be configured with different values of -N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1. +* init "description" -## location tracking + Initializes git-annex with a descripotion of the git repository. + This is an optional, but recommended step. -git-annex keeps track of in which repositories it last saw a file's content. -This location tracking information is stored in `.git-annex/$key.log`. -Repositories record their UUID and the date when they get or drop -a file's content. (Git is configured to use a union merge for this file, -so the lines may be in arbitrary order, but it will never conflict.) +* fix [path ...] -This location tracking information is useful if you have multiple -repositories, and not all are always accessible. For example, perhaps one -is on a home file server, and you are away from home. Then git-annex can -tell you what git remote it needs access to in order to get a file: + Fixes up symlinks that have become broken to again point to annexed content. + This is useful to run if you have been moving the symlinks around. - # git annex get myfile - get myfile (need access to one of these remotes: home) - git-annex: get myfile failed +# OPTIONS -Another way the location tracking comes in handy is if you put repositories -on removable USB drives, that might be archived away offline in a safe -place. In this sort of case, you probably don't have a git remotes -configured for every USB drive. So git-annex may have to resort to talking -about repository UUIDs. If you have previously used "git annex init" -to attach descriptions to those repositories, it will include their -descriptions to help you with finding them: +* --force - # git annex get myfile - get myfile (No available git remotes have the file.) - It has been seen before in these repositories: - c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 - e1938fee-d95b-11df-96cc-002170d25c55 - git-annex: get myfile failed + Force unsafe actions, such as dropping a file's content when no other + source of it can be verified to still exist. Use with care. -## symlink farming commit hook +## CONFIGURATION -git-annex does use a lot of symlinks. Specicially, relative symlinks, -that are checked into git. To allow you to move those around without -annoyance, git-annex can run as a post-commit hook. This way, you can `git mv` -a symlink to an annexed file, and as soon as you commit, it will be fixed -up. +Like other git commands, git-annex is configured via `.git/config`. -`git annex init` tries to set up a post-commit hook that is itself a symlink -back to git-annex. If you want to have your own shell script in the post-commit -hook, just make it call `git annex` with no parameters. git-annex will detect -when it's run from a git hook and do the necessary fixups. - -## configuration - -* `annex.uuid` -- a unique UUID for this repository +* `annex.uuid` -- a unique UUID for this repository (automatically set) * `annex.numcopies` -- number of copies of files to keep across all repositories (default: 1) * `annex.backends` -- space-separated list of names of @@ -176,6 +123,24 @@ when it's run from a git hook and do the necessary fixups. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. -## contact +# FILES -Joey Hess +These files are used, in your git repository: + +`.git/annex/` contains the annexed file contents that are currently +available. Annexed files in your git repository symlink to that content. + +`.git-annex/uuid.log` is used to map between repository UUID and +decscriptions. You may edit it. + +`.git-annex/*.log` is where git-annex records its content tracking +information. These files should be committed to git. + +`.git-annex/.gitattributes` is configured to use git's union merge driver +to avoid conflicts when merging files in the `.git-annex` directory. + +# AUTHOR + +Joey Hess + +Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD). Edit with care diff --git a/doc/index.mdwn b/doc/index.mdwn index 8d2cb1ef57..df42eabc11 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -11,12 +11,19 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. -* [[man page|git-annex]] * **[[download]]** * [[news]] * [[bugs]] * [[contact]] +## documentation + +* [[man page|git-annex]] +* [[key-value backends|backends]] for data storage +* [[location_tracking]] reminds you where git-annex has seen files +* git-annex prevents accidential data loss by [[tracking copies|copies]] + of your files + ---- git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn new file mode 100644 index 0000000000..a7d5c150b1 --- /dev/null +++ b/doc/location_tracking.mdwn @@ -0,0 +1,28 @@ +git-annex keeps track of in which repositories it last saw a file's content. +This location tracking information is stored in `.git-annex/$key.log`. +Repositories record their UUID and the date when they get or drop +a file's content. (Git is configured to use a union merge for this file, +so the lines may be in arbitrary order, but it will never conflict.) + +This location tracking information is useful if you have multiple +repositories, and not all are always accessible. For example, perhaps one +is on a home file server, and you are away from home. Then git-annex can +tell you what git remote it needs access to in order to get a file: + + # git annex get myfile + get myfile(not available) + I was unable to access these remotes: home + +Another way the location tracking comes in handy is if you put repositories +on removable USB drives, that might be archived away offline in a safe +place. In this sort of case, you probably don't have a git remotes +configured for every USB drive. So git-annex may have to resort to talking +about repository UUIDs. If you have previously used "git annex init" +to attach descriptions to those repositories, it will include their +descriptions to help you with finding them: + + # git annex get myfile + get myfile (not available) + Try making some of these repositories available: + c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 + e1938fee-d95b-11df-96cc-002170d25c55 From 05539c773ea1246de253e1373ca1711412b91503 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:01:48 -0400 Subject: [PATCH 0228/2835] split TODO --- TODO | 21 --------------------- debian/docs | 3 +-- doc/bugs/backendchecksum.mdwn | 1 + doc/bugs/dotdot_problem.mdwn | 2 ++ doc/bugs/file_copy_progress_bar.mdwn | 3 +++ doc/bugs/fsck.mdwn | 1 + doc/bugs/gitrm.mdwn | 2 ++ doc/bugs/network_remotes.mdwn | 3 +++ doc/bugs/pushpull.mdwn | 2 ++ 9 files changed, 15 insertions(+), 23 deletions(-) delete mode 100644 TODO create mode 100644 doc/bugs/backendchecksum.mdwn create mode 100644 doc/bugs/dotdot_problem.mdwn create mode 100644 doc/bugs/file_copy_progress_bar.mdwn create mode 100644 doc/bugs/fsck.mdwn create mode 100644 doc/bugs/gitrm.mdwn create mode 100644 doc/bugs/network_remotes.mdwn create mode 100644 doc/bugs/pushpull.mdwn diff --git a/TODO b/TODO deleted file mode 100644 index a804597b88..0000000000 --- a/TODO +++ /dev/null @@ -1,21 +0,0 @@ -* bug: cannot "git annex ../foo" (GitRepo.relative is buggy and - git-ls-files also refuses w/o --full-name, which would need other changes) - -* --push/--pull should take a reponame and files, and push those files - to that repo; dropping them from the current repo - -* how to handle git mv file? -> git annex fix -> run automatically on commit - -* how to handle git rm file? (should try to drop keys that have no - referring file, if it seems safe..) - -* add a git annex fsck that finds keys that have no referring file - -* Support for remote git repositories (ssh:// specifically can be made to - work, although the other end probably needs to have git-annex installed..) - -* Find a way to copy a file with a progress bar, while still preserving - stat. Easiest way might be to use pv and fix up the permissions etc - after? - -* finish BackendChecksum diff --git a/debian/docs b/debian/docs index d6fa57dd7b..1936cc1d44 100644 --- a/debian/docs +++ b/debian/docs @@ -1,2 +1 @@ -doc/*.mdwn -TODO +html diff --git a/doc/bugs/backendchecksum.mdwn b/doc/bugs/backendchecksum.mdwn new file mode 100644 index 0000000000..40ff868c22 --- /dev/null +++ b/doc/bugs/backendchecksum.mdwn @@ -0,0 +1 @@ +This backend is not finished. diff --git a/doc/bugs/dotdot_problem.mdwn b/doc/bugs/dotdot_problem.mdwn new file mode 100644 index 0000000000..9d247a9c09 --- /dev/null +++ b/doc/bugs/dotdot_problem.mdwn @@ -0,0 +1,2 @@ +cannot "git annex ../foo" (GitRepo.relative is buggy and +git-ls-files also refuses w/o --full-name, which would need other changes) diff --git a/doc/bugs/file_copy_progress_bar.mdwn b/doc/bugs/file_copy_progress_bar.mdwn new file mode 100644 index 0000000000..cd4ea33b78 --- /dev/null +++ b/doc/bugs/file_copy_progress_bar.mdwn @@ -0,0 +1,3 @@ +Find a way to copy a file with a progress bar, while still preserving +stat. Easiest way might be to use pv and fix up the permissions etc +after? diff --git a/doc/bugs/fsck.mdwn b/doc/bugs/fsck.mdwn new file mode 100644 index 0000000000..308a1cb63a --- /dev/null +++ b/doc/bugs/fsck.mdwn @@ -0,0 +1 @@ +add a git annex fsck that finds keys that have no referring file diff --git a/doc/bugs/gitrm.mdwn b/doc/bugs/gitrm.mdwn new file mode 100644 index 0000000000..d771aa32a7 --- /dev/null +++ b/doc/bugs/gitrm.mdwn @@ -0,0 +1,2 @@ +how to handle git rm file? (should try to drop keys that have no +referring file, if it seems safe..) diff --git a/doc/bugs/network_remotes.mdwn b/doc/bugs/network_remotes.mdwn new file mode 100644 index 0000000000..be43ee20be --- /dev/null +++ b/doc/bugs/network_remotes.mdwn @@ -0,0 +1,3 @@ +Support for remote git repositories (ssh:// specifically can be made to +work, although the other end probably needs to have git-annex +installed..) diff --git a/doc/bugs/pushpull.mdwn b/doc/bugs/pushpull.mdwn new file mode 100644 index 0000000000..47da2107f2 --- /dev/null +++ b/doc/bugs/pushpull.mdwn @@ -0,0 +1,2 @@ +--push/--pull should take a reponame and files, and push those files + to that repo; dropping them from the current repo From b1d22b3229a1cc86e0df8d4899b44d3e44e960d0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:07:33 -0400 Subject: [PATCH 0229/2835] update --- doc/bugs/using_url_backend.mdwn | 9 +++++++++ doc/index.mdwn | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 doc/bugs/using_url_backend.mdwn diff --git a/doc/bugs/using_url_backend.mdwn b/doc/bugs/using_url_backend.mdwn new file mode 100644 index 0000000000..a0d447c6e2 --- /dev/null +++ b/doc/bugs/using_url_backend.mdwn @@ -0,0 +1,9 @@ +There is no way to `git annex add` a file using the URL [[backend|backends]]. + +For now, we have to manually make the symlink. Something like this: + + ln -s .git/annex/URL:http:%%www.example.com%foo.tar.gz + +Note the escaping of slashes. + +A `git annex register ` command could do this.. diff --git a/doc/index.mdwn b/doc/index.mdwn index df42eabc11..285f84f132 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -18,7 +18,7 @@ revision control. ## documentation -* [[man page|git-annex]] +* [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage * [[location_tracking]] reminds you where git-annex has seen files * git-annex prevents accidential data loss by [[tracking copies|copies]] From d6911f57b76a6469484076dc991adb39d8d9b8e0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:17:29 -0400 Subject: [PATCH 0230/2835] update --- Makefile | 2 +- debian/control | 2 +- debian/copyright | 5 + debian/manpages | 1 + doc/GPL | 339 +++++++++++++++++++++++++++++++++++++++++++++ doc/git-annex.mdwn | 4 +- doc/index.mdwn | 1 + 7 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 debian/copyright create mode 100644 debian/manpages create mode 100644 doc/GPL diff --git a/Makefile b/Makefile index f18f076da9..39f5ba8ad7 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ endif docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ - --no-usedirs + --no-usedirs --disable-plugin=openid clean: rm -rf build git-annex git-annex.1 diff --git a/debian/control b/debian/control index 846844c32a..4e3ad01bdb 100644 --- a/debian/control +++ b/debian/control @@ -5,7 +5,7 @@ Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-de Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex -Homepage: http://kitenet.net/~joey/code/git-annex/ +Homepage: http://git-annex.branchable.com/ Package: git-annex Architecture: any diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000000..5d0ae13c87 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,5 @@ +Files: * +Copyright: © 2010 Joey Hess +License: GPL-2+ + The full text of the GPL is distributed as doc/GPL in this package's + source, or in /usr/share/common-licenses/GPL on Debian systems. diff --git a/debian/manpages b/debian/manpages new file mode 100644 index 0000000000..ca34203aa0 --- /dev/null +++ b/debian/manpages @@ -0,0 +1 @@ +git-annex.1 diff --git a/doc/GPL b/doc/GPL new file mode 100644 index 0000000000..d159169d10 --- /dev/null +++ b/doc/GPL @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 25cf6f776c..09b245497c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -87,7 +87,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. any more. In that case you should use `git annex drop` instead, and you can also `git rm` the file. -* init "description" +* init description Initializes git-annex with a descripotion of the git repository. This is an optional, but recommended step. @@ -143,4 +143,6 @@ to avoid conflicts when merging files in the `.git-annex` directory. Joey Hess + + Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD). Edit with care diff --git a/doc/index.mdwn b/doc/index.mdwn index 285f84f132..3541847fa4 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -23,6 +23,7 @@ revision control. * [[location_tracking]] reminds you where git-annex has seen files * git-annex prevents accidential data loss by [[tracking copies|copies]] of your files +* git-annex is Free Software, licensed under the [[GPL]]. ---- From d53519fade3baf9dfb3a92adcfa3c03d852171d2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:30:17 -0400 Subject: [PATCH 0231/2835] probably won't use SHA module --- Backend/SHA1.hs | 2 -- debian/control | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 2143a6af59..caece6b78c 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -3,8 +3,6 @@ module Backend.SHA1 (backend) where -import Data.Digest.Pure.SHA - import qualified Backend.File import TypeInternals diff --git a/debian/control b/debian/control index 4e3ad01bdb..83bc8c82bf 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-dev, ikiwiki +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, ikiwiki Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From 0f153765b7552054cb459730b95477f8f4f1ae21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:32:40 -0400 Subject: [PATCH 0232/2835] update --- INSTALL | 8 +------- doc/download.mdwn | 2 ++ doc/index.mdwn | 1 + 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/INSTALL b/INSTALL index a7fc7f6f31..f024541c1a 100644 --- a/INSTALL +++ b/INSTALL @@ -1,7 +1 @@ -To build and use git-annex, you will need: - -* ghc -* These haskell libraries: MissingH SHA -* uuid - -Then just run make; make install +See doc/install.mdwn for installation instructions. diff --git a/doc/download.mdwn b/doc/download.mdwn index 2ceb73193a..664f46ed95 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -3,3 +3,5 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` There are no binary packages yet, but you can build Debian packages from the source tree with `dpkg-buildpackage`. + +Next: [[install]] diff --git a/doc/index.mdwn b/doc/index.mdwn index 3541847fa4..e30326853e 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -12,6 +12,7 @@ etc that are associated with annexed files but that benefit from full revision control. * **[[download]]** +* [[install]] * [[news]] * [[bugs]] * [[contact]] From e7572f9249f0e4c3f757bb8da889a41f53fd9e34 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:32:50 -0400 Subject: [PATCH 0233/2835] add --- doc/install.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/install.mdwn diff --git a/doc/install.mdwn b/doc/install.mdwn new file mode 100644 index 0000000000..cc6fb6fb35 --- /dev/null +++ b/doc/install.mdwn @@ -0,0 +1,7 @@ +To build and use git-annex, you will need: + +* The Haskell Platform: +* MissingH: +* uuid: + +Then just [[download]] git-annex and run: `make; make install` From 9d5b8ebab0e247c5c9c05a5216dcd4c638299190 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 19:28:29 -0400 Subject: [PATCH 0234/2835] update --- Makefile | 3 ++- doc/bugs.mdwn | 2 +- doc/bugs/done.mdwn | 3 ++- doc/index.mdwn | 18 ++++++++++++++++++ doc/not.mdwn | 10 ++++++++++ doc/templates/bare.tmpl | 1 + doc/use_case/Alice.mdwn | 18 ++++++++++++++++++ doc/use_case/Bob.mdwn | 18 ++++++++++++++++++ 8 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 doc/not.mdwn create mode 100644 doc/templates/bare.tmpl create mode 100644 doc/use_case/Alice.mdwn create mode 100644 doc/use_case/Bob.mdwn diff --git a/Makefile b/Makefile index 39f5ba8ad7..d35e82ad55 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,8 @@ endif docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ - --no-usedirs --disable-plugin=openid + --no-usedirs --disable-plugin=openid --plugin=sidebar \ + --underlaydir=/dev/null clean: rm -rf build git-annex git-annex.1 diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn index dd2a9b4035..2786e5bf74 100644 --- a/doc/bugs.mdwn +++ b/doc/bugs.mdwn @@ -1,4 +1,4 @@ This is git-annex's bug list. Link bugs to [[bugs/done]] when done. [[!inline pages="./bugs/* and !./bugs/done and !link(done) -and !*/Discussion" actions=yes postform=yes show=0]] +and !*/Discussion" actions=yes postform=yes show=0 archive=yes]] diff --git a/doc/bugs/done.mdwn b/doc/bugs/done.mdwn index ad332e2a26..a35d427198 100644 --- a/doc/bugs/done.mdwn +++ b/doc/bugs/done.mdwn @@ -1,3 +1,4 @@ recently fixed [[bugs]] -[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10]] +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10 +archive=yes]] diff --git a/doc/index.mdwn b/doc/index.mdwn index e30326853e..b3a871627f 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -11,11 +11,28 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. +[[!sidebar content=""" * **[[download]]** * [[install]] * [[news]] * [[bugs]] * [[contact]] +"""]] + + +## sample use cases + + + + + + +
[[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]]
+ +If that describes you, or if you're some from column A and some from column +B, then git-annex may be the tool you've been looking for to expand from +keeping all your small important files in git, to managing your large +files with git. ## documentation @@ -24,6 +41,7 @@ revision control. * [[location_tracking]] reminds you where git-annex has seen files * git-annex prevents accidential data loss by [[tracking copies|copies]] of your files +* [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. ---- diff --git a/doc/not.mdwn b/doc/not.mdwn new file mode 100644 index 0000000000..2697a9b1fd --- /dev/null +++ b/doc/not.mdwn @@ -0,0 +1,10 @@ +[[!meta title="what git-annex is not"]] + +* git-annex is not a backup system. It may be a useful component of an + [[archival|use_case/bob]] system, or a way to deliver files to a backup + system. + + For a backup system that uses git, take a look at + [bup](http://github.com/apenwarr/bup). + +* probably several other things.. diff --git a/doc/templates/bare.tmpl b/doc/templates/bare.tmpl new file mode 100644 index 0000000000..2d476b716f --- /dev/null +++ b/doc/templates/bare.tmpl @@ -0,0 +1 @@ + diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn new file mode 100644 index 0000000000..c42eb3a74c --- /dev/null +++ b/doc/use_case/Alice.mdwn @@ -0,0 +1,18 @@ +### The Nomad + +Alice is always on the move, often with her trusty netbook and a small +handheld terabyte USB drive, or a smaller USB keydrive. She has a server +out there on the net. All these things can have different files on them, +but Alice no longer has to deal with the tedious process of keeping them +manually in sync. + +When she has 1 bar on her cell, Alice queues up interesting files on her +server for later. At a coffee shop, she has git-annex download them to her +USB drive. High in the sky or in a remote cabin, she catches up on +podcasts, videos, and games, first letting git-annex copy them from +her USB drive to the netbook (this saves battery power). + +When she's done, she tells git-annex which to keep and which to remove. +They're all removed from her netbook to save space, and Alice knowns +that next time she syncs up to the net, her changes will be synced back +to her server. diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn new file mode 100644 index 0000000000..a5dc01b373 --- /dev/null +++ b/doc/use_case/Bob.mdwn @@ -0,0 +1,18 @@ +### The Archivist + +Bob has many drives to archive his data, most of them kept offline, in a +safe place. + +With git-annex, Bob has a single directory tree that includes all +his files, even if their content is being stored offline. He can +reorganize his files using that tree, committing new versions to git, +without worry about accidentially deleting anything. + +When Bob needs access to some files, git-annex can tell him which drive(s) +they're on, and easily make them available. Indeed, every drive knows what +is on every other drive. + +Run in a cron job, git-annex adds new files to achival drives at night. It +also helps Bob keep track of intentional, and unintentional copies of +files, and logs information he can use to decide when it's time to duplicate +the content of old drives. From b08b45815dffeadd5f5fb2492bb4e5c36b921aee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 19:30:08 -0400 Subject: [PATCH 0235/2835] update --- doc/not.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index 2697a9b1fd..a91a6a7a04 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -7,4 +7,8 @@ For a backup system that uses git, take a look at [bup](http://github.com/apenwarr/bup). +* git-annex is not unison, but if you're finding unison's checksumming + too slow, or its strict mirroring of everything to both places too + limiting, then git-annex could be a useful alternative. + * probably several other things.. From 0b84d02b8427a7bb2fb2da5e9abf606f0e67dca9 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 19 Oct 2010 23:35:13 +0000 Subject: [PATCH 0236/2835] initial commit --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..b84c806686 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.ikiwiki +/recentchanges From 972639d85c663855dd0c7476b732dcb319efdb2e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 19:35:56 -0400 Subject: [PATCH 0237/2835] update --- doc/not.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/not.mdwn b/doc/not.mdwn index a91a6a7a04..affcb57f11 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -11,4 +11,6 @@ too slow, or its strict mirroring of everything to both places too limiting, then git-annex could be a useful alternative. -* probably several other things.. +* git-annex is not some flaky script that was quickly thrown together. + I wrote it in Haskell because I wanted it to be solid and to compile + down to a binary. From 54db046c5ae2e7fce73fcd5ce4da278b5f8b445c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 19:38:14 -0400 Subject: [PATCH 0238/2835] clean --- .gitignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index b84c806686..0000000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.ikiwiki -/recentchanges From cdfea4debc86d49fb4003fc4b27e76d056d318fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:07:50 -0400 Subject: [PATCH 0239/2835] add walkthrough --- doc/index.mdwn | 3 +- doc/walkthrough.mdwn | 112 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 doc/walkthrough.mdwn diff --git a/doc/index.mdwn b/doc/index.mdwn index b3a871627f..de5fe55a3c 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -11,6 +11,8 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. +To get a feel for it, see the [[walkthrough]]. + [[!sidebar content=""" * **[[download]]** * [[install]] @@ -19,7 +21,6 @@ revision control. * [[contact]] """]] - ## sample use cases diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn new file mode 100644 index 0000000000..056361e13b --- /dev/null +++ b/doc/walkthrough.mdwn @@ -0,0 +1,112 @@ +## creating a repository + +This is very straightforward. Just tell it a description of the repository. + + # mkdir ~/annex + # cd ~/annex + # git init + # git annex init "my laptop" + +## adding a remote + +This could be a USB drive, or a sshfs or NFS mount to a file server, for +example. + + # sudo mount /media/usb + # cd /media/usb + # git clone ~/annex + # cd annex + # git annex init "portable USB drive" + # git remote add home ~/annex + # cd ~/annex + # git remote add usbdrive /media/usb + +There was nothing git-annex specific about that, except telling it the name +of the new repository created on the USB drive. + +Notice that both repos are set up as remotes of the other one. This lets +either get annexed files from the other. + +## adding files + + # cd ~/annex + # cp /tmp/big_file . + # cp /tmp/debian.iso . + # git annex add . + add big_file ok + add debian.iso ok + # git commit -a -m added + +Notice you commit at the end, this checks in git-annex's record of the +files but not thier actual, large, content. + +## transferring files around + +Let's copy everything in the laptop's home annex to the USB drive. + + # cd /media/usb/annex + # git pull home master + # git annex get . + get big_file (copying from home...) ok + get debian.iso (copying from home...) ok + +Notice that you had to git pull from home first, this lets git-annex know +what has changed in home, and so it knows about the files you added and +can get them. + +## transferring files: When things go wrong + +After a while, you'll have serveral annexes, with different file contents. +You don't have to try to keep all that straight; git-annex does +[[location_tracking] for you. If you ask it to get a file and the drive +or file server is not accessible, it will let you know what it needs to get +it: + + # git annex get video/hackity_hack_and_kaxxt.mov + get video/_why_hackity_hack_and_kaxxt.mov (not available) + I was unable to access these remotes: server + Try making some of these repositories available: + 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + failed + # sudo mount /media/usb + # git annex get video/hackity_hack_and_kaxxt.mov + get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok + # git commit -a -m "got a video I want to rewatch on the plane" + +## removing files + +You can always drop files safely. Git-annex checks that some other annex +has the file before removing it. + + # git annex drop debian.iso + drop iso/Debian_5.0.iso ok + # git commit -a -m "freed up space" + +## removing files: When things go wrong + +Before dropping a file, git-annex wants to be able to look at other +remotes, and verify that they still have a file. After all, it could +have been dropped from them too. If the remotes are not mounted/available, +you'll see something like this. + + # git annex drop important_file other.iso + drop important_file (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + I was unable to access these remotes: usbdrive + Try making some of these repositories available: + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + (Use --force to override this check, or adjust annex.numcopies.) + failed + drop other.iso (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + No other repository is known to contain the file. + (Use --force to override this check, or adjust annex.numcopies.) + failed + +Here you might --force it to drop `important_file` if you trust your backup. +But `other.iso` looks to have never been copied to anywhere else, so if +it's something you want to hold onto, you'd need to transfer it to +some other repository before dropping it. From bbac7178e8798751d0c2a2b8ee2f4f119aaece6c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:10:45 -0400 Subject: [PATCH 0240/2835] on bare --- doc/walkthrough.mdwn | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 056361e13b..24c3d71368 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -21,11 +21,16 @@ example. # cd ~/annex # git remote add usbdrive /media/usb -There was nothing git-annex specific about that, except telling it the name +This is all standard ad-hoc distributed git repository setup. Or you +could have added a centralized bare repository on a server if you prefer +doing things that way. + +Anyway, the only git-annex specific part is telling it the name of the new repository created on the USB drive. Notice that both repos are set up as remotes of the other one. This lets -either get annexed files from the other. +either get annexed files from the other. You'll want to do that even +if you are using a centralized bare repository. ## adding files From a7cc89f1eb96add4ad18812f47721e9e6c652580 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:19:52 -0400 Subject: [PATCH 0241/2835] fix --- doc/walkthrough.mdwn | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 24c3d71368..e5cbc7df19 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -45,6 +45,21 @@ if you are using a centralized bare repository. Notice you commit at the end, this checks in git-annex's record of the files but not thier actual, large, content. +## renaming files + + # cd ~/annex + # git mv big_file my_cool_big_file + # mkdir iso + # git mv debian.iso iso + # git annex fix . + fix iso/debian.iso ok + # git commit -m moved + +You can use any normal git operations to move files around, or even +make copies or delete them. `git-annex fix` needs to be run if a file +is moved into a different directory, in order to fix up the symlink +pointing to the file's content. + ## transferring files around Let's copy everything in the laptop's home annex to the USB drive. From 14dd4dc316d8d30b972f51b72459ca6e63f44f39 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:22:37 -0400 Subject: [PATCH 0242/2835] tweak --- doc/walkthrough.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index e5cbc7df19..fb824c995a 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -67,8 +67,8 @@ Let's copy everything in the laptop's home annex to the USB drive. # cd /media/usb/annex # git pull home master # git annex get . - get big_file (copying from home...) ok - get debian.iso (copying from home...) ok + get my_cool_big_file (copying from home...) ok + get iso/debian.iso (copying from home...) ok Notice that you had to git pull from home first, this lets git-annex know what has changed in home, and so it knows about the files you added and @@ -100,7 +100,7 @@ it: You can always drop files safely. Git-annex checks that some other annex has the file before removing it. - # git annex drop debian.iso + # git annex drop iso/debian.iso drop iso/Debian_5.0.iso ok # git commit -a -m "freed up space" From 277b0f0de10d0ca10007c61255eb5a8486e702f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:27:07 -0400 Subject: [PATCH 0243/2835] add --- doc/bugs/{backendchecksum.mdwn => backendSHA1.mdwn} | 0 doc/news/prerelease.mdwn | 9 +++++++++ 2 files changed, 9 insertions(+) rename doc/bugs/{backendchecksum.mdwn => backendSHA1.mdwn} (100%) create mode 100644 doc/news/prerelease.mdwn diff --git a/doc/bugs/backendchecksum.mdwn b/doc/bugs/backendSHA1.mdwn similarity index 100% rename from doc/bugs/backendchecksum.mdwn rename to doc/bugs/backendSHA1.mdwn diff --git a/doc/news/prerelease.mdwn b/doc/news/prerelease.mdwn new file mode 100644 index 0000000000..3ecdf01241 --- /dev/null +++ b/doc/news/prerelease.mdwn @@ -0,0 +1,9 @@ +git-annex is not yet formally released. If you [[download]] it from git, +everything described on this web site should already work, and I hope work +well, with these exceptions: + +* The SHA1 backend is incomplete. + +I reserve the right to change its data formats before the first release, +though I already have enough drives using it that I'd probably have to +write a transition tool anyway! From a069ed164fa9da63f62d0b4f63bde61b9117630f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:29:38 -0400 Subject: [PATCH 0244/2835] sign --- doc/news/prerelease.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/news/prerelease.mdwn b/doc/news/prerelease.mdwn index 3ecdf01241..3a1227ff0a 100644 --- a/doc/news/prerelease.mdwn +++ b/doc/news/prerelease.mdwn @@ -6,4 +6,4 @@ well, with these exceptions: I reserve the right to change its data formats before the first release, though I already have enough drives using it that I'd probably have to -write a transition tool anyway! +write a transition tool anyway! --[[Joey]] From bed33d540ca43a616fca491ed0f197aefcb0edb2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:36:08 -0400 Subject: [PATCH 0245/2835] fix --- doc/copies.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/copies.mdwn b/doc/copies.mdwn index ff66f4e8ae..aec10ab7af 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -1,4 +1,4 @@ -The WORM and SHA1 key-value [[backends|backend]] store data inside +The WORM and SHA1 key-value [[backends]] store data inside your git repository's `.git` directory, not in some external data store. It's important that data not get lost by an ill-considered `git annex drop` From b79a53eb242f6b46772457e893fe2b95684ed111 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 21:25:59 -0400 Subject: [PATCH 0246/2835] forgot git :) --- doc/install.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/install.mdwn b/doc/install.mdwn index cc6fb6fb35..e217799a32 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -1,5 +1,6 @@ To build and use git-annex, you will need: +* git: * The Haskell Platform: * MissingH: * uuid: From ee10027a246aa9c5fcd9503314242c65734e8d8a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Oct 2010 11:56:36 -0400 Subject: [PATCH 0247/2835] 3 --- debian/copyright | 2 +- doc/GPL | 833 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 585 insertions(+), 250 deletions(-) diff --git a/debian/copyright b/debian/copyright index 5d0ae13c87..55638653c1 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,5 +1,5 @@ Files: * Copyright: © 2010 Joey Hess -License: GPL-2+ +License: GPL-3+ The full text of the GPL is distributed as doc/GPL in this package's source, or in /usr/share/common-licenses/GPL on Debian systems. diff --git a/doc/GPL b/doc/GPL index d159169d10..94a9ed024d 100644 --- a/doc/GPL +++ b/doc/GPL @@ -1,281 +1,622 @@ GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 + Version 3, 29 June 2007 - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + TERMS AND CONDITIONS - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". + 0. Definitions. -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. + "This License" refers to version 3 of the GNU General Public License. - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. + A "covered work" means either the unmodified Program or a work based +on the Program. - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. + 1. Source Code. -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. + The Corresponding Source for a work in source code form is that +same work. -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. + 2. Basic Permissions. - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of this License. - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. + 13. Use with the GNU Affero General Public License. -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. + 14. Revised Versions of this License. - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. - NO WARRANTY + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. + 15. Disclaimer of Warranty. - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. END OF TERMS AND CONDITIONS @@ -287,15 +628,15 @@ free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least +state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) - This program is free software; you can redistribute it and/or modify + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or + the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, @@ -303,37 +644,31 @@ the "copyright" line and a pointer to where the full notice is found. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + You should have received a copy of the GNU General Public License + along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. From d42f8a9f392e92a8145fb9d68aad4d2028a6210b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Oct 2010 12:07:24 -0400 Subject: [PATCH 0248/2835] autocommit on init --- Commands.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Commands.hs b/Commands.hs index 2addf714e8..1cc046c03a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -220,6 +220,7 @@ initCmd description = do describeUUID u description log <- uuidLog liftIO $ Git.run g ["add", log] + liftIO $ Git.run g ["commit", "-m", "git annex init", log] liftIO $ putStrLn "description set" -- helpers From 4e3f4e9b0528e1b7f087ffc56ec9da0e88a1cc36 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Oct 2010 12:16:49 -0400 Subject: [PATCH 0249/2835] typo --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index fb824c995a..0da11a8b31 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -78,7 +78,7 @@ can get them. After a while, you'll have serveral annexes, with different file contents. You don't have to try to keep all that straight; git-annex does -[[location_tracking] for you. If you ask it to get a file and the drive +[[location_tracking]] for you. If you ask it to get a file and the drive or file server is not accessible, it will let you know what it needs to get it: From a68e36f518589bd15fea32da273ad6fd2f288bb5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Oct 2010 12:54:40 -0400 Subject: [PATCH 0250/2835] entry for the prerelease --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 998754777e..85d105e1b5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -git-annex (0.01) UNRELEASED; urgency=low +git-annex (0.01) unstable; urgency=low - * First release + * First prerelease. - -- Joey Hess Thu, 09 Sep 2010 08:24:58 -0400 + -- Joey Hess Wed, 20 Oct 2010 12:54:24 -0400 From 19fde4960dc1d6c8c05efd0f5b4293c2fb52ebf9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:30:16 -0400 Subject: [PATCH 0251/2835] new fromkey subcommand, for registering urls, etc had to redo Annex monad's flag storage --- Annex.hs | 27 ++++++---- Backend.hs | 19 ++++--- Backend/File.hs | 2 +- Commands.hs | 93 ++++++++++++++++++++++++++------- Core.hs | 5 +- Makefile | 2 +- Remotes.hs | 4 +- TypeInternals.hs | 19 ++++--- Types.hs | 5 +- doc/bugs/using_url_backend.mdwn | 2 + doc/git-annex.mdwn | 21 +++++++- doc/walkthrough.mdwn | 31 +++++++++++ git-annex.hs | 4 +- 13 files changed, 179 insertions(+), 55 deletions(-) diff --git a/Annex.hs b/Annex.hs index b68e513553..d021f936ef 100644 --- a/Annex.hs +++ b/Annex.hs @@ -10,10 +10,12 @@ module Annex ( supportedBackends, flagIsSet, flagChange, + flagGet, Flag(..) ) where import Control.Monad.State +import qualified Data.Map as M import qualified GitRepo as Git import Types @@ -27,7 +29,7 @@ new gitrepo allbackends = do Internals.repo = gitrepo, Internals.backends = [], Internals.supportedBackends = allbackends, - Internals.flags = [] + Internals.flags = M.empty } (_,s') <- Annex.run s (prep gitrepo) return s' @@ -63,15 +65,20 @@ supportedBackends :: Annex [Backend] supportedBackends = do state <- get return (Internals.supportedBackends state) -flagIsSet :: Flag -> Annex Bool -flagIsSet flag = do +flagIsSet :: FlagName -> Annex Bool +flagIsSet name = do state <- get - return $ elem flag $ Internals.flags state -flagChange :: Flag -> Bool -> Annex () -flagChange flag set = do + case (M.lookup name $ Internals.flags state) of + Just (FlagBool True) -> return True + _ -> return False +flagChange :: FlagName -> Flag -> Annex () +flagChange name val = do state <- get - let f = filter (/= flag) $ Internals.flags state - if (set) - then put state { Internals.flags = (flag:f) } - else put state { Internals.flags = f } + put state { Internals.flags = M.insert name val $ Internals.flags state } return () +flagGet :: FlagName -> Annex String +flagGet name = do + state <- get + case (M.lookup name $ Internals.flags state) of + Just (FlagString s) -> return s + _ -> return "" diff --git a/Backend.hs b/Backend.hs index a427234d7c..b8def21cd8 100644 --- a/Backend.hs +++ b/Backend.hs @@ -14,6 +14,7 @@ - -} module Backend ( + list, storeFileKey, retrieveKeyFile, removeKey, @@ -36,24 +37,28 @@ import Types import qualified TypeInternals as Internals {- List of backends in the order to try them when storing a new key. -} -backendList :: Annex [Backend] -backendList = do - l <- Annex.backends +list :: Annex [Backend] +list = do + l <- Annex.backends -- list is cached here if (0 < length l) then return l else do all <- Annex.supportedBackends g <- Annex.gitRepo let l = parseBackendList all $ Git.configGet g "annex.backends" "" - Annex.backendsChange l - return l + backendflag <- Annex.flagGet "backend" + let l' = if (0 < length backendflag) + then (lookupBackendName all backendflag):l + else l + Annex.backendsChange $ l' + return l' where parseBackendList all s = if (length s == 0) then all else map (lookupBackendName all) $ words s -{- Looks up a backend in the list of supportedBackends -} +{- Looks up a backend in a list -} lookupBackendName :: [Backend] -> String -> Backend lookupBackendName all s = if ((length matches) /= 1) @@ -66,7 +71,7 @@ storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) storeFileKey file = do g <- Annex.gitRepo let relfile = Git.relative g file - b <- backendList + b <- list storeFileKey' b file relfile storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do diff --git a/Backend/File.hs b/Backend/File.hs index 4ea25daa73..d1ed1ec64e 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -107,7 +107,7 @@ showTriedRemotes remotes = - error if not. -} checkRemoveKey :: Key -> Annex (Bool) checkRemoveKey key = do - force <- Annex.flagIsSet Force + force <- Annex.flagIsSet "force" if (force) then return True else do diff --git a/Commands.hs b/Commands.hs index 1cc046c03a..59915f59c2 100644 --- a/Commands.hs +++ b/Commands.hs @@ -8,6 +8,7 @@ import System.Posix.Files import System.Directory import System.Path import Data.String.Utils +import Control.Monad (filterM) import List import IO @@ -23,7 +24,8 @@ import Core import qualified Remotes import qualified TypeInternals -data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString +data CmdWants = FilesInGit | FilesNotInGit | FilesMissing | + RepoName | Description data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), @@ -41,26 +43,49 @@ cmds = [ "indicate content of files not currently wanted") , (Command "unannex" unannexCmd FilesInGit "undo accidential add command") - , (Command "init" initCmd SingleString + , (Command "init" initCmd Description "initialize git-annex with repository description") , (Command "fix" fixCmd FilesInGit "fix up files' symlinks to point to annexed content") + , (Command "fromkey" fromKeyCmd FilesMissing + "adds a file using a specific key") ] +-- Each dashed command-line option results in generation of an action +-- in the Annex monad that performs the necessary setting. +options :: [OptDescr (Annex ())] options = [ - Option ['f'] ["force"] (NoArg Force) "allow actions that may lose annexed data" + Option ['f'] ["force"] + (NoArg (Annex.flagChange "force" $ FlagBool True)) + "allow actions that may lose annexed data" + , Option ['b'] ["backend"] + (ReqArg (\s -> Annex.flagChange "backend" $ FlagString s) "NAME") + "specify default key-value backend to use" + , Option ['k'] ["key"] + (ReqArg (\s -> Annex.flagChange "key" $ FlagString s) "KEY") + "specify a key to use" ] -header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) ++ " [path ...]" +header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) +usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where cmddescs = unlines $ map (\c -> indent $ showcmd c) cmds showcmd c = (cmdname c) ++ - (take (10 - (length (cmdname c))) $ repeat ' ') ++ + (pad 10 (cmdname c)) ++ + (descWanted (cmdwants c)) ++ + (pad 13 (descWanted (cmdwants c))) ++ (cmddesc c) indent l = " " ++ l + pad n s = take (n - (length s)) $ repeat ' ' + +{- Generate descrioptions of wanted parameters for subcommands. -} +descWanted :: CmdWants -> String +descWanted Description = "DESCRIPTION" +descWanted RepoName = "REPO" +descWanted _ = "PATH ..." {- Finds the type of parameters a command wants, from among the passed - parameter list. -} @@ -71,14 +96,23 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files -findWanted SingleString params _ = do +findWanted FilesMissing params repo = do + files <- liftIO $ filterM missing params + return $ files + where + missing f = do + e <- doesFileExist f + if (e) then return False else return True +findWanted Description params _ = do return $ [unwords params] findWanted RepoName params _ = do return $ params -{- Parses command line and returns a list of flags and a list of - - actions to be run in the Annex monad. -} -parseCmd :: [String] -> AnnexState -> IO ([Flag], [Annex ()]) +{- Parses command line and returns two lists of actions to be + - run in the Annex monad. The first actions configure it + - according to command line options, while the second actions + - handle subcommands. -} +parseCmd :: [String] -> AnnexState -> IO ([Annex ()], [Annex ()]) parseCmd argv state = do (flags, params) <- getopt case (length params) of @@ -100,7 +134,7 @@ parseCmd argv state = do {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () -addCmd file = inBackend file $ do +addCmd file = notInBackend file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return () @@ -125,9 +159,9 @@ addCmd file = inBackend file $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () -unannexCmd file = notinBackend file $ \(key, backend) -> do +unannexCmd file = inBackend file $ \(key, backend) -> do showStart "unannex" file - Annex.flagChange Force True -- force backend to always remove + Annex.flagChange "force" $ FlagBool True -- force backend to always remove Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo @@ -145,7 +179,7 @@ unannexCmd file = notinBackend file $ \(key, backend) -> do {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () -getCmd file = notinBackend file $ \(key, backend) -> do +getCmd file = inBackend file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return () @@ -167,7 +201,7 @@ getCmd file = notinBackend file $ \(key, backend) -> do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} dropCmd :: FilePath -> Annex () -dropCmd file = notinBackend file $ \(key, backend) -> do +dropCmd file = inBackend file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return () -- no-op @@ -192,8 +226,8 @@ dropCmd file = notinBackend file $ \(key, backend) -> do else return () {- Fixes the symlink to an annexed file. -} -fixCmd :: String -> Annex () -fixCmd file = notinBackend file $ \(key, backend) -> do +fixCmd :: FilePath -> Annex () +fixCmd file = inBackend file $ \(key, backend) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) @@ -223,13 +257,36 @@ initCmd description = do liftIO $ Git.run g ["commit", "-m", "git annex init", log] liftIO $ putStrLn "description set" +{- Adds a file pointing at a manually-specified key -} +fromKeyCmd :: FilePath -> Annex () +fromKeyCmd file = do + keyname <- Annex.flagGet "key" + if (0 == length keyname) + then error "please specify the key with --key" + else return () + backends <- Backend.list + let key = genKey (backends !! 0) keyname + + inbackend <- Backend.hasKey key + if (not inbackend) + then error $ "key ("++keyname++") is not present in backend" + else return () + + link <- calcGitLink file key + showStart "fromkey" file + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ createSymbolicLink link file + g <- Annex.gitRepo + liftIO $ Git.run g ["add", file] + showEndOk + -- helpers -inBackend file a = do +notInBackend file a = do r <- Backend.lookupFile file case (r) of Just v -> return () Nothing -> a -notinBackend file a = do +inBackend file a = do r <- Backend.lookupFile file case (r) of Just v -> a v diff --git a/Core.hs b/Core.hs index 8dc4bff6fc..4941dc26bf 100644 --- a/Core.hs +++ b/Core.hs @@ -18,9 +18,8 @@ import qualified Annex import Utility {- Sets up a git repo for git-annex. -} -startup :: [Flag] -> Annex () -startup flags = do - mapM (\f -> Annex.flagChange f True) flags +startup :: Annex () +startup = do g <- Annex.gitRepo liftIO $ gitAttributes g prepUUID diff --git a/Makefile b/Makefile index d35e82ad55..5f8bb50122 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ - --underlaydir=/dev/null + --underlaydir=/dev/null --disable-plugin=shortcut clean: rm -rf build git-annex git-annex.1 diff --git a/Remotes.hs b/Remotes.hs index a0894f418b..07aafe51b7 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -36,7 +36,7 @@ withKey key = do -- mounted at their location). So unless it happens to find all -- remotes, try harder, loading the remotes' configs. remotes <- reposByUUID allremotes uuids - remotesread <- Annex.flagIsSet RemotesRead + remotesread <- Annex.flagIsSet "remotesread" if ((length allremotes /= length remotes) && not remotesread) then tryharder allremotes uuids else return remotes @@ -46,7 +46,7 @@ withKey key = do eitherremotes <- mapM tryGitConfigRead allremotes let allremotes' = map fromEither eitherremotes remotes' <- reposByUUID allremotes' uuids - Annex.flagChange RemotesRead True + Annex.flagChange "remotesread" $ FlagBool True return remotes' {- Cost Ordered list of remotes. -} diff --git a/TypeInternals.hs b/TypeInternals.hs index 4a9d2653e1..6d1c72d2ea 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -7,12 +7,15 @@ module TypeInternals where import Control.Monad.State (StateT) import Data.String.Utils +import qualified Data.Map as M import qualified GitRepo as Git -data Flag = - Force | -- command-line flags - RemotesRead -- indicates that remote repo configs have been read +-- command-line flags +type FlagName = String +data Flag = + FlagBool Bool | + FlagString String deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, @@ -21,7 +24,7 @@ data AnnexState = AnnexState { repo :: Git.Repo, backends :: [Backend], supportedBackends :: [Backend], - flags :: [Flag] + flags :: M.Map FlagName Flag } deriving (Show) -- git-annex's monad @@ -32,6 +35,10 @@ type KeyFrag = String type BackendName = String data Key = Key (BackendName, KeyFrag) deriving (Eq) +-- constructs a key in a backend +genKey :: Backend -> KeyFrag -> Key +genKey b f = Key (name b,f) + -- show a key to convert it to a string; the string includes the -- name of the backend to avoid collisions between key strings instance Show Key where @@ -48,10 +55,6 @@ instance Read Key where backendName :: Key -> BackendName backendName (Key (b,k)) = b --- pulls the key fragment out -keyFrag :: Key -> KeyFrag -keyFrag (Key (b,k)) = k - -- this structure represents a key-value backend data Backend = Backend { -- name of this backend diff --git a/Types.hs b/Types.hs index 2284d92674..50597962ce 100644 --- a/Types.hs +++ b/Types.hs @@ -5,9 +5,10 @@ module Types ( AnnexState, Backend, Key, + genKey, backendName, - keyFrag, - Flag(..), + FlagName, + Flag(..) ) where import TypeInternals diff --git a/doc/bugs/using_url_backend.mdwn b/doc/bugs/using_url_backend.mdwn index a0d447c6e2..1f3cd56281 100644 --- a/doc/bugs/using_url_backend.mdwn +++ b/doc/bugs/using_url_backend.mdwn @@ -7,3 +7,5 @@ For now, we have to manually make the symlink. Something like this: Note the escaping of slashes. A `git annex register ` command could do this.. + +[[done]] diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 09b245497c..81c229c513 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -4,7 +4,7 @@ git-annex - manage files with git, without checking their contents in # SYNOPSIS -git annex subcommand [path ...] +git annex subcommand [params ...] # DESCRIPTION @@ -97,6 +97,16 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Fixes up symlinks that have become broken to again point to annexed content. This is useful to run if you have been moving the symlinks around. +* fromkey file + + This can be used to maually set up a file to link to a specified key + in the key-value backend. How you determine an existing key in the backend + varies. For the URL backend, the key is just a URL to the content. + + Example: + + git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + # OPTIONS * --force @@ -104,6 +114,15 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Force unsafe actions, such as dropping a file's content when no other source of it can be verified to still exist. Use with care. +* --backend=name + + Specify the default key-value backend to use, adding it to the front + of the list normally configured by `annex.backends`. + +* --key=name + + Specifies a key to operate on, for use with the addkey subcommand. + ## CONFIGURATION Like other git commands, git-annex is configured via `.git/config`. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 0da11a8b31..7018a839e9 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -1,3 +1,7 @@ +A walkthrough of the basic features of git-annex. + +[[!toc]] + ## creating a repository This is very straightforward. Just tell it a description of the repository. @@ -130,3 +134,30 @@ Here you might --force it to drop `important_file` if you trust your backup. But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. + +## using other backends: manually adding a remote URL + +git-annex has multiple key-value [[backends]]. So far this walkthrough has +demonstrated the default, WORM (Write Once, Read Many) backend. + +Another handy backend is the URL backend, which can fetch file's content +from remote URLs. Here's how to set up some files in your repository +that use this backend: + + # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + add somefile ok + # git commit -m "added a file from the Internet Archive" + +Now you if you ask git-annex to get that file, it will download it, +and cache it locally, until you have it drop it. + + # git annex get somefile + get somefile (downloading) + #########################################################################100.0% + ok + +You can always drop files downloaded by the URL backend. It is assumed +that the URL is stable; no local backup is kept. + + # git annex drop somefile + drop somefile (ok) diff --git a/git-annex.hs b/git-annex.hs index 71a21379dc..602f672c5b 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -15,8 +15,8 @@ main = do args <- getArgs gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends - (flags, actions) <- parseCmd args state - tryRun state $ [startup flags] ++ actions ++ [shutdown] + (configure, actions) <- parseCmd args state + tryRun state $ [startup] ++ configure ++ actions ++ [shutdown] {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). From fd5d6403f05ed346e5660ccefb7254ff062505cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:33:04 -0400 Subject: [PATCH 0252/2835] typo --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 7018a839e9..e25c36fb40 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -47,7 +47,7 @@ if you are using a centralized bare repository. # git commit -a -m added Notice you commit at the end, this checks in git-annex's record of the -files but not thier actual, large, content. +files but not their actual, large, content. ## renaming files From 42eab98f56c0641467441d1d868f850a1e7ef81c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:35:04 -0400 Subject: [PATCH 0253/2835] typo --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index e25c36fb40..7114cceae2 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -145,7 +145,7 @@ from remote URLs. Here's how to set up some files in your repository that use this backend: # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile - add somefile ok + fromkey somefile ok # git commit -m "added a file from the Internet Archive" Now you if you ask git-annex to get that file, it will download it, From 409285d8e18fb27d4c1723693d6ac22d67ebec08 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:36:01 -0400 Subject: [PATCH 0254/2835] clarify --- doc/walkthrough.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 7114cceae2..4069fb87bf 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -135,7 +135,7 @@ But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. -## using other backends: manually adding a remote URL +## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has demonstrated the default, WORM (Write Once, Read Many) backend. @@ -149,7 +149,7 @@ that use this backend: # git commit -m "added a file from the Internet Archive" Now you if you ask git-annex to get that file, it will download it, -and cache it locally, until you have it drop it. +and cache it locally. # git annex get somefile get somefile (downloading) From e8e397036f8f04c4ef088d5a4a1b12b37b3a6118 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:38:14 -0400 Subject: [PATCH 0255/2835] changelog --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 85d105e1b5..1c36652302 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.02) UNRELEASED; urgency=low + + * New fromkey subcommand, for registering urls, etc. + + -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 + git-annex (0.01) unstable; urgency=low * First prerelease. From 514b98ff5574fe09ec365d5881684c16f752df6c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 17:59:32 -0400 Subject: [PATCH 0256/2835] document move subcommand --- Commands.hs | 38 +++++++++++++++++++++++++------------- doc/git-annex.mdwn | 33 ++++++++++++++++++++++++--------- doc/walkthrough.mdwn | 24 ++++++++++++++++++++---- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/Commands.hs b/Commands.hs index 59915f59c2..62eb08e83d 100644 --- a/Commands.hs +++ b/Commands.hs @@ -24,8 +24,7 @@ import Core import qualified Remotes import qualified TypeInternals -data CmdWants = FilesInGit | FilesNotInGit | FilesMissing | - RepoName | Description +data CmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), @@ -41,10 +40,12 @@ cmds = [ "make content of annexed files available") , (Command "drop" dropCmd FilesInGit "indicate content of files not currently wanted") - , (Command "unannex" unannexCmd FilesInGit - "undo accidential add command") + , (Command "move" moveCmd FilesInGit + "transfer content of files to another repository") , (Command "init" initCmd Description "initialize git-annex with repository description") + , (Command "unannex" unannexCmd FilesInGit + "undo accidential add command") , (Command "fix" fixCmd FilesInGit "fix up files' symlinks to point to annexed content") , (Command "fromkey" fromKeyCmd FilesMissing @@ -55,16 +56,18 @@ cmds = [ -- in the Annex monad that performs the necessary setting. options :: [OptDescr (Annex ())] options = [ - Option ['f'] ["force"] - (NoArg (Annex.flagChange "force" $ FlagBool True)) + Option ['f'] ["force"] (NoArg (storebool "force" True)) "allow actions that may lose annexed data" - , Option ['b'] ["backend"] - (ReqArg (\s -> Annex.flagChange "backend" $ FlagString s) "NAME") + , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") "specify default key-value backend to use" - , Option ['k'] ["key"] - (ReqArg (\s -> Annex.flagChange "key" $ FlagString s) "KEY") + , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") "specify a key to use" + , Option ['r'] ["repository"] (ReqArg (storestring "repository") "REPOSITORY") + "specify a repository" ] + where + storebool n b = Annex.flagChange n $ FlagBool b + storestring n s = Annex.flagChange n $ FlagString s header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) @@ -84,7 +87,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Generate descrioptions of wanted parameters for subcommands. -} descWanted :: CmdWants -> String descWanted Description = "DESCRIPTION" -descWanted RepoName = "REPO" descWanted _ = "PATH ..." {- Finds the type of parameters a command wants, from among the passed @@ -105,8 +107,6 @@ findWanted FilesMissing params repo = do if (e) then return False else return True findWanted Description params _ = do return $ [unwords params] -findWanted RepoName params _ = do - return $ params {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -198,6 +198,18 @@ getCmd file = inBackend file $ \(key, backend) -> do else do showEndFail +{- Moves the content of an annexed file to another repository, + - removing it from the current repository, and updates locationlog + - information on both. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. + -} +moveCmd :: FilePath -> Annex () +moveCmd file = inBackend file $ \(key, backend) -> do + error "TODO" + {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} dropCmd :: FilePath -> Annex () diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 81c229c513..86ac9c6359 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -48,11 +48,12 @@ content from the key-value store. add iso/Debian_5.0.iso ok # git commit -a -m "saving Debian CD for later" - # git annex push usbdrive iso - error: push not yet implemented! # git annex drop iso drop iso/Debian_5.0.iso ok # git commit -a -m "freed up space" + + # git annex move video --to=usbdrive + move iso/Debian_5.0.iso (to usbdrive...) ok # SUBCOMMANDS @@ -77,8 +78,22 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Drops the content of annexed files from this repository. - git-annex may refuse to drop a content if the backend does not think - it is safe to do so. + git-annex may refuse to drop content if the backend does not think + it is safe to do so, typically because of the setting of annex.numcopies. + +* move [path ...] + + Moves the content of annexed files from the current repository to + another one. Use with the --to option. + + Note that unlike drop, this does not honor annex.numcopies. + A file's content can be moved even if there are insufficient + copies to allow it to be dropped. + +* init description + + Initializes git-annex with a descripotion of the git repository. + This is an optional, but recommended step. * unannex [path ...] @@ -87,11 +102,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. any more. In that case you should use `git annex drop` instead, and you can also `git rm` the file. -* init description - - Initializes git-annex with a descripotion of the git repository. - This is an optional, but recommended step. - * fix [path ...] Fixes up symlinks that have become broken to again point to annexed content. @@ -123,6 +133,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a key to operate on, for use with the addkey subcommand. +* --to=repository + + Specifies a git repository that content will be sent to. + It can be specified by a path, url, or remote name. + ## CONFIGURATION Like other git commands, git-annex is configured via `.git/config`. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 4069fb87bf..7dbe62bdc2 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -64,9 +64,14 @@ make copies or delete them. `git-annex fix` needs to be run if a file is moved into a different directory, in order to fix up the symlink pointing to the file's content. -## transferring files around +## getting file content -Let's copy everything in the laptop's home annex to the USB drive. +A repository does not always have all annexed file contents available. +When you need the content of a file, you can use "git annex get" to +make it available. + +We can use this to copy everything in the laptop's home annex to the +USB drive. # cd /media/usb/annex # git pull home master @@ -75,8 +80,8 @@ Let's copy everything in the laptop's home annex to the USB drive. get iso/debian.iso (copying from home...) ok Notice that you had to git pull from home first, this lets git-annex know -what has changed in home, and so it knows about the files you added and -can get them. +what has changed in home, and so it knows about the files present there and +can get them. See below for an easier way. ## transferring files: When things go wrong @@ -135,6 +140,17 @@ But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. +## moving file content to another repository + +Often you will want to transfer some file contents from a repository to +some other one, and then drop it from the first repository. For example, +your laptop's disk is getting full; time to move some files to an external +disk. Doing that by hand is possible, but a bit of a pain. `git annex move` +makes it very easy. + + # git annex move my_cool_big_file --to usbdrive + move my_cool_big_file (to usbdrive...) ok + ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has From 014f7f650de9e272628fd5031c8c9a00b1eb69ae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 19:27:29 -0400 Subject: [PATCH 0257/2835] crazy idea --- doc/bugs/add_a_git_backend.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/bugs/add_a_git_backend.mdwn diff --git a/doc/bugs/add_a_git_backend.mdwn b/doc/bugs/add_a_git_backend.mdwn new file mode 100644 index 0000000000..91a5001cc9 --- /dev/null +++ b/doc/bugs/add_a_git_backend.mdwn @@ -0,0 +1,6 @@ +There should be a backend where the file content is stored.. in a git +repository! + +This way, you know your annexed content is safe & versioned, but you only +have to deal with the pain of git with large files in one place, and can +use all of git-annex's features everywhere else. From 8da596feff4f402fec08b6fd3815fd32e9770af6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 12:38:20 -0400 Subject: [PATCH 0258/2835] support reading config over ssh --- GitRepo.hs | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 5b0e68cd62..e8504a8410 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -56,12 +56,11 @@ data Repo = -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String } | RemoteRepo { - url :: String, - top :: FilePath, + url :: URI, config :: Map String String, remotes :: [Repo], remoteName :: Maybe String - } deriving (Show, Read, Eq) + } deriving (Show, Eq) {- Local Repo constructor. -} repoFromPath :: FilePath -> Repo @@ -77,13 +76,11 @@ repoFromPath dir = repoFromUrl :: String -> Repo repoFromUrl url = RemoteRepo { - url = url, - top = path url, + url = fromJust $ parseURI url, config = Map.empty, remotes = [], remoteName = Nothing } - where path url = uriPath $ fromJust $ parseURI url {- User-visible description of a git repo. -} repoDescribe repo = @@ -91,7 +88,7 @@ repoDescribe repo = then fromJust $ remoteName repo else if (repoIsLocal repo) then top repo - else url repo + else show (url repo) {- Constructs and returns an updated version of a repo with - different remotes list. -} @@ -105,8 +102,8 @@ repoRemoteName r = then fromJust $ remoteName r else "" -{- Some code needs to vary between remote and local repos, or bare and - - non-bare, these functions help with that. -} +{- Some code needs to vary between remote and local repos, + - or bare and non-bare, these functions help with that. -} repoIsLocal repo = case (repo) of LocalRepo {} -> True RemoteRepo {} -> False @@ -116,6 +113,10 @@ assertlocal repo action = then action else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ " not supported" +assertssh repo action = + case (uriScheme $ url repo) of + "ssh:" -> action + _ -> error $ "unsupported remote repo type " ++ (show $ url repo) bare :: Repo -> Bool bare repo = if (member b (config repo)) @@ -193,16 +194,24 @@ notInRepo repo location = do {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo -configRead repo = assertlocal repo $ do +configRead repo = if (repoIsLocal repo) {- Cannot use pipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} - cwd <- getCurrentDirectory - bracket_ (changeWorkingDirectory (top repo)) - (\_ -> changeWorkingDirectory cwd) $ - pOpen ReadFromPipe "git" ["config", "--list"] $ \h -> do - val <- hGetContentsStrict h - let r = repo { config = configParse val } - return r { remotes = configRemotes r } + then do + cwd <- getCurrentDirectory + bracket_ (changeWorkingDirectory (top repo)) + (\_ -> changeWorkingDirectory cwd) $ + pOpen ReadFromPipe "git" ["config", "--list"] proc + else assertssh repo $ do + pOpen ReadFromPipe "ssh" [sshhost, sshcommand] proc + where + sshhost = (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) + where a = fromJust $ uriAuthority $ url repo + sshcommand = "cd '" ++ (uriPath $ url repo) ++ "' && git config --list" + proc h = do + val <- hGetContentsStrict h + let r = repo { config = configParse val } + return r { remotes = configRemotes r } {- Calculates a list of a repo's configured remotes, by parsing its config. -} configRemotes :: Repo -> [Repo] From 9f13f3ac5e7d20df91cb57af5e630fd48776d775 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 13:21:43 -0400 Subject: [PATCH 0259/2835] support using scp for ssh remotes --- Backend/File.hs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index d1ed1ec64e..d28b927287 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -73,7 +73,7 @@ copyKeyFile key file = do case (result) of Left err -> trycopy full rs Right r' -> do - showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." + showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." liftIO $ copyFromRemote r' key file {- Tries to copy a file from a remote. -} @@ -81,11 +81,15 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool copyFromRemote r key file = do if (Git.repoIsLocal r) then getlocal - else getremote + else if (Git.repoIsSsh r) + then getssh + else error "copying from non-ssh repo not supported" where - getlocal = boolSystem "cp" ["-a", location, file] - getremote = return False -- TODO implement get from remote location = annexLocation r key + getlocal = boolSystem "cp" ["-a", location, file] + getssh = do + liftIO $ putStrLn "" -- make way for scp progress bar + boolSystem "scp" [location, file] showLocations :: Key -> Annex () showLocations key = do @@ -130,10 +134,10 @@ checkRemoveKey key = do Right True -> findcopies need (have+1) rs bad Right False -> findcopies need have rs bad Left _ -> findcopies need have rs (r:bad) - remoteHasKey r all = do + remoteHasKey remote all = do -- To check if a remote has a key, construct a new -- Annex monad and query its backend. - a <- Annex.new r all + a <- Annex.new remote all (result, _) <- Annex.run a (Backend.hasKey key) return result notEnoughCopies need have bad = do From 897bf49b4eddba41b5ca36210ccf36f34df84e01 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 13:40:19 -0400 Subject: [PATCH 0260/2835] support ssh repo in workTree --- GitRepo.hs | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index e8504a8410..80a3722cec 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -12,6 +12,7 @@ module GitRepo ( repoFromUrl, repoIsLocal, repoIsRemote, + repoIsSsh, repoDescribe, workTree, dir, @@ -108,15 +109,21 @@ repoIsLocal repo = case (repo) of LocalRepo {} -> True RemoteRepo {} -> False repoIsRemote repo = not $ repoIsLocal repo +repoIsSsh repo = repoIsRemote repo && (uriScheme $ url repo) == "ssh:" assertlocal repo action = if (repoIsLocal repo) then action else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ " not supported" -assertssh repo action = - case (uriScheme $ url repo) of - "ssh:" -> action - _ -> error $ "unsupported remote repo type " ++ (show $ url repo) +assertremote repo action = + if (repoIsRemote repo) + then action + else error $ "acting on local git repo " ++ (repoDescribe repo) ++ + " not supported" +assertssh repo action = + if (repoIsSsh repo) + then action + else error $ "unsupported url " ++ (show $ url repo) bare :: Repo -> Bool bare repo = if (member b (config repo)) @@ -135,14 +142,21 @@ attributes repo = assertlocal repo $ do {- Path to a repository's .git directory, relative to its topdir. -} dir :: Repo -> String -dir repo = assertlocal repo $ - if (bare repo) - then "" - else ".git" +dir repo = if (bare repo) then "" else ".git" {- Path to a repository's --work-tree. -} workTree :: Repo -> FilePath -workTree repo = top repo +workTree repo = + if (repoIsLocal) repo + then top repo + else assertssh repo $ (remoteHost repo) ++ ":" ++ (uriPath $ url repo) + +{- Hostname for a remote repo. (May include a username and/or port too.) -} +remoteHost :: Repo -> String +remoteHost repo = assertremote repo $ + (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) + where + a = fromJust $ uriAuthority $ url repo {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. @@ -203,10 +217,8 @@ configRead repo = if (repoIsLocal repo) (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] proc else assertssh repo $ do - pOpen ReadFromPipe "ssh" [sshhost, sshcommand] proc + pOpen ReadFromPipe "ssh" [remoteHost repo, sshcommand] proc where - sshhost = (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) - where a = fromJust $ uriAuthority $ url repo sshcommand = "cd '" ++ (uriPath $ url repo) ++ "' && git config --list" proc h = do val <- hGetContentsStrict h From 46ac66a4380e238865f213ff28411283a4efbbbd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 14:05:30 -0400 Subject: [PATCH 0261/2835] fix remote vs remote wording confusion --- GitRepo.hs | 56 ++++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 80a3722cec..ea9e8a8b76 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -10,8 +10,7 @@ module GitRepo ( repoFromCwd, repoFromPath, repoFromUrl, - repoIsLocal, - repoIsRemote, + repoIsUrl, repoIsSsh, repoDescribe, workTree, @@ -46,17 +45,16 @@ import Maybe import Utility -{- A git repository can be on local disk or remote. Not to be confused - - with a git repo's configured remotes, some of which may be on local - - disk. -} +{- There are two types of repositories; those on local disk and those + - accessed via an URL. -} data Repo = - LocalRepo { + Repo { top :: FilePath, config :: Map String String, remotes :: [Repo], -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String - } | RemoteRepo { + } | UrlRepo { url :: URI, config :: Map String String, remotes :: [Repo], @@ -66,7 +64,7 @@ data Repo = {- Local Repo constructor. -} repoFromPath :: FilePath -> Repo repoFromPath dir = - LocalRepo { + Repo { top = dir, config = Map.empty, remotes = [], @@ -76,7 +74,7 @@ repoFromPath dir = {- Remote Repo constructor. Throws exception on invalid url. -} repoFromUrl :: String -> Repo repoFromUrl url = - RemoteRepo { + UrlRepo { url = fromJust $ parseURI url, config = Map.empty, remotes = [], @@ -87,7 +85,7 @@ repoFromUrl url = repoDescribe repo = if (isJust $ remoteName repo) then fromJust $ remoteName repo - else if (repoIsLocal repo) + else if (not $ repoIsUrl repo) then top repo else show (url repo) @@ -103,20 +101,19 @@ repoRemoteName r = then fromJust $ remoteName r else "" -{- Some code needs to vary between remote and local repos, +{- Some code needs to vary between URL and normal repos, - or bare and non-bare, these functions help with that. -} -repoIsLocal repo = case (repo) of - LocalRepo {} -> True - RemoteRepo {} -> False -repoIsRemote repo = not $ repoIsLocal repo -repoIsSsh repo = repoIsRemote repo && (uriScheme $ url repo) == "ssh:" -assertlocal repo action = - if (repoIsLocal repo) +repoIsUrl repo = case (repo) of + UrlRepo {} -> True + Repo {} -> False +repoIsSsh repo = repoIsUrl repo && (uriScheme $ url repo) == "ssh:" +assertLocal repo action = + if (not $ repoIsUrl repo) then action else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ " not supported" -assertremote repo action = - if (repoIsRemote repo) +assertUrl repo action = + if (repoIsUrl repo) then action else error $ "acting on local git repo " ++ (repoDescribe repo) ++ " not supported" @@ -135,7 +132,7 @@ bare repo = {- Path to a repository's gitattributes file. -} attributes :: Repo -> String -attributes repo = assertlocal repo $ do +attributes repo = assertLocal repo $ do if (bare repo) then (top repo) ++ "/info/.gitattributes" else (top repo) ++ "/.gitattributes" @@ -147,13 +144,13 @@ dir repo = if (bare repo) then "" else ".git" {- Path to a repository's --work-tree. -} workTree :: Repo -> FilePath workTree repo = - if (repoIsLocal) repo + if (not $ repoIsUrl repo) then top repo else assertssh repo $ (remoteHost repo) ++ ":" ++ (uriPath $ url repo) {- Hostname for a remote repo. (May include a username and/or port too.) -} remoteHost :: Repo -> String -remoteHost repo = assertremote repo $ +remoteHost repo = assertUrl repo $ (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) where a = fromJust $ uriAuthority $ url repo @@ -175,19 +172,19 @@ relative repo file = drop (length absrepo) absfile {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: Repo -> [String] -> [String] -gitCommandLine repo params = assertlocal repo $ +gitCommandLine repo params = assertLocal repo $ -- force use of specified repo via --git-dir and --work-tree ["--git-dir="++(top repo)++"/"++(dir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () -run repo params = assertlocal repo $ do +run repo params = assertLocal repo $ do r <- safeSystem "git" (gitCommandLine repo params) return () {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String -pipeRead repo params = assertlocal repo $ do +pipeRead repo params = assertLocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do ret <- hGetContentsStrict h return ret @@ -208,10 +205,11 @@ notInRepo repo location = do {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo -configRead repo = if (repoIsLocal repo) - {- Cannot use pipeRead because it relies on the config having - been already read. Instead, chdir to the repo. -} +configRead repo = + if (not $ repoIsUrl repo) then do + {- Cannot use pipeRead because it relies on the config having + been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory bracket_ (changeWorkingDirectory (top repo)) (\_ -> changeWorkingDirectory cwd) $ From 9ec5d90b6a28e28f6349635d33df0000ce9203ef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 14:28:47 -0400 Subject: [PATCH 0262/2835] avoid reading configs for URL remotes every time --- Backend/File.hs | 2 +- Remotes.hs | 67 +++++++++++++++++++++++++++++++--------------- debian/changelog | 1 + doc/git-annex.mdwn | 2 ++ 4 files changed, 50 insertions(+), 22 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index d28b927287..b99a064ae3 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -79,7 +79,7 @@ copyKeyFile key file = do {- Tries to copy a file from a remote. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool copyFromRemote r key file = do - if (Git.repoIsLocal r) + if (not $ Git.repoIsUrl r) then getlocal else if (Git.repoIsSsh r) then getssh diff --git a/Remotes.hs b/Remotes.hs index 07aafe51b7..cb8f4d1312 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -8,6 +8,7 @@ module Remotes ( import Control.Exception import Control.Monad.State (liftIO) +import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils import Data.Either.Utils @@ -20,6 +21,7 @@ import qualified Annex import LocationLog import Locations import UUID +import Core {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -31,23 +33,36 @@ withKey key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost - -- This only uses cached data, so may not include new remotes - -- or remotes whose uuid has changed (eg by a different drive being - -- mounted at their location). So unless it happens to find all - -- remotes, try harder, loading the remotes' configs. - remotes <- reposByUUID allremotes uuids + -- To determine if a remote has a key, its UUID needs to be known. + -- The locally cached UIIDs of remotes can fall out of date if + -- eg, a different drive is mounted at the same location. + -- But, reading the config of remotes can be expensive, so make + -- sure we only do it once per git-annex run. remotesread <- Annex.flagIsSet "remotesread" - if ((length allremotes /= length remotes) && not remotesread) - then tryharder allremotes uuids - else return remotes + if (remotesread) + then reposByUUID allremotes uuids + else do + -- We assume that it's cheap to read the config + -- of non-URL remotes, so that is done each time. + -- But reading the config of an URL remote is + -- only done when there is no cached UUID value. + let cheap = filter (not . Git.repoIsUrl) allremotes + let expensive = filter Git.repoIsUrl allremotes + doexpensive <- filterM cachedUUID expensive + if (0 < length doexpensive) + then showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." + else return () + let todo = cheap ++ doexpensive + if (0 < length todo) + then do + e <- mapM tryGitConfigRead todo + Annex.flagChange "remotesread" $ FlagBool True + withKey key + else reposByUUID allremotes uuids where - tryharder allremotes uuids = do - -- more expensive; read each remote's config - eitherremotes <- mapM tryGitConfigRead allremotes - let allremotes' = map fromEither eitherremotes - remotes' <- reposByUUID allremotes' uuids - Annex.flagChange "remotesread" $ FlagBool True - return remotes' + cachedUUID r = do + u <- getUUID r + return $ 0 == length u {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] @@ -55,10 +70,11 @@ remotesByCost = do g <- Annex.gitRepo reposByCost $ Git.remotes g -{- Orders a list of git repos by cost. -} +{- Orders a list of git repos by cost, and throws out ignored ones. -} reposByCost :: [Git.Repo] -> Annex [Git.Repo] reposByCost l = do - costpairs <- mapM costpair l + notignored <- filterM repoNotIgnored l + costpairs <- mapM costpair notignored return $ fst $ unzip $ sortBy bycost $ costpairs where costpair r = do @@ -76,13 +92,22 @@ repoCost r = do g <- Annex.gitRepo if ((length $ config g r) > 0) then return $ read $ config g r - else if (Git.repoIsLocal r) - then return 100 - else return 200 + else if (Git.repoIsUrl r) + then return 200 + else return 100 where config g r = Git.configGet g (configkey r) "" configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" +{- Checks if a repo should be ignored. -} +repoNotIgnored :: Git.Repo -> Annex Bool +repoNotIgnored r = do + g <- Annex.gitRepo + return ("true" /= config g r) + where + config g r = Git.configGet g (configkey r) "" + configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" + {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the - config for a specified remote, and updates state. If successful, it @@ -95,7 +120,7 @@ tryGitConfigRead r = do -- for other reasons; catch all possible exceptions result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) case (result) of - Left err -> return $ Left r + Left e -> return $ Left r Right r' -> do g <- Annex.gitRepo let l = Git.remotes g diff --git a/debian/changelog b/debian/changelog index 1c36652302..2800e54a33 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.02) UNRELEASED; urgency=low * New fromkey subcommand, for registering urls, etc. + * Can scp annexed files from remotes. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 86ac9c6359..a7c6b9e48c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -154,6 +154,8 @@ Like other git commands, git-annex is configured via `.git/config`. repositories. Note that other factors may be configured when pushing files to repositories, in particular, whether the repository is on a filesystem with sufficient free space. +* `remote..annex-ignore` -- If set to "true", prevents git-annex + from ever using this remote. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. From 91e6625eb56671472abd9532a5635f541d025a60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 14:57:02 -0400 Subject: [PATCH 0263/2835] add shellEscape ugly, but sometimes necessary There is a haskell shell-escape module, but it is not packaged in Debian --- Utility.hs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index 09b973002f..8bffde0577 100644 --- a/Utility.hs +++ b/Utility.hs @@ -7,7 +7,8 @@ module Utility ( parentDir, relPathCwdToDir, relPathDirToDir, - boolSystem + boolSystem, + shellEscape ) where import System.IO @@ -108,3 +109,9 @@ boolSystem command params = do ExitFailure e -> if Just e == cast sigINT then error $ command ++ "interrupted" else return False + +{- Escapes a filename to be safely able to be exposed to the shell. -} +shellEscape f = "'" ++ quote ++ "'" + where + -- replace ' with '"'"' + quote = join "'\"'\"'" $ split "'" f From aafb63edb13ac87eb5d741c75b90de6115f06452 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:06:14 -0400 Subject: [PATCH 0264/2835] support checking network remotes when dropping --- Backend/File.hs | 19 +++++++++++++++---- GitRepo.hs | 35 ++++++++++++++++++++++------------- debian/changelog | 3 ++- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index b99a064ae3..15d8f4a263 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -44,9 +44,18 @@ mustProvide = error "must provide this field" dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True -{- Just check if the .git/annex/ file for the key exists. -} +{- Just check if the .git/annex/ file for the key exists. + - + - But, if running against a remote annex, need to use ssh to do it. -} checkKeyFile :: Key -> Annex Bool -checkKeyFile k = inAnnex k +checkKeyFile k = do + g <- Annex.gitRepo + if (not $ Git.repoIsUrl g) + then inAnnex k + else do + showNote ("checking " ++ Git.repoDescribe g ++ "...") + liftIO $ boolSystem "ssh" [Git.urlHost g, + "test -e " ++ (shellEscape $ annexLocation g k)] {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} @@ -85,11 +94,13 @@ copyFromRemote r key file = do then getssh else error "copying from non-ssh repo not supported" where - location = annexLocation r key getlocal = boolSystem "cp" ["-a", location, file] getssh = do liftIO $ putStrLn "" -- make way for scp progress bar - boolSystem "scp" [location, file] + -- TODO double-shell-quote path for scp + boolSystem "scp" [sshlocation, file] + location = annexLocation r key + sshlocation = (Git.urlHost r) ++ ":" ++ location showLocations :: Key -> Annex () showLocations key = do diff --git a/GitRepo.hs b/GitRepo.hs index ea9e8a8b76..553e91fec0 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -16,6 +16,8 @@ module GitRepo ( workTree, dir, relative, + urlPath, + urlHost, configGet, configMap, configRead, @@ -110,7 +112,7 @@ repoIsSsh repo = repoIsUrl repo && (uriScheme $ url repo) == "ssh:" assertLocal repo action = if (not $ repoIsUrl repo) then action - else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ + else error $ "acting on URL git repo " ++ (repoDescribe repo) ++ " not supported" assertUrl repo action = if (repoIsUrl repo) @@ -137,23 +139,18 @@ attributes repo = assertLocal repo $ do then (top repo) ++ "/info/.gitattributes" else (top repo) ++ "/.gitattributes" -{- Path to a repository's .git directory, relative to its topdir. -} +{- Path to a repository's .git directory, relative to its workTree. -} dir :: Repo -> String dir repo = if (bare repo) then "" else ".git" -{- Path to a repository's --work-tree. -} +{- Path to a repository's --work-tree, that is, its top. + - + - Note that for URL repositories, this is relative to the urlHost -} workTree :: Repo -> FilePath workTree repo = if (not $ repoIsUrl repo) then top repo - else assertssh repo $ (remoteHost repo) ++ ":" ++ (uriPath $ url repo) - -{- Hostname for a remote repo. (May include a username and/or port too.) -} -remoteHost :: Repo -> String -remoteHost repo = assertUrl repo $ - (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) - where - a = fromJust $ uriAuthority $ url repo + else urlPath repo {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. @@ -170,6 +167,18 @@ relative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo +{- Hostname of an URL repo. (May include a username and/or port too.) -} +urlHost :: Repo -> String +urlHost repo = assertUrl repo $ + (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) + where + a = fromJust $ uriAuthority $ url repo + +{- Path of an URL repo. -} +urlPath :: Repo -> String +urlPath repo = assertUrl repo $ + uriPath $ url repo + {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo params = assertLocal repo $ @@ -215,9 +224,9 @@ configRead repo = (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] proc else assertssh repo $ do - pOpen ReadFromPipe "ssh" [remoteHost repo, sshcommand] proc + pOpen ReadFromPipe "ssh" [urlHost repo, sshcommand] proc where - sshcommand = "cd '" ++ (uriPath $ url repo) ++ "' && git config --list" + sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ " && git config --list" proc h = do val <- hGetContentsStrict h let r = repo { config = configParse val } diff --git a/debian/changelog b/debian/changelog index 2800e54a33..3d791ffe57 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,8 @@ git-annex (0.02) UNRELEASED; urgency=low * New fromkey subcommand, for registering urls, etc. - * Can scp annexed files from remotes. + * Can scp annexed files from remote hosts, and check remote hosts for + file content when dropping files. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 From 599cb15f305468bd5d5d23b7950018c8a767cc4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:08:53 -0400 Subject: [PATCH 0265/2835] update --- debian/changelog | 2 ++ doc/bugs/network_remotes.mdwn | 2 ++ doc/git-annex.mdwn | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 3d791ffe57..68f9f02cde 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,8 @@ git-annex (0.02) UNRELEASED; urgency=low * New fromkey subcommand, for registering urls, etc. * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. + * Add remote.annex-ignore git config setting to allow completly disabling + a given remote. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 diff --git a/doc/bugs/network_remotes.mdwn b/doc/bugs/network_remotes.mdwn index be43ee20be..42efa832f5 100644 --- a/doc/bugs/network_remotes.mdwn +++ b/doc/bugs/network_remotes.mdwn @@ -1,3 +1,5 @@ Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) + +[[done]], at least get and put work.. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index a7c6b9e48c..37100fceef 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -92,7 +92,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * init description - Initializes git-annex with a descripotion of the git repository. + Initializes git-annex with a description of the git repository. This is an optional, but recommended step. * unannex [path ...] From 1e40562b0284d81e0509e61369b28c7f6b14152a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:10:21 -0400 Subject: [PATCH 0266/2835] todo --- doc/bugs/rsync.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/bugs/rsync.mdwn diff --git a/doc/bugs/rsync.mdwn b/doc/bugs/rsync.mdwn new file mode 100644 index 0000000000..75e0175c86 --- /dev/null +++ b/doc/bugs/rsync.mdwn @@ -0,0 +1,2 @@ +Transferring a file from a ssh:// remote should use rsync to allow resuming +of a prior transfer. From 5b4272aba3ee04c89c8f2be509377e7dd68cc5f6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:12:48 -0400 Subject: [PATCH 0267/2835] update --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5f8bb50122..921eb70670 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,8 @@ docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ - --underlaydir=/dev/null --disable-plugin=shortcut + --underlaydir=/dev/null --disable-plugin=shortcut \ + --disable-plugin=smiley clean: rm -rf build git-annex git-annex.1 From 26005d23bae2ab213fda694d2744ed239e412cbe Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:21:23 -0400 Subject: [PATCH 0268/2835] separate remote names with commas --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index cb8f4d1312..0e7dd31eaf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -25,7 +25,7 @@ import Core {- Human visible list of remotes. -} list :: [Git.Repo] -> String -list remotes = join " " $ map Git.repoDescribe remotes +list remotes = join ", " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} withKey :: Key -> Annex [Git.Repo] From e6048b0389f8da9bd1e56e2f1f7d18abc773f03f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:38:31 -0400 Subject: [PATCH 0269/2835] document using ssh remotes --- debian/control | 2 +- doc/walkthrough.mdwn | 46 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/debian/control b/debian/control index 83bc8c82bf..d8abc487cb 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Homepage: http://git-annex.branchable.com/ Package: git-annex Architecture: any Section: utils -Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid +Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 7dbe62bdc2..5c91f2a9c6 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -13,8 +13,8 @@ This is very straightforward. Just tell it a description of the repository. ## adding a remote -This could be a USB drive, or a sshfs or NFS mount to a file server, for -example. +Like any other git repository, git-annex repositories have remotes. +Let's start by adding a USB drive as a remote. # sudo mount /media/usb # cd /media/usb @@ -25,11 +25,8 @@ example. # cd ~/annex # git remote add usbdrive /media/usb -This is all standard ad-hoc distributed git repository setup. Or you -could have added a centralized bare repository on a server if you prefer -doing things that way. - -Anyway, the only git-annex specific part is telling it the name +This is all standard ad-hoc distributed git repository setup. +The only git-annex specific part is telling it the name of the new repository created on the USB drive. Notice that both repos are set up as remotes of the other one. This lets @@ -151,6 +148,41 @@ makes it very easy. # git annex move my_cool_big_file --to usbdrive move my_cool_big_file (to usbdrive...) ok +## using ssh remotes + +So far git-annex has been used with a remote repository on a USB drive. +But it can also be used with a remote that is truely remote, a host +accessed by ssh. + +Say you have a desktop on the same network as your laptop and want +to clone the laptop's annex to it: + + # git clone ssh://mylaptop/home/me/annex ~/annex + # cd ~/annex + # git annex init "my desktop" + +Now you can get files and they will be transferred by `scp`: + + # git annex get my_cool_big_file + get my_cool_big_file (getting UUIDs for origin...) (copying from origin...) + my_cool_big_file 100% 2159 2.1KB/s 00:00 + ok + +When you drop files, git-annex will ssh over to the remote and make +sure the file's content is still there before removing it locally: + + # git annex drop my_cool_big_file + drop my_cool_big_file (checking origin..) ok + +Note that normally git-annex prefers to use non-ssh remotes, like +a USB drive, before ssh remotes. They are assumed to be faster/cheaper to +access, if available. There is a annex-cost setting you can configure in +`.git/config` to adjust which repositories it prefers. See +[[the_man_page|git-annex]] for details. + +Also, note that you need full shell access for this to work -- +git-annex needs to be able to ssh in and run commands. + ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has From 6f2a6a42d1fce3d5aa8e0b585829f4004c498165 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:42:59 -0400 Subject: [PATCH 0270/2835] real output --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 5c91f2a9c6..41e0332402 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -165,7 +165,7 @@ Now you can get files and they will be transferred by `scp`: # git annex get my_cool_big_file get my_cool_big_file (getting UUIDs for origin...) (copying from origin...) - my_cool_big_file 100% 2159 2.1KB/s 00:00 + WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 ok When you drop files, git-annex will ssh over to the remote and make From 3dbba26275d1c7ae895962b7c66e7774a716cc91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:56:57 -0400 Subject: [PATCH 0271/2835] reorg --- Backend/File.hs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 15d8f4a263..797d487479 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -102,21 +102,6 @@ copyFromRemote r key file = do location = annexLocation r key sshlocation = (Git.urlHost r) ++ ":" ++ location -showLocations :: Key -> Annex () -showLocations key = do - g <- Annex.gitRepo - u <- getUUID g - uuids <- liftIO $ keyLocations g key - let uuidsf = filter (\v -> v /= u) uuids - ppuuids <- prettyPrintUUIDs uuidsf - if (0 < length uuidsf) - then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids - else showLongNote $ "No other repository is known to contain the file." - -showTriedRemotes remotes = - showLongNote $ "I was unable to access these remotes: " ++ - (Remotes.list remotes) - {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an - error if not. -} @@ -163,3 +148,18 @@ checkRemoveKey key = do return False unsafe = showNote "unsafe" hint = showLongNote $ "(Use --force to override this check, or adjust annex.numcopies.)" + +showLocations :: Key -> Annex () +showLocations key = do + g <- Annex.gitRepo + u <- getUUID g + uuids <- liftIO $ keyLocations g key + let uuidsf = filter (\v -> v /= u) uuids + ppuuids <- prettyPrintUUIDs uuidsf + if (0 < length uuidsf) + then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids + else showLongNote $ "No other repository is known to contain the file." + +showTriedRemotes remotes = + showLongNote $ "I was unable to access these remotes: " ++ + (Remotes.list remotes) From 8ff3c73556704f94dcf80a1ddbed9430a1579a54 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:59:13 -0400 Subject: [PATCH 0272/2835] bug --- doc/bugs/scp_interrupt_to_background.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/bugs/scp_interrupt_to_background.mdwn diff --git a/doc/bugs/scp_interrupt_to_background.mdwn b/doc/bugs/scp_interrupt_to_background.mdwn new file mode 100644 index 0000000000..700c81c650 --- /dev/null +++ b/doc/bugs/scp_interrupt_to_background.mdwn @@ -0,0 +1,2 @@ +When getting a file with scp, SIGINT is blocked, exposing the git +subcommand fork to background bug again. From ff38e49eb453ccfd58ce0e424aeca97389ab0100 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 20:35:39 -0400 Subject: [PATCH 0273/2835] --from/--to can be used to limit the remote repository that git-annex uses. --- Commands.hs | 6 ++++-- Remotes.hs | 17 ++++++++++++----- debian/changelog | 2 ++ doc/git-annex.mdwn | 9 ++++++++- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/Commands.hs b/Commands.hs index 62eb08e83d..3e7447c5b1 100644 --- a/Commands.hs +++ b/Commands.hs @@ -62,8 +62,10 @@ options = [ "specify default key-value backend to use" , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") "specify a key to use" - , Option ['r'] ["repository"] (ReqArg (storestring "repository") "REPOSITORY") - "specify a repository" + , Option ['t'] ["to"] (ReqArg (storestring "repository") "REPOSITORY") + "specify a repository to transfer content to" + , Option ['f'] ["from"] (ReqArg (storestring "repository") "REPOSITORY") + "specify a repository to transfer content from" ] where storebool n b = Annex.flagChange n $ FlagBool b diff --git a/Remotes.hs b/Remotes.hs index 0e7dd31eaf..8418a42253 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -70,7 +70,7 @@ remotesByCost = do g <- Annex.gitRepo reposByCost $ Git.remotes g -{- Orders a list of git repos by cost, and throws out ignored ones. -} +{- Orders a list of git repos by cost. Throws out ignored ones. -} reposByCost :: [Git.Repo] -> Annex [Git.Repo] reposByCost l = do notignored <- filterM repoNotIgnored l @@ -99,14 +99,21 @@ repoCost r = do config g r = Git.configGet g (configkey r) "" configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" -{- Checks if a repo should be ignored. -} +{- Checks if a repo should be ignored, based either on annex-ignore + - setting, or on command-line options. Allows command-line to override + - annex-ignore. -} repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do g <- Annex.gitRepo - return ("true" /= config g r) + name <- Annex.flagGet "repository" + if (not $ null name) + then return $ match name + else return $ notignored g where - config g r = Git.configGet g (configkey r) "" - configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" + match name = name == Git.repoRemoteName r + notignored g = "true" /= config g + config g = Git.configGet g configkey "" + configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the diff --git a/debian/changelog b/debian/changelog index 68f9f02cde..0a27cd12ed 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ git-annex (0.02) UNRELEASED; urgency=low file content when dropping files. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. + * --from/--to can be used to limit the remote repository that git-annex + uses. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 37100fceef..e67d9092c5 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -133,10 +133,17 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a key to operate on, for use with the addkey subcommand. +* --from=repository + + Specifies a repository that content will be retrieved from. + It should be specified using the name of a configured git remote. + + This can be used to limit the repository used by 'git annex get'. + * --to=repository Specifies a git repository that content will be sent to. - It can be specified by a path, url, or remote name. + It should be specified using the name of a configured git remote. ## CONFIGURATION From f4e2dde8a8ceaf689ec5391174b53cb1b213ea8b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 20:47:14 -0400 Subject: [PATCH 0274/2835] fix perl refugee code --- Backend.hs | 6 +++--- Backend/File.hs | 10 +++++----- Commands.hs | 10 +++++----- Remotes.hs | 8 ++++---- UUID.hs | 2 +- Utility.hs | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Backend.hs b/Backend.hs index b8def21cd8..c70b7b707b 100644 --- a/Backend.hs +++ b/Backend.hs @@ -40,21 +40,21 @@ import qualified TypeInternals as Internals list :: Annex [Backend] list = do l <- Annex.backends -- list is cached here - if (0 < length l) + if (not $ null l) then return l else do all <- Annex.supportedBackends g <- Annex.gitRepo let l = parseBackendList all $ Git.configGet g "annex.backends" "" backendflag <- Annex.flagGet "backend" - let l' = if (0 < length backendflag) + let l' = if (not $ null backendflag) then (lookupBackendName all backendflag):l else l Annex.backendsChange $ l' return l' where parseBackendList all s = - if (length s == 0) + if (null s) then all else map (lookupBackendName all) $ words s diff --git a/Backend/File.hs b/Backend/File.hs index 797d487479..3396db3e58 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -62,7 +62,7 @@ checkKeyFile k = do copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do remotes <- Remotes.withKey key - if (0 == length remotes) + if (null remotes) then do showNote "not available" showLocations key @@ -142,7 +142,7 @@ checkRemoveKey key = do "Could only verify the existence of " ++ (show have) ++ " out of " ++ (show need) ++ " necessary copies" - if (0 /= length bad) then showTriedRemotes bad else return () + if (not $ null bad) then showTriedRemotes bad else return () showLocations key hint return False @@ -156,9 +156,9 @@ showLocations key = do uuids <- liftIO $ keyLocations g key let uuidsf = filter (\v -> v /= u) uuids ppuuids <- prettyPrintUUIDs uuidsf - if (0 < length uuidsf) - then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids - else showLongNote $ "No other repository is known to contain the file." + if (null uuidsf) + then showLongNote $ "No other repository is known to contain the file." + else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ diff --git a/Commands.hs b/Commands.hs index 3e7447c5b1..cb923d5e59 100644 --- a/Commands.hs +++ b/Commands.hs @@ -117,9 +117,9 @@ findWanted Description params _ = do parseCmd :: [String] -> AnnexState -> IO ([Annex ()], [Annex ()]) parseCmd argv state = do (flags, params) <- getopt - case (length params) of - 0 -> error usage - _ -> case (lookupCmd (params !! 0)) of + if (null params) + then error usage + else case (lookupCmd (params !! 0)) of [] -> error usage [Command _ action want _] -> do f <- findWanted want (drop 1 params) @@ -258,7 +258,7 @@ fixCmd file = inBackend file $ \(key, backend) -> do {- Stores description for the repository. -} initCmd :: String -> Annex () initCmd description = do - if (0 == length description) + if (null description) then error $ "please specify a description of this repository\n" ++ usage @@ -275,7 +275,7 @@ initCmd description = do fromKeyCmd :: FilePath -> Annex () fromKeyCmd file = do keyname <- Annex.flagGet "key" - if (0 == length keyname) + if (null keyname) then error "please specify the key with --key" else return () backends <- Backend.list diff --git a/Remotes.hs b/Remotes.hs index 8418a42253..48ab9ef8c0 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -49,11 +49,11 @@ withKey key = do let cheap = filter (not . Git.repoIsUrl) allremotes let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive - if (0 < length doexpensive) + if (not $ null doexpensive) then showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." else return () let todo = cheap ++ doexpensive - if (0 < length todo) + if (not $ null todo) then do e <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True @@ -62,7 +62,7 @@ withKey key = do where cachedUUID r = do u <- getUUID r - return $ 0 == length u + return $ null u {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] @@ -90,7 +90,7 @@ reposByCost l = do repoCost :: Git.Repo -> Annex Int repoCost r = do g <- Annex.gitRepo - if ((length $ config g r) > 0) + if (not $ null $ config g r) then return $ read $ config g r else if (Git.repoIsUrl r) then return 200 diff --git a/UUID.hs b/UUID.hs index 47d305c4fb..a7783d6140 100644 --- a/UUID.hs +++ b/UUID.hs @@ -103,7 +103,7 @@ prettyPrintUUIDs uuids = do return $ unwords $ map (\u -> "\t"++(prettify m u)++"\n") uuids where prettify m u = - if (0 < (length $ findlog m u)) + if (not $ null $ findlog m u) then u ++ " -- " ++ (findlog m u) else u findlog m u = M.findWithDefault "" u m diff --git a/Utility.hs b/Utility.hs index 8bffde0577..8e620c64cd 100644 --- a/Utility.hs +++ b/Utility.hs @@ -45,7 +45,7 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s {- Returns the parent directory of a path. Parent of / is "" -} parentDir :: String -> String parentDir dir = - if length dirs > 0 + if (not $ null dirs) then slash ++ (join s $ take ((length dirs) - 1) dirs) else "" where @@ -81,7 +81,7 @@ relPathCwdToDir dir = do -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = - if (0 < length path) + if (not $ null path) then addTrailingPathSeparator path else "" where From 2fbbbd34bcf8e925f84d96510eb063bdfbfe3f9b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 12:35:10 -0400 Subject: [PATCH 0275/2835] store from and to repositories separately --- Commands.hs | 4 ++-- Remotes.hs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Commands.hs b/Commands.hs index cb923d5e59..8be930bcc1 100644 --- a/Commands.hs +++ b/Commands.hs @@ -62,9 +62,9 @@ options = [ "specify default key-value backend to use" , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storestring "repository") "REPOSITORY") + , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") "specify a repository to transfer content to" - , Option ['f'] ["from"] (ReqArg (storestring "repository") "REPOSITORY") + , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") "specify a repository to transfer content from" ] where diff --git a/Remotes.hs b/Remotes.hs index 48ab9ef8c0..f24da2c222 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -105,7 +105,9 @@ repoCost r = do repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do g <- Annex.gitRepo - name <- Annex.flagGet "repository" + fromName <- Annex.flagGet "fromrepository" + toName <- Annex.flagGet "torepository" + let name = if (not $ null fromName) then fromName else toName if (not $ null name) then return $ match name else return $ notignored g From 5a91543be33719d6da7b53c4c449be8f75481375 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 12:41:13 -0400 Subject: [PATCH 0276/2835] update --- doc/git-annex.mdwn | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e67d9092c5..522be75700 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -48,11 +48,11 @@ content from the key-value store. add iso/Debian_5.0.iso ok # git commit -a -m "saving Debian CD for later" - # git annex drop iso - drop iso/Debian_5.0.iso ok + # git annex drop iso/Debian_4.0.iso + drop iso/Debian_4.0.iso ok # git commit -a -m "freed up space" - # git annex move video --to=usbdrive + # git annex move iso --to=usbdrive move iso/Debian_5.0.iso (to usbdrive...) ok # SUBCOMMANDS @@ -83,12 +83,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * move [path ...] - Moves the content of annexed files from the current repository to - another one. Use with the --to option. + When used with the --to option, moves the content of annexed files from + the current repository to the specified one. - Note that unlike drop, this does not honor annex.numcopies. - A file's content can be moved even if there are insufficient - copies to allow it to be dropped. + When used with the --from option, moves the content of annexed files + from the specified repository to the current one. * init description @@ -138,8 +137,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a repository that content will be retrieved from. It should be specified using the name of a configured git remote. - This can be used to limit the repository used by 'git annex get'. - * --to=repository Specifies a git repository that content will be sent to. From 9dfbf40d1a8493ec191f8e79410ed9d2a9508141 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 13:18:47 -0400 Subject: [PATCH 0277/2835] reorg remote key presense checking code Also, it now checks if a key is inAnnex, ie, cached in .git/annex, not if it is present in a remote. For the File Backend, these are equivilant, not so for other backends. --- Backend/File.hs | 29 ++++++----------------------- Core.hs | 12 ++++++++++-- Remotes.hs | 31 ++++++++++++++++++++++++------- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 3396db3e58..dbd0674286 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -44,24 +44,15 @@ mustProvide = error "must provide this field" dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True -{- Just check if the .git/annex/ file for the key exists. - - - - But, if running against a remote annex, need to use ssh to do it. -} +{- Just check if the .git/annex/ file for the key exists. -} checkKeyFile :: Key -> Annex Bool -checkKeyFile k = do - g <- Annex.gitRepo - if (not $ Git.repoIsUrl g) - then inAnnex k - else do - showNote ("checking " ++ Git.repoDescribe g ++ "...") - liftIO $ boolSystem "ssh" [Git.urlHost g, - "test -e " ++ (shellEscape $ annexLocation g k)] +checkKeyFile k = inAnnex k {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do - remotes <- Remotes.withKey key + remotes <- Remotes.keyPossibilities key if (null remotes) then do showNote "not available" @@ -97,7 +88,6 @@ copyFromRemote r key file = do getlocal = boolSystem "cp" ["-a", location, file] getssh = do liftIO $ putStrLn "" -- make way for scp progress bar - -- TODO double-shell-quote path for scp boolSystem "scp" [sshlocation, file] location = annexLocation r key sshlocation = (Git.urlHost r) ++ ":" ++ location @@ -112,7 +102,7 @@ checkRemoveKey key = do then return True else do g <- Annex.gitRepo - remotes <- Remotes.withKey key + remotes <- Remotes.keyPossibilities key let numcopies = read $ Git.configGet g config "1" if (numcopies > length remotes) then notEnoughCopies numcopies (length remotes) [] @@ -124,18 +114,11 @@ checkRemoveKey key = do then return True else notEnoughCopies need have bad findcopies need have (r:rs) bad = do - all <- Annex.supportedBackends - result <- liftIO $ ((try $ remoteHasKey r all)::IO (Either SomeException Bool)) - case (result) of + haskey <- Remotes.inAnnex r key + case (haskey) of Right True -> findcopies need (have+1) rs bad Right False -> findcopies need have rs bad Left _ -> findcopies need have rs (r:bad) - remoteHasKey remote all = do - -- To check if a remote has a key, construct a new - -- Annex monad and query its backend. - a <- Annex.new remote all - (result, _) <- Annex.run a (Backend.hasKey key) - return result notEnoughCopies need have bad = do unsafe showLongNote $ diff --git a/Core.hs b/Core.hs index 4941dc26bf..da05823bb3 100644 --- a/Core.hs +++ b/Core.hs @@ -62,11 +62,19 @@ gitAttributes repo = do Git.run repo ["commit", "-m", "git-annex setup", attributes] -{- Checks if a given key is currently present in the annexLocation -} +{- Checks if a given key is currently present in the annexLocation. + - + - This can be run against a remote repository to check the key there. -} inAnnex :: Key -> Annex Bool inAnnex key = do g <- Annex.gitRepo - liftIO $ doesFileExist $ annexLocation g key + if (not $ Git.repoIsUrl g) + then liftIO $ doesFileExist $ annexLocation g key + else do + showNote ("checking " ++ Git.repoDescribe g ++ "...") + liftIO $ boolSystem "ssh" [Git.urlHost g, + "test -e " ++ + (shellEscape $ annexLocation g key)] {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath diff --git a/Remotes.hs b/Remotes.hs index f24da2c222..13f66aae23 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -2,8 +2,9 @@ module Remotes ( list, - withKey, - tryGitConfigRead + keyPossibilities, + tryGitConfigRead, + inAnnex ) where import Control.Exception @@ -18,18 +19,19 @@ import Maybe import Types import qualified GitRepo as Git import qualified Annex +import qualified Backend import LocationLog import Locations import UUID -import Core +import qualified Core {- Human visible list of remotes. -} list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -withKey :: Key -> Annex [Git.Repo] -withKey key = do +keyPossibilities :: Key -> Annex [Git.Repo] +keyPossibilities key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost @@ -50,20 +52,35 @@ withKey key = do let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive if (not $ null doexpensive) - then showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." + then Core.showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." else return () let todo = cheap ++ doexpensive if (not $ null todo) then do e <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True - withKey key + keyPossibilities key else reposByUUID allremotes uuids where cachedUUID r = do u <- getUUID r return $ null u +{- Checks if a given remote has the content for a key inAnnex. + - + - This is done by constructing a new Annex monad using the remote. + - + - If the remote cannot be accessed, returns a Left error. + -} +inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) +inAnnex remote key = do + a <- liftIO $ Annex.new remote [] + liftIO $ ((try $ check a)::IO (Either IOException Bool)) + where + check a = do + (result, _) <- Annex.run a (Core.inAnnex key) + return result + {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] remotesByCost = do From 03bcb8d8b3033726eb4a5b1122fbadc1adb6b3a5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 13:59:47 -0400 Subject: [PATCH 0278/2835] better message display --- Core.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core.hs b/Core.hs index da05823bb3..176aa2bf38 100644 --- a/Core.hs +++ b/Core.hs @@ -97,11 +97,11 @@ logStatus key status = do {- Output logging -} showStart :: String -> String -> Annex () showStart command file = do - liftIO $ putStr $ command ++ " " ++ file + liftIO $ putStr $ command ++ " " ++ file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () showNote s = do - liftIO $ putStr $ " (" ++ s ++ ")" + liftIO $ putStr $ "(" ++ s ++ ") " liftIO $ hFlush stdout showLongNote :: String -> Annex () showLongNote s = do @@ -110,7 +110,7 @@ showLongNote s = do indent s = join "\n" $ map (\l -> " " ++ l) $ lines s showEndOk :: Annex () showEndOk = do - liftIO $ putStrLn " ok" + liftIO $ putStrLn "ok" showEndFail :: Annex () showEndFail = do liftIO $ putStrLn "\nfailed" From 08236e780fd0047b1dbab2ef7c50d96be8709cab Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:14:36 -0400 Subject: [PATCH 0279/2835] reorg --- Backend/File.hs | 29 ++++------------------------- Remotes.hs | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index dbd0674286..14b4b9dae5 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -66,31 +66,10 @@ copyKeyFile key file = do showLocations key return False trycopy full (r:rs) = do - -- annexLocation needs the git config to have been - -- read for a remote, so do that now, - -- if it hasn't been already - result <- Remotes.tryGitConfigRead r - case (result) of - Left err -> trycopy full rs - Right r' -> do - showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." - liftIO $ copyFromRemote r' key file - -{- Tries to copy a file from a remote. -} -copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool -copyFromRemote r key file = do - if (not $ Git.repoIsUrl r) - then getlocal - else if (Git.repoIsSsh r) - then getssh - else error "copying from non-ssh repo not supported" - where - getlocal = boolSystem "cp" ["-a", location, file] - getssh = do - liftIO $ putStrLn "" -- make way for scp progress bar - boolSystem "scp" [sshlocation, file] - location = annexLocation r key - sshlocation = (Git.urlHost r) ++ ":" ++ location + copied <- Remotes.copyFromRemote r key file + if (copied) + then return True + else trycopy full rs {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an diff --git a/Remotes.hs b/Remotes.hs index 13f66aae23..a4b358c779 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -4,7 +4,9 @@ module Remotes ( list, keyPossibilities, tryGitConfigRead, - inAnnex + inAnnex, + commandLineRemote, + copyFromRemote ) where import Control.Exception @@ -20,10 +22,11 @@ import Types import qualified GitRepo as Git import qualified Annex import qualified Backend +import qualified Core import LocationLog import Locations import UUID -import qualified Core +import Utility {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -134,6 +137,22 @@ repoNotIgnored r = do config g = Git.configGet g configkey "" configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" +{- Returns the remote specified by --from or --to, may fail with error. -} +commandLineRemote :: Annex Git.Repo +commandLineRemote = do + fromName <- Annex.flagGet "fromrepository" + toName <- Annex.flagGet "torepository" + let name = if (not $ null fromName) then fromName else toName + if (null name) + then error "no remote specified" + else do + g <- Annex.gitRepo + let match = filter (\r -> name == Git.repoRemoteName r) $ + Git.remotes g + if (null match) + then error $ "there is no git remote named \"" ++ name ++ "\"" + else return $ match !! 0 + {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the - config for a specified remote, and updates state. If successful, it @@ -161,3 +180,27 @@ tryGitConfigRead r = do if ((Git.repoRemoteName old) == (Git.repoRemoteName new)) then new:(exchange ls new) else old:(exchange ls new) + +{- Tries to copy a file from a remote. -} +copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool +copyFromRemote r key file = do + -- annexLocation needs the git config to have been read for a remote, + -- so do that now if it hasn't been already + result <- tryGitConfigRead r + case (result) of + Left err -> return False + Right r' -> copy r' + where + copy r = do + Core.showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." + if (not $ Git.repoIsUrl r) + then getlocal + else if (Git.repoIsSsh r) + then getssh + else error "copying from non-ssh repo not supported" + getlocal = liftIO $ boolSystem "cp" ["-a", location, file] + getssh = do + liftIO $ putStrLn "" -- make way for scp progress bar + liftIO $ boolSystem "scp" [sshlocation, file] + location = annexLocation r key + sshlocation = (Git.urlHost r) ++ ":" ++ location From 5601a8db02e516d77d62fe18972a882ebad53b29 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:14:50 -0400 Subject: [PATCH 0280/2835] update --- doc/walkthrough.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 41e0332402..c9eba0a575 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -166,7 +166,7 @@ Now you can get files and they will be transferred by `scp`: # git annex get my_cool_big_file get my_cool_big_file (getting UUIDs for origin...) (copying from origin...) WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 - ok + ok When you drop files, git-annex will ssh over to the remote and make sure the file's content is still there before removing it locally: @@ -202,7 +202,7 @@ and cache it locally. # git annex get somefile get somefile (downloading) #########################################################################100.0% - ok + ok You can always drop files downloaded by the URL backend. It is assumed that the URL is stable; no local backup is kept. From fcd30ce99251cf8bd98ab25fb73f1fcb5b081a4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:26:23 -0400 Subject: [PATCH 0281/2835] bugfix --- Remotes.hs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index a4b358c779..c1cab73c62 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -198,9 +198,10 @@ copyFromRemote r key file = do else if (Git.repoIsSsh r) then getssh else error "copying from non-ssh repo not supported" - getlocal = liftIO $ boolSystem "cp" ["-a", location, file] - getssh = do - liftIO $ putStrLn "" -- make way for scp progress bar - liftIO $ boolSystem "scp" [sshlocation, file] - location = annexLocation r key - sshlocation = (Git.urlHost r) ++ ":" ++ location + where + getlocal = liftIO $ boolSystem "cp" ["-a", location, file] + getssh = do + liftIO $ putStrLn "" -- make way for scp progress bar + liftIO $ boolSystem "scp" [sshlocation, file] + location = annexLocation r key + sshlocation = (Git.urlHost r) ++ ":" ++ location From 4c7248c77998d011f8978a32328b8f3817d7acbc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:26:38 -0400 Subject: [PATCH 0282/2835] factored out getViaTmp --- Core.hs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Core.hs b/Core.hs index 176aa2bf38..acb4489c8e 100644 --- a/Core.hs +++ b/Core.hs @@ -94,6 +94,24 @@ logStatus key status = do u <- getUUID g liftIO $ logChange g key u status +{- Runs an action, passing it a temporary filename to download, + - and if the action succeeds, moves the temp file into + - the annex as a key's content. -} +getViaTmp :: Key -> (FilePath -> Annex (Bool)) -> Annex () +getViaTmp key action = do + g <- Annex.gitRepo + let dest = annexLocation g key + let tmp = (annexTmpLocation g) ++ (keyFile key) + liftIO $ createDirectoryIfMissing True (parentDir tmp) + success <- action tmp + if (success) + then do + liftIO $ renameFile tmp dest + logStatus key ValuePresent + showEndOk + else do + showEndFail + {- Output logging -} showStart :: String -> String -> Annex () showStart command file = do From f05ed818f9e8e49b646805402be928f9c89c9a7f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:27:04 -0400 Subject: [PATCH 0283/2835] implemented 1/4th of move subcommand --- Commands.hs | 115 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 34 deletions(-) diff --git a/Commands.hs b/Commands.hs index 8be930bcc1..011481bd8f 100644 --- a/Commands.hs +++ b/Commands.hs @@ -41,7 +41,7 @@ cmds = [ , (Command "drop" dropCmd FilesInGit "indicate content of files not currently wanted") , (Command "move" moveCmd FilesInGit - "transfer content of files to another repository") + "transfer content of files to/from another repository") , (Command "init" initCmd Description "initialize git-annex with repository description") , (Command "unannex" unannexCmd FilesInGit @@ -63,9 +63,9 @@ options = [ , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") "specify a key to use" , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") - "specify a repository to transfer content to" + "specify to where to transfer content" , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") - "specify a repository to transfer content from" + "specify from where to transfer content" ] where storebool n b = Annex.flagChange n $ FlagBool b @@ -136,7 +136,7 @@ parseCmd argv state = do {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () -addCmd file = notInBackend file $ do +addCmd file = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return () @@ -161,7 +161,7 @@ addCmd file = notInBackend file $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () -unannexCmd file = inBackend file $ \(key, backend) -> do +unannexCmd file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file Annex.flagChange "force" $ FlagBool True -- force backend to always remove Backend.removeKey backend key @@ -181,41 +181,18 @@ unannexCmd file = inBackend file $ \(key, backend) -> do {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () -getCmd file = inBackend file $ \(key, backend) -> do +getCmd file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return () else do showStart "get" file - g <- Annex.gitRepo - let dest = annexLocation g key - let tmp = (annexTmpLocation g) ++ (keyFile key) - liftIO $ createDirectoryIfMissing True (parentDir tmp) - success <- Backend.retrieveKeyFile backend key tmp - if (success) - then do - liftIO $ renameFile tmp dest - logStatus key ValuePresent - showEndOk - else do - showEndFail - -{- Moves the content of an annexed file to another repository, - - removing it from the current repository, and updates locationlog - - information on both. - - - - Note that unlike drop, this does not honor annex.numcopies. - - A file's content can be moved even if there are insufficient copies to - - allow it to be dropped. - -} -moveCmd :: FilePath -> Annex () -moveCmd file = inBackend file $ \(key, backend) -> do - error "TODO" + getViaTmp key (Backend.retrieveKeyFile backend key) {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} dropCmd :: FilePath -> Annex () -dropCmd file = inBackend file $ \(key, backend) -> do +dropCmd file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return () -- no-op @@ -241,7 +218,7 @@ dropCmd file = inBackend file $ \(key, backend) -> do {- Fixes the symlink to an annexed file. -} fixCmd :: FilePath -> Annex () -fixCmd file = inBackend file $ \(key, backend) -> do +fixCmd file = isAnnexed file $ \(key, backend) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) @@ -294,13 +271,83 @@ fromKeyCmd file = do liftIO $ Git.run g ["add", file] showEndOk +{- Move a file either --to or --from a repository. + - + - This only operates on the cached file content; it does not involve + - moving data in the key-value backend. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. + -} +moveCmd :: FilePath -> Annex () +moveCmd file = do + fromName <- Annex.flagGet "fromrepository" + toName <- Annex.flagGet "torepository" + case (fromName, toName) of + ("", "") -> error "specify either --from or --to" + ("", to) -> moveTo file + (from, "") -> moveFrom file + (_, _) -> error "only one of --from or --to can be specified" + +{- Moves the content of an annexed file to another repository, + - removing it from the current repository, and updates locationlog + - information on both. + - + - If the destination already has the content, it is still removed + - from the current repository. + -} +moveTo :: FilePath -> Annex () +moveTo file = isAnnexed file $ \(key, backend) -> do + ishere <- inAnnex key + if (not ishere) + then return () -- not here, so nothing to do + else do + showStart "move" file + remote <- Remotes.commandLineRemote + isthere <- Remotes.inAnnex remote key + case isthere of + Left err -> error (show err) + Right True -> removeit + Right False -> moveit + where + moveit = do + error $ "TODO move" ++ file + removeit = do + error $ "TODO remove" ++ file + +{- Moves the content of an annexed file from another repository to the current + - repository and updates locationlog information on both. + - + - If the current repository already has the content, it is still removed + - from the other repository. + -} +moveFrom :: FilePath -> Annex () +moveFrom file = isAnnexed file $ \(key, backend) -> do + showStart "move" file -- have to show this before checking remote + ishere <- inAnnex key + remote <- Remotes.commandLineRemote + isthere <- Remotes.inAnnex remote key + case (ishere, isthere) of + (_, Left err) -> error (show err) + (_, Right False) -> showEndFail + (False, Right True) -> moveit remote key + (True, Right True) -> removeit remote key + where + moveit remote key = do + getViaTmp key (Remotes.copyFromRemote remote key) + removeit remote key + removeit remote key = do + error $ "TODO remove" ++ file + showEndOk + -- helpers -notInBackend file a = do +notAnnexed file a = do r <- Backend.lookupFile file case (r) of Just v -> return () Nothing -> a -inBackend file a = do +isAnnexed file a = do r <- Backend.lookupFile file case (r) of Just v -> a v From 3cf16c9883d63247cceb1f8cf2ce41d43b7a214e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:58:14 -0400 Subject: [PATCH 0284/2835] incomplete --- Commands.hs | 11 ++++++----- Remotes.hs | 42 ++++++++++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Commands.hs b/Commands.hs index 011481bd8f..ffa3576cf9 100644 --- a/Commands.hs +++ b/Commands.hs @@ -308,12 +308,13 @@ moveTo file = isAnnexed file $ \(key, backend) -> do isthere <- Remotes.inAnnex remote key case isthere of Left err -> error (show err) - Right True -> removeit - Right False -> moveit + Right False -> moveit remote key + Right True -> removeit remote key where - moveit = do - error $ "TODO move" ++ file - removeit = do + moveit remote key = do + Remotes.copyToRemote remote key + removeit remote key + removeit remote key = do error $ "TODO remove" ++ file {- Moves the content of an annexed file from another repository to the current diff --git a/Remotes.hs b/Remotes.hs index c1cab73c62..4cfcfdffdc 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -6,7 +6,8 @@ module Remotes ( tryGitConfigRead, inAnnex, commandLineRemote, - copyFromRemote + copyFromRemote, + copyToRemote ) where import Control.Exception @@ -70,13 +71,11 @@ keyPossibilities key = do return $ null u {- Checks if a given remote has the content for a key inAnnex. - - - - This is done by constructing a new Annex monad using the remote. - - - If the remote cannot be accessed, returns a Left error. -} inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) inAnnex remote key = do + -- the check needs to run in an Annex monad using the remote a <- liftIO $ Annex.new remote [] liftIO $ ((try $ check a)::IO (Either IOException Bool)) where @@ -181,7 +180,7 @@ tryGitConfigRead r = do then new:(exchange ls new) else old:(exchange ls new) -{- Tries to copy a file from a remote. -} +{- Tries to copy a key's content from a remote to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file = do -- annexLocation needs the git config to have been read for a remote, @@ -189,13 +188,13 @@ copyFromRemote r key file = do result <- tryGitConfigRead r case (result) of Left err -> return False - Right r' -> copy r' + Right from -> copy from where - copy r = do - Core.showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." - if (not $ Git.repoIsUrl r) + copy from = do + Core.showNote $ "copying from " ++ (Git.repoDescribe from) ++ "..." + if (not $ Git.repoIsUrl from) then getlocal - else if (Git.repoIsSsh r) + else if (Git.repoIsSsh from) then getssh else error "copying from non-ssh repo not supported" where @@ -203,5 +202,24 @@ copyFromRemote r key file = do getssh = do liftIO $ putStrLn "" -- make way for scp progress bar liftIO $ boolSystem "scp" [sshlocation, file] - location = annexLocation r key - sshlocation = (Git.urlHost r) ++ ":" ++ location + location = annexLocation from key + sshlocation = (Git.urlHost from) ++ ":" ++ location + +{- Tries to copy a key's content to a remote. -} +copyToRemote :: Git.Repo -> Key -> Annex Bool +copyToRemote r key = do + g <- Annex.gitRepo + Core.showNote $ "copying to " ++ (Git.repoDescribe r) ++ "..." + if (not $ Git.repoIsUrl r) + then sendlocal g + else if (Git.repoIsSsh r) + then sendssh g + else error "copying to non-ssh repo not supported" + where + sendlocal g = liftIO $ boolSystem "cp" ["-a", location g, file] + sendssh g = do + liftIO $ putStrLn "" -- make way for scp progress bar + liftIO $ boolSystem "scp" [location g, sshlocation] + location g = annexLocation g key + sshlocation = (Git.urlHost r) ++ ":" ++ file + file = error "TODO" From fed1d3c1f534e01414292cd99724e15f5b19cda7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 22:25:07 -0400 Subject: [PATCH 0285/2835] bugfix Annex.new can throw an IO exception if a remote git repo is not available. --- Remotes.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 4cfcfdffdc..aec38a363e 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -76,10 +76,10 @@ keyPossibilities key = do inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) inAnnex remote key = do -- the check needs to run in an Annex monad using the remote - a <- liftIO $ Annex.new remote [] - liftIO $ ((try $ check a)::IO (Either IOException Bool)) + liftIO $ ((try $ check)::IO (Either IOException Bool)) where - check a = do + check = do + a <- Annex.new remote [] (result, _) <- Annex.run a (Core.inAnnex key) return result From 81f71e57b9ac68b8d79c30fb27a22c5e3941fcee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 11:47:45 -0400 Subject: [PATCH 0286/2835] reorg --- doc/index.mdwn | 1 + doc/todo.mdwn | 4 ++++ doc/{bugs => todo}/add_a_git_backend.mdwn | 0 doc/{bugs => todo}/backendSHA1.mdwn | 0 doc/{bugs => todo}/branching.mdwn | 0 doc/todo/done.mdwn | 4 ++++ doc/{bugs => todo}/file_copy_progress_bar.mdwn | 0 doc/{bugs => todo}/fsck.mdwn | 0 doc/{bugs => todo}/gitrm.mdwn | 0 doc/{bugs => todo}/network_remotes.mdwn | 0 doc/todo/parallel_possibilities.mdwn | 13 +++++++++++++ doc/{bugs => todo}/pushpull.mdwn | 0 doc/{bugs => todo}/rsync.mdwn | 0 doc/{bugs => todo}/symlink_farming_commit_hook.mdwn | 0 doc/{bugs => todo}/using_url_backend.mdwn | 0 15 files changed, 22 insertions(+) create mode 100644 doc/todo.mdwn rename doc/{bugs => todo}/add_a_git_backend.mdwn (100%) rename doc/{bugs => todo}/backendSHA1.mdwn (100%) rename doc/{bugs => todo}/branching.mdwn (100%) create mode 100644 doc/todo/done.mdwn rename doc/{bugs => todo}/file_copy_progress_bar.mdwn (100%) rename doc/{bugs => todo}/fsck.mdwn (100%) rename doc/{bugs => todo}/gitrm.mdwn (100%) rename doc/{bugs => todo}/network_remotes.mdwn (100%) create mode 100644 doc/todo/parallel_possibilities.mdwn rename doc/{bugs => todo}/pushpull.mdwn (100%) rename doc/{bugs => todo}/rsync.mdwn (100%) rename doc/{bugs => todo}/symlink_farming_commit_hook.mdwn (100%) rename doc/{bugs => todo}/using_url_backend.mdwn (100%) diff --git a/doc/index.mdwn b/doc/index.mdwn index de5fe55a3c..ae5dbf85e5 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -18,6 +18,7 @@ To get a feel for it, see the [[walkthrough]]. * [[install]] * [[news]] * [[bugs]] +* [[todo]] * [[contact]] """]] diff --git a/doc/todo.mdwn b/doc/todo.mdwn new file mode 100644 index 0000000000..79552298b6 --- /dev/null +++ b/doc/todo.mdwn @@ -0,0 +1,4 @@ +This is git-annex's todo list. Link items to [[todo/done]] when done. + +[[!inline pages="./todo/* and !./todo/done and !link(done) +and !*/Discussion" actions=yes postform=yes show=0 archive=yes]] diff --git a/doc/bugs/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn similarity index 100% rename from doc/bugs/add_a_git_backend.mdwn rename to doc/todo/add_a_git_backend.mdwn diff --git a/doc/bugs/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn similarity index 100% rename from doc/bugs/backendSHA1.mdwn rename to doc/todo/backendSHA1.mdwn diff --git a/doc/bugs/branching.mdwn b/doc/todo/branching.mdwn similarity index 100% rename from doc/bugs/branching.mdwn rename to doc/todo/branching.mdwn diff --git a/doc/todo/done.mdwn b/doc/todo/done.mdwn new file mode 100644 index 0000000000..e7c98081b7 --- /dev/null +++ b/doc/todo/done.mdwn @@ -0,0 +1,4 @@ +recently fixed [[todo]] items. + +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10 +archive=yes]] diff --git a/doc/bugs/file_copy_progress_bar.mdwn b/doc/todo/file_copy_progress_bar.mdwn similarity index 100% rename from doc/bugs/file_copy_progress_bar.mdwn rename to doc/todo/file_copy_progress_bar.mdwn diff --git a/doc/bugs/fsck.mdwn b/doc/todo/fsck.mdwn similarity index 100% rename from doc/bugs/fsck.mdwn rename to doc/todo/fsck.mdwn diff --git a/doc/bugs/gitrm.mdwn b/doc/todo/gitrm.mdwn similarity index 100% rename from doc/bugs/gitrm.mdwn rename to doc/todo/gitrm.mdwn diff --git a/doc/bugs/network_remotes.mdwn b/doc/todo/network_remotes.mdwn similarity index 100% rename from doc/bugs/network_remotes.mdwn rename to doc/todo/network_remotes.mdwn diff --git a/doc/todo/parallel_possibilities.mdwn b/doc/todo/parallel_possibilities.mdwn new file mode 100644 index 0000000000..178f95021b --- /dev/null +++ b/doc/todo/parallel_possibilities.mdwn @@ -0,0 +1,13 @@ +One of my reasons for using haskell was that it provides the possibility of +some parallell processing. Although since git-annex hits the filesystem +heavily and mostly runs other git commands, maybe not a whole lot. + +Anyway, each git-annex command is broken down into a series of independant +actions, which has some potential for parallelism. + +Probably they would need to be split further. Each action currently has 3 +distinct phases, basically "check", "do", and "record". If the check action +returned a do action that returned a record action, then it could easily +make sense to parallelize the check actions and start on the do actions +(which probably won't parallelize well) while they are still being +generated, and possibly parallelize the record actions at the end. diff --git a/doc/bugs/pushpull.mdwn b/doc/todo/pushpull.mdwn similarity index 100% rename from doc/bugs/pushpull.mdwn rename to doc/todo/pushpull.mdwn diff --git a/doc/bugs/rsync.mdwn b/doc/todo/rsync.mdwn similarity index 100% rename from doc/bugs/rsync.mdwn rename to doc/todo/rsync.mdwn diff --git a/doc/bugs/symlink_farming_commit_hook.mdwn b/doc/todo/symlink_farming_commit_hook.mdwn similarity index 100% rename from doc/bugs/symlink_farming_commit_hook.mdwn rename to doc/todo/symlink_farming_commit_hook.mdwn diff --git a/doc/bugs/using_url_backend.mdwn b/doc/todo/using_url_backend.mdwn similarity index 100% rename from doc/bugs/using_url_backend.mdwn rename to doc/todo/using_url_backend.mdwn From e29210d1dddb79abc0f93fc5175add8e10455688 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 14:10:38 -0400 Subject: [PATCH 0287/2835] update --- Commands.hs | 17 ++++++++++++++--- Core.hs | 6 +++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Commands.hs b/Commands.hs index ffa3576cf9..8c9dca9adb 100644 --- a/Commands.hs +++ b/Commands.hs @@ -86,7 +86,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' -{- Generate descrioptions of wanted parameters for subcommands. -} +{- Generate descriptions of wanted parameters for subcommands. -} descWanted :: CmdWants -> String descWanted Description = "DESCRIPTION" descWanted _ = "PATH ..." @@ -187,7 +187,10 @@ getCmd file = isAnnexed file $ \(key, backend) -> do then return () else do showStart "get" file - getViaTmp key (Backend.retrieveKeyFile backend key) + ok <- getViaTmp key (Backend.retrieveKeyFile backend key) + if (ok) + then showEndOk + else showEndFail {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} @@ -315,7 +318,15 @@ moveTo file = isAnnexed file $ \(key, backend) -> do Remotes.copyToRemote remote key removeit remote key removeit remote key = do - error $ "TODO remove" ++ file + error "TODO: drop key from local" + -- Update local location log; key is present + -- there and missing here. + logStatus key ValueMissing + u <- getUUID remote + liftIO $ logChange remote key u ValuePresent + -- Propigate location log to remote. + error "TODO: update remote locationlog" + showEndOk {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. diff --git a/Core.hs b/Core.hs index acb4489c8e..1d887792ad 100644 --- a/Core.hs +++ b/Core.hs @@ -97,7 +97,7 @@ logStatus key status = do {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into - the annex as a key's content. -} -getViaTmp :: Key -> (FilePath -> Annex (Bool)) -> Annex () +getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool getViaTmp key action = do g <- Annex.gitRepo let dest = annexLocation g key @@ -108,9 +108,9 @@ getViaTmp key action = do then do liftIO $ renameFile tmp dest logStatus key ValuePresent - showEndOk + return True else do - showEndFail + return False {- Output logging -} showStart :: String -> String -> Annex () From 7fe4bfa20fc9e6ced0b0e933891becb0546b79bb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 15:44:27 -0400 Subject: [PATCH 0288/2835] split commands into 3 phases I feel like I just leveled up in Haskell. --- Commands.hs | 393 ++++++++++++++++----------- Core.hs | 2 + doc/todo/parallel_possibilities.mdwn | 11 +- 3 files changed, 236 insertions(+), 170 deletions(-) diff --git a/Commands.hs b/Commands.hs index 8c9dca9adb..f4286abf5b 100644 --- a/Commands.hs +++ b/Commands.hs @@ -24,31 +24,66 @@ import Core import qualified Remotes import qualified TypeInternals -data CmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description -data Command = Command { - cmdname :: String, - cmdaction :: (String -> Annex ()), - cmdwants :: CmdWants, - cmddesc :: String +{- A subcommand runs in three stages. Each stage can return the next stage + - to run. + - + - 1. The start stage is run before anything is printed about the + - subcommand, and can early abort it if the input does not make sense. + - It should run quickly and should not modify Annex state. + - + - 2. The perform stage is run after a message is printed about the subcommand + - being run. + - + - 3. The cleanup stage is run only if the do stage succeeds, and it returns + - the overall success/fail of the subcommand. + -} +type SubCmdStart = String -> Annex (Maybe SubCmdPerform) +type SubCmdPerform = Annex (Maybe SubCmdCleanup) +type SubCmdCleanup = Annex Bool + +{- Runs a subcommand through its three stages. -} +doSubCmd :: String -> SubCmdStart -> String -> Annex () +doSubCmd cmdname start param = do + res <- start param :: Annex (Maybe SubCmdPerform) + case (res) of + Nothing -> return () + Just perform -> do + showStart cmdname param + res <- perform :: Annex (Maybe SubCmdCleanup) + case (res) of + Nothing -> showEndFail + Just cleanup -> do + res <- cleanup + if (res) + then showEndOk + else showEndFail + + +data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description +data SubCommand = Command { + subcmdname :: String, + subcmdaction :: SubCmdStart, + subcmdwants :: SubCmdWants, + subcmddesc :: String } -cmds :: [Command] -cmds = [ - (Command "add" addCmd FilesNotInGit +subCmds :: [SubCommand] +subCmds = [ + (Command "add" addStart FilesNotInGit "add files to annex") - , (Command "get" getCmd FilesInGit + , (Command "get" getStart FilesInGit "make content of annexed files available") - , (Command "drop" dropCmd FilesInGit + , (Command "drop" dropStart FilesInGit "indicate content of files not currently wanted") - , (Command "move" moveCmd FilesInGit + , (Command "move" moveStart FilesInGit "transfer content of files to/from another repository") - , (Command "init" initCmd Description + , (Command "init" initStart Description "initialize git-annex with repository description") - , (Command "unannex" unannexCmd FilesInGit + , (Command "unannex" unannexStart FilesInGit "undo accidential add command") - , (Command "fix" fixCmd FilesInGit + , (Command "fix" fixStart FilesInGit "fix up files' symlinks to point to annexed content") - , (Command "fromkey" fromKeyCmd FilesMissing + , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") ] @@ -71,29 +106,29 @@ options = [ storebool n b = Annex.flagChange n $ FlagBool b storestring n s = Annex.flagChange n $ FlagString s -header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) +header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where - cmddescs = unlines $ map (\c -> indent $ showcmd c) cmds + cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds showcmd c = - (cmdname c) ++ - (pad 10 (cmdname c)) ++ - (descWanted (cmdwants c)) ++ - (pad 13 (descWanted (cmdwants c))) ++ - (cmddesc c) + (subcmdname c) ++ + (pad 10 (subcmdname c)) ++ + (descWanted (subcmdwants c)) ++ + (pad 13 (descWanted (subcmdwants c))) ++ + (subcmddesc c) indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' {- Generate descriptions of wanted parameters for subcommands. -} -descWanted :: CmdWants -> String +descWanted :: SubCmdWants -> String descWanted Description = "DESCRIPTION" descWanted _ = "PATH ..." -{- Finds the type of parameters a command wants, from among the passed +{- Finds the type of parameters a subcommand wants, from among the passed - parameter list. -} -findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] +findWanted :: SubCmdWants -> [String] -> Git.Repo -> IO [String] findWanted FilesNotInGit params repo = do files <- mapM (Git.notInRepo repo) params return $ foldl (++) [] files @@ -121,139 +156,154 @@ parseCmd argv state = do then error usage else case (lookupCmd (params !! 0)) of [] -> error usage - [Command _ action want _] -> do + [Command name action want _] -> do f <- findWanted want (drop 1 params) (TypeInternals.repo state) - return (flags, map action $ filter notstate f) + return (flags, map (doSubCmd name action) $ + filter notstate f) where -- never include files from the state directory notstate f = stateLoc /= take (length stateLoc) f getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) - lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds + lookupCmd cmd = filter (\c -> cmd == subcmdname c) subCmds -{- Annexes a file, storing it in a backend, and then moving it into - - the annex directory and setting up the symlink pointing to its content. -} -addCmd :: FilePath -> Annex () -addCmd file = notAnnexed file $ do +{- The add subcommand annexes a file, storing it in a backend, and then + - moving it into the annex directory and setting up the symlink pointing + - to its content. -} +addStart :: FilePath -> Annex (Maybe SubCmdPerform) +addStart file = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) - then return () - else do - showStart "add" file - g <- Annex.gitRepo - stored <- Backend.storeFileKey file - case (stored) of - Nothing -> showEndFail - Just (key, backend) -> do - logStatus key ValuePresent - setup g key - where - setup g key = do - let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - liftIO $ renameFile file dest - link <- calcGitLink file key - liftIO $ createSymbolicLink link file - liftIO $ Git.run g ["add", file] - showEndOk + then return Nothing + else return $ Just $ addPerform file +addPerform :: FilePath -> Annex (Maybe SubCmdCleanup) +addPerform file = do + g <- Annex.gitRepo + stored <- Backend.storeFileKey file + case (stored) of + Nothing -> return Nothing + Just (key, backend) -> return $ Just $ addCleanup file key +addCleanup :: FilePath -> Key -> Annex Bool +addCleanup file key = do + logStatus key ValuePresent + g <- Annex.gitRepo + let dest = annexLocation g key + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ renameFile file dest + link <- calcGitLink file key + liftIO $ createSymbolicLink link file + liftIO $ Git.run g ["add", file] + return True -{- Undo addCmd. -} -unannexCmd :: FilePath -> Annex () -unannexCmd file = isAnnexed file $ \(key, backend) -> do - showStart "unannex" file - Annex.flagChange "force" $ FlagBool True -- force backend to always remove +{- The unannex subcommand undoes an add. -} +unannexStart :: FilePath -> Annex (Maybe SubCmdPerform) +unannexStart file = isAnnexed file $ \(key, backend) -> do + return $ Just $ unannexPerform file key backend +unannexPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) +unannexPerform file key backend = do + -- force backend to always remove + Annex.flagChange "force" $ FlagBool True Backend.removeKey backend key + return $ Just $ unannexCleanup file key +unannexCleanup :: FilePath -> Key -> Annex Bool +unannexCleanup file key = do logStatus key ValueMissing g <- Annex.gitRepo let src = annexLocation g key - moveout g src - where - moveout g src = do - liftIO $ removeFile file - liftIO $ Git.run g ["rm", "--quiet", file] - -- git rm deletes empty directories; - -- put them back - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ renameFile src file - showEndOk + liftIO $ removeFile file + liftIO $ Git.run g ["rm", "--quiet", file] + -- git rm deletes empty directories; put them back + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ renameFile src file + return True {- Gets an annexed file from one of the backends. -} -getCmd :: FilePath -> Annex () -getCmd file = isAnnexed file $ \(key, backend) -> do +getStart :: FilePath -> Annex (Maybe SubCmdPerform) +getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) - then return () - else do - showStart "get" file - ok <- getViaTmp key (Backend.retrieveKeyFile backend key) - if (ok) - then showEndOk - else showEndFail + then return Nothing + else return $ Just $ getPerform file key backend +getPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) +getPerform file key backend = do + ok <- getViaTmp key (Backend.retrieveKeyFile backend key) + if (ok) + then return $ Just $ return True + else return Nothing {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropCmd :: FilePath -> Annex () -dropCmd file = isAnnexed file $ \(key, backend) -> do +dropStart :: FilePath -> Annex (Maybe SubCmdPerform) +dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) - then return () -- no-op - else do - showStart "drop" file - success <- Backend.removeKey backend key - if (success) - then do - cleanup key - showEndOk - else showEndFail - where - cleanup key = do - logStatus key ValueMissing - inannex <- inAnnex key - if (inannex) - then do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc - return () - else return () + then return Nothing + else return $ Just $ dropPerform file key backend +dropPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) +dropPerform file key backend = do + success <- Backend.removeKey backend key + if (success) + then return $ Just $ dropCleanup key + else return Nothing +dropCleanup :: Key -> Annex Bool +dropCleanup key = do + logStatus key ValueMissing + inannex <- inAnnex key + if (inannex) + then do + g <- Annex.gitRepo + let loc = annexLocation g key + liftIO $ removeFile loc + return True + else return True {- Fixes the symlink to an annexed file. -} -fixCmd :: FilePath -> Annex () -fixCmd file = isAnnexed file $ \(key, backend) -> do +fixStart :: FilePath -> Annex (Maybe SubCmdPerform) +fixStart file = isAnnexed file $ \(key, backend) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) - then return () - else do - showStart "fix" file - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ removeFile file - liftIO $ createSymbolicLink link file - g <- Annex.gitRepo - liftIO $ Git.run g ["add", file] - showEndOk + then return Nothing + else return $ Just $ fixPerform file link +fixPerform :: FilePath -> FilePath -> Annex (Maybe SubCmdCleanup) +fixPerform file link = do + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + g <- Annex.gitRepo + liftIO $ Git.run g ["add", file] + return $ Just $ fixCleanup +fixCleanup :: Annex Bool +fixCleanup = do + return True {- Stores description for the repository. -} -initCmd :: String -> Annex () -initCmd description = do +initStart :: String -> Annex (Maybe SubCmdPerform) +initStart description = do if (null description) then error $ "please specify a description of this repository\n" ++ usage - else do - g <- Annex.gitRepo - u <- getUUID g - describeUUID u description - log <- uuidLog - liftIO $ Git.run g ["add", log] - liftIO $ Git.run g ["commit", "-m", "git annex init", log] - liftIO $ putStrLn "description set" + else return $ Just $ initPerform description +initPerform :: String -> Annex (Maybe SubCmdCleanup) +initPerform description = do + g <- Annex.gitRepo + u <- getUUID g + describeUUID u description + return $ Just $ initCleanup +initCleanup :: Annex Bool +initCleanup = do + g <- Annex.gitRepo + log <- uuidLog + liftIO $ Git.run g ["add", log] + liftIO $ Git.run g ["commit", "-m", "git annex init", log] + return True {- Adds a file pointing at a manually-specified key -} -fromKeyCmd :: FilePath -> Annex () -fromKeyCmd file = do +fromKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) +fromKeyStart file = do keyname <- Annex.flagGet "key" if (null keyname) then error "please specify the key with --key" @@ -264,33 +314,31 @@ fromKeyCmd file = do inbackend <- Backend.hasKey key if (not inbackend) then error $ "key ("++keyname++") is not present in backend" - else return () - + else return $ Just $ fromKeyPerform file key +fromKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +fromKeyPerform file key = do link <- calcGitLink file key - showStart "fromkey" file liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file + return $ Just $ fromKeyCleanup file +fromKeyCleanup :: FilePath -> Annex Bool +fromKeyCleanup file = do g <- Annex.gitRepo liftIO $ Git.run g ["add", file] - showEndOk + return True {- Move a file either --to or --from a repository. - - This only operates on the cached file content; it does not involve - - moving data in the key-value backend. - - - - Note that unlike drop, this does not honor annex.numcopies. - - A file's content can be moved even if there are insufficient copies to - - allow it to be dropped. - -} -moveCmd :: FilePath -> Annex () -moveCmd file = do + - moving data in the key-value backend. -} +moveStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveStart file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" case (fromName, toName) of ("", "") -> error "specify either --from or --to" - ("", to) -> moveTo file - (from, "") -> moveFrom file + ("", to) -> moveToStart file + (from, "") -> moveFromStart file (_, _) -> error "only one of --from or --to can be specified" {- Moves the content of an annexed file to another repository, @@ -299,34 +347,42 @@ moveCmd file = do - - If the destination already has the content, it is still removed - from the current repository. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. -} -moveTo :: FilePath -> Annex () -moveTo file = isAnnexed file $ \(key, backend) -> do +moveToStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveToStart file = isAnnexed file $ \(key, backend) -> do ishere <- inAnnex key if (not ishere) - then return () -- not here, so nothing to do - else do - showStart "move" file - remote <- Remotes.commandLineRemote - isthere <- Remotes.inAnnex remote key - case isthere of - Left err -> error (show err) - Right False -> moveit remote key - Right True -> removeit remote key + then return Nothing -- not here, so nothing to do + else return $ Just $ moveToPerform file key +moveToPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +moveToPerform file key = do + -- checking the remote is expensive, so not done in the start step + remote <- Remotes.commandLineRemote + isthere <- Remotes.inAnnex remote key + case isthere of + Left err -> error (show err) + Right False -> moveit remote key + Right True -> removeit remote key where moveit remote key = do Remotes.copyToRemote remote key removeit remote key removeit remote key = do error "TODO: drop key from local" - -- Update local location log; key is present - -- there and missing here. - logStatus key ValueMissing - u <- getUUID remote - liftIO $ logChange remote key u ValuePresent - -- Propigate location log to remote. - error "TODO: update remote locationlog" - showEndOk + return $ Just $ moveToCleanup remote key +moveToCleanup :: Git.Repo -> Key -> Annex Bool +moveToCleanup remote key = do + -- Update local location log; key is present there and missing here. + logStatus key ValueMissing + u <- getUUID remote + liftIO $ logChange remote key u ValuePresent + -- Propigate location log to remote. + error "TODO: update remote locationlog" + return True {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. @@ -334,33 +390,42 @@ moveTo file = isAnnexed file $ \(key, backend) -> do - If the current repository already has the content, it is still removed - from the other repository. -} -moveFrom :: FilePath -> Annex () -moveFrom file = isAnnexed file $ \(key, backend) -> do - showStart "move" file -- have to show this before checking remote - ishere <- inAnnex key +moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveFromStart file = isAnnexed file $ \(key, backend) -> do + return $ Just $ moveFromPerform file key +moveFromPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +moveFromPerform file key = do + -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key + ishere <- inAnnex key case (ishere, isthere) of (_, Left err) -> error (show err) - (_, Right False) -> showEndFail + (_, Right False) -> return Nothing -- not in remote; fail (False, Right True) -> moveit remote key (True, Right True) -> removeit remote key where moveit remote key = do - getViaTmp key (Remotes.copyFromRemote remote key) - removeit remote key + ok <- getViaTmp key (Remotes.copyFromRemote remote key) + if (ok) + then removeit remote key + else return Nothing -- fail removeit remote key = do error $ "TODO remove" ++ file - showEndOk + return $ Just moveFromCleanup +moveFromCleanup :: Annex Bool +moveFromCleanup = do + error "update location logs" + return True -- helpers notAnnexed file a = do r <- Backend.lookupFile file case (r) of - Just v -> return () + Just v -> return Nothing Nothing -> a isAnnexed file a = do r <- Backend.lookupFile file case (r) of Just v -> a v - Nothing -> return () + Nothing -> return Nothing diff --git a/Core.hs b/Core.hs index 1d887792ad..a97bf5090d 100644 --- a/Core.hs +++ b/Core.hs @@ -110,6 +110,8 @@ getViaTmp key action = do logStatus key ValuePresent return True else do + -- the tmp file is left behind, in case caller wants + -- to resume its transfer return False {- Output logging -} diff --git a/doc/todo/parallel_possibilities.mdwn b/doc/todo/parallel_possibilities.mdwn index 178f95021b..15e5171ca3 100644 --- a/doc/todo/parallel_possibilities.mdwn +++ b/doc/todo/parallel_possibilities.mdwn @@ -5,9 +5,8 @@ heavily and mostly runs other git commands, maybe not a whole lot. Anyway, each git-annex command is broken down into a series of independant actions, which has some potential for parallelism. -Probably they would need to be split further. Each action currently has 3 -distinct phases, basically "check", "do", and "record". If the check action -returned a do action that returned a record action, then it could easily -make sense to parallelize the check actions and start on the do actions -(which probably won't parallelize well) while they are still being -generated, and possibly parallelize the record actions at the end. +Each action has 3 distinct phases, basically "check", "perform", and +"cleanup". The perform actions are not parellizable; the cleanup may be, +and the check should be easily parallelizable, although they may access the +disk or run minor git query commands, so would probably not want to run +too many of them at once. From 1f3088fb945bbe241e5309a961a5b2df0fa9f93a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 15:46:53 -0400 Subject: [PATCH 0289/2835] wording --- Commands.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Commands.hs b/Commands.hs index f4286abf5b..cda34d1bdf 100644 --- a/Commands.hs +++ b/Commands.hs @@ -25,17 +25,17 @@ import qualified Remotes import qualified TypeInternals {- A subcommand runs in three stages. Each stage can return the next stage - - to run. + - to run. - - 1. The start stage is run before anything is printed about the - subcommand, and can early abort it if the input does not make sense. - It should run quickly and should not modify Annex state. - - 2. The perform stage is run after a message is printed about the subcommand - - being run. + - being run, and it should be where the bulk of the work happens. - - - 3. The cleanup stage is run only if the do stage succeeds, and it returns - - the overall success/fail of the subcommand. + - 3. The cleanup stage is run only if the perform stage succeeds, and it + - returns the overall success/fail of the subcommand. -} type SubCmdStart = String -> Annex (Maybe SubCmdPerform) type SubCmdPerform = Annex (Maybe SubCmdCleanup) From 3f0de706dd37f6b50db224bef19139eb780afef0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 15:49:52 -0400 Subject: [PATCH 0290/2835] comments --- Commands.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index cda34d1bdf..9c35c22e18 100644 --- a/Commands.hs +++ b/Commands.hs @@ -59,14 +59,16 @@ doSubCmd cmdname start param = do else showEndFail +{- A subcommand can broadly want one of several kinds of input parameters. + - This allows a first stage of filtering before starting a subcommand. -} data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description + data SubCommand = Command { subcmdname :: String, subcmdaction :: SubCmdStart, subcmdwants :: SubCmdWants, subcmddesc :: String } - subCmds :: [SubCommand] subCmds = [ (Command "add" addStart FilesNotInGit @@ -108,6 +110,7 @@ options = [ header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) +{- Usage message with lists of options and subcommands. -} usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where From 1aa19422ac8748eeff219ac4f46df166dae783c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 17:17:03 -0400 Subject: [PATCH 0291/2835] git annex move --from remote almost working --- Commands.hs | 69 ++++++++++++++++++++++++++++---------------------- LocationLog.hs | 4 ++- Locations.hs | 3 +++ Remotes.hs | 44 +++++++++++++++++++++++++++++++- 4 files changed, 88 insertions(+), 32 deletions(-) diff --git a/Commands.hs b/Commands.hs index 9c35c22e18..cf05164636 100644 --- a/Commands.hs +++ b/Commands.hs @@ -367,25 +367,28 @@ moveToPerform file key = do remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key case isthere of - Left err -> error (show err) - Right False -> moveit remote key - Right True -> removeit remote key - where - moveit remote key = do - Remotes.copyToRemote remote key - removeit remote key - removeit remote key = do - error "TODO: drop key from local" - return $ Just $ moveToCleanup remote key + Left err -> do + showNote $ show err + return Nothing + Right False -> do + ok <- Remotes.copyToRemote remote key + if (ok) + then return $ Just $ moveToCleanup remote key + else return Nothing -- failed + Right True -> return $ Just $ moveToCleanup remote key moveToCleanup :: Git.Repo -> Key -> Annex Bool moveToCleanup remote key = do - -- Update local location log; key is present there and missing here. - logStatus key ValueMissing - u <- getUUID remote - liftIO $ logChange remote key u ValuePresent - -- Propigate location log to remote. - error "TODO: update remote locationlog" - return True + -- cleanup on the local side is the same as done for the drop subcommand + ok <- dropCleanup key + if (not ok) + then return False + else do + -- Record that the key is present on the remote. + u <- getUUID remote + liftIO $ logChange remote key u ValuePresent + -- Propigate location log to remote. + error "TODO: update remote locationlog" + return True {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. @@ -403,22 +406,28 @@ moveFromPerform file key = do isthere <- Remotes.inAnnex remote key ishere <- inAnnex key case (ishere, isthere) of - (_, Left err) -> error (show err) - (_, Right False) -> return Nothing -- not in remote; fail - (False, Right True) -> moveit remote key - (True, Right True) -> removeit remote key - where - moveit remote key = do + (_, Left err) -> do + showNote $ show err + return Nothing + (_, Right False) -> do + showNote $ "not present in " ++ (Git.repoDescribe remote) + return Nothing + (False, Right True) -> do + -- copy content from remote ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) - then removeit remote key + then return $ Just $ moveFromCleanup remote key else return Nothing -- fail - removeit remote key = do - error $ "TODO remove" ++ file - return $ Just moveFromCleanup -moveFromCleanup :: Annex Bool -moveFromCleanup = do - error "update location logs" + (True, Right True) -> do + -- the content is already here, just remove from remote + return $ Just $ moveFromCleanup remote key +moveFromCleanup :: Git.Repo -> Key -> Annex Bool +moveFromCleanup remote key = do + Remotes.removeRemoteFile remote $ annexLocation remote key + -- Record that the key is not on the remote. + u <- getUUID remote + liftIO $ logChange remote key u ValueMissing + Remotes.updateRemoteLogStatus remote key return True -- helpers diff --git a/LocationLog.hs b/LocationLog.hs index 785b3330db..9ec71ce232 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -19,7 +19,9 @@ module LocationLog ( LogStatus(..), logChange, - keyLocations + keyLocations, + logFile, + readLog ) where import Data.Time.Clock.POSIX diff --git a/Locations.hs b/Locations.hs index 18d416eb4a..92918a7e0c 100644 --- a/Locations.hs +++ b/Locations.hs @@ -28,6 +28,9 @@ gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc - - - That allows deriving the key and backend by looking at the symlink to it. + - + - Note that even if the repo is a bare repo, the annex is put in a .git + - sub -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = diff --git a/Remotes.hs b/Remotes.hs index aec38a363e..67ebd75f97 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -7,7 +7,9 @@ module Remotes ( inAnnex, commandLineRemote, copyFromRemote, - copyToRemote + copyToRemote, + removeRemoteFile, + updateRemoteLogStatus ) where import Control.Exception @@ -16,8 +18,11 @@ import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils import Data.Either.Utils +import System.Cmd.Utils +import System.Directory import List import Maybe +import IO (hPutStrLn) import Types import qualified GitRepo as Git @@ -223,3 +228,40 @@ copyToRemote r key = do location g = annexLocation g key sshlocation = (Git.urlHost r) ++ ":" ++ file file = error "TODO" + +{- Removes a file from a remote. -} +removeRemoteFile :: Git.Repo -> FilePath -> Annex () +removeRemoteFile r file = do + if (not $ Git.repoIsUrl r) + then liftIO $ removeFile file + else if (Git.repoIsSsh r) + then do + ok <- liftIO $ boolSystem "ssh" + [Git.urlHost r, "rm -f " ++ + (shellEscape file)] + if (ok) + then return () + else error "failed to remove file from remote" + else error "removing file from non-ssh repo not supported" + +{- Update's a remote's location log for a key, by merging the local + - location log into it. -} +updateRemoteLogStatus :: Git.Repo -> Key -> Annex () +updateRemoteLogStatus r key = do + -- To merge, just append data to the remote's + -- log. Since the log is timestamped, the presumably newer + -- information from the local will superscede the older + -- information in the remote's log. + -- TODO: remote log locking + let mergecmd = "cat >> " ++ (shellEscape $ logFile r key) ++ " && " ++ + "cd " ++ (shellEscape $ Git.workTree r) ++ " && " ++ + "git add " ++ (shellEscape $ gitStateDir r) + let shellcmd = if (not $ Git.repoIsUrl r) + then pOpen WriteToPipe "sh" ["-c", mergecmd] + else if (Git.repoIsSsh r) + then pOpen WriteToPipe "ssh" [Git.urlHost r, mergecmd] + else error "updating non-ssh repo not supported" + g <- Annex.gitRepo + liftIO $ shellcmd $ \h -> do + lines <- readLog $ logFile g key + hPutStrLn h $ unlines $ map show lines From 8beed17168aab12bb4045b6d8635b37503d5099b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 17:31:07 -0400 Subject: [PATCH 0292/2835] drop incomplete bare repo support Added a bug about it. Now git annex move --from works fully --- Locations.hs | 14 +++++++------- Remotes.hs | 34 +++++++++++++--------------------- doc/bugs/bare_git_repos.mdwn | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 doc/bugs/bare_git_repos.mdwn diff --git a/Locations.hs b/Locations.hs index 92918a7e0c..c2c747e582 100644 --- a/Locations.hs +++ b/Locations.hs @@ -28,22 +28,22 @@ gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc - - - That allows deriving the key and backend by looking at the symlink to it. - - - - Note that even if the repo is a bare repo, the annex is put in a .git - - sub -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = (Git.workTree r) ++ "/" ++ (annexLocationRelative r key) -{- Annexed file's location relative to git's working tree. -} +{- Annexed file's location relative to git's working tree. + - + - Note: Assumes repo is NOT bare.-} annexLocationRelative :: Git.Repo -> Key -> FilePath -annexLocationRelative r key = Git.dir r ++ "/annex/" ++ (keyFile key) +annexLocationRelative r key = ".git/annex/" ++ (keyFile key) {- .git-annex/tmp is used for temp files - -} + - + - Note: Assumes repo is NOT bare. -} annexTmpLocation :: Git.Repo -> FilePath -annexTmpLocation r = (Git.workTree r) ++ "/" ++ Git.dir r ++ "/annex/tmp/" +annexTmpLocation r = (Git.workTree r) ++ ".git/annex/tmp/" {- Converts a key into a filename fragment. - diff --git a/Remotes.hs b/Remotes.hs index 67ebd75f97..c9c65babeb 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -188,27 +188,19 @@ tryGitConfigRead r = do {- Tries to copy a key's content from a remote to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file = do - -- annexLocation needs the git config to have been read for a remote, - -- so do that now if it hasn't been already - result <- tryGitConfigRead r - case (result) of - Left err -> return False - Right from -> copy from + Core.showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." + if (not $ Git.repoIsUrl r) + then getlocal + else if (Git.repoIsSsh r) + then getssh + else error "copying from non-ssh repo not supported" where - copy from = do - Core.showNote $ "copying from " ++ (Git.repoDescribe from) ++ "..." - if (not $ Git.repoIsUrl from) - then getlocal - else if (Git.repoIsSsh from) - then getssh - else error "copying from non-ssh repo not supported" - where - getlocal = liftIO $ boolSystem "cp" ["-a", location, file] - getssh = do - liftIO $ putStrLn "" -- make way for scp progress bar - liftIO $ boolSystem "scp" [sshlocation, file] - location = annexLocation from key - sshlocation = (Git.urlHost from) ++ ":" ++ location + getlocal = liftIO $ boolSystem "cp" ["-a", location, file] + getssh = do + liftIO $ putStrLn "" -- make way for scp progress bar + liftIO $ boolSystem "scp" [sshlocation, file] + location = annexLocation r key + sshlocation = (Git.urlHost r) ++ ":" ++ location {- Tries to copy a key's content to a remote. -} copyToRemote :: Git.Repo -> Key -> Annex Bool @@ -255,7 +247,7 @@ updateRemoteLogStatus r key = do -- TODO: remote log locking let mergecmd = "cat >> " ++ (shellEscape $ logFile r key) ++ " && " ++ "cd " ++ (shellEscape $ Git.workTree r) ++ " && " ++ - "git add " ++ (shellEscape $ gitStateDir r) + "git add " ++ (shellEscape $ stateLoc) let shellcmd = if (not $ Git.repoIsUrl r) then pOpen WriteToPipe "sh" ["-c", mergecmd] else if (Git.repoIsSsh r) diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn new file mode 100644 index 0000000000..e24d7a7ee3 --- /dev/null +++ b/doc/bugs/bare_git_repos.mdwn @@ -0,0 +1,14 @@ +It would be nice if git-annex could be used in bare git repos. +However, that is not currently supported. Problems include: + +* git-annex often does not read a git repo's config before touching it, + so it doesn't know if the repo is bare or not + (reading the config when operating on ssh repos would be a pain and SLOW; + I had some of that code in as of 1aa19422ac8748eeff219ac4f46df166dae783c5, + but ripped it all out) +* .. which results in creating `.git/annex` in a bare repo, which mightily + confuses git (so it will complain that the bare repo is not + a git repo at all!) +* `.git-annex/` needs to have state recorded to it and committed, and that + is not possible with a bare repo. (If [[todo/branching]] were done, + that might be fixed.) From d0a9cdadafca1ee0da100a993b23e8a063f86bf8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 18:32:29 -0400 Subject: [PATCH 0293/2835] add dropkey subcommand and --quiet Needed for better git annex move --from --- Commands.hs | 51 +++++++++++++++++++++++++++++++++++-------- Core.hs | 14 +++++++----- GitRepo.hs | 2 +- Remotes.hs | 54 +++++++++++++++------------------------------- TypeInternals.hs | 9 ++++---- Types.hs | 1 + doc/git-annex.mdwn | 12 +++++++++++ 7 files changed, 87 insertions(+), 56 deletions(-) diff --git a/Commands.hs b/Commands.hs index cf05164636..2b8da585e3 100644 --- a/Commands.hs +++ b/Commands.hs @@ -61,7 +61,8 @@ doSubCmd cmdname start param = do {- A subcommand can broadly want one of several kinds of input parameters. - This allows a first stage of filtering before starting a subcommand. -} -data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description +data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing + | Description | Keys data SubCommand = Command { subcmdname :: String, @@ -87,6 +88,8 @@ subCmds = [ "fix up files' symlinks to point to annexed content") , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") + , (Command "dropkey" fromKeyStart Keys + "drops cached content for specified keys") ] -- Each dashed command-line option results in generation of an action @@ -95,6 +98,8 @@ options :: [OptDescr (Annex ())] options = [ Option ['f'] ["force"] (NoArg (storebool "force" True)) "allow actions that may lose annexed data" + , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) + "avoid verbose output" , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") "specify default key-value backend to use" , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") @@ -127,6 +132,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Generate descriptions of wanted parameters for subcommands. -} descWanted :: SubCmdWants -> String descWanted Description = "DESCRIPTION" +descWanted Keys = "KEY ..." descWanted _ = "PATH ..." {- Finds the type of parameters a subcommand wants, from among the passed @@ -147,6 +153,7 @@ findWanted FilesMissing params repo = do if (e) then return False else return True findWanted Description params _ = do return $ [unwords params] +findWanted Keys params _ = return params {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -243,9 +250,9 @@ dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return Nothing - else return $ Just $ dropPerform file key backend -dropPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) -dropPerform file key backend = do + else return $ Just $ dropPerform key backend +dropPerform :: Key -> Backend -> Annex (Maybe SubCmdCleanup) +dropPerform key backend = do success <- Backend.removeKey backend key if (success) then return $ Just $ dropCleanup key @@ -262,6 +269,29 @@ dropCleanup key = do return True else return True +{- Drops cached content for a key. -} +dropKeyStart :: String -> Annex (Maybe SubCmdPerform) +dropKeyStart keyname = do + backends <- Backend.list + let key = genKey (backends !! 0) keyname + present <- inAnnex key + force <- Annex.flagIsSet "force" + if (not present) + then return Nothing + else if (not force) + then error "dropkey is can cause data loss; use --force if you're sure you want to do this" + else return $ Just $ dropKeyPerform key +dropKeyPerform :: Key -> Annex (Maybe SubCmdCleanup) +dropKeyPerform key = do + g <- Annex.gitRepo + let loc = annexLocation g key + liftIO $ removeFile loc + return $ Just $ dropKeyCleanup key +dropKeyCleanup :: Key -> Annex Bool +dropKeyCleanup key = do + logStatus key ValueMissing + return True + {- Fixes the symlink to an annexed file. -} fixStart :: FilePath -> Annex (Maybe SubCmdPerform) fixStart file = isAnnexed file $ \(key, backend) -> do @@ -423,11 +453,14 @@ moveFromPerform file key = do return $ Just $ moveFromCleanup remote key moveFromCleanup :: Git.Repo -> Key -> Annex Bool moveFromCleanup remote key = do - Remotes.removeRemoteFile remote $ annexLocation remote key - -- Record that the key is not on the remote. - u <- getUUID remote - liftIO $ logChange remote key u ValueMissing - Remotes.updateRemoteLogStatus remote key + -- Force drop content from the remote. + Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", + "--backend=" ++ (backendName key), + keyName key] + -- Record locally that the key is not on the remote. + remoteuuid <- getUUID remote + g <- Annex.gitRepo + liftIO $ logChange g key remoteuuid ValueMissing return True -- helpers diff --git a/Core.hs b/Core.hs index a97bf5090d..8717aee818 100644 --- a/Core.hs +++ b/Core.hs @@ -115,22 +115,26 @@ getViaTmp key action = do return False {- Output logging -} +verbose :: Annex () -> Annex () +verbose a = do + q <- Annex.flagIsSet "quiet" + if (q) then return () else a showStart :: String -> String -> Annex () -showStart command file = do +showStart command file = verbose $ do liftIO $ putStr $ command ++ " " ++ file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () -showNote s = do +showNote s = verbose $ do liftIO $ putStr $ "(" ++ s ++ ") " liftIO $ hFlush stdout showLongNote :: String -> Annex () -showLongNote s = do +showLongNote s = verbose $ do liftIO $ putStr $ "\n" ++ (indent s) where indent s = join "\n" $ map (\l -> " " ++ l) $ lines s showEndOk :: Annex () -showEndOk = do +showEndOk = verbose $ do liftIO $ putStrLn "ok" showEndFail :: Annex () -showEndFail = do +showEndFail = verbose $ do liftIO $ putStrLn "\nfailed" diff --git a/GitRepo.hs b/GitRepo.hs index 553e91fec0..ee1bdba344 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -156,7 +156,7 @@ workTree repo = - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} relative :: Repo -> String -> String -relative repo file = drop (length absrepo) absfile +relative repo file = assertLocal repo $ drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file diff --git a/Remotes.hs b/Remotes.hs index c9c65babeb..985199e1cd 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -8,11 +8,11 @@ module Remotes ( commandLineRemote, copyFromRemote, copyToRemote, - removeRemoteFile, - updateRemoteLogStatus + runCmd ) where -import Control.Exception +import IO (bracket_) +import Control.Exception hiding (bracket_) import Control.Monad.State (liftIO) import Control.Monad (filterM) import qualified Data.Map as Map @@ -20,9 +20,9 @@ import Data.String.Utils import Data.Either.Utils import System.Cmd.Utils import System.Directory +import System.Posix.Directory import List import Maybe -import IO (hPutStrLn) import Types import qualified GitRepo as Git @@ -221,39 +221,19 @@ copyToRemote r key = do sshlocation = (Git.urlHost r) ++ ":" ++ file file = error "TODO" -{- Removes a file from a remote. -} -removeRemoteFile :: Git.Repo -> FilePath -> Annex () -removeRemoteFile r file = do +{- Runs a command in a remote. -} +runCmd :: Git.Repo -> String -> [String] -> Annex Bool +runCmd r command params = do if (not $ Git.repoIsUrl r) - then liftIO $ removeFile file + then do + cwd <- liftIO $ getCurrentDirectory + liftIO $ bracket_ (changeWorkingDirectory (Git.workTree r)) + (\_ -> changeWorkingDirectory cwd) $ + boolSystem command params else if (Git.repoIsSsh r) then do - ok <- liftIO $ boolSystem "ssh" - [Git.urlHost r, "rm -f " ++ - (shellEscape file)] - if (ok) - then return () - else error "failed to remove file from remote" - else error "removing file from non-ssh repo not supported" - -{- Update's a remote's location log for a key, by merging the local - - location log into it. -} -updateRemoteLogStatus :: Git.Repo -> Key -> Annex () -updateRemoteLogStatus r key = do - -- To merge, just append data to the remote's - -- log. Since the log is timestamped, the presumably newer - -- information from the local will superscede the older - -- information in the remote's log. - -- TODO: remote log locking - let mergecmd = "cat >> " ++ (shellEscape $ logFile r key) ++ " && " ++ - "cd " ++ (shellEscape $ Git.workTree r) ++ " && " ++ - "git add " ++ (shellEscape $ stateLoc) - let shellcmd = if (not $ Git.repoIsUrl r) - then pOpen WriteToPipe "sh" ["-c", mergecmd] - else if (Git.repoIsSsh r) - then pOpen WriteToPipe "ssh" [Git.urlHost r, mergecmd] - else error "updating non-ssh repo not supported" - g <- Annex.gitRepo - liftIO $ shellcmd $ \h -> do - lines <- readLog $ logFile g key - hPutStrLn h $ unlines $ map show lines + liftIO $ boolSystem "ssh" [Git.urlHost r, + "cd " ++ (shellEscape $ Git.workTree r) ++ + " && " ++ command ++ " " ++ + unwords params] + else error "running command in non-ssh repo not supported" diff --git a/TypeInternals.hs b/TypeInternals.hs index 6d1c72d2ea..188f5e534b 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -31,12 +31,12 @@ data AnnexState = AnnexState { type Annex = StateT AnnexState IO -- annexed filenames are mapped through a backend into keys -type KeyFrag = String +type KeyName = String type BackendName = String -data Key = Key (BackendName, KeyFrag) deriving (Eq) +data Key = Key (BackendName, KeyName) deriving (Eq) -- constructs a key in a backend -genKey :: Backend -> KeyFrag -> Key +genKey :: Backend -> KeyName -> Key genKey b f = Key (name b,f) -- show a key to convert it to a string; the string includes the @@ -51,9 +51,10 @@ instance Read Key where b = l !! 0 k = join ":" $ drop 1 l --- pulls the backend name out backendName :: Key -> BackendName backendName (Key (b,k)) = b +keyName :: Key -> KeyName +keyName (Key (b,k)) = k -- this structure represents a key-value backend data Backend = Backend { diff --git a/Types.hs b/Types.hs index 50597962ce..c3d6467a3f 100644 --- a/Types.hs +++ b/Types.hs @@ -7,6 +7,7 @@ module Types ( Key, genKey, backendName, + keyName, FlagName, Flag(..) ) where diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 522be75700..e7057afeea 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -116,6 +116,13 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile +* dropkey [key ...] + + Drops the cached data for the specified keys from this repository. + + This can be used to drop content for arbitrary keys, which do not need + to have a file in the git repository pointing at them. + # OPTIONS * --force @@ -123,6 +130,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Force unsafe actions, such as dropping a file's content when no other source of it can be verified to still exist. Use with care. +* --quiet + + Avoid the default verbose logging of what is done; only show errors + and progress displays. + * --backend=name Specify the default key-value backend to use, adding it to the front From 3b6b9ab4e1cbe3e174418e032795a598ab11fb8e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 18:33:59 -0400 Subject: [PATCH 0294/2835] typo --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index 2b8da585e3..78e1ab32cf 100644 --- a/Commands.hs +++ b/Commands.hs @@ -88,7 +88,7 @@ subCmds = [ "fix up files' symlinks to point to annexed content") , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") - , (Command "dropkey" fromKeyStart Keys + , (Command "dropkey" dropKeyStart Keys "drops cached content for specified keys") ] From 47892ced883b96c3a9c2903aa8a59b3b8a2f1731 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 18:36:41 -0400 Subject: [PATCH 0295/2835] new bug --- doc/bugs/error_propigation.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/error_propigation.mdwn diff --git a/doc/bugs/error_propigation.mdwn b/doc/bugs/error_propigation.mdwn new file mode 100644 index 0000000000..0a0b38f5ee --- /dev/null +++ b/doc/bugs/error_propigation.mdwn @@ -0,0 +1,3 @@ +If a subcommand fails w/o throwing an error, no error is propigated to the +git-annex exit code. With --quiet, this makes it look like the command +succeeded. From e87287c11b81ea6f339628bcbebfb239d0ccadd0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 19:17:11 -0400 Subject: [PATCH 0296/2835] fix failure propigation --- Commands.hs | 26 ++++++++++++++++++-------- Core.hs | 7 +++++-- doc/bugs/error_propigation.mdwn | 2 +- git-annex.hs | 5 +++-- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Commands.hs b/Commands.hs index 78e1ab32cf..729eae1240 100644 --- a/Commands.hs +++ b/Commands.hs @@ -42,21 +42,27 @@ type SubCmdPerform = Annex (Maybe SubCmdCleanup) type SubCmdCleanup = Annex Bool {- Runs a subcommand through its three stages. -} -doSubCmd :: String -> SubCmdStart -> String -> Annex () +doSubCmd :: String -> SubCmdStart -> String -> Annex Bool doSubCmd cmdname start param = do res <- start param :: Annex (Maybe SubCmdPerform) case (res) of - Nothing -> return () + Nothing -> return True Just perform -> do showStart cmdname param res <- perform :: Annex (Maybe SubCmdCleanup) case (res) of - Nothing -> showEndFail + Nothing -> do + showEndFail + return False Just cleanup -> do res <- cleanup if (res) - then showEndOk - else showEndFail + then do + showEndOk + return True + else do + showEndFail + return False {- A subcommand can broadly want one of several kinds of input parameters. @@ -159,7 +165,7 @@ findWanted Keys params _ = return params - run in the Annex monad. The first actions configure it - according to command line options, while the second actions - handle subcommands. -} -parseCmd :: [String] -> AnnexState -> IO ([Annex ()], [Annex ()]) +parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) parseCmd argv state = do (flags, params) <- getopt if (null params) @@ -169,8 +175,12 @@ parseCmd argv state = do [Command name action want _] -> do f <- findWanted want (drop 1 params) (TypeInternals.repo state) - return (flags, map (doSubCmd name action) $ - filter notstate f) + let actions = map (doSubCmd name action) $ + filter notstate f + let configactions = map (\f -> do + f + return True) flags + return (configactions, actions) where -- never include files from the state directory notstate f = stateLoc /= take (length stateLoc) f diff --git a/Core.hs b/Core.hs index 8717aee818..0d95e382b0 100644 --- a/Core.hs +++ b/Core.hs @@ -18,14 +18,15 @@ import qualified Annex import Utility {- Sets up a git repo for git-annex. -} -startup :: Annex () +startup :: Annex Bool startup = do g <- Annex.gitRepo liftIO $ gitAttributes g prepUUID + return True {- When git-annex is done, it runs this. -} -shutdown :: Annex () +shutdown :: Annex Bool shutdown = do g <- Annex.gitRepo @@ -38,6 +39,8 @@ shutdown = do then liftIO $ removeDirectoryRecursive $ tmp else return () + return True + {- configure git to use union merge driver on state files, if it is not - already -} gitAttributes :: Git.Repo -> IO () diff --git a/doc/bugs/error_propigation.mdwn b/doc/bugs/error_propigation.mdwn index 0a0b38f5ee..25998907e8 100644 --- a/doc/bugs/error_propigation.mdwn +++ b/doc/bugs/error_propigation.mdwn @@ -1,3 +1,3 @@ If a subcommand fails w/o throwing an error, no error is propigated to the git-annex exit code. With --quiet, this makes it look like the command -succeeded. +succeeded. [[done]] diff --git a/git-annex.hs b/git-annex.hs index 602f672c5b..d7b26cd968 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -27,7 +27,7 @@ main = do - or more likely I missed an easy way to do it. So, I have to laboriously - thread AnnexState through this function. -} -tryRun :: AnnexState -> [Annex ()] -> IO () +tryRun :: AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions tryRun' state errnum (a:as) = do result <- try $ Annex.run state a @@ -35,7 +35,8 @@ tryRun' state errnum (a:as) = do Left err -> do showErr err tryRun' state (errnum + 1) as - Right (_,state') -> tryRun' state' errnum as + Right (True,state') -> tryRun' state' errnum as + Right (False,state') -> tryRun' state' (errnum + 1) as tryRun' state errnum [] = do if (errnum > 0) then error $ (show errnum) ++ " failed" From a8fbd5d91fc56ebedb08614bb89db2d383c9a0ad Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 19:34:44 -0400 Subject: [PATCH 0297/2835] speed up git annex move --from Avoid extra ssh to check if the remote has the key, just trust the location log (and propigate error if it's wrong). Quick exit when asked to move files that are not on the remote, so this is now suitable to be used on a big directory. --- Commands.hs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index 729eae1240..d201b6f799 100644 --- a/Commands.hs +++ b/Commands.hs @@ -438,32 +438,27 @@ moveToCleanup remote key = do -} moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) moveFromStart file = isAnnexed file $ \(key, backend) -> do - return $ Just $ moveFromPerform file key + g <- Annex.gitRepo + remote <- Remotes.commandLineRemote + l <- Remotes.keyPossibilities key + if (elem remote l) + then return $ Just $ moveFromPerform file key + else return Nothing moveFromPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) moveFromPerform file key = do - -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote - isthere <- Remotes.inAnnex remote key ishere <- inAnnex key - case (ishere, isthere) of - (_, Left err) -> do - showNote $ show err - return Nothing - (_, Right False) -> do - showNote $ "not present in " ++ (Git.repoDescribe remote) - return Nothing - (False, Right True) -> do + if (ishere) + then return $ Just $ moveFromCleanup remote key + else do -- copy content from remote ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail - (True, Right True) -> do - -- the content is already here, just remove from remote - return $ Just $ moveFromCleanup remote key moveFromCleanup :: Git.Repo -> Key -> Annex Bool moveFromCleanup remote key = do - -- Force drop content from the remote. + showNote $ "dropping from " ++ (Git.repoDescribe remote) ++ "..." Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), keyName key] From a0e8ba37c69a7ce69a6bca42b4e4a20d046b3566 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 19:38:59 -0400 Subject: [PATCH 0298/2835] changelog --- debian/changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 0a27cd12ed..292931d0f3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,12 +1,15 @@ git-annex (0.02) UNRELEASED; urgency=low + * New move subcommand, that can move files from or to a remote. * New fromkey subcommand, for registering urls, etc. * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. - * --from/--to can be used to limit the remote repository that git-annex + * --from/--to can be used to control the remote repository that git-annex uses. + * --quiet can be used to avoid verbose output + * New plumbing-level dropkey subcommand. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 From fec9f611df1a5e973f4847ac71fe85bd85abdff4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:19:08 -0400 Subject: [PATCH 0299/2835] add setkey subcommand And finished implementing move --to --- Commands.hs | 65 ++++++++++++++++++++++++++++++++-------------- Core.hs | 4 ++- Remotes.hs | 30 ++++++++++----------- debian/changelog | 2 +- doc/git-annex.mdwn | 11 +++++++- 5 files changed, 75 insertions(+), 37 deletions(-) diff --git a/Commands.hs b/Commands.hs index d201b6f799..3c64083918 100644 --- a/Commands.hs +++ b/Commands.hs @@ -68,7 +68,7 @@ doSubCmd cmdname start param = do {- A subcommand can broadly want one of several kinds of input parameters. - This allows a first stage of filtering before starting a subcommand. -} data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing - | Description | Keys + | Description | Keys | Tempfile data SubCommand = Command { subcmdname :: String, @@ -95,7 +95,9 @@ subCmds = [ , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") , (Command "dropkey" dropKeyStart Keys - "drops cached content for specified keys") + "drops annexed content for specified keys") + , (Command "setkey" setKeyStart Tempfile + "sets annexed content for a key using a temp file") ] -- Each dashed command-line option results in generation of an action @@ -159,7 +161,7 @@ findWanted FilesMissing params repo = do if (e) then return False else return True findWanted Description params _ = do return $ [unwords params] -findWanted Keys params _ = return params +findWanted _ params _ = return params {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -302,6 +304,29 @@ dropKeyCleanup key = do logStatus key ValueMissing return True +{- Sets cached content for a key. -} +setKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) +setKeyStart tmpfile = do + keyname <- Annex.flagGet "key" + if (null keyname) + then error "please specify the key with --key" + else return () + backends <- Backend.list + let key = genKey (backends !! 0) keyname + return $ Just $ setKeyPerform tmpfile key +setKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +setKeyPerform tmpfile key = do + g <- Annex.gitRepo + let loc = annexLocation g key + ok <- liftIO $ boolSystem "mv" [tmpfile, loc] + if (not ok) + then error "mv failed!" + else return $ Just $ setKeyCleanup tmpfile key +setKeyCleanup :: FilePath -> Key -> Annex Bool +setKeyCleanup tmpfile key = do + logStatus key ValuePresent + return True + {- Fixes the symlink to an annexed file. -} fixStart :: FilePath -> Annex (Maybe SubCmdPerform) fixStart file = isAnnexed file $ \(key, backend) -> do @@ -411,24 +436,26 @@ moveToPerform file key = do showNote $ show err return Nothing Right False -> do - ok <- Remotes.copyToRemote remote key + let tmpfile = (annexTmpLocation remote) ++ (keyFile key) + ok <- Remotes.copyToRemote remote key tmpfile if (ok) - then return $ Just $ moveToCleanup remote key + then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed - Right True -> return $ Just $ moveToCleanup remote key -moveToCleanup :: Git.Repo -> Key -> Annex Bool -moveToCleanup remote key = do - -- cleanup on the local side is the same as done for the drop subcommand - ok <- dropCleanup key - if (not ok) - then return False - else do - -- Record that the key is present on the remote. - u <- getUUID remote - liftIO $ logChange remote key u ValuePresent - -- Propigate location log to remote. - error "TODO: update remote locationlog" - return True + Right True -> return $ Just $ dropCleanup key +moveToCleanup :: Git.Repo -> Key -> FilePath -> Annex Bool +moveToCleanup remote key tmpfile = do + -- Tell remote to use the transferred content. + Remotes.runCmd remote "git-annex" ["setkey", "--quiet", + "--backend=" ++ (backendName key), + "--key=" ++ keyName key, + tmpfile] + -- Record that the key is present on the remote. + g <- Annex.gitRepo + remoteuuid <- getUUID remote + liftIO $ logChange g key remoteuuid ValuePresent + -- Cleanup on the local side is the same as done for the + -- drop subcommand. + dropCleanup key {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. diff --git a/Core.hs b/Core.hs index 0d95e382b0..881b668e0d 100644 --- a/Core.hs +++ b/Core.hs @@ -32,12 +32,14 @@ shutdown = do liftIO $ Git.run g ["add", gitStateDir g] - -- clean up any files left in the temp directory + -- clean up any files left in the temp directory, but leave + -- the tmp directory itself let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp if (exists) then liftIO $ removeDirectoryRecursive $ tmp else return () + liftIO $ createDirectoryIfMissing True tmp return True diff --git a/Remotes.hs b/Remotes.hs index 985199e1cd..1d5992704a 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -195,31 +195,31 @@ copyFromRemote r key file = do then getssh else error "copying from non-ssh repo not supported" where - getlocal = liftIO $ boolSystem "cp" ["-a", location, file] + getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] getssh = do liftIO $ putStrLn "" -- make way for scp progress bar - liftIO $ boolSystem "scp" [sshlocation, file] - location = annexLocation r key - sshlocation = (Git.urlHost r) ++ ":" ++ location + liftIO $ boolSystem "scp" [sshLocation r keyloc, file] + keyloc = annexLocation r key -{- Tries to copy a key's content to a remote. -} -copyToRemote :: Git.Repo -> Key -> Annex Bool -copyToRemote r key = do +{- Tries to copy a key's content to a file on a remote. -} +copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool +copyToRemote r key file = do g <- Annex.gitRepo + let keyloc = annexLocation g key Core.showNote $ "copying to " ++ (Git.repoDescribe r) ++ "..." if (not $ Git.repoIsUrl r) - then sendlocal g + then putlocal keyloc else if (Git.repoIsSsh r) - then sendssh g + then putssh keyloc else error "copying to non-ssh repo not supported" where - sendlocal g = liftIO $ boolSystem "cp" ["-a", location g, file] - sendssh g = do + putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] + putssh src = do liftIO $ putStrLn "" -- make way for scp progress bar - liftIO $ boolSystem "scp" [location g, sshlocation] - location g = annexLocation g key - sshlocation = (Git.urlHost r) ++ ":" ++ file - file = error "TODO" + liftIO $ boolSystem "scp" [src, sshLocation r file] + +sshLocation :: Git.Repo -> FilePath -> FilePath +sshLocation r file = (Git.urlHost r) ++ ":" ++ file {- Runs a command in a remote. -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool diff --git a/debian/changelog b/debian/changelog index 292931d0f3..46ad2ac5db 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,7 +9,7 @@ git-annex (0.02) UNRELEASED; urgency=low * --from/--to can be used to control the remote repository that git-annex uses. * --quiet can be used to avoid verbose output - * New plumbing-level dropkey subcommand. + * New plumbing-level dropkey and setkey subcommands. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e7057afeea..cba634f205 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -118,11 +118,20 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * dropkey [key ...] - Drops the cached data for the specified keys from this repository. + Drops the annexed data for the specified keys from this repository. This can be used to drop content for arbitrary keys, which do not need to have a file in the git repository pointing at them. +* setkey file + + Sets the annxed data for a key to the content of the specified file, + and then removes the file. + + Example: + + git annex setkey --key=1287765018:3 /tmp/file + # OPTIONS * --force From 4a69cb8ade2dc4572c0e8cada6bc30f38ed6bc30 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:20:17 -0400 Subject: [PATCH 0300/2835] done --- doc/todo/pushpull.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/pushpull.mdwn b/doc/todo/pushpull.mdwn index 47da2107f2..6828b35b2f 100644 --- a/doc/todo/pushpull.mdwn +++ b/doc/todo/pushpull.mdwn @@ -1,2 +1,4 @@ --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo + +[[done]] (move --from/--to) From 0a4235e26e0bce14ee8d65de9e0480d23b87ed87 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:22:37 -0400 Subject: [PATCH 0301/2835] update --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 46ad2ac5db..fd1fddafda 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,15 +1,15 @@ git-annex (0.02) UNRELEASED; urgency=low - * New move subcommand, that can move files from or to a remote. - * New fromkey subcommand, for registering urls, etc. * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. + * New move subcommand, that can move files from or to a remote. + * New fromkey subcommand, for registering urls, etc. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. * --from/--to can be used to control the remote repository that git-annex uses. * --quiet can be used to avoid verbose output - * New plumbing-level dropkey and setkey subcommands. + * New plumbing-level dropkey and addkey subcommands. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 From 0788c12ffe9c07104ca70f4dc0636ed0f6879ad2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:45:50 -0400 Subject: [PATCH 0302/2835] improve git annex move in walkthrough --- doc/walkthrough.mdwn | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index c9eba0a575..15d19db2b2 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -78,7 +78,7 @@ USB drive. Notice that you had to git pull from home first, this lets git-annex know what has changed in home, and so it knows about the files present there and -can get them. See below for an easier way. +can get them. ## transferring files: When things go wrong @@ -137,17 +137,6 @@ But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. -## moving file content to another repository - -Often you will want to transfer some file contents from a repository to -some other one, and then drop it from the first repository. For example, -your laptop's disk is getting full; time to move some files to an external -disk. Doing that by hand is possible, but a bit of a pain. `git annex move` -makes it very easy. - - # git annex move my_cool_big_file --to usbdrive - move my_cool_big_file (to usbdrive...) ok - ## using ssh remotes So far git-annex has been used with a remote repository on a USB drive. @@ -183,6 +172,21 @@ access, if available. There is a annex-cost setting you can configure in Also, note that you need full shell access for this to work -- git-annex needs to be able to ssh in and run commands. +## moving file content to another repository + +Often you will want to move some file contents from a repository to some +other one. For example, your laptop's disk is getting full; time to move +some files to an external disk before moving another file from home to your +laptop. Doing that by hand (by using `git annex get` and `git annex drop`) +is possible, but a bit of a pain. `git annex move` makes it very easy. + + # git annex move my_cool_big_file --to usbdrive + move my_cool_big_file (copying to usbdrive...) ok + # git annex move video/hackity_hack_and_kaxxt.mov --from home + move video/hackity_hack_and_kaxxt.mov (copying from home...)a + WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 + (dropping from home...) ok + ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has From 99eaf41da57819ed49e554b09a61b74ac91c7ddb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:48:32 -0400 Subject: [PATCH 0303/2835] better messages --- Commands.hs | 4 ++-- Remotes.hs | 2 -- doc/git-annex.mdwn | 2 +- doc/walkthrough.mdwn | 6 +++--- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Commands.hs b/Commands.hs index 3c64083918..2941d619e1 100644 --- a/Commands.hs +++ b/Commands.hs @@ -436,6 +436,7 @@ moveToPerform file key = do showNote $ show err return Nothing Right False -> do + Core.showNote $ "moving to " ++ (Git.repoDescribe r) ++ "..." let tmpfile = (annexTmpLocation remote) ++ (keyFile key) ok <- Remotes.copyToRemote remote key tmpfile if (ok) @@ -478,14 +479,13 @@ moveFromPerform file key = do if (ishere) then return $ Just $ moveFromCleanup remote key else do - -- copy content from remote + Core.showNote $ "moving from " ++ (Git.repoDescribe r) ++ "..." ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail moveFromCleanup :: Git.Repo -> Key -> Annex Bool moveFromCleanup remote key = do - showNote $ "dropping from " ++ (Git.repoDescribe remote) ++ "..." Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), keyName key] diff --git a/Remotes.hs b/Remotes.hs index 1d5992704a..81a4af47ac 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -188,7 +188,6 @@ tryGitConfigRead r = do {- Tries to copy a key's content from a remote to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file = do - Core.showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." if (not $ Git.repoIsUrl r) then getlocal else if (Git.repoIsSsh r) @@ -206,7 +205,6 @@ copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyToRemote r key file = do g <- Annex.gitRepo let keyloc = annexLocation g key - Core.showNote $ "copying to " ++ (Git.repoDescribe r) ++ "..." if (not $ Git.repoIsUrl r) then putlocal keyloc else if (Git.repoIsSsh r) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index cba634f205..3645d37f2a 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -53,7 +53,7 @@ content from the key-value store. # git commit -a -m "freed up space" # git annex move iso --to=usbdrive - move iso/Debian_5.0.iso (to usbdrive...) ok + move iso/Debian_5.0.iso (moving to usbdrive...) ok # SUBCOMMANDS diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 15d19db2b2..ce68a77f7a 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -181,11 +181,11 @@ laptop. Doing that by hand (by using `git annex get` and `git annex drop`) is possible, but a bit of a pain. `git annex move` makes it very easy. # git annex move my_cool_big_file --to usbdrive - move my_cool_big_file (copying to usbdrive...) ok + move my_cool_big_file (moving to usbdrive...) ok # git annex move video/hackity_hack_and_kaxxt.mov --from home - move video/hackity_hack_and_kaxxt.mov (copying from home...)a + move video/hackity_hack_and_kaxxt.mov (moving from home...) WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 - (dropping from home...) ok + ok ## using the URL backend From 5d4ff035ba0f5a9bd9c61def3effd68bfeb02448 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:52:03 -0400 Subject: [PATCH 0304/2835] bugfix --- Commands.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Commands.hs b/Commands.hs index 2941d619e1..da4d840ae1 100644 --- a/Commands.hs +++ b/Commands.hs @@ -436,7 +436,7 @@ moveToPerform file key = do showNote $ show err return Nothing Right False -> do - Core.showNote $ "moving to " ++ (Git.repoDescribe r) ++ "..." + Core.showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." let tmpfile = (annexTmpLocation remote) ++ (keyFile key) ok <- Remotes.copyToRemote remote key tmpfile if (ok) @@ -479,7 +479,7 @@ moveFromPerform file key = do if (ishere) then return $ Just $ moveFromCleanup remote key else do - Core.showNote $ "moving from " ++ (Git.repoDescribe r) ++ "..." + Core.showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) then return $ Just $ moveFromCleanup remote key From eea140c90dadfc8cc5ac4aec65c9d710f859c310 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 21:06:31 -0400 Subject: [PATCH 0305/2835] handle better the case of a disconnected drive remote --- Backend/File.hs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 14b4b9dae5..4b7d7917c0 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,6 +15,7 @@ import System.IO import System.Cmd import System.Cmd.Utils import Control.Exception +import System.Directory import List import Maybe @@ -66,10 +67,27 @@ copyKeyFile key file = do showLocations key return False trycopy full (r:rs) = do - copied <- Remotes.copyFromRemote r key file - if (copied) - then return True + probablythere <- probablyPresent r + if (probablythere) + then do + showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." + copied <- Remotes.copyFromRemote r key file + if (copied) + then return True + else trycopy full rs else trycopy full rs + probablyPresent r = do + -- This check is to avoid an ugly message if a + -- remote is a drive that is not mounted. + -- Avoid checking inAnnex for ssh remotes because + -- that is unnecessarily slow, and the locationlog + -- should be trusted. (If the ssh remote is down + -- or really lacks the file, it's ok to show + -- an ugly message before going on to the next + -- remote.) + if (not $ Git.repoIsUrl r) + then liftIO $ doesFileExist $ annexLocation r key + else return True {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an From 68e2687661cdc83197e78fd56a10803890401159 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 21:15:29 -0400 Subject: [PATCH 0306/2835] short-circuit as soon as enough remotes are verified to have a file --- Backend/File.hs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 4b7d7917c0..4273ba36be 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -111,11 +111,14 @@ checkRemoveKey key = do then return True else notEnoughCopies need have bad findcopies need have (r:rs) bad = do - haskey <- Remotes.inAnnex r key - case (haskey) of - Right True -> findcopies need (have+1) rs bad - Right False -> findcopies need have rs bad - Left _ -> findcopies need have rs (r:bad) + if (have >= need) + then return True + else do + haskey <- Remotes.inAnnex r key + case (haskey) of + Right True -> findcopies need (have+1) rs bad + Right False -> findcopies need have rs bad + Left _ -> findcopies need have rs (r:bad) notEnoughCopies need have bad = do unsafe showLongNote $ From 4cda7b6e7c2f08c99b0c4c14bb86e691903a985b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 21:35:45 -0400 Subject: [PATCH 0307/2835] bugfix --- Locations.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Locations.hs b/Locations.hs index c2c747e582..970f89aa8d 100644 --- a/Locations.hs +++ b/Locations.hs @@ -43,7 +43,7 @@ annexLocationRelative r key = ".git/annex/" ++ (keyFile key) - - Note: Assumes repo is NOT bare. -} annexTmpLocation :: Git.Repo -> FilePath -annexTmpLocation r = (Git.workTree r) ++ ".git/annex/tmp/" +annexTmpLocation r = (Git.workTree r) ++ "/.git/annex/tmp/" {- Converts a key into a filename fragment. - From ef26076a5a3df9b8740883e3f7b3b68585b47ad5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Oct 2010 15:59:50 -0400 Subject: [PATCH 0308/2835] add git queue to Annex monad not used anywhere just yet.. --- Annex.hs | 42 ++++++++++++++++++++++++++----- Core.hs | 9 +++++++ GitQueue.hs | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ TypeInternals.hs | 4 ++- 4 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 GitQueue.hs diff --git a/Annex.hs b/Annex.hs index d021f936ef..1963d19c69 100644 --- a/Annex.hs +++ b/Annex.hs @@ -11,25 +11,28 @@ module Annex ( flagIsSet, flagChange, flagGet, - Flag(..) + Flag(..), + queue, + queueGet ) where import Control.Monad.State import qualified Data.Map as M import qualified GitRepo as Git +import qualified GitQueue import Types import qualified TypeInternals as Internals -{- Create and returns an Annex state object for the specified git repo. - -} +{- Create and returns an Annex state object for the specified git repo. -} new :: Git.Repo -> [Backend] -> IO AnnexState new gitrepo allbackends = do let s = Internals.AnnexState { Internals.repo = gitrepo, Internals.backends = [], Internals.supportedBackends = allbackends, - Internals.flags = M.empty + Internals.flags = M.empty, + Internals.repoqueue = GitQueue.empty } (_,s') <- Annex.run s (prep gitrepo) return s' @@ -39,46 +42,73 @@ new gitrepo allbackends = do gitrepo' <- liftIO $ Git.configRead gitrepo Annex.gitRepoChange gitrepo' --- performs an action in the Annex monad +{- performs an action in the Annex monad -} run state action = runStateT (action) state --- Annex monad state accessors +{- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo gitRepo = do state <- get return (Internals.repo state) + +{- Changes the git repository being acted on. -} gitRepoChange :: Git.Repo -> Annex () gitRepoChange r = do state <- get put state { Internals.repo = r } return () + +{- Returns the backends being used. -} backends :: Annex [Backend] backends = do state <- get return (Internals.backends state) + +{- Sets the backends to use. -} backendsChange :: [Backend] -> Annex () backendsChange b = do state <- get put state { Internals.backends = b } return () + +{- Returns the full list of supported backends. -} supportedBackends :: Annex [Backend] supportedBackends = do state <- get return (Internals.supportedBackends state) + +{- Return True if a Bool flag is set. -} flagIsSet :: FlagName -> Annex Bool flagIsSet name = do state <- get case (M.lookup name $ Internals.flags state) of Just (FlagBool True) -> return True _ -> return False + +{- Sets the value of a flag. -} flagChange :: FlagName -> Flag -> Annex () flagChange name val = do state <- get put state { Internals.flags = M.insert name val $ Internals.flags state } return () + +{- Gets the value of a String flag (or "" if there is no such String flag) -} flagGet :: FlagName -> Annex String flagGet name = do state <- get case (M.lookup name $ Internals.flags state) of Just (FlagString s) -> return s _ -> return "" + +{- Adds a git command to the queue. -} +queue :: String -> [String] -> FilePath -> Annex () +queue subcommand params file = do + state <- get + let q = Internals.repoqueue state + put state { Internals.repoqueue = GitQueue.add q subcommand params file } + +{- Returns the queue. -} +queueGet :: Annex GitQueue.Queue +queueGet = do + state <- get + return (Internals.repoqueue state) diff --git a/Core.hs b/Core.hs index 881b668e0d..4c7c9205e6 100644 --- a/Core.hs +++ b/Core.hs @@ -14,6 +14,7 @@ import Locations import LocationLog import UUID import qualified GitRepo as Git +import qualified GitQueue import qualified Annex import Utility @@ -30,6 +31,14 @@ shutdown :: Annex Bool shutdown = do g <- Annex.gitRepo + -- Runs all queued git commands. + q <- Annex.queueGet + if (q == GitQueue.empty) + then return () + else do + liftIO $ putStrLn "Recording state in git..." + liftIO $ GitQueue.run g q + liftIO $ Git.run g ["add", gitStateDir g] -- clean up any files left in the temp directory, but leave diff --git a/GitQueue.hs b/GitQueue.hs new file mode 100644 index 0000000000..b7210ccb59 --- /dev/null +++ b/GitQueue.hs @@ -0,0 +1,64 @@ +{- git repository command queues + -} + +module GitQueue ( + Queue, + empty, + add, + run +) where + +import qualified Data.Map as M + +import qualified GitRepo as Git + +{- An action to perform in a git repository. The file to act on + - is not included, and must be able to be appended after the params. -} +data Action = Action { + subcommand :: String, + params :: [String] + } deriving (Show, Eq, Ord) + +{- A queue of actions to perform (in any order) on a git repository, + - with lists of files to perform them on. This allows coalescing + - similar git commands. -} +type Queue = M.Map Action [FilePath] + +{- Constructor for empty queue. -} +empty :: Queue +empty = M.empty + +{- Adds an action to a queue. -} +add :: Queue -> String -> [String] -> FilePath -> Queue +add queue subcommand params file = M.insertWith (++) action [file] queue + where + action = Action subcommand params + +{- Runs a queue on a git repository. -} +run :: Git.Repo -> Queue -> IO () +run repo queue = do + mapM (\(k, v) -> runAction repo k v) $ M.toList queue + return () + +{- Runs an Action on a list of files in a git repository. + - + - Complicated by commandline length limits. -} +runAction :: Git.Repo -> Action -> [FilePath] -> IO () +runAction repo action files = do + xargs [] 0 files + where + arg_max = 2048 -- TODO get better ARG_MAX + maxlen = arg_max - cmdlen + c = (subcommand action):(params action) + cmdlen = (length "git") + + (foldl (\a b -> a + b + 1) 1 $ map length c) + xargs collect _ [] = exec collect + xargs collect len (f:fs) = do + let len' = len + 1 + length f + if (len' >= maxlen) + then do + exec collect + xargs [f] (length f) fs + else xargs (f:collect) len' fs + exec [] = return () + exec fs = Git.run repo $ c ++ fs diff --git a/TypeInternals.hs b/TypeInternals.hs index 188f5e534b..d4404fd390 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -10,6 +10,7 @@ import Data.String.Utils import qualified Data.Map as M import qualified GitRepo as Git +import qualified GitQueue -- command-line flags type FlagName = String @@ -24,7 +25,8 @@ data AnnexState = AnnexState { repo :: Git.Repo, backends :: [Backend], supportedBackends :: [Backend], - flags :: M.Map FlagName Flag + flags :: M.Map FlagName Flag, + repoqueue :: GitQueue.Queue } deriving (Show) -- git-annex's monad From 24ee4439d4783cde7102f4ffd857c521367ce16f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Oct 2010 16:15:29 -0400 Subject: [PATCH 0309/2835] use git command queue --- Commands.hs | 20 ++++++++++---------- Core.hs | 5 ++--- LocationLog.hs | 6 ++++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index da4d840ae1..6018ed8223 100644 --- a/Commands.hs +++ b/Commands.hs @@ -216,7 +216,7 @@ addCleanup file key = do liftIO $ renameFile file dest link <- calcGitLink file key liftIO $ createSymbolicLink link file - liftIO $ Git.run g ["add", file] + Annex.queue "add" [] file return True {- The unannex subcommand undoes an add. -} @@ -340,11 +340,10 @@ fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file - g <- Annex.gitRepo - liftIO $ Git.run g ["add", file] - return $ Just $ fixCleanup -fixCleanup :: Annex Bool -fixCleanup = do + return $ Just $ fixCleanup file +fixCleanup :: FilePath -> Annex Bool +fixCleanup file = do + Annex.queue "add" [] file return True {- Stores description for the repository. -} @@ -391,8 +390,7 @@ fromKeyPerform file key = do return $ Just $ fromKeyCleanup file fromKeyCleanup :: FilePath -> Annex Bool fromKeyCleanup file = do - g <- Annex.gitRepo - liftIO $ Git.run g ["add", file] + Annex.queue "add" [] file return True {- Move a file either --to or --from a repository. @@ -453,7 +451,8 @@ moveToCleanup remote key tmpfile = do -- Record that the key is present on the remote. g <- Annex.gitRepo remoteuuid <- getUUID remote - liftIO $ logChange g key remoteuuid ValuePresent + log <- liftIO $ logChange g key remoteuuid ValuePresent + Annex.queue "add" [] log -- Cleanup on the local side is the same as done for the -- drop subcommand. dropCleanup key @@ -492,7 +491,8 @@ moveFromCleanup remote key = do -- Record locally that the key is not on the remote. remoteuuid <- getUUID remote g <- Annex.gitRepo - liftIO $ logChange g key remoteuuid ValueMissing + log <- liftIO $ logChange g key remoteuuid ValueMissing + Annex.queue "add" [] log return True -- helpers diff --git a/Core.hs b/Core.hs index 4c7c9205e6..27baba28e6 100644 --- a/Core.hs +++ b/Core.hs @@ -39,8 +39,6 @@ shutdown = do liftIO $ putStrLn "Recording state in git..." liftIO $ GitQueue.run g q - liftIO $ Git.run g ["add", gitStateDir g] - -- clean up any files left in the temp directory, but leave -- the tmp directory itself let tmp = annexTmpLocation g @@ -106,7 +104,8 @@ logStatus :: Key -> LogStatus -> Annex () logStatus key status = do g <- Annex.gitRepo u <- getUUID g - liftIO $ logChange g key u status + log <- liftIO $ logChange g key u status + Annex.queue "add" [] log {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into diff --git a/LocationLog.hs b/LocationLog.hs index 9ec71ce232..d027c4b809 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -82,12 +82,14 @@ instance Read LogLine where undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] -{- Log a change in the presence of a key's value in a repository. -} -logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO () +{- Log a change in the presence of a key's value in a repository, + - and returns the filename of the logfile. -} +logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO (FilePath) logChange repo key uuid status = do log <- logNow status uuid ls <- readLog logfile writeLog logfile (compactLog $ log:ls) + return logfile where logfile = logFile repo key From 5b4fa4aeca9c67dc1d40dee9782a4806d7702b4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 13:12:02 -0400 Subject: [PATCH 0310/2835] use xargs --- Core.hs | 2 +- GitQueue.hs | 28 +++++++++++----------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/Core.hs b/Core.hs index 27baba28e6..13177588d0 100644 --- a/Core.hs +++ b/Core.hs @@ -36,7 +36,7 @@ shutdown = do if (q == GitQueue.empty) then return () else do - liftIO $ putStrLn "Recording state in git..." + verbose $ liftIO $ putStrLn "Recording state in git..." liftIO $ GitQueue.run g q -- clean up any files left in the temp directory, but leave diff --git a/GitQueue.hs b/GitQueue.hs index b7210ccb59..34a89f17b7 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -1,4 +1,4 @@ -{- git repository command queues +{- git repository command queue -} module GitQueue ( @@ -9,6 +9,9 @@ module GitQueue ( ) where import qualified Data.Map as M +import System.IO +import System.Cmd.Utils +import Data.String.Utils import qualified GitRepo as Git @@ -45,20 +48,11 @@ run repo queue = do - Complicated by commandline length limits. -} runAction :: Git.Repo -> Action -> [FilePath] -> IO () runAction repo action files = do - xargs [] 0 files + if (null files) + then return () + else runxargs where - arg_max = 2048 -- TODO get better ARG_MAX - maxlen = arg_max - cmdlen - c = (subcommand action):(params action) - cmdlen = (length "git") + - (foldl (\a b -> a + b + 1) 1 $ map length c) - xargs collect _ [] = exec collect - xargs collect len (f:fs) = do - let len' = len + 1 + length f - if (len' >= maxlen) - then do - exec collect - xargs [f] (length f) fs - else xargs (f:collect) len' fs - exec [] = return () - exec fs = Git.run repo $ c ++ fs + runxargs = pOpen WriteToPipe "xargs" + (["-0", "git", subcommand action] ++ (params action)) + feedxargs + feedxargs h = hPutStr h $ join "\0" files From 4e7c27f58bc8af8be50bca82523ad18c1dfe2a4e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 13:39:45 -0400 Subject: [PATCH 0311/2835] tweak --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index 81a4af47ac..8f4fcf0f0c 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -61,7 +61,7 @@ keyPossibilities key = do let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive if (not $ null doexpensive) - then Core.showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." + then Core.showNote $ "getting UUID for " ++ (list doexpensive) ++ "..." else return () let todo = cheap ++ doexpensive if (not $ null todo) From 3281a1cb19e25f964ca9b91877a194fcb216ca5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 13:55:28 -0400 Subject: [PATCH 0312/2835] don't try to set up .gitattributes every time; only do it on git annex init --- Commands.hs | 3 ++- Core.hs | 1 - doc/git-annex.mdwn | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Commands.hs b/Commands.hs index 6018ed8223..015a2d822a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -346,7 +346,7 @@ fixCleanup file = do Annex.queue "add" [] file return True -{- Stores description for the repository. -} +{- Stores description for the repository etc. -} initStart :: String -> Annex (Maybe SubCmdPerform) initStart description = do if (null description) @@ -359,6 +359,7 @@ initPerform description = do g <- Annex.gitRepo u <- getUUID g describeUUID u description + liftIO $ gitAttributes g return $ Just $ initCleanup initCleanup :: Annex Bool initCleanup = do diff --git a/Core.hs b/Core.hs index 13177588d0..80bf56cc40 100644 --- a/Core.hs +++ b/Core.hs @@ -22,7 +22,6 @@ import Utility startup :: Annex Bool startup = do g <- Annex.gitRepo - liftIO $ gitAttributes g prepUUID return True diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3645d37f2a..417ff7e58d 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -91,7 +91,8 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * init description - Initializes git-annex with a description of the git repository. + Initializes git-annex with a description of the git repository, + and sets up `.gitattributes` and the pre-commit hook. This is an optional, but recommended step. * unannex [path ...] @@ -106,6 +107,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Fixes up symlinks that have become broken to again point to annexed content. This is useful to run if you have been moving the symlinks around. + You do not normally need to run this by hand since `git-annex init` + installs a pre-commit hook that automatically fixes up symlinks when + they are committed. + * fromkey file This can be used to maually set up a file to link to a specified key From 563484e1354878df4a6877e4506af0d70e41b13e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 14:33:44 -0400 Subject: [PATCH 0313/2835] pre-commit hook --- Commands.hs | 20 +++++++++++++++++--- Core.hs | 15 +++++++++++++++ debian/changelog | 3 +++ doc/git-annex.mdwn | 12 ++++++++---- doc/todo/symlink_farming_commit_hook.mdwn | 2 ++ doc/walkthrough.mdwn | 18 ++++++++++-------- 6 files changed, 55 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index 015a2d822a..b6bc6dfc22 100644 --- a/Commands.hs +++ b/Commands.hs @@ -68,7 +68,7 @@ doSubCmd cmdname start param = do {- A subcommand can broadly want one of several kinds of input parameters. - This allows a first stage of filtering before starting a subcommand. -} data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing - | Description | Keys | Tempfile + | Description | Keys | Tempfile | FilesToBeCommitted data SubCommand = Command { subcmdname :: String, @@ -91,7 +91,9 @@ subCmds = [ , (Command "unannex" unannexStart FilesInGit "undo accidential add command") , (Command "fix" fixStart FilesInGit - "fix up files' symlinks to point to annexed content") + "fix up symlinks to point to annexed content") + , (Command "pre-commit" fixStart FilesToBeCommitted + "fix up symlinks before they are committed") , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") , (Command "dropkey" dropKeyStart Keys @@ -130,7 +132,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds showcmd c = (subcmdname c) ++ - (pad 10 (subcmdname c)) ++ + (pad 11 (subcmdname c)) ++ (descWanted (subcmdwants c)) ++ (pad 13 (descWanted (subcmdwants c))) ++ (subcmddesc c) @@ -161,6 +163,17 @@ findWanted FilesMissing params repo = do if (e) then return False else return True findWanted Description params _ = do return $ [unwords params] +findWanted FilesToBeCommitted params repo = do + files <- mapM gitcached params + return $ foldl (++) [] files + where + gitcached p = do + -- ask git for files staged for commit that + -- are being added, moved, or changed (but not deleted) + fs0 <- Git.pipeRead repo ["diff", "--cached", + "--name-only", "--diff-filter=ACMRT", + "-z", "HEAD", p] + return $ filter (not . null) $ split "\0" fs0 findWanted _ params _ = return params {- Parses command line and returns two lists of actions to be @@ -360,6 +373,7 @@ initPerform description = do u <- getUUID g describeUUID u description liftIO $ gitAttributes g + liftIO $ gitPreCommitHook g return $ Just $ initCleanup initCleanup :: Annex Bool initCleanup = do diff --git a/Core.hs b/Core.hs index 80bf56cc40..254bcec517 100644 --- a/Core.hs +++ b/Core.hs @@ -73,6 +73,21 @@ gitAttributes repo = do Git.run repo ["commit", "-m", "git-annex setup", attributes] +{- set up a git pre-commit hook, if one is not already present -} +gitPreCommitHook :: Git.Repo -> IO () +gitPreCommitHook repo = do + let hook = (Git.workTree repo) ++ "/" ++ (Git.dir repo) ++ + "/hooks/pre-commit" + exists <- doesFileExist hook + if (exists) + then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" + else do + writeFile hook $ "#!/bin/sh\n" ++ + "# automatically configured by git-annex\n" ++ + "git annex pre-commit .\n" + p <- getPermissions hook + setPermissions hook $ p {executable = True} + {- Checks if a given key is currently present in the annexLocation. - - This can be run against a remote repository to check the key there. -} diff --git a/debian/changelog b/debian/changelog index fd1fddafda..d0922c1c5b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ git-annex (0.02) UNRELEASED; urgency=low file content when dropping files. * New move subcommand, that can move files from or to a remote. * New fromkey subcommand, for registering urls, etc. + * git-annex init will now set up a pre-commit hook that fixes up symlinks + before they are committed, to ensure that moving symlinks around does not + break them. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. * --from/--to can be used to control the remote repository that git-annex diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 417ff7e58d..f7bb64988d 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -107,9 +107,13 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Fixes up symlinks that have become broken to again point to annexed content. This is useful to run if you have been moving the symlinks around. - You do not normally need to run this by hand since `git-annex init` - installs a pre-commit hook that automatically fixes up symlinks when - they are committed. +* pre-commit [path ...] + + Fixes up symlinks that are staged as part of a commit, to ensure they + point to annexed content. + + This is meant to be called from git's pre-commit hook. `git annex init` + automatically creates a pre-commit hook using this. * fromkey file @@ -202,7 +206,7 @@ decscriptions. You may edit it. `.git-annex/*.log` is where git-annex records its content tracking information. These files should be committed to git. -`.git-annex/.gitattributes` is configured to use git's union merge driver +`.gitattributes` is configured to use git's union merge driver to avoid conflicts when merging files in the `.git-annex` directory. # AUTHOR diff --git a/doc/todo/symlink_farming_commit_hook.mdwn b/doc/todo/symlink_farming_commit_hook.mdwn index af03beb70e..3e93cb34b8 100644 --- a/doc/todo/symlink_farming_commit_hook.mdwn +++ b/doc/todo/symlink_farming_commit_hook.mdwn @@ -10,3 +10,5 @@ up. back to git-annex. If you want to have your own shell script in the post-commit hook, just make it call `git annex` with no parameters. git-annex will detect when it's run from a git hook and do the necessary fixups. + +[[done]] diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index ce68a77f7a..b49a00f195 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -43,23 +43,25 @@ if you are using a centralized bare repository. add debian.iso ok # git commit -a -m added -Notice you commit at the end, this checks in git-annex's record of the -files but not their actual, large, content. +When you add a file to the annex and commit it, only a symlink to +the annexed content is committed. The content itself is stored in +git-annex's backend. ## renaming files # cd ~/annex # git mv big_file my_cool_big_file # mkdir iso - # git mv debian.iso iso - # git annex fix . - fix iso/debian.iso ok + # git mv debian.iso iso/ # git commit -m moved You can use any normal git operations to move files around, or even -make copies or delete them. `git-annex fix` needs to be run if a file -is moved into a different directory, in order to fix up the symlink -pointing to the file's content. +make copies or delete them. + +Notice that, since annexed files are represented by symlinks, +the symlink will break when the file is moved into a subdirectory. +But, git-annex will fix this up for you when you commit -- +it has a pre-commit hook that watches for and corrects broken symlinks. ## getting file content From 3874b978ab36993fe3df8b5749a6d9c53eec2bfa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 14:39:21 -0400 Subject: [PATCH 0314/2835] update --- debian/changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index d0922c1c5b..c3255ba271 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,17 +2,20 @@ git-annex (0.02) UNRELEASED; urgency=low * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. - * New move subcommand, that can move files from or to a remote. + * New move subcommand, that makes it easy to move file contents from + or to a remote. * New fromkey subcommand, for registering urls, etc. * git-annex init will now set up a pre-commit hook that fixes up symlinks before they are committed, to ensure that moving symlinks around does not break them. + * More intelligent and fast staging of modified files; git add coalescing. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. * --from/--to can be used to control the remote repository that git-annex uses. * --quiet can be used to avoid verbose output * New plumbing-level dropkey and addkey subcommands. + * Lots of bug fixes. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 From 7c65a18f1f748f575a5ad0ecc271f76b85a76c92 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 14:40:50 -0400 Subject: [PATCH 0315/2835] doc pointer --- doc/git-annex.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index f7bb64988d..03e6ce444f 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -209,6 +209,14 @@ information. These files should be committed to git. `.gitattributes` is configured to use git's union merge driver to avoid conflicts when merging files in the `.git-annex` directory. +# SEE ALSO + +Most of git-annex's documentation is available on its web site, + + +If git-annex is installed from a package, a copy of its documentation +should be included, in, for example, `/usr/share/doc/git-annex/` + # AUTHOR Joey Hess From d92730bef6cc40cdd96ff24957a9c2d2bc3cb730 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 14:48:59 -0400 Subject: [PATCH 0316/2835] tweaks --- doc/walkthrough.mdwn | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index b49a00f195..ab0067470d 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -21,7 +21,7 @@ Let's start by adding a USB drive as a remote. # git clone ~/annex # cd annex # git annex init "portable USB drive" - # git remote add home ~/annex + # git remote add laptop ~/annex # cd ~/annex # git remote add usbdrive /media/usb @@ -29,9 +29,9 @@ This is all standard ad-hoc distributed git repository setup. The only git-annex specific part is telling it the name of the new repository created on the USB drive. -Notice that both repos are set up as remotes of the other one. This lets +Notice that both repos are set up as remotes of one another. This lets either get annexed files from the other. You'll want to do that even -if you are using a centralized bare repository. +if you are using git in a more centralized fashion. ## adding files @@ -69,17 +69,17 @@ A repository does not always have all annexed file contents available. When you need the content of a file, you can use "git annex get" to make it available. -We can use this to copy everything in the laptop's home annex to the +We can use this to copy everything in the laptop's annex to the USB drive. # cd /media/usb/annex - # git pull home master + # git pull laptop master # git annex get . - get my_cool_big_file (copying from home...) ok - get iso/debian.iso (copying from home...) ok + get my_cool_big_file (copying from laptop...) ok + get iso/debian.iso (copying from laptop...) ok -Notice that you had to git pull from home first, this lets git-annex know -what has changed in home, and so it knows about the files present there and +Notice that you had to git pull from laptop first, this lets git-annex know +what has changed in laptop, and so it knows about the files present there and can get them. ## transferring files: When things go wrong @@ -92,7 +92,7 @@ it: # git annex get video/hackity_hack_and_kaxxt.mov get video/_why_hackity_hack_and_kaxxt.mov (not available) - I was unable to access these remotes: server + I was unable to access these remotes: usbdrive, server Try making some of these repositories available: 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive @@ -141,9 +141,9 @@ some other repository before dropping it. ## using ssh remotes -So far git-annex has been used with a remote repository on a USB drive. -But it can also be used with a remote that is truely remote, a host -accessed by ssh. +So far in this walkthrough, git-annex has been used with a remote +repository on a USB drive. But it can also be used with a git remote +that is truely remote, a host accessed by ssh. Say you have a desktop on the same network as your laptop and want to clone the laptop's annex to it: @@ -155,7 +155,7 @@ to clone the laptop's annex to it: Now you can get files and they will be transferred by `scp`: # git annex get my_cool_big_file - get my_cool_big_file (getting UUIDs for origin...) (copying from origin...) + get my_cool_big_file (getting UUID for origin...) (copying from origin...) WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 ok @@ -174,18 +174,19 @@ access, if available. There is a annex-cost setting you can configure in Also, note that you need full shell access for this to work -- git-annex needs to be able to ssh in and run commands. -## moving file content to another repository +## moving file content between repositories Often you will want to move some file contents from a repository to some other one. For example, your laptop's disk is getting full; time to move -some files to an external disk before moving another file from home to your -laptop. Doing that by hand (by using `git annex get` and `git annex drop`) -is possible, but a bit of a pain. `git annex move` makes it very easy. +some files to an external disk before moving another file from a file +server to your laptop. Doing that by hand (by using `git annex get` and +`git annex drop`) is possible, but a bit of a pain. `git annex move` +makes it very easy. # git annex move my_cool_big_file --to usbdrive move my_cool_big_file (moving to usbdrive...) ok - # git annex move video/hackity_hack_and_kaxxt.mov --from home - move video/hackity_hack_and_kaxxt.mov (moving from home...) + # git annex move video/hackity_hack_and_kaxxt.mov --from fileserver + move video/hackity_hack_and_kaxxt.mov (moving from fileserver...) WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 ok From 7bd7cca39932858a08dd50a15e5a6f2dc9daa6c0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 15:00:41 -0400 Subject: [PATCH 0317/2835] some work on the sha1 backend; still incomplete --- Backend/SHA1.hs | 12 +++++++++++- GitRepo.hs | 4 ++-- doc/todo/backendSHA1.mdwn | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index caece6b78c..9e8d6df6dd 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -3,6 +3,11 @@ module Backend.SHA1 (backend) where +import Control.Monad.State +import Data.String.Utils +import System.Cmd.Utils +import System.IO + import qualified Backend.File import TypeInternals @@ -13,4 +18,9 @@ backend = Backend.File.backend { -- checksum the file to get its key keyValue :: FilePath -> Annex (Maybe Key) -keyValue k = error "SHA1 keyValue unimplemented" -- TODO +keyValue file = liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do + line <- hGetLine h + let bits = split " " line + if (null bits) + then error "sha1sum parse error" + else return $ Just $ Key ((name backend), bits !! 0) diff --git a/GitRepo.hs b/GitRepo.hs index ee1bdba344..9418102355 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -195,8 +195,8 @@ run repo params = assertLocal repo $ do pipeRead :: Repo -> [String] -> IO String pipeRead repo params = assertLocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - ret <- hGetContentsStrict h - return ret + ret <- hGetContentsStrict h + return ret {- Passed a location, recursively scans for all files that - are checked into git at that location. -} diff --git a/doc/todo/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn index 40ff868c22..fa9728af64 100644 --- a/doc/todo/backendSHA1.mdwn +++ b/doc/todo/backendSHA1.mdwn @@ -1 +1,5 @@ This backend is not finished. + +In particular, while files can be added using it, git-annex will not notice +when their content changes, and will not create a new key for the new sha1 +of the net content. From 46f9525351e51f3cc3069181fae5f2ba22f869c1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 15:02:05 -0400 Subject: [PATCH 0318/2835] warning about sha1 --- doc/backends.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/backends.mdwn b/doc/backends.mdwn index d3ccaec491..cd587726c0 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -17,5 +17,6 @@ to store different files' contents in a given repository. * `SHA1` -- This backend stores the file's content in `.git/annex/`, with a name based on its sha1 checksum. This backend allows modifications of files to be tracked. Its need to generate checksums - can make it slower for large files. + can make it slower for large files. **Warning** this backend is not ready + for use. * `URL` -- This backend downloads the file's content from an external URL. From e44c7d113605941285015f0953d1e8be562a848b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 15:08:46 -0400 Subject: [PATCH 0319/2835] update --- debian/rules | 2 +- doc/git-annex.mdwn | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/rules b/debian/rules index e0a209a72c..9079120a80 100755 --- a/debian/rules +++ b/debian/rules @@ -4,4 +4,4 @@ # Not intended for use by anyone except the author. announcedir: - @echo ${HOME}/src/joeywiki/code/git-annex/news + @echo ${HOME}/src/git-annex/doc/news diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 03e6ce444f..3c48e9dc30 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -172,7 +172,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a git repository that content will be sent to. It should be specified using the name of a configured git remote. -## CONFIGURATION +# CONFIGURATION Like other git commands, git-annex is configured via `.git/config`. @@ -188,7 +188,7 @@ Like other git commands, git-annex is configured via `.git/config`. repositories. Note that other factors may be configured when pushing files to repositories, in particular, whether the repository is on a filesystem with sufficient free space. -* `remote..annex-ignore` -- If set to "true", prevents git-annex +* `remote..annex-ignore` -- If set to `true`, prevents git-annex from ever using this remote. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. From d47fb4c11ef75e4029c8da1f3ed2e68f86e62486 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 15:14:59 -0400 Subject: [PATCH 0320/2835] symlinks --- CHANGELOG | 1 + GPL | 1 + INSTALL | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) create mode 120000 CHANGELOG create mode 120000 GPL mode change 100644 => 120000 INSTALL diff --git a/CHANGELOG b/CHANGELOG new file mode 120000 index 0000000000..d526672ce2 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1 @@ +debian/changelog \ No newline at end of file diff --git a/GPL b/GPL new file mode 120000 index 0000000000..9539e93f72 --- /dev/null +++ b/GPL @@ -0,0 +1 @@ +doc/GPL \ No newline at end of file diff --git a/INSTALL b/INSTALL deleted file mode 100644 index f024541c1a..0000000000 --- a/INSTALL +++ /dev/null @@ -1 +0,0 @@ -See doc/install.mdwn for installation instructions. diff --git a/INSTALL b/INSTALL new file mode 120000 index 0000000000..67566818f0 --- /dev/null +++ b/INSTALL @@ -0,0 +1 @@ +doc/install.mdwn \ No newline at end of file From 833d4b342e7909e6770edb19fbc58d781e922205 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 16:53:54 -0400 Subject: [PATCH 0321/2835] copyright statements --- Annex.hs | 7 ++++++- Backend.hs | 6 +++++- Backend/File.hs | 4 ++++ Backend/SHA1.hs | 6 +++++- Backend/URL.hs | 6 +++++- Backend/WORM.hs | 6 +++++- BackendList.hs | 6 +++++- Commands.hs | 7 ++++++- Core.hs | 7 ++++++- GitQueue.hs | 4 ++++ GitRepo.hs | 3 +++ LocationLog.hs | 4 ++++ Locations.hs | 4 ++++ Remotes.hs | 7 ++++++- TypeInternals.hs | 4 ++++ Types.hs | 7 ++++++- UUID.hs | 3 +++ Utility.hs | 4 ++++ debian/changelog | 4 ++-- git-annex.hs | 7 ++++++- 20 files changed, 93 insertions(+), 13 deletions(-) diff --git a/Annex.hs b/Annex.hs index 1963d19c69..0e8ec2b7bc 100644 --- a/Annex.hs +++ b/Annex.hs @@ -1,4 +1,9 @@ -{- git-annex monad -} +{- git-annex monad + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Annex ( new, diff --git a/Backend.hs b/Backend.hs index c70b7b707b..b7c52ddec4 100644 --- a/Backend.hs +++ b/Backend.hs @@ -11,7 +11,11 @@ - - Multiple pluggable backends are supported, and more than one can be used - to store different files' contents in a given repository. - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Backend ( list, diff --git a/Backend/File.hs b/Backend/File.hs index 4273ba36be..4b9a3b45b5 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -6,6 +6,10 @@ - - This is an abstract backend; getKey has to be implemented to complete - it. + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Backend.File (backend) where diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 9e8d6df6dd..f6daeffec4 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -1,5 +1,9 @@ {- git-annex "SHA1" backend - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Backend.SHA1 (backend) where diff --git a/Backend/URL.hs b/Backend/URL.hs index c9b6ab6df8..fd55ddf01e 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -1,5 +1,9 @@ {- git-annex "URL" backend - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Backend.URL (backend) where diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 0588ddaf83..b5ae11807e 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -1,5 +1,9 @@ {- git-annex "WORM" backend -- Write Once, Read Many - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Backend.WORM (backend) where diff --git a/BackendList.hs b/BackendList.hs index 25f3ae5eac..e628e3a612 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -1,5 +1,9 @@ {- git-annex backend list - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module BackendList (allBackends) where diff --git a/Commands.hs b/Commands.hs index b6bc6dfc22..6974b697c3 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,4 +1,9 @@ -{- git-annex command line -} +{- git-annex command line + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Commands (parseCmd) where diff --git a/Core.hs b/Core.hs index 254bcec517..e0993a53e7 100644 --- a/Core.hs +++ b/Core.hs @@ -1,4 +1,9 @@ -{- git-annex core functions -} +{- git-annex core functions + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Core where diff --git a/GitQueue.hs b/GitQueue.hs index 34a89f17b7..6a68edb25b 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -1,4 +1,8 @@ {- git repository command queue + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module GitQueue ( diff --git a/GitRepo.hs b/GitRepo.hs index 9418102355..229b76847f 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -3,6 +3,9 @@ - This is written to be completely independant of git-annex and should be - suitable for other uses. - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module GitRepo ( diff --git a/LocationLog.hs b/LocationLog.hs index d027c4b809..10a6377080 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -14,6 +14,10 @@ - - Git is configured to use a union merge for this file, - so the lines may be in arbitrary order, but it will never conflict. + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module LocationLog ( diff --git a/Locations.hs b/Locations.hs index 970f89aa8d..49ee878c85 100644 --- a/Locations.hs +++ b/Locations.hs @@ -1,4 +1,8 @@ {- git-annex file locations + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Locations ( diff --git a/Remotes.hs b/Remotes.hs index 8f4fcf0f0c..665de38ae5 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -1,4 +1,9 @@ -{- git-annex remote repositories -} +{- git-annex remote repositories + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Remotes ( list, diff --git a/TypeInternals.hs b/TypeInternals.hs index d4404fd390..f45be4760c 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -1,6 +1,10 @@ {- git-annex internal data types - - Most things should not need this, using Types and/or Annex instead. + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module TypeInternals where diff --git a/Types.hs b/Types.hs index c3d6467a3f..b94a4170af 100644 --- a/Types.hs +++ b/Types.hs @@ -1,4 +1,9 @@ -{- git-annex abstract data types -} +{- git-annex abstract data types + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Types ( Annex, diff --git a/UUID.hs b/UUID.hs index a7783d6140..79b2b55fa2 100644 --- a/UUID.hs +++ b/UUID.hs @@ -3,6 +3,9 @@ - Each git repository used by git-annex has an annex.uuid setting that - uniquely identifies that repository. - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module UUID ( diff --git a/Utility.hs b/Utility.hs index 8e620c64cd..ab90c51608 100644 --- a/Utility.hs +++ b/Utility.hs @@ -1,4 +1,8 @@ {- git-annex utility functions + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Utility ( diff --git a/debian/changelog b/debian/changelog index c3255ba271..72ae91c02e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.02) UNRELEASED; urgency=low +git-annex (0.02) unstable; urgency=low * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. @@ -17,7 +17,7 @@ git-annex (0.02) UNRELEASED; urgency=low * New plumbing-level dropkey and addkey subcommands. * Lots of bug fixes. - -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 + -- Joey Hess Wed, 27 Oct 2010 16:39:29 -0400 git-annex (0.01) unstable; urgency=low diff --git a/git-annex.hs b/git-annex.hs index d7b26cd968..e9e7ff0273 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,4 +1,9 @@ -{- git-annex main program -} +{- git-annex main program + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} import IO (try) import System.IO From 27d28b0bf2f3b723f404407d1814c38a3c37076d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 16:57:36 -0400 Subject: [PATCH 0322/2835] add news item for git-annex 0.02 --- doc/news/version_0.02.mdwn | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/news/version_0.02.mdwn diff --git a/doc/news/version_0.02.mdwn b/doc/news/version_0.02.mdwn new file mode 100644 index 0000000000..689124ab4e --- /dev/null +++ b/doc/news/version_0.02.mdwn @@ -0,0 +1,18 @@ +git-annex 0.02 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Can scp annexed files from remote hosts, and check remote hosts for + file content when dropping files. + * New move subcommand, that makes it easy to move file contents from + or to a remote. + * New fromkey subcommand, for registering urls, etc. + * git-annex init will now set up a pre-commit hook that fixes up symlinks + before they are committed, to ensure that moving symlinks around does not + break them. + * More intelligent and fast staging of modified files; git add coalescing. + * Add remote.annex-ignore git config setting to allow completly disabling + a given remote. + * --from/--to can be used to control the remote repository that git-annex + uses. + * --quiet can be used to avoid verbose output + * New plumbing-level dropkey and addkey subcommands. + * Lots of bug fixes."""]] \ No newline at end of file From 2de5f51a57f011654007829850e7b24992497e8d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 19:10:28 -0400 Subject: [PATCH 0323/2835] xargs needed --- doc/install.mdwn | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index e217799a32..9e3b385c5c 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -1,8 +1,9 @@ To build and use git-annex, you will need: -* git: +* `git`: * The Haskell Platform: * MissingH: -* uuid: +* `uuid`: +* `xargs`: Then just [[download]] git-annex and run: `make; make install` From 1118b4a6466f3453f5c517ff8eadbfbd1a4895f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 19:27:01 -0400 Subject: [PATCH 0324/2835] idea --- doc/todo/union_mounting.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/todo/union_mounting.mdwn diff --git a/doc/todo/union_mounting.mdwn b/doc/todo/union_mounting.mdwn new file mode 100644 index 0000000000..c42a055021 --- /dev/null +++ b/doc/todo/union_mounting.mdwn @@ -0,0 +1,10 @@ +It should be possible to union mount annexes. So if multiple drives have +content, an annex mounting them both would have available all the +content from all the drives. + +This could be done by just making .git/annex/KEY link to the actual content +on the mounted annex. + +(Need to make sure the [[copy_tracking|copies]] code does not +confused and think the symlink is a copy of the content.. Also need to make +sure that code that writes to .git/annex does not follow symlinks.)) From ca667f5612d0d13f1aaf6ca973681a9448ed3864 Mon Sep 17 00:00:00 2001 From: Eugen_Paiuc Date: Thu, 28 Oct 2010 03:07:02 +0000 Subject: [PATCH 0325/2835] --- doc/install.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index 9e3b385c5c..e49100f786 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,5 +5,5 @@ To build and use git-annex, you will need: * MissingH: * `uuid`: * `xargs`: - -Then just [[download]] git-annex and run: `make; make install` +* `ikiwiki`: <> +* Then just [[download]] git-annex and run: `make; make install` From 9c7b3dce9e8f964ed60dd45bca580a46ff8a5ed5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 12:15:21 -0400 Subject: [PATCH 0326/2835] tweaks --- GitRepo.hs | 18 +++++++++++------- Remotes.hs | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 229b76847f..0e87c9526d 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -24,6 +24,7 @@ module GitRepo ( configGet, configMap, configRead, + configTrue, run, pipeRead, attributes, @@ -47,6 +48,7 @@ import Data.String.Utils import Data.Map as Map hiding (map, split) import Network.URI import Maybe +import Char import Utility @@ -127,13 +129,11 @@ assertssh repo action = then action else error $ "unsupported url " ++ (show $ url repo) bare :: Repo -> Bool -bare repo = - if (member b (config repo)) - then ("true" == fromJust (Map.lookup b (config repo))) - else error $ "it is not known if git repo " ++ (repoDescribe repo) ++ +bare repo = case Map.lookup "core.bare" $ config repo of + Just v -> configTrue v + Nothing -> error $ "it is not known if git repo " ++ + (repoDescribe repo) ++ " is a bare repository; config not read" - where - b = "core.bare" {- Path to a repository's gitattributes file. -} attributes :: Repo -> String @@ -173,7 +173,7 @@ relative repo file = assertLocal repo $ drop (length absrepo) absfile {- Hostname of an URL repo. (May include a username and/or port too.) -} urlHost :: Repo -> String urlHost repo = assertUrl repo $ - (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) + uriUserInfo a ++ uriRegName a ++ uriPort a where a = fromJust $ uriAuthority $ url repo @@ -235,6 +235,10 @@ configRead repo = let r = repo { config = configParse val } return r { remotes = configRemotes r } +{- Checks if a string fron git config is a true value. -} +configTrue :: String -> Bool +configTrue s = map toLower s == "true" + {- Calculates a list of a repo's configured remotes, by parsing its config. -} configRemotes :: Repo -> [Repo] configRemotes repo = map construct remotes diff --git a/Remotes.hs b/Remotes.hs index 665de38ae5..bee98a6f3d 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -139,10 +139,10 @@ repoNotIgnored r = do let name = if (not $ null fromName) then fromName else toName if (not $ null name) then return $ match name - else return $ notignored g + else return $ not $ ignored g where match name = name == Git.repoRemoteName r - notignored g = "true" /= config g + ignored g = Git.configTrue $ config g config g = Git.configGet g configkey "" configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" From 045b051ec10023afc2e62895032527d5b5130495 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 12:40:05 -0400 Subject: [PATCH 0327/2835] got rid of almost all 'return ()' --- Annex.hs | 3 --- Backend/File.hs | 5 +++-- Commands.hs | 9 +++------ Core.hs | 23 +++++++++-------------- GitQueue.hs | 5 ++--- GitRepo.hs | 3 +-- Remotes.hs | 7 ++++--- UUID.hs | 14 ++++---------- git-annex.hs | 11 ++++------- 9 files changed, 30 insertions(+), 50 deletions(-) diff --git a/Annex.hs b/Annex.hs index 0e8ec2b7bc..60ae91708b 100644 --- a/Annex.hs +++ b/Annex.hs @@ -61,7 +61,6 @@ gitRepoChange :: Git.Repo -> Annex () gitRepoChange r = do state <- get put state { Internals.repo = r } - return () {- Returns the backends being used. -} backends :: Annex [Backend] @@ -74,7 +73,6 @@ backendsChange :: [Backend] -> Annex () backendsChange b = do state <- get put state { Internals.backends = b } - return () {- Returns the full list of supported backends. -} supportedBackends :: Annex [Backend] @@ -95,7 +93,6 @@ flagChange :: FlagName -> Flag -> Annex () flagChange name val = do state <- get put state { Internals.flags = M.insert name val $ Internals.flags state } - return () {- Gets the value of a String flag (or "" if there is no such String flag) -} flagGet :: FlagName -> Annex String diff --git a/Backend/File.hs b/Backend/File.hs index 4b9a3b45b5..b45354752f 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -129,7 +129,7 @@ checkRemoveKey key = do "Could only verify the existence of " ++ (show have) ++ " out of " ++ (show need) ++ " necessary copies" - if (not $ null bad) then showTriedRemotes bad else return () + showTriedRemotes bad showLocations key hint return False @@ -146,7 +146,8 @@ showLocations key = do if (null uuidsf) then showLongNote $ "No other repository is known to contain the file." else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids - + +showTriedRemotes [] = return () showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ (Remotes.list remotes) diff --git a/Commands.hs b/Commands.hs index 6974b697c3..a46b9fb163 100644 --- a/Commands.hs +++ b/Commands.hs @@ -14,6 +14,7 @@ import System.Directory import System.Path import Data.String.Utils import Control.Monad (filterM) +import Monad (when) import List import IO @@ -326,9 +327,7 @@ dropKeyCleanup key = do setKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) setKeyStart tmpfile = do keyname <- Annex.flagGet "key" - if (null keyname) - then error "please specify the key with --key" - else return () + when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname return $ Just $ setKeyPerform tmpfile key @@ -392,9 +391,7 @@ initCleanup = do fromKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) fromKeyStart file = do keyname <- Annex.flagGet "key" - if (null keyname) - then error "please specify the key with --key" - else return () + when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname diff --git a/Core.hs b/Core.hs index e0993a53e7..cf97768c73 100644 --- a/Core.hs +++ b/Core.hs @@ -13,6 +13,7 @@ import System.Directory import Control.Monad.State (liftIO) import System.Path import Data.String.Utils +import Monad (when, unless) import Types import Locations @@ -37,19 +38,15 @@ shutdown = do -- Runs all queued git commands. q <- Annex.queueGet - if (q == GitQueue.empty) - then return () - else do - verbose $ liftIO $ putStrLn "Recording state in git..." - liftIO $ GitQueue.run g q + unless (q == GitQueue.empty) $ do + verbose $ liftIO $ putStrLn "Recording state in git..." + liftIO $ GitQueue.run g q -- clean up any files left in the temp directory, but leave -- the tmp directory itself let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp - if (exists) - then liftIO $ removeDirectoryRecursive $ tmp - else return () + when (exists) $ liftIO $ removeDirectoryRecursive $ tmp liftIO $ createDirectoryIfMissing True tmp return True @@ -65,11 +62,9 @@ gitAttributes repo = do commit else do content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () + when (all (/= attrLine) (lines content)) $ do + appendFile attributes $ attrLine ++ "\n" + commit where attrLine = stateLoc ++ "*.log merge=union" attributes = Git.attributes repo @@ -150,7 +145,7 @@ getViaTmp key action = do verbose :: Annex () -> Annex () verbose a = do q <- Annex.flagIsSet "quiet" - if (q) then return () else a + unless q a showStart :: String -> String -> Annex () showStart command file = verbose $ do liftIO $ putStr $ command ++ " " ++ file ++ " " diff --git a/GitQueue.hs b/GitQueue.hs index 6a68edb25b..09b8037e64 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -16,6 +16,7 @@ import qualified Data.Map as M import System.IO import System.Cmd.Utils import Data.String.Utils +import Monad (unless) import qualified GitRepo as Git @@ -52,9 +53,7 @@ run repo queue = do - Complicated by commandline length limits. -} runAction :: Git.Repo -> Action -> [FilePath] -> IO () runAction repo action files = do - if (null files) - then return () - else runxargs + unless (null files) runxargs where runxargs = pOpen WriteToPipe "xargs" (["-0", "git", subcommand action] ++ (params action)) diff --git a/GitRepo.hs b/GitRepo.hs index 0e87c9526d..d0fac96c1a 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -191,8 +191,7 @@ gitCommandLine repo params = assertLocal repo $ {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () run repo params = assertLocal repo $ do - r <- safeSystem "git" (gitCommandLine repo params) - return () + safeSystem "git" (gitCommandLine repo params) {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String diff --git a/Remotes.hs b/Remotes.hs index bee98a6f3d..02d4dc9c27 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -28,6 +28,7 @@ import System.Directory import System.Posix.Directory import List import Maybe +import Monad (when, unless) import Types import qualified GitRepo as Git @@ -65,9 +66,9 @@ keyPossibilities key = do let cheap = filter (not . Git.repoIsUrl) allremotes let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive - if (not $ null doexpensive) - then Core.showNote $ "getting UUID for " ++ (list doexpensive) ++ "..." - else return () + unless (null doexpensive) $ do + Core.showNote $ "getting UUID for " ++ + (list doexpensive) ++ "..." let todo = cheap ++ doexpensive if (not $ null todo) then do diff --git a/UUID.hs b/UUID.hs index 79b2b55fa2..f2235e4b62 100644 --- a/UUID.hs +++ b/UUID.hs @@ -63,10 +63,7 @@ getUUID r = do where uncached r = Git.configGet r "annex.uuid" "" cached r g = Git.configGet g (cachekey r) "" - updatecache g r u = do - if (g /= r) - then setConfig (cachekey r) u - else return () + updatecache g r u = when (g /= r) $ setConfig (cachekey r) u cachekey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} @@ -74,11 +71,9 @@ prepUUID :: Annex () prepUUID = do g <- Annex.gitRepo u <- getUUID g - if ("" == u) - then do - uuid <- liftIO $ genUUID - setConfig configkey uuid - else return () + when ("" == u) $ do + uuid <- liftIO $ genUUID + setConfig configkey uuid {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () @@ -88,7 +83,6 @@ setConfig key value = do -- re-read git config and update the repo's state g' <- liftIO $ Git.configRead g Annex.gitRepoChange g' - return () {- Filters a list of repos to ones that have listed UUIDs. -} reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] diff --git a/git-annex.hs b/git-annex.hs index e9e7ff0273..5011fade2b 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -8,6 +8,7 @@ import IO (try) import System.IO import System.Environment +import Monad import qualified Annex import Types @@ -42,12 +43,8 @@ tryRun' state errnum (a:as) = do tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' state errnum [] = do - if (errnum > 0) - then error $ (show errnum) ++ " failed" - else return () +tryRun' state errnum [] = + when (errnum > 0) $ error $ (show errnum) ++ " failed" {- Exception pretty-printing. -} -showErr e = do - hPutStrLn stderr $ "git-annex: " ++ (show e) - return () +showErr e = hPutStrLn stderr $ "git-annex: " ++ (show e) From 7109e20e5d2026c25a42a2784c1e3a430a67d3cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 12:52:40 -0400 Subject: [PATCH 0328/2835] tweak --- GitRepo.hs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index d0fac96c1a..4bad8a50d3 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -150,10 +150,8 @@ dir repo = if (bare repo) then "" else ".git" - - Note that for URL repositories, this is relative to the urlHost -} workTree :: Repo -> FilePath -workTree repo = - if (not $ repoIsUrl repo) - then top repo - else urlPath repo +workTree r@(UrlRepo { }) = urlPath r +workTree (Repo { top = p }) = p {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. @@ -186,7 +184,8 @@ urlPath repo = assertUrl repo $ gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo params = assertLocal repo $ -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(top repo)++"/"++(dir repo), "--work-tree="++(top repo)] ++ params + ["--git-dir="++(top repo)++"/"++(dir repo), + "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () From 3e02977814e77b47e4db020e9b0dedadad1b6e7e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 13:40:10 -0400 Subject: [PATCH 0329/2835] took Josh's asvice and unified the Repo data types & used pattern matching more --- GitRepo.hs | 154 +++++++++++++++++++++++++---------------------------- 1 file changed, 73 insertions(+), 81 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 4bad8a50d3..cd2c80691b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -54,47 +54,37 @@ import Utility {- There are two types of repositories; those on local disk and those - accessed via an URL. -} -data Repo = +data RepoLocation = Dir FilePath | Url URI + deriving (Show, Eq) + +data Repo = Repo { + location :: RepoLocation, + config :: Map String String, + remotes :: [Repo], + -- remoteName holds the name used for this repo in remotes + remoteName :: Maybe String +} deriving (Show, Eq) + +newFrom l = Repo { - top :: FilePath, - config :: Map String String, - remotes :: [Repo], - -- remoteName holds the name used for this repo in remotes - remoteName :: Maybe String - } | UrlRepo { - url :: URI, - config :: Map String String, - remotes :: [Repo], - remoteName :: Maybe String - } deriving (Show, Eq) + location = l, + config = Map.empty, + remotes = [], + remoteName = Nothing + } {- Local Repo constructor. -} repoFromPath :: FilePath -> Repo -repoFromPath dir = - Repo { - top = dir, - config = Map.empty, - remotes = [], - remoteName = Nothing - } +repoFromPath dir = newFrom $ Dir dir {- Remote Repo constructor. Throws exception on invalid url. -} repoFromUrl :: String -> Repo -repoFromUrl url = - UrlRepo { - url = fromJust $ parseURI url, - config = Map.empty, - remotes = [], - remoteName = Nothing - } +repoFromUrl url = newFrom $ Url $ fromJust $ parseURI url {- User-visible description of a git repo. -} -repoDescribe repo = - if (isJust $ remoteName repo) - then fromJust $ remoteName repo - else if (not $ repoIsUrl repo) - then top repo - else show (url repo) +repoDescribe Repo { remoteName = Just name } = name +repoDescribe Repo { location = Url url } = show url +repoDescribe Repo { location = Dir dir } = dir {- Constructs and returns an updated version of a repo with - different remotes list. -} @@ -103,17 +93,19 @@ remotesAdd repo rs = repo { remotes = rs } {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} -repoRemoteName r = - if (isJust $ remoteName r) - then fromJust $ remoteName r - else "" +repoRemoteName Repo { remoteName = Just name } = name +repoRemoteName _ = "" {- Some code needs to vary between URL and normal repos, - or bare and non-bare, these functions help with that. -} -repoIsUrl repo = case (repo) of - UrlRepo {} -> True - Repo {} -> False -repoIsSsh repo = repoIsUrl repo && (uriScheme $ url repo) == "ssh:" +repoIsUrl Repo { location = Url _ } = True +repoIsUrl _ = False + +repoIsSsh Repo { location = Url url } + | uriScheme url == "ssh:" = True + | otherwise = False +repoIsSsh _ = False + assertLocal repo action = if (not $ repoIsUrl repo) then action @@ -124,10 +116,10 @@ assertUrl repo action = then action else error $ "acting on local git repo " ++ (repoDescribe repo) ++ " not supported" -assertssh repo action = +assertSsh repo action = if (repoIsSsh repo) then action - else error $ "unsupported url " ++ (show $ url repo) + else error $ "unsupported url in repo " ++ (repoDescribe repo) bare :: Repo -> Bool bare repo = case Map.lookup "core.bare" $ config repo of Just v -> configTrue v @@ -137,55 +129,56 @@ bare repo = case Map.lookup "core.bare" $ config repo of {- Path to a repository's gitattributes file. -} attributes :: Repo -> String -attributes repo = assertLocal repo $ do - if (bare repo) - then (top repo) ++ "/info/.gitattributes" - else (top repo) ++ "/.gitattributes" +attributes repo + | bare repo = (workTree repo) ++ "/info/.gitattributes" + | otherwise = (workTree repo) ++ "/.gitattributes" {- Path to a repository's .git directory, relative to its workTree. -} dir :: Repo -> String -dir repo = if (bare repo) then "" else ".git" +dir repo + | bare repo = "" + | otherwise = ".git" {- Path to a repository's --work-tree, that is, its top. - - - Note that for URL repositories, this is relative to the urlHost -} + - Note that for URL repositories, this is the path on the remote host. -} workTree :: Repo -> FilePath -workTree r@(UrlRepo { }) = urlPath r -workTree (Repo { top = p }) = p +workTree r@(Repo { location = Url _ }) = urlPath r +workTree (Repo { location = Dir d }) = d {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} relative :: Repo -> String -> String -relative repo file = assertLocal repo $ drop (length absrepo) absfile +relative repo@(Repo { location = Dir d }) file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file - absrepo = case (absNormPath "/" (top repo)) of + absrepo = case (absNormPath "/" d) of Just f -> f ++ "/" - Nothing -> error $ "bad repo" ++ (top repo) + Nothing -> error $ "bad repo" ++ (repoDescribe repo) absfile = case (secureAbsNormPath absrepo file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo +relative repo file = assertLocal repo $ error "internal" {- Hostname of an URL repo. (May include a username and/or port too.) -} urlHost :: Repo -> String -urlHost repo = assertUrl repo $ - uriUserInfo a ++ uriRegName a ++ uriPort a - where - a = fromJust $ uriAuthority $ url repo +urlHost Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a + where a = fromJust $ uriAuthority $ u +urlHost repo = assertUrl repo $ error "internal" {- Path of an URL repo. -} urlPath :: Repo -> String -urlPath repo = assertUrl repo $ - uriPath $ url repo +urlPath Repo { location = Url u } = uriPath u +urlPath repo = assertUrl repo $ error "internal" {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: Repo -> [String] -> [String] -gitCommandLine repo params = assertLocal repo $ +gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(top repo)++"/"++(dir repo), - "--work-tree="++(top repo)] ++ params + ["--git-dir="++d++"/"++(dir repo), "--work-tree="++d] ++ params +gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () @@ -215,23 +208,23 @@ notInRepo repo location = do {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo -configRead repo = - if (not $ repoIsUrl repo) - then do - {- Cannot use pipeRead because it relies on the config having - been already read. Instead, chdir to the repo. -} - cwd <- getCurrentDirectory - bracket_ (changeWorkingDirectory (top repo)) - (\_ -> changeWorkingDirectory cwd) $ - pOpen ReadFromPipe "git" ["config", "--list"] proc - else assertssh repo $ do - pOpen ReadFromPipe "ssh" [urlHost repo, sshcommand] proc +configRead repo@(Repo { location = Dir d }) = do + {- Cannot use pipeRead because it relies on the config having + been already read. Instead, chdir to the repo. -} + cwd <- getCurrentDirectory + bracket_ (changeWorkingDirectory d) + (\_ -> changeWorkingDirectory cwd) $ + pOpen ReadFromPipe "git" ["config", "--list"] $ + hConfigRead repo +configRead repo = assertSsh repo $ do + pOpen ReadFromPipe "ssh" [urlHost repo, sshcommand] $ hConfigRead repo where - sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ " && git config --list" - proc h = do - val <- hGetContentsStrict h - let r = repo { config = configParse val } - return r { remotes = configRemotes r } + sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ + " && git config --list" +hConfigRead repo h = do + val <- hGetContentsStrict h + let r = repo { config = configParse val } + return r { remotes = configRemotes r } {- Checks if a string fron git config is a true value. -} configTrue :: String -> Bool @@ -246,9 +239,8 @@ configRemotes repo = map construct remotes isremote k = (startswith "remote." k) && (endswith ".url" k) remotename k = (split "." k) !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } - gen v = if (isURI v) - then repoFromUrl v - else repoFromPath v + gen v | isURI v = repoFromUrl v + | otherwise = repoFromPath v {- Parses git config --list output into a config map. -} configParse :: String -> Map.Map String String From 5c2c652d7dd6346990143f10f9a7d520496cc871 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 13:47:10 -0400 Subject: [PATCH 0330/2835] Fix support for file:// remotes. --- GitRepo.hs | 5 ++++- debian/changelog | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index cd2c80691b..fd69ec21ae 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -79,7 +79,10 @@ repoFromPath dir = newFrom $ Dir dir {- Remote Repo constructor. Throws exception on invalid url. -} repoFromUrl :: String -> Repo -repoFromUrl url = newFrom $ Url $ fromJust $ parseURI url +repoFromUrl url + | startswith "file://" url = repoFromPath $ uriPath u + | otherwise = newFrom $ Url u + where u = fromJust $ parseURI url {- User-visible description of a git repo. -} repoDescribe Repo { remoteName = Just name } = name diff --git a/debian/changelog b/debian/changelog index 72ae91c02e..ac1ed0ab40 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.03) UNRELEASED; urgency=low + + * Fix support for file:// remotes. + + -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 + git-annex (0.02) unstable; urgency=low * Can scp annexed files from remote hosts, and check remote hosts for From ecfbc01ff8b55d4cbe02c02c604264a627e759bf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 14:04:22 -0400 Subject: [PATCH 0331/2835] Add --verbose --- Commands.hs | 2 ++ debian/changelog | 1 + doc/git-annex.mdwn | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/Commands.hs b/Commands.hs index a46b9fb163..ac17c34d1a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -116,6 +116,8 @@ options = [ "allow actions that may lose annexed data" , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (storebool "quiet" False)) + "allow verbose output" , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") "specify default key-value backend to use" , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") diff --git a/debian/changelog b/debian/changelog index ac1ed0ab40..c7297d1c3f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Fix support for file:// remotes. + * Add --verbose -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3c48e9dc30..47fbb37601 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -153,6 +153,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Avoid the default verbose logging of what is done; only show errors and progress displays. +* --verbose + + Enable verbose logging. + * --backend=name Specify the default key-value backend to use, adding it to the front From 694a33e91b290373649594688ad880203d140dcb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 14:20:02 -0400 Subject: [PATCH 0332/2835] syntax tweaks --- Commands.hs | 45 +++++++++++++++++++++------------------------ LocationLog.hs | 2 +- Remotes.hs | 18 ++++++++---------- 3 files changed, 30 insertions(+), 35 deletions(-) diff --git a/Commands.hs b/Commands.hs index ac17c34d1a..41e9ad54dc 100644 --- a/Commands.hs +++ b/Commands.hs @@ -14,7 +14,7 @@ import System.Directory import System.Path import Data.String.Utils import Control.Monad (filterM) -import Monad (when) +import Monad (when, unless) import List import IO @@ -168,7 +168,7 @@ findWanted FilesMissing params repo = do where missing f = do e <- doesFileExist f - if (e) then return False else return True + return $ not e findWanted Description params _ = do return $ [unwords params] findWanted FilesToBeCommitted params repo = do @@ -191,19 +191,18 @@ findWanted _ params _ = return params parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) parseCmd argv state = do (flags, params) <- getopt - if (null params) - then error usage - else case (lookupCmd (params !! 0)) of - [] -> error usage - [Command name action want _] -> do - f <- findWanted want (drop 1 params) - (TypeInternals.repo state) - let actions = map (doSubCmd name action) $ - filter notstate f - let configactions = map (\f -> do - f - return True) flags - return (configactions, actions) + when (null params) $ error usage + case lookupCmd (params !! 0) of + [] -> error usage + [Command name action want _] -> do + f <- findWanted want (drop 1 params) + (TypeInternals.repo state) + let actions = map (doSubCmd name action) $ + filter notstate f + let configactions = map (\f -> do + f + return True) flags + return (configactions, actions) where -- never include files from the state directory notstate f = stateLoc /= take (length stateLoc) f @@ -273,7 +272,7 @@ getPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) getPerform file key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) - then return $ Just $ return True + then return $ Just $ return True -- no cleanup needed else return Nothing {- Indicates a file's content is not wanted anymore, and should be removed @@ -368,11 +367,9 @@ fixCleanup file = do {- Stores description for the repository etc. -} initStart :: String -> Annex (Maybe SubCmdPerform) initStart description = do - if (null description) - then error $ - "please specify a description of this repository\n" ++ - usage - else return $ Just $ initPerform description + when (null description) $ error $ + "please specify a description of this repository\n" ++ usage + return $ Just $ initPerform description initPerform :: String -> Annex (Maybe SubCmdCleanup) initPerform description = do g <- Annex.gitRepo @@ -398,9 +395,9 @@ fromKeyStart file = do let key = genKey (backends !! 0) keyname inbackend <- Backend.hasKey key - if (not inbackend) - then error $ "key ("++keyname++") is not present in backend" - else return $ Just $ fromKeyPerform file key + unless (inbackend) $ error $ + "key ("++keyname++") is not present in backend" + return $ Just $ fromKeyPerform file key fromKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) fromKeyPerform file key = do link <- calcGitLink file key diff --git a/LocationLog.hs b/LocationLog.hs index 10a6377080..f92dee652d 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -163,6 +163,6 @@ mapLog map log = then Map.insert (uuid log) log map else map where - better = case (Map.lookup (uuid log) map) of + better = case Map.lookup (uuid log) map of Just l -> (date l <= date log) Nothing -> True diff --git a/Remotes.hs b/Remotes.hs index 02d4dc9c27..a432e1b5d6 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -153,15 +153,13 @@ commandLineRemote = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" let name = if (not $ null fromName) then fromName else toName - if (null name) - then error "no remote specified" - else do - g <- Annex.gitRepo - let match = filter (\r -> name == Git.repoRemoteName r) $ - Git.remotes g - if (null match) - then error $ "there is no git remote named \"" ++ name ++ "\"" - else return $ match !! 0 + when (null name) $ error "no remote specified" + g <- Annex.gitRepo + let match = filter (\r -> name == Git.repoRemoteName r) $ + Git.remotes g + when (null match) $ error $ + "there is no git remote named \"" ++ name ++ "\"" + return $ match !! 0 {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the @@ -187,7 +185,7 @@ tryGitConfigRead r = do where exchange [] new = [] exchange (old:ls) new = - if ((Git.repoRemoteName old) == (Git.repoRemoteName new)) + if (Git.repoRemoteName old == Git.repoRemoteName new) then new:(exchange ls new) else old:(exchange ls new) From fde01e52f382d7903749609301f99d78791c2f2c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 12:38:41 -0400 Subject: [PATCH 0333/2835] Fix SIGINT handling. --- Utility.hs | 3 +-- debian/changelog | 1 + doc/bugs/scp_interrupt_to_background.mdwn | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Utility.hs b/Utility.hs index ab90c51608..338aca7a39 100644 --- a/Utility.hs +++ b/Utility.hs @@ -19,7 +19,6 @@ import System.IO import System.Cmd import System.Exit import System.Posix.Signals -import Data.Typeable import System.Posix.IO import Data.String.Utils import System.Path @@ -110,7 +109,7 @@ boolSystem command params = do r <- rawSystem command params case r of ExitSuccess -> return True - ExitFailure e -> if Just e == cast sigINT + ExitFailure e -> if toInteger e == toInteger sigINT then error $ command ++ "interrupted" else return False diff --git a/debian/changelog b/debian/changelog index c7297d1c3f..77e01482fd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Fix support for file:// remotes. * Add --verbose + * Fix SIGINT handling. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/bugs/scp_interrupt_to_background.mdwn b/doc/bugs/scp_interrupt_to_background.mdwn index 700c81c650..381f5cd736 100644 --- a/doc/bugs/scp_interrupt_to_background.mdwn +++ b/doc/bugs/scp_interrupt_to_background.mdwn @@ -1,2 +1,2 @@ When getting a file with scp, SIGINT is blocked, exposing the git -subcommand fork to background bug again. +subcommand fork to background bug again. [[done]] From e3030196b6616690d365597470a7123476444e57 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 13:57:22 -0400 Subject: [PATCH 0334/2835] really fix SIGINT handling Have to completly avoid SIGINT being trapped, which means going very low-level. --- Utility.hs | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/Utility.hs b/Utility.hs index 338aca7a39..233825b651 100644 --- a/Utility.hs +++ b/Utility.hs @@ -16,8 +16,10 @@ module Utility ( ) where import System.IO -import System.Cmd +import System.Cmd.Utils import System.Exit +import System.Posix.Process +import System.Posix.Process.Internals import System.Posix.Signals import System.Posix.IO import Data.String.Utils @@ -101,17 +103,30 @@ relPathDirToDir from to = {- Run a system command, and returns True or False - if it succeeded or failed. - - - An error is thrown if the command exits due to SIGINT, - - to propigate ctrl-c. + - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -} boolSystem :: FilePath -> [String] -> IO Bool boolSystem command params = do - r <- rawSystem command params - case r of - ExitSuccess -> return True - ExitFailure e -> if toInteger e == toInteger sigINT - then error $ command ++ "interrupted" - else return False + -- Going low-level because all the high-level system functions + -- block SIGINT etc. We need to block SIGCHLD, but allow + -- SIGINT to do its default program termination. + let sigset = addSignal sigCHLD emptySignalSet + oldint <- installHandler sigINT Default Nothing + oldset <- getSignalMask + blockSignals sigset + childpid <- forkProcess $ childaction oldint oldset + mps <- getProcessStatus True False childpid + restoresignals oldint oldset + case mps of + Just (Exited ExitSuccess) -> return True + _ -> return False + where + restoresignals oldint oldset = do + installHandler sigINT oldint Nothing + setSignalMask oldset + childaction oldint oldset = do + restoresignals oldint oldset + executeFile command True params Nothing {- Escapes a filename to be safely able to be exposed to the shell. -} shellEscape f = "'" ++ quote ++ "'" From fa04c36fbe6bd9e936a74230bc8e887a90250bfd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 13:59:48 -0400 Subject: [PATCH 0335/2835] ikiwiki is not really needed --- doc/install.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index e49100f786..ec9ea92c7f 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,5 +5,7 @@ To build and use git-annex, you will need: * MissingH: * `uuid`: * `xargs`: -* `ikiwiki`: <> * Then just [[download]] git-annex and run: `make; make install` + +([Ikiwiki](http://ikiwiki.info) is needed to build the documentation, +but that will be skipped if it is not installed.) From d92f186fc4bc34e0999a6e47f15e54717e0ab206 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 14:07:26 -0400 Subject: [PATCH 0336/2835] convert safeSystem to boolSystem to fix ctrl-c handling --- Backend/URL.hs | 8 ++------ GitRepo.hs | 6 ++++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index fd55ddf01e..7f0bd66733 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -7,7 +7,6 @@ module Backend.URL (backend) where -import Control.Exception import Control.Monad.State (liftIO) import Data.String.Utils import System.Cmd @@ -16,6 +15,7 @@ import System.Exit import TypeInternals import Core +import Utility backend = Backend { name = "URL", @@ -42,10 +42,6 @@ downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" liftIO $ putStrLn "" -- make way for curl progress bar - result <- liftIO $ (try curl::IO (Either SomeException ())) - case result of - Left err -> return False - Right succ -> return True + liftIO $ boolSystem "curl" ["-#", "-o", file, url] where - curl = safeSystem "curl" ["-#", "-o", file, url] url = join ":" $ drop 1 $ split ":" $ show key diff --git a/GitRepo.hs b/GitRepo.hs index fd69ec21ae..b9980826ca 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -35,6 +35,7 @@ module GitRepo ( notInRepo ) where +import Monad (when, unless) import Directory import System import System.Directory @@ -183,10 +184,11 @@ gitCommandLine repo@(Repo { location = Dir d} ) params = ["--git-dir="++d++"/"++(dir repo), "--work-tree="++d] ++ params gitCommandLine repo _ = assertLocal repo $ error "internal" -{- Runs git in the specified repo. -} +{- Runs git in the specified repo, throwing an error if it fails. -} run :: Repo -> [String] -> IO () run repo params = assertLocal repo $ do - safeSystem "git" (gitCommandLine repo params) + ok <- boolSystem "git" (gitCommandLine repo params) + unless (ok) $ error $ "git " ++ (show params) ++ " failed" {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String From 7c0777c60d5381cc16342483747e1fc9c92e41e0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 14:10:55 -0400 Subject: [PATCH 0337/2835] avoid unnessary newlines before progress in quiet mode --- Backend/URL.hs | 2 +- Core.hs | 2 ++ Remotes.hs | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index 7f0bd66733..384f933ebf 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -41,7 +41,7 @@ dummyOk url = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" - liftIO $ putStrLn "" -- make way for curl progress bar + showProgress -- make way for curl progress bar liftIO $ boolSystem "curl" ["-#", "-o", file, url] where url = join ":" $ drop 1 $ split ":" $ show key diff --git a/Core.hs b/Core.hs index cf97768c73..8cdc6979ab 100644 --- a/Core.hs +++ b/Core.hs @@ -154,6 +154,8 @@ showNote :: String -> Annex () showNote s = verbose $ do liftIO $ putStr $ "(" ++ s ++ ") " liftIO $ hFlush stdout +showProgress :: Annex () +showProgress = verbose $ liftIO $ putStr $ "\n" showLongNote :: String -> Annex () showLongNote s = verbose $ do liftIO $ putStr $ "\n" ++ (indent s) diff --git a/Remotes.hs b/Remotes.hs index a432e1b5d6..66395adcda 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -39,6 +39,7 @@ import LocationLog import Locations import UUID import Utility +import qualified Core {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -200,7 +201,7 @@ copyFromRemote r key file = do where getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] getssh = do - liftIO $ putStrLn "" -- make way for scp progress bar + Core.showProgress -- make way for scp progress bar liftIO $ boolSystem "scp" [sshLocation r keyloc, file] keyloc = annexLocation r key @@ -217,7 +218,7 @@ copyToRemote r key file = do where putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] putssh src = do - liftIO $ putStrLn "" -- make way for scp progress bar + Core.showProgress -- make way for scp progress bar liftIO $ boolSystem "scp" [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath From c9347693d78a1287cc8e3d066642931f28523263 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 17:26:26 -0400 Subject: [PATCH 0338/2835] factor out stagedFiles --- Commands.hs | 10 +--------- GitRepo.hs | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index 41e9ad54dc..2af8874e5c 100644 --- a/Commands.hs +++ b/Commands.hs @@ -172,16 +172,8 @@ findWanted FilesMissing params repo = do findWanted Description params _ = do return $ [unwords params] findWanted FilesToBeCommitted params repo = do - files <- mapM gitcached params + files <- mapM (Git.stagedFiles repo) params return $ foldl (++) [] files - where - gitcached p = do - -- ask git for files staged for commit that - -- are being added, moved, or changed (but not deleted) - fs0 <- Git.pipeRead repo ["diff", "--cached", - "--name-only", "--diff-filter=ACMRT", - "-z", "HEAD", p] - return $ filter (not . null) $ split "\0" fs0 findWanted _ params _ = return params {- Parses command line and returns two lists of actions to be diff --git a/GitRepo.hs b/GitRepo.hs index b9980826ca..355600248e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -32,7 +32,8 @@ module GitRepo ( remotesAdd, repoRemoteName, inRepo, - notInRepo + notInRepo, + stagedFiles ) where import Monad (when, unless) @@ -46,7 +47,7 @@ import System.Cmd.Utils import System.IO import IO (bracket_) import Data.String.Utils -import Data.Map as Map hiding (map, split) +import qualified Data.Map as Map hiding (map, split) import Network.URI import Maybe import Char @@ -60,7 +61,7 @@ data RepoLocation = Dir FilePath | Url URI data Repo = Repo { location :: RepoLocation, - config :: Map String String, + config :: Map.Map String String, remotes :: [Repo], -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String @@ -211,6 +212,14 @@ notInRepo repo location = do s <- pipeRead repo ["ls-files", "--others", "--exclude-standard", location] return $ lines s +{- Passed a location, returns a list of the files that are staged for + - commit that are being added, moved, or changed (but not deleted). -} +stagedFiles :: Repo -> FilePath -> IO [FilePath] +stagedFiles repo location = do + fs0 <- pipeRead repo ["diff", "--cached", "--name-only", + "--diff-filter=ACMRT", "-z", "HEAD", location] + return $ filter (not . null) $ split "\0" fs0 + {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo configRead repo@(Repo { location = Dir d }) = do @@ -239,8 +248,8 @@ configTrue s = map toLower s == "true" configRemotes :: Repo -> [Repo] configRemotes repo = map construct remotes where - remotes = toList $ filter $ config repo - filter = filterWithKey (\k _ -> isremote k) + remotes = Map.toList $ filter $ config repo + filter = Map.filterWithKey (\k _ -> isremote k) isremote k = (startswith "remote." k) && (endswith ".url" k) remotename k = (split "." k) !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } @@ -263,7 +272,7 @@ configGet repo key defaultValue = Map.findWithDefault defaultValue key (config repo) {- Access to raw config Map -} -configMap :: Repo -> Map String String +configMap :: Repo -> Map.Map String String configMap repo = config repo {- Finds the current git repository, which may be in a parent directory. -} From 8e158b7cec7baa77af511e77a6251d4954fb26d8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 17:37:05 -0400 Subject: [PATCH 0339/2835] use -z with git-ls-files, to support files with odd chars --- GitRepo.hs | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 355600248e..c8ba77133c 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -201,24 +201,30 @@ pipeRead repo params = assertLocal repo $ do {- Passed a location, recursively scans for all files that - are checked into git at that location. -} inRepo :: Repo -> FilePath -> IO [FilePath] -inRepo repo location = do - s <- pipeRead repo ["ls-files", "--cached", "--exclude-standard", location] - return $ lines s +inRepo repo location = pipeNullSplit repo + ["ls-files", "--cached", "--exclude-standard", "-z", location] {- Passed a location, recursively scans for all files that are not checked - into git, and not gitignored. -} notInRepo :: Repo -> FilePath -> IO [FilePath] -notInRepo repo location = do - s <- pipeRead repo ["ls-files", "--others", "--exclude-standard", location] - return $ lines s +notInRepo repo location = pipeNullSplit repo + ["ls-files", "--others", "--exclude-standard", "-z", location] -{- Passed a location, returns a list of the files that are staged for - - commit that are being added, moved, or changed (but not deleted). -} +{- Passed a location, returns a list of the files, staged for + - commit, that are being added, moved, or changed (but not deleted). -} stagedFiles :: Repo -> FilePath -> IO [FilePath] -stagedFiles repo location = do - fs0 <- pipeRead repo ["diff", "--cached", "--name-only", - "--diff-filter=ACMRT", "-z", "HEAD", location] - return $ filter (not . null) $ split "\0" fs0 +stagedFiles repo location = pipeNullSplit repo + ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", + "HEAD", location] + +{- Reads null terminated output of a git command (as enabled by the -z + - parameter), and splits it into a list of files. -} +pipeNullSplit :: Repo -> [String] -> IO [FilePath] +pipeNullSplit repo params = do + fs0 <- pipeRead repo params + return $ split0 fs0 + where + split0 s = filter (not . null) $ split "\0" s {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo From c88d4939453845efee04da811d64aa41046f9c11 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 17:38:12 -0400 Subject: [PATCH 0340/2835] changelog --- debian/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/changelog b/debian/changelog index 77e01482fd..4759f7e515 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Fix support for file:// remotes. * Add --verbose * Fix SIGINT handling. + * Fix handling of files with unusual characters in their name. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 From d1fd2c14285adf59f1c4219f0b24fb96b54a9c6c Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sat, 30 Oct 2010 17:25:38 +0000 Subject: [PATCH 0341/2835] --- doc/bugs/building_on_lenny.mdwn | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 doc/bugs/building_on_lenny.mdwn diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn new file mode 100644 index 0000000000..3b24e0ecf0 --- /dev/null +++ b/doc/bugs/building_on_lenny.mdwn @@ -0,0 +1,37 @@ +hi, + +I am trying to build git annex on lenny. + +I checked out the latest from git c88d4939453845efee04da811d64aa41046f9c11, +installed all the packages (some from backports) as required by dpkg-buildpackage + +Then I get this: + + ... + mkdir -p build + ghc -odir build -hidir build --make git-annex + [ 1 of 19] Compiling Utility ( Utility.hs, build/Utility.o ) + [ 2 of 19] Compiling GitRepo ( GitRepo.hs, build/GitRepo.o ) + [ 3 of 19] Compiling GitQueue ( GitQueue.hs, build/GitQueue.o ) + [ 4 of 19] Compiling TypeInternals ( TypeInternals.hs, build/TypeInternals.o ) + [ 5 of 19] Compiling Types ( Types.hs, build/Types.o ) + [ 6 of 19] Compiling Annex ( Annex.hs, build/Annex.o ) + [ 7 of 19] Compiling Locations ( Locations.hs, build/Locations.o ) + [ 8 of 19] Compiling UUID ( UUID.hs, build/UUID.o ) + [ 9 of 19] Compiling LocationLog ( LocationLog.hs, build/LocationLog.o ) + [10 of 19] Compiling Core ( Core.hs, build/Core.o ) + [11 of 19] Compiling Backend.URL ( Backend/URL.hs, build/Backend/URL.o ) + [12 of 19] Compiling Backend ( Backend.hs, build/Backend.o ) + + Backend.hs:114:50: + Not in scope: type constructor or class `SomeException' + make[1]: *** [git-annex] Error 1 + make[1]: Leaving directory `/home/cstamas/tmp/git-annex' + dh_auto_build: make -j1 returned exit code 2 + make: *** [build] Error 2 + dpkg-buildpackage: failure: debian/rules build gave error exit status 2 + +I will try to check the mentioned file for error, but I do not know how to program in haskell. + +Thanks for your help! + From 83715949c95be7f50e6ec07e3a7356b2b1f7cad1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 15:03:34 -0400 Subject: [PATCH 0342/2835] response --- doc/bugs/building_on_lenny.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index 3b24e0ecf0..c0e45912ab 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -35,3 +35,6 @@ I will try to check the mentioned file for error, but I do not know how to progr Thanks for your help! +> Newer versions of ghc changed their exception handling types, and +> I coded git-annex to use the new style and not the old. gch6 6.12 will +> work. I do not think there is a backport available though. --[[Joey]] From 23da029b75417fc6acac795204e2579b13011ab0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 15:10:10 -0400 Subject: [PATCH 0343/2835] Support building with Debian stable's ghc. --- Backend.hs | 1 + Makefile | 2 +- Portability.hs | 9 +++++++++ Remotes.hs | 1 + debian/changelog | 1 + doc/bugs/building_on_lenny.mdwn | 3 +++ 6 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Portability.hs diff --git a/Backend.hs b/Backend.hs index b7c52ddec4..59df37fc4d 100644 --- a/Backend.hs +++ b/Backend.hs @@ -39,6 +39,7 @@ import qualified Annex import Utility import Types import qualified TypeInternals as Internals +import Portability {- List of backends in the order to try them when storing a new key. -} list :: Annex [Backend] diff --git a/Makefile b/Makefile index 921eb70670..3ffa29e3b3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: git-annex docs git-annex: mkdir -p build - ghc -odir build -hidir build --make git-annex + ghc -cpp -odir build -hidir build --make git-annex install: install -d $(DESTDIR)/usr/bin diff --git a/Portability.hs b/Portability.hs new file mode 100644 index 0000000000..9cb386a9be --- /dev/null +++ b/Portability.hs @@ -0,0 +1,9 @@ +{- git-annex - Nasty portability workarounds. -} +module Portability where + +-- old ghc does not know about SomeException. +-- http://haskell.1045720.n5.nabble.com/Help-using-catch-in-6-10-td3127921.html#a3127921 +-- This needs ghc -cpp +#if __GLASGOW_HASKELL__ < 610 +type SomeException = Exception +#endif diff --git a/Remotes.hs b/Remotes.hs index 66395adcda..32016b7758 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -40,6 +40,7 @@ import Locations import UUID import Utility import qualified Core +import Portability {- Human visible list of remotes. -} list :: [Git.Repo] -> String diff --git a/debian/changelog b/debian/changelog index 4759f7e515..d6cc73e49e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Add --verbose * Fix SIGINT handling. * Fix handling of files with unusual characters in their name. + * Support building with Debian stable's ghc. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index c0e45912ab..cf5a20a7cf 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -38,3 +38,6 @@ Thanks for your help! > Newer versions of ghc changed their exception handling types, and > I coded git-annex to use the new style and not the old. gch6 6.12 will > work. I do not think there is a backport available though. --[[Joey]] +> +> Ok, found and deployed a workaround. It is not tested. Let me know how it +> works for you. --[[Joey]] From 128eaa7073f7399ffcb4ae7ccd503b00cd4225bc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 15:11:11 -0400 Subject: [PATCH 0344/2835] note --- Portability.hs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Portability.hs b/Portability.hs index 9cb386a9be..3c2be0e5fc 100644 --- a/Portability.hs +++ b/Portability.hs @@ -2,8 +2,13 @@ module Portability where -- old ghc does not know about SomeException. +-- -- http://haskell.1045720.n5.nabble.com/Help-using-catch-in-6-10-td3127921.html#a3127921 +-- -- This needs ghc -cpp +-- +-- This would be better, but is not packaged in Debian yet. +-- http://hackage.haskell.org/package/extensible-exceptions #if __GLASGOW_HASKELL__ < 610 type SomeException = Exception #endif From b4a218e0784cebfba45fc2ffba81557915565fbd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 15:15:04 -0400 Subject: [PATCH 0345/2835] meh --- Portability.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Portability.hs b/Portability.hs index 3c2be0e5fc..d864d3b938 100644 --- a/Portability.hs +++ b/Portability.hs @@ -7,7 +7,7 @@ module Portability where -- -- This needs ghc -cpp -- --- This would be better, but is not packaged in Debian yet. +-- This would be better, but then users of old ghc would need to install it. -- http://hackage.haskell.org/package/extensible-exceptions #if __GLASGOW_HASKELL__ < 610 type SomeException = Exception From 2be74a60df229c0e887b9f6c30ea1195a64e83a0 Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sat, 30 Oct 2010 19:40:17 +0000 Subject: [PATCH 0346/2835] reply to joey --- doc/bugs/building_on_lenny.mdwn | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index cf5a20a7cf..d394182500 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -33,7 +33,7 @@ Then I get this: I will try to check the mentioned file for error, but I do not know how to program in haskell. -Thanks for your help! +Thanks for your help! --[[cstamas]] > Newer versions of ghc changed their exception handling types, and > I coded git-annex to use the new style and not the old. gch6 6.12 will @@ -41,3 +41,19 @@ Thanks for your help! > > Ok, found and deployed a workaround. It is not tested. Let me know how it > works for you. --[[Joey]] +>> +>> I did a git pull and now I get: + + mkdir -p build + ghc -cpp -odir build -hidir build --make git-annex + [ 1 of 20] Compiling Portability ( Portability.hs, build/Portability.o ) + + Portability.hs:13:21: + Not in scope: type constructor or class `Exception' + make[1]: *** [git-annex] Error 1 + make[1]: Leaving directory `/home/cstamas/tmp/git-annex' + dh_auto_build: make -j1 returned exit code 2 + make: *** [build] Error 2 + dpkg-buildpackage: failure: debian/rules build gave error exit status 2 + +>> --[[cstamas]] From 8edc7a0e434f9574e1f46682acb3f5c5321e825e Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sat, 30 Oct 2010 19:46:39 +0000 Subject: [PATCH 0347/2835] formatting fix --- doc/bugs/building_on_lenny.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index d394182500..07b359e855 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -41,7 +41,7 @@ Thanks for your help! --[[cstamas]] > > Ok, found and deployed a workaround. It is not tested. Let me know how it > works for you. --[[Joey]] ->> + >> I did a git pull and now I get: mkdir -p build From fad1616e68b2219a7e1a6c4b7e1cfe8485ad8c2b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 17:26:45 -0400 Subject: [PATCH 0348/2835] build fix --- Portability.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Portability.hs b/Portability.hs index d864d3b938..91c8334cc9 100644 --- a/Portability.hs +++ b/Portability.hs @@ -1,5 +1,7 @@ {- git-annex - Nasty portability workarounds. -} -module Portability where +module Portability (SomeException) where + +import Control.Exception -- old ghc does not know about SomeException. -- From 583e8118d4e77fc4f76474717a821c56e3cf772d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 17:29:11 -0400 Subject: [PATCH 0349/2835] ok, let's just use extensible-exceptions --- Backend.hs | 3 +-- Portability.hs | 16 ---------------- Remotes.hs | 3 +-- 3 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 Portability.hs diff --git a/Backend.hs b/Backend.hs index 59df37fc4d..00b2833e03 100644 --- a/Backend.hs +++ b/Backend.hs @@ -27,7 +27,7 @@ module Backend ( ) where import Control.Monad.State -import Control.Exception +import Control.Exception.Extensible import System.Directory import System.FilePath import Data.String.Utils @@ -39,7 +39,6 @@ import qualified Annex import Utility import Types import qualified TypeInternals as Internals -import Portability {- List of backends in the order to try them when storing a new key. -} list :: Annex [Backend] diff --git a/Portability.hs b/Portability.hs deleted file mode 100644 index 91c8334cc9..0000000000 --- a/Portability.hs +++ /dev/null @@ -1,16 +0,0 @@ -{- git-annex - Nasty portability workarounds. -} -module Portability (SomeException) where - -import Control.Exception - --- old ghc does not know about SomeException. --- --- http://haskell.1045720.n5.nabble.com/Help-using-catch-in-6-10-td3127921.html#a3127921 --- --- This needs ghc -cpp --- --- This would be better, but then users of old ghc would need to install it. --- http://hackage.haskell.org/package/extensible-exceptions -#if __GLASGOW_HASKELL__ < 610 -type SomeException = Exception -#endif diff --git a/Remotes.hs b/Remotes.hs index 32016b7758..27bd39ead7 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -17,7 +17,7 @@ module Remotes ( ) where import IO (bracket_) -import Control.Exception hiding (bracket_) +import Control.Exception.Extensible hiding (bracket_) import Control.Monad.State (liftIO) import Control.Monad (filterM) import qualified Data.Map as Map @@ -40,7 +40,6 @@ import Locations import UUID import Utility import qualified Core -import Portability {- Human visible list of remotes. -} list :: [Git.Repo] -> String From 765c9fa82d958259d7beaf8edc51f06a0db31a00 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 17:38:20 -0400 Subject: [PATCH 0350/2835] meh --- doc/bugs/building_on_lenny.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index 07b359e855..388db2010d 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -57,3 +57,9 @@ Thanks for your help! --[[cstamas]] dpkg-buildpackage: failure: debian/rules build gave error exit status 2 >> --[[cstamas]] + +>>> Ok well, I'm not going to try to reimplement all of +>>> Control.Exception.Extensible so I've made it use it. You will have to +>>> figure out how to install that library yourself though, I don't know +>>> how to use cabal with such an old ghc. Library is here: +>>> --[[Joey]] From 465fb0ebc4e9173606303a719cc753b45e133b20 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 20:43:05 -0400 Subject: [PATCH 0351/2835] link to ask.debian.net --- doc/bugs/building_on_lenny.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index 388db2010d..654b1283e0 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -62,4 +62,6 @@ Thanks for your help! --[[cstamas]] >>> Control.Exception.Extensible so I've made it use it. You will have to >>> figure out how to install that library yourself though, I don't know >>> how to use cabal with such an old ghc. Library is here: ->>> --[[Joey]] +>>> +>>> and I asked how to get it on stable here: +>>> --[[Joey]] From 963bfa967369511bbe3261860c3609b5ae9554ec Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 22:47:34 -0400 Subject: [PATCH 0352/2835] cpp not needed --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3ffa29e3b3..921eb70670 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: git-annex docs git-annex: mkdir -p build - ghc -cpp -odir build -hidir build --make git-annex + ghc -odir build -hidir build --make git-annex install: install -d $(DESTDIR)/usr/bin From c2651d64bc8a384f8669879c051577ee61da27ed Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 23:19:33 -0400 Subject: [PATCH 0353/2835] Fixed memory leak; git-annex no longer reads the whole file list from git before starting, and will be much faster with large repos. --- GitRepo.hs | 16 +++++++++++++++- debian/changelog | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index c8ba77133c..b6670e2ebe 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -27,6 +27,7 @@ module GitRepo ( configTrue, run, pipeRead, + hPipeRead, attributes, remotes, remotesAdd, @@ -198,6 +199,17 @@ pipeRead repo params = assertLocal repo $ do ret <- hGetContentsStrict h return ret +{- Like pipeRead, but does not read output strictly; recommended + - for git commands that produce a lot of output that will be processed + - lazily. + - + - ONLY AFTER the string has been read completely, You must call either + - getProcessStatus or forceSuccess on the PipeHandle. Zombies will result + - otherwise.-} +hPipeRead :: Repo -> [String] -> IO (PipeHandle, String) +hPipeRead repo params = assertLocal repo $ do + pipeFrom "git" (gitCommandLine repo params) + {- Passed a location, recursively scans for all files that - are checked into git at that location. -} inRepo :: Repo -> FilePath -> IO [FilePath] @@ -221,7 +233,9 @@ stagedFiles repo location = pipeNullSplit repo - parameter), and splits it into a list of files. -} pipeNullSplit :: Repo -> [String] -> IO [FilePath] pipeNullSplit repo params = do - fs0 <- pipeRead repo params + -- XXX handle is left open, this is ok for git-annex, but may need + -- to be cleaned up for other uses. + (handle, fs0) <- hPipeRead repo params return $ split0 fs0 where split0 s = filter (not . null) $ split "\0" s diff --git a/debian/changelog b/debian/changelog index d6cc73e49e..70e9052231 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ git-annex (0.03) UNRELEASED; urgency=low * Fix SIGINT handling. * Fix handling of files with unusual characters in their name. * Support building with Debian stable's ghc. + * Fixed memory leak; git-annex no longer reads the whole file list + from git before starting, and will be much faster with large repos. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 From d5a0c16298a625d80288276a92dcfebf1cb9b3de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 23:34:40 -0400 Subject: [PATCH 0354/2835] more space saving by not locking location log for read Actions that need to read all the location logs, like "git annex get .", were still using a lot of memory, and profiling pointed at the location log reading as the problem. Not locking them for read, and thus avoiding the strict reading fixes the problem, although I don't quite understand why. (Oddly, -sstderr profiling did not show the memory as used, though top showed dozens of MB being used.) Anyway, it's fine to not lock location logs for read, since the log format and parser should be safe if a partial read of a file being written happens. Note that that could easily happen anyway, if doing a git pull, etc, especially if git needs to union merge in changes from elsewhere. The worst that will happen is git-annex could get a bad or out of date idea about locations and refuse to eg, --drop something. --- LocationLog.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index f92dee652d..20e777c567 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -104,8 +104,7 @@ readLog file = do exists <- doesFileExist file if exists then do - s <- withFileLocked file ReadMode $ \h -> - hGetContentsStrict h + s <- readFile file -- filter out any unparsable lines return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s From 59672d32edd9cc30942a9200670b3e1f817d26aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 01:06:58 -0400 Subject: [PATCH 0355/2835] write to tmp file Writing to a tmp file means no locking is needed, and it fixes a bug introduced by the last commit, which made log files be read lazily, so they could still be open when written, which breaks due to haskell's internal locking. --- LocationLog.hs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 20e777c567..14ae88abc3 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -35,6 +35,7 @@ import qualified Data.Map as Map import System.IO import System.Directory import Data.Char +import System.Posix.Process import qualified GitRepo as Git import Utility @@ -111,19 +112,14 @@ readLog file = do else do return [] -{- Adds a LogLine to a log file -} -appendLog :: FilePath -> LogLine -> IO () -appendLog file line = do - createDirectoryIfMissing True (parentDir file) - withFileLocked file AppendMode $ \h -> - hPutStrLn h $ show line - {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () writeLog file lines = do + pid <- getProcessID + let tmpfile = file ++ ".tmp" ++ show pid createDirectoryIfMissing True (parentDir file) - withFileLocked file WriteMode $ \h -> - hPutStr h $ unlines $ map show lines + writeFile tmpfile $ unlines $ map show lines + renameFile tmpfile file {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> UUID -> IO LogLine From 6a9a9bd5a316f143964574e5ed12eb69b1d17c41 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 01:51:50 -0400 Subject: [PATCH 0356/2835] another memory optimisation This time memory leaked if lots of UUIDs needed to be pretty-printed, as in a get or drop of many files. Essentially the same strict read buffering problem that affected the LocationLog underneath though. uuidMap really could stand to be cached, as the uuid log is read many times in this case. But it is a fairly edge case. --- UUID.hs | 10 ++++++---- Utility.hs | 16 ---------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/UUID.hs b/UUID.hs index f2235e4b62..20f85bae12 100644 --- a/UUID.hs +++ b/UUID.hs @@ -26,6 +26,7 @@ import System.Cmd.Utils import System.IO import System.Directory import qualified Data.Map as M +import System.Posix.Process import qualified GitRepo as Git import Types @@ -111,8 +112,11 @@ describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m log <- uuidLog + pid <- liftIO $ getProcessID + let tmplog = log ++ ".tmp" ++ show pid liftIO $ createDirectoryIfMissing True (parentDir log) - liftIO $ withFileLocked log WriteMode (\h -> hPutStr h $ serialize m') + liftIO $ writeFile tmplog $ serialize m' + liftIO $ renameFile tmplog log where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m @@ -120,9 +124,7 @@ describeUUID uuid desc = do uuidMap :: Annex (M.Map UUID String) uuidMap = do log <- uuidLog - s <- liftIO $ catch - (withFileLocked log ReadMode $ \h -> hGetContentsStrict h) - (\error -> return "") + s <- liftIO $ catch (readFile log) (\error -> return "") return $ M.fromList $ map (\l -> pair l) $ lines s where pair l = diff --git a/Utility.hs b/Utility.hs index 233825b651..e7b4b510b8 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,7 +6,6 @@ -} module Utility ( - withFileLocked, hGetContentsStrict, parentDir, relPathCwdToDir, @@ -28,21 +27,6 @@ import System.IO.HVFS import System.FilePath import System.Directory -{- Let's just say that Haskell makes reading/writing a file with - - file locking excessively difficult. -} -withFileLocked file mode action = do - -- TODO: find a way to use bracket here - handle <- openFile file mode - lockfd <- handleToFd handle -- closes handle - waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) - handle' <- fdToHandle lockfd - ret <- action handle' - hClose handle' - return ret - where - lockType ReadMode = ReadLock - lockType _ = WriteLock - {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s From bcfd0d908a3e52ca61aa295ddebf7b1898224ab1 Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sun, 31 Oct 2010 16:06:00 +0000 Subject: [PATCH 0357/2835] building on squeeze --- doc/bugs/building_on_lenny.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index 654b1283e0..fee13717ff 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -65,3 +65,15 @@ Thanks for your help! --[[cstamas]] >>> >>> and I asked how to get it on stable here: >>> --[[Joey]] + +>>>> I made some effort with cabal on lenny. I can install (and I did it) cabal +>>>> from squeeze as dependencies are ok. Then I installed extensible +>>>> exceptions, but it places it in some local dir that git-annex's installer +>>>> (or ghc itself) does not know about. +>>>> +>>>> Later I realized that *only* for the compilation ghc6 and its friends are +>>>> needed. So I built the package on my other machine running squeeze. Then +>>>> resulting deb packages cleanly installs on lenny +>>>> +>>>> For me this is OK. Thanks! --[[cstamas]] + From 081517b6e9111536cd67c1f501adf07497862ce5 Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sun, 31 Oct 2010 16:10:22 +0000 Subject: [PATCH 0358/2835] add 'done' link --- doc/bugs/building_on_lenny.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index fee13717ff..48386bde47 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -77,3 +77,4 @@ Thanks for your help! --[[cstamas]] >>>> >>>> For me this is OK. Thanks! --[[cstamas]] +[[done]] From dc12ce762e521a5db052346eb67590ca62e4f2f6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 14:23:51 -0400 Subject: [PATCH 0359/2835] -Wall clean --- Remotes.hs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 27bd39ead7..b9c1b48f3d 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -22,19 +22,14 @@ import Control.Monad.State (liftIO) import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils -import Data.Either.Utils -import System.Cmd.Utils import System.Directory import System.Posix.Directory import List -import Maybe import Monad (when, unless) import Types import qualified GitRepo as Git import qualified Annex -import qualified Backend -import qualified Core import LocationLog import Locations import UUID @@ -73,7 +68,7 @@ keyPossibilities key = do let todo = cheap ++ doexpensive if (not $ null todo) then do - e <- mapM tryGitConfigRead todo + _ <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True keyPossibilities key else reposByUUID allremotes uuids @@ -121,14 +116,14 @@ reposByCost l = do repoCost :: Git.Repo -> Annex Int repoCost r = do g <- Annex.gitRepo - if (not $ null $ config g r) - then return $ read $ config g r + if (not $ null $ config g) + then return $ read $ config g else if (Git.repoIsUrl r) then return 200 else return 100 where - config g r = Git.configGet g (configkey r) "" - configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" + config g = Git.configGet g configkey "" + configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" {- Checks if a repo should be ignored, based either on annex-ignore - setting, or on command-line options. Allows command-line to override @@ -174,7 +169,7 @@ tryGitConfigRead r = do -- for other reasons; catch all possible exceptions result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) case (result) of - Left e -> return $ Left r + Left _ -> return $ Left r Right r' -> do g <- Annex.gitRepo let l = Git.remotes g @@ -184,7 +179,7 @@ tryGitConfigRead r = do return $ Right r' else return $ Right r -- config already read where - exchange [] new = [] + exchange [] _ = [] exchange (old:ls) new = if (Git.repoRemoteName old == Git.repoRemoteName new) then new:(exchange ls new) From 1576c48c80e4806b6021ec66f0dc645cf0a83486 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 14:32:18 -0400 Subject: [PATCH 0360/2835] more Wall cleaning --- Annex.hs | 5 +++-- Makefile | 2 +- git-annex.hs | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Annex.hs b/Annex.hs index 60ae91708b..63eef51589 100644 --- a/Annex.hs +++ b/Annex.hs @@ -39,15 +39,16 @@ new gitrepo allbackends = do Internals.flags = M.empty, Internals.repoqueue = GitQueue.empty } - (_,s') <- Annex.run s (prep gitrepo) + (_,s') <- Annex.run s prep return s' where - prep gitrepo = do + prep = do -- read git config and update state gitrepo' <- liftIO $ Git.configRead gitrepo Annex.gitRepoChange gitrepo' {- performs an action in the Annex monad -} +run :: AnnexState -> StateT AnnexState IO a -> IO (a, AnnexState) run state action = runStateT (action) state {- Returns the git repository being acted on -} diff --git a/Makefile b/Makefile index 921eb70670..ed262333a3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: git-annex docs git-annex: mkdir -p build - ghc -odir build -hidir build --make git-annex + ghc -Wall -odir build -hidir build --make git-annex install: install -d $(DESTDIR)/usr/bin diff --git a/git-annex.hs b/git-annex.hs index 5011fade2b..e9e0d6f027 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -17,6 +17,7 @@ import Commands import qualified GitRepo as Git import BackendList +main :: IO () main = do args <- getArgs gitrepo <- Git.repoFromCwd @@ -35,6 +36,7 @@ main = do -} tryRun :: AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions +tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () tryRun' state errnum (a:as) = do result <- try $ Annex.run state a case (result) of @@ -43,8 +45,9 @@ tryRun' state errnum (a:as) = do tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' state errnum [] = +tryRun' _ errnum [] = when (errnum > 0) $ error $ (show errnum) ++ " failed" {- Exception pretty-printing. -} +showErr :: (Show a) => a -> IO () showErr e = hPutStrLn stderr $ "git-annex: " ++ (show e) From 2d893b3331f5515b179d7541c9b4cb1f30162fce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 14:39:53 -0400 Subject: [PATCH 0361/2835] more Wall cleaning --- Backend.hs | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/Backend.hs b/Backend.hs index 00b2833e03..d75c2a761e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,15 +28,12 @@ module Backend ( import Control.Monad.State import Control.Exception.Extensible -import System.Directory import System.FilePath -import Data.String.Utils import System.Posix.Files import Locations import qualified GitRepo as Git import qualified Annex -import Utility import Types import qualified TypeInternals as Internals @@ -47,28 +44,28 @@ list = do if (not $ null l) then return l else do - all <- Annex.supportedBackends + bs <- Annex.supportedBackends g <- Annex.gitRepo - let l = parseBackendList all $ Git.configGet g "annex.backends" "" + let defaults = parseBackendList bs $ Git.configGet g "annex.backends" "" backendflag <- Annex.flagGet "backend" let l' = if (not $ null backendflag) - then (lookupBackendName all backendflag):l - else l + then (lookupBackendName bs backendflag):defaults + else defaults Annex.backendsChange $ l' return l' where - parseBackendList all s = + parseBackendList bs s = if (null s) - then all - else map (lookupBackendName all) $ words s + then bs + else map (lookupBackendName bs) $ words s {- Looks up a backend in a list -} lookupBackendName :: [Backend] -> String -> Backend -lookupBackendName all s = +lookupBackendName bs s = if ((length matches) /= 1) then error $ "unknown backend " ++ s else matches !! 0 - where matches = filter (\b -> s == Internals.name b) all + where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) @@ -77,10 +74,11 @@ storeFileKey file = do let relfile = Git.relative g file b <- list storeFileKey' b file relfile +storeFileKey' :: [Backend] -> FilePath -> FilePath -> Annex (Maybe (Key, Backend)) storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do - try <- (Internals.getKey b) relfile - case (try) of + result <- (Internals.getKey b) relfile + case (result) of Nothing -> nextbackend Just key -> do stored <- (Internals.storeFileKey b) file key @@ -103,23 +101,23 @@ removeKey backend key = (Internals.removeKey backend) key {- Checks if a backend has its key. -} hasKey :: Key -> Annex Bool hasKey key = do - all <- Annex.supportedBackends - (Internals.hasKey (lookupBackendName all $ backendName key)) key + bs <- Annex.supportedBackends + (Internals.hasKey (lookupBackendName bs $ backendName key)) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) lookupFile file = do - all <- Annex.supportedBackends - result <- liftIO $ (try (lookup all)::IO (Either SomeException (Maybe (Key, Backend)))) + bs <- Annex.supportedBackends + result <- liftIO $ (try (find bs)::IO (Either SomeException (Maybe (Key, Backend)))) case (result) of - Left err -> return Nothing - Right succ -> return succ + Left _ -> return Nothing + Right val -> return val where - lookup all = do + find bs = do l <- readSymbolicLink file - return $ Just $ pair all $ takeFileName l - pair all file = (k, b) + return $ Just $ pair bs $ takeFileName l + pair bs f = (k, b) where - k = fileKey file - b = lookupBackendName all $ backendName k + k = fileKey f + b = lookupBackendName bs $ backendName k From 40729e4bfdf4fe9753705a5a7a93cf1e0012a92c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 14:40:34 -0400 Subject: [PATCH 0362/2835] more Wall cleaning --- BackendList.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/BackendList.hs b/BackendList.hs index e628e3a612..59ec026d91 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -11,6 +11,9 @@ module BackendList (allBackends) where import qualified Backend.WORM import qualified Backend.SHA1 import qualified Backend.URL +import TypeInternals + +allBackends :: [Backend] allBackends = [ Backend.WORM.backend , Backend.SHA1.backend From 28b5a9fa2048f4e6425cc1d8dbe13b0e4c36a15b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:09:50 -0400 Subject: [PATCH 0363/2835] changelog --- Commands.hs | 146 +++++++++++++++++++++++++++------------------------- 1 file changed, 76 insertions(+), 70 deletions(-) diff --git a/Commands.hs b/Commands.hs index 2af8874e5c..fb76e5502f 100644 --- a/Commands.hs +++ b/Commands.hs @@ -11,12 +11,9 @@ import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory -import System.Path import Data.String.Utils import Control.Monad (filterM) import Monad (when, unless) -import List -import IO import qualified GitRepo as Git import qualified Annex @@ -50,19 +47,19 @@ type SubCmdCleanup = Annex Bool {- Runs a subcommand through its three stages. -} doSubCmd :: String -> SubCmdStart -> String -> Annex Bool doSubCmd cmdname start param = do - res <- start param :: Annex (Maybe SubCmdPerform) - case (res) of + startres <- start param :: Annex (Maybe SubCmdPerform) + case (startres) of Nothing -> return True Just perform -> do showStart cmdname param - res <- perform :: Annex (Maybe SubCmdCleanup) - case (res) of + performres <- perform :: Annex (Maybe SubCmdCleanup) + case (performres) of Nothing -> do showEndFail return False Just cleanup -> do - res <- cleanup - if (res) + cleanupres <- cleanup + if (cleanupres) then do showEndOk return True @@ -76,7 +73,7 @@ doSubCmd cmdname start param = do data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description | Keys | Tempfile | FilesToBeCommitted -data SubCommand = Command { +data SubCommand = SubCommand { subcmdname :: String, subcmdaction :: SubCmdStart, subcmdwants :: SubCmdWants, @@ -84,27 +81,27 @@ data SubCommand = Command { } subCmds :: [SubCommand] subCmds = [ - (Command "add" addStart FilesNotInGit + (SubCommand "add" addStart FilesNotInGit "add files to annex") - , (Command "get" getStart FilesInGit + , (SubCommand "get" getStart FilesInGit "make content of annexed files available") - , (Command "drop" dropStart FilesInGit + , (SubCommand "drop" dropStart FilesInGit "indicate content of files not currently wanted") - , (Command "move" moveStart FilesInGit + , (SubCommand "move" moveStart FilesInGit "transfer content of files to/from another repository") - , (Command "init" initStart Description + , (SubCommand "init" initStart Description "initialize git-annex with repository description") - , (Command "unannex" unannexStart FilesInGit + , (SubCommand "unannex" unannexStart FilesInGit "undo accidential add command") - , (Command "fix" fixStart FilesInGit + , (SubCommand "fix" fixStart FilesInGit "fix up symlinks to point to annexed content") - , (Command "pre-commit" fixStart FilesToBeCommitted + , (SubCommand "pre-commit" fixStart FilesToBeCommitted "fix up symlinks before they are committed") - , (Command "fromkey" fromKeyStart FilesMissing + , (SubCommand "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") - , (Command "dropkey" dropKeyStart Keys + , (SubCommand "dropkey" dropKeyStart Keys "drops annexed content for specified keys") - , (Command "setkey" setKeyStart Tempfile + , (SubCommand "setkey" setKeyStart Tempfile "sets annexed content for a key using a temp file") ] @@ -131,6 +128,7 @@ options = [ storebool n b = Annex.flagChange n $ FlagBool b storestring n s = Annex.flagChange n $ FlagString s +header :: String header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) {- Usage message with lists of options and subcommands. -} @@ -162,7 +160,7 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files -findWanted FilesMissing params repo = do +findWanted FilesMissing params _ = do files <- liftIO $ filterM missing params return $ files where @@ -186,15 +184,17 @@ parseCmd argv state = do when (null params) $ error usage case lookupCmd (params !! 0) of [] -> error usage - [Command name action want _] -> do - f <- findWanted want (drop 1 params) + [SubCommand { subcmdname = name, subcmdaction = action, + subcmdwants = want, subcmddesc = _ }] -> do + files <- findWanted want (drop 1 params) (TypeInternals.repo state) let actions = map (doSubCmd name action) $ - filter notstate f - let configactions = map (\f -> do - f + filter notstate files + let configactions = map (\flag -> do + flag return True) flags return (configactions, actions) + _ -> error "internal error: multiple matching subcommands" where -- never include files from the state directory notstate f = stateLoc /= take (length stateLoc) f @@ -214,11 +214,10 @@ addStart file = notAnnexed file $ do else return $ Just $ addPerform file addPerform :: FilePath -> Annex (Maybe SubCmdCleanup) addPerform file = do - g <- Annex.gitRepo stored <- Backend.storeFileKey file case (stored) of Nothing -> return Nothing - Just (key, backend) -> return $ Just $ addCleanup file key + Just (key, _) -> return $ Just $ addCleanup file key addCleanup :: FilePath -> Key -> Annex Bool addCleanup file key = do logStatus key ValuePresent @@ -239,8 +238,10 @@ unannexPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) unannexPerform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True - Backend.removeKey backend key - return $ Just $ unannexCleanup file key + ok <- Backend.removeKey backend key + if (ok) + then return $ Just $ unannexCleanup file key + else return Nothing unannexCleanup :: FilePath -> Key -> Annex Bool unannexCleanup file key = do logStatus key ValueMissing @@ -259,9 +260,9 @@ getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return Nothing - else return $ Just $ getPerform file key backend -getPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) -getPerform file key backend = do + else return $ Just $ getPerform key backend +getPerform :: Key -> Backend -> Annex (Maybe SubCmdCleanup) +getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) then return $ Just $ return True -- no cleanup needed @@ -331,15 +332,15 @@ setKeyPerform tmpfile key = do ok <- liftIO $ boolSystem "mv" [tmpfile, loc] if (not ok) then error "mv failed!" - else return $ Just $ setKeyCleanup tmpfile key -setKeyCleanup :: FilePath -> Key -> Annex Bool -setKeyCleanup tmpfile key = do + else return $ Just $ setKeyCleanup key +setKeyCleanup :: Key -> Annex Bool +setKeyCleanup key = do logStatus key ValuePresent return True {- Fixes the symlink to an annexed file. -} fixStart :: FilePath -> Annex (Maybe SubCmdPerform) -fixStart file = isAnnexed file $ \(key, backend) -> do +fixStart file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) @@ -373,9 +374,9 @@ initPerform description = do initCleanup :: Annex Bool initCleanup = do g <- Annex.gitRepo - log <- uuidLog - liftIO $ Git.run g ["add", log] - liftIO $ Git.run g ["commit", "-m", "git annex init", log] + logfile <- uuidLog + liftIO $ Git.run g ["add", logfile] + liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] return True {- Adds a file pointing at a manually-specified key -} @@ -411,9 +412,9 @@ moveStart file = do toName <- Annex.flagGet "torepository" case (fromName, toName) of ("", "") -> error "specify either --from or --to" - ("", to) -> moveToStart file - (from, "") -> moveFromStart file - (_, _) -> error "only one of --from or --to can be specified" + ("", _) -> moveToStart file + (_ , "") -> moveFromStart file + (_ , _) -> error "only one of --from or --to can be specified" {- Moves the content of an annexed file to another repository, - removing it from the current repository, and updates locationlog @@ -427,13 +428,13 @@ moveStart file = do - allow it to be dropped. -} moveToStart :: FilePath -> Annex (Maybe SubCmdPerform) -moveToStart file = isAnnexed file $ \(key, backend) -> do +moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) then return Nothing -- not here, so nothing to do - else return $ Just $ moveToPerform file key -moveToPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) -moveToPerform file key = do + else return $ Just $ moveToPerform key +moveToPerform :: Key -> Annex (Maybe SubCmdCleanup) +moveToPerform key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key @@ -452,18 +453,21 @@ moveToPerform file key = do moveToCleanup :: Git.Repo -> Key -> FilePath -> Annex Bool moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. - Remotes.runCmd remote "git-annex" ["setkey", "--quiet", + ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", "--backend=" ++ (backendName key), "--key=" ++ keyName key, tmpfile] - -- Record that the key is present on the remote. - g <- Annex.gitRepo - remoteuuid <- getUUID remote - log <- liftIO $ logChange g key remoteuuid ValuePresent - Annex.queue "add" [] log - -- Cleanup on the local side is the same as done for the - -- drop subcommand. - dropCleanup key + if ok + then do + -- Record that the key is present on the remote. + g <- Annex.gitRepo + remoteuuid <- getUUID remote + logfile <- liftIO $ logChange g key remoteuuid ValuePresent + Annex.queue "add" [] logfile + -- Cleanup on the local side is the same as done for the + -- drop subcommand. + dropCleanup key + else return False {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. @@ -472,15 +476,14 @@ moveToCleanup remote key tmpfile = do - from the other repository. -} moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) -moveFromStart file = isAnnexed file $ \(key, backend) -> do - g <- Annex.gitRepo +moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key if (elem remote l) - then return $ Just $ moveFromPerform file key + then return $ Just $ moveFromPerform key else return Nothing -moveFromPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) -moveFromPerform file key = do +moveFromPerform :: Key -> Annex (Maybe SubCmdCleanup) +moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key if (ishere) @@ -493,22 +496,25 @@ moveFromPerform file key = do else return Nothing -- fail moveFromCleanup :: Git.Repo -> Key -> Annex Bool moveFromCleanup remote key = do - Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", + ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), keyName key] - -- Record locally that the key is not on the remote. - remoteuuid <- getUUID remote - g <- Annex.gitRepo - log <- liftIO $ logChange g key remoteuuid ValueMissing - Annex.queue "add" [] log - return True + when ok $ do + -- Record locally that the key is not on the remote. + remoteuuid <- getUUID remote + g <- Annex.gitRepo + logfile <- liftIO $ logChange g key remoteuuid ValueMissing + Annex.queue "add" [] logfile + return ok -- helpers +notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) notAnnexed file a = do r <- Backend.lookupFile file case (r) of - Just v -> return Nothing + Just _ -> return Nothing Nothing -> a +isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) isAnnexed file a = do r <- Backend.lookupFile file case (r) of From 96fa6b89acdd77498140790553a42c81250816ed Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:12:56 -0400 Subject: [PATCH 0364/2835] more Wall cleaning --- Core.hs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Core.hs b/Core.hs index 8cdc6979ab..5f799161cf 100644 --- a/Core.hs +++ b/Core.hs @@ -7,7 +7,6 @@ module Core where -import Maybe import System.IO import System.Directory import Control.Monad.State (liftIO) @@ -27,7 +26,6 @@ import Utility {- Sets up a git repo for git-annex. -} startup :: Annex Bool startup = do - g <- Annex.gitRepo prepUUID return True @@ -118,8 +116,8 @@ logStatus :: Key -> LogStatus -> Annex () logStatus key status = do g <- Annex.gitRepo u <- getUUID g - log <- liftIO $ logChange g key u status - Annex.queue "add" [] log + logfile <- liftIO $ logChange g key u status + Annex.queue "add" [] logfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into @@ -158,9 +156,9 @@ showProgress :: Annex () showProgress = verbose $ liftIO $ putStr $ "\n" showLongNote :: String -> Annex () showLongNote s = verbose $ do - liftIO $ putStr $ "\n" ++ (indent s) + liftIO $ putStr $ "\n" ++ indented where - indent s = join "\n" $ map (\l -> " " ++ l) $ lines s + indented = join "\n" $ map (\l -> " " ++ l) $ lines s showEndOk :: Annex () showEndOk = verbose $ do liftIO $ putStrLn "ok" From 435ec21d58f9c7ddb622f290648b31da2c9a8f1e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:25:55 -0400 Subject: [PATCH 0365/2835] bugfix: really run GitQueue against specified repo, not necessarily pwd --- GitQueue.hs | 12 ++++++------ GitRepo.hs | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/GitQueue.hs b/GitQueue.hs index 09b8037e64..632d1d3910 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -23,8 +23,8 @@ import qualified GitRepo as Git {- An action to perform in a git repository. The file to act on - is not included, and must be able to be appended after the params. -} data Action = Action { - subcommand :: String, - params :: [String] + getSubcommand :: String, + getParams :: [String] } deriving (Show, Eq, Ord) {- A queue of actions to perform (in any order) on a git repository, @@ -45,7 +45,7 @@ add queue subcommand params file = M.insertWith (++) action [file] queue {- Runs a queue on a git repository. -} run :: Git.Repo -> Queue -> IO () run repo queue = do - mapM (\(k, v) -> runAction repo k v) $ M.toList queue + _ <- mapM (\(k, v) -> runAction repo k v) $ M.toList queue return () {- Runs an Action on a list of files in a git repository. @@ -55,7 +55,7 @@ runAction :: Git.Repo -> Action -> [FilePath] -> IO () runAction repo action files = do unless (null files) runxargs where - runxargs = pOpen WriteToPipe "xargs" - (["-0", "git", subcommand action] ++ (params action)) - feedxargs + runxargs = pOpen WriteToPipe "xargs" ("-0":gitcmd) feedxargs + gitcmd = ["git"] ++ Git.gitCommandLine repo + ((getSubcommand action):(getParams action)) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index b6670e2ebe..9ecd3923aa 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -25,6 +25,7 @@ module GitRepo ( configMap, configRead, configTrue, + gitCommandLine, run, pipeRead, hPipeRead, From aa05859410b43da72c3165b7915d8c917bc015ca Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:38:47 -0400 Subject: [PATCH 0366/2835] more Wall cleaning --- Core.hs | 2 +- GitRepo.hs | 57 ++++++++++++++++++++++++++++++------------------------ 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/Core.hs b/Core.hs index 5f799161cf..c5026c6a5f 100644 --- a/Core.hs +++ b/Core.hs @@ -74,7 +74,7 @@ gitAttributes repo = do {- set up a git pre-commit hook, if one is not already present -} gitPreCommitHook :: Git.Repo -> IO () gitPreCommitHook repo = do - let hook = (Git.workTree repo) ++ "/" ++ (Git.dir repo) ++ + let hook = (Git.workTree repo) ++ "/" ++ (Git.gitDir repo) ++ "/hooks/pre-commit" exists <- doesFileExist hook if (exists) diff --git a/GitRepo.hs b/GitRepo.hs index 9ecd3923aa..874b5c3c9b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -17,7 +17,7 @@ module GitRepo ( repoIsSsh, repoDescribe, workTree, - dir, + gitDir, relative, urlPath, urlHost, @@ -38,17 +38,14 @@ module GitRepo ( stagedFiles ) where -import Monad (when, unless) +import Monad (unless) import Directory -import System -import System.Directory import System.Posix.Directory import System.Path -import System.Cmd import System.Cmd.Utils -import System.IO import IO (bracket_) import Data.String.Utils +import System.IO import qualified Data.Map as Map hiding (map, split) import Network.URI import Maybe @@ -69,6 +66,7 @@ data Repo = Repo { remoteName :: Maybe String } deriving (Show, Eq) +newFrom :: RepoLocation -> Repo newFrom l = Repo { location = l, @@ -89,6 +87,7 @@ repoFromUrl url where u = fromJust $ parseURI url {- User-visible description of a git repo. -} +repoDescribe :: Repo -> String repoDescribe Repo { remoteName = Just name } = name repoDescribe Repo { location = Url url } = show url repoDescribe Repo { location = Dir dir } = dir @@ -100,29 +99,35 @@ remotesAdd repo rs = repo { remotes = rs } {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} +repoRemoteName :: Repo -> String repoRemoteName Repo { remoteName = Just name } = name repoRemoteName _ = "" {- Some code needs to vary between URL and normal repos, - or bare and non-bare, these functions help with that. -} +repoIsUrl :: Repo -> Bool repoIsUrl Repo { location = Url _ } = True repoIsUrl _ = False +repoIsSsh :: Repo -> Bool repoIsSsh Repo { location = Url url } | uriScheme url == "ssh:" = True | otherwise = False repoIsSsh _ = False +assertLocal :: Repo -> a -> a assertLocal repo action = if (not $ repoIsUrl repo) then action else error $ "acting on URL git repo " ++ (repoDescribe repo) ++ " not supported" +assertUrl :: Repo -> a -> a assertUrl repo action = if (repoIsUrl repo) then action else error $ "acting on local git repo " ++ (repoDescribe repo) ++ " not supported" +assertSsh :: Repo -> a -> a assertSsh repo action = if (repoIsSsh repo) then action @@ -141,8 +146,8 @@ attributes repo | otherwise = (workTree repo) ++ "/.gitattributes" {- Path to a repository's .git directory, relative to its workTree. -} -dir :: Repo -> String -dir repo +gitDir :: Repo -> String +gitDir repo | bare repo = "" | otherwise = ".git" @@ -167,7 +172,7 @@ relative repo@(Repo { location = Dir d }) file = drop (length absrepo) absfile absfile = case (secureAbsNormPath absrepo file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -relative repo file = assertLocal repo $ error "internal" +relative repo _ = assertLocal repo $ error "internal" {- Hostname of an URL repo. (May include a username and/or port too.) -} urlHost :: Repo -> String @@ -184,7 +189,7 @@ urlPath repo = assertUrl repo $ error "internal" gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++d++"/"++(dir repo), "--work-tree="++d] ++ params + ["--git-dir="++d++"/"++(gitDir repo), "--work-tree="++d] ++ params gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo, throwing an error if it fails. -} @@ -214,21 +219,21 @@ hPipeRead repo params = assertLocal repo $ do {- Passed a location, recursively scans for all files that - are checked into git at that location. -} inRepo :: Repo -> FilePath -> IO [FilePath] -inRepo repo location = pipeNullSplit repo - ["ls-files", "--cached", "--exclude-standard", "-z", location] +inRepo repo l = pipeNullSplit repo + ["ls-files", "--cached", "--exclude-standard", "-z", l] {- Passed a location, recursively scans for all files that are not checked - into git, and not gitignored. -} notInRepo :: Repo -> FilePath -> IO [FilePath] -notInRepo repo location = pipeNullSplit repo - ["ls-files", "--others", "--exclude-standard", "-z", location] +notInRepo repo l = pipeNullSplit repo + ["ls-files", "--others", "--exclude-standard", "-z", l] {- Passed a location, returns a list of the files, staged for - commit, that are being added, moved, or changed (but not deleted). -} stagedFiles :: Repo -> FilePath -> IO [FilePath] -stagedFiles repo location = pipeNullSplit repo +stagedFiles repo l = pipeNullSplit repo ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", - "HEAD", location] + "HEAD", l] {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} @@ -236,7 +241,7 @@ pipeNullSplit :: Repo -> [String] -> IO [FilePath] pipeNullSplit repo params = do -- XXX handle is left open, this is ok for git-annex, but may need -- to be cleaned up for other uses. - (handle, fs0) <- hPipeRead repo params + (_, fs0) <- hPipeRead repo params return $ split0 fs0 where split0 s = filter (not . null) $ split "\0" s @@ -256,6 +261,7 @@ configRead repo = assertSsh repo $ do where sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ " && git config --list" +hConfigRead :: Repo -> Handle -> IO Repo hConfigRead repo h = do val <- hGetContentsStrict h let r = repo { config = configParse val } @@ -267,10 +273,10 @@ configTrue s = map toLower s == "true" {- Calculates a list of a repo's configured remotes, by parsing its config. -} configRemotes :: Repo -> [Repo] -configRemotes repo = map construct remotes +configRemotes repo = map construct remotepairs where - remotes = Map.toList $ filter $ config repo - filter = Map.filterWithKey (\k _ -> isremote k) + remotepairs = Map.toList $ filterremotes $ config repo + filterremotes = Map.filterWithKey (\k _ -> isremote k) isremote k = (startswith "remote." k) && (endswith ".url" k) remotename k = (split "." k) !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } @@ -314,14 +320,15 @@ seekUp dir want = do "" -> return Nothing d -> seekUp d want +isRepoTop :: FilePath -> IO Bool isRepoTop dir = do - r <- isRepo dir - b <- isBareRepo dir + r <- isRepo + b <- isBareRepo return (r || b) where - isRepo dir = gitSignature dir ".git" ".git/config" - isBareRepo dir = gitSignature dir "objects" "config" - gitSignature dir subdir file = do + isRepo = gitSignature ".git" ".git/config" + isBareRepo = gitSignature "objects" "config" + gitSignature subdir file = do s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) f <- (doesFileExist (dir ++ "/" ++ file)) return (s && f) From b2c28c1ac0700eadc8689cdfb6f065d5147108bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:50:07 -0400 Subject: [PATCH 0367/2835] more Wall cleaning --- LocationLog.hs | 61 +++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 14ae88abc3..9a9dad1333 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -32,9 +32,7 @@ import Data.Time.Clock.POSIX import Data.Time import System.Locale import qualified Data.Map as Map -import System.IO import System.Directory -import Data.Char import System.Posix.Process import qualified GitRepo as Git @@ -63,8 +61,7 @@ instance Read LogStatus where readsPrec _ _ = [(Undefined, "")] instance Show LogLine where - show (LogLine date status uuid) = unwords - [(show date), (show status), uuid] + show (LogLine d s u) = unwords [show d, show s, u] instance Read LogLine where -- This parser is robust in that even unparsable log lines are @@ -74,26 +71,25 @@ instance Read LogLine where if (length w == 3) then case (pdate) of Just v -> good v - Nothing -> undefined - else undefined + Nothing -> bad + else bad where w = words string - date = w !! 0 - status = read $ w !! 1 - uuid = w !! 2 - pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime + s = read $ w !! 1 + u = w !! 2 + pdate = (parseTime defaultTimeLocale "%s%Qs" $ w !! 0) :: Maybe UTCTime - good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status uuid - undefined = ret $ LogLine (0) Undefined "" + good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s u + bad = ret $ LogLine (0) Undefined "" ret v = [(v, "")] {- Log a change in the presence of a key's value in a repository, - and returns the filename of the logfile. -} logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO (FilePath) -logChange repo key uuid status = do - log <- logNow status uuid +logChange repo key u s = do + line <- logNow s u ls <- readLog logfile - writeLog logfile (compactLog $ log:ls) + writeLog logfile (compactLog $ line:ls) return logfile where logfile = logFile repo key @@ -114,18 +110,18 @@ readLog file = do {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () -writeLog file lines = do +writeLog file ls = do pid <- getProcessID let tmpfile = file ++ ".tmp" ++ show pid createDirectoryIfMissing True (parentDir file) - writeFile tmpfile $ unlines $ map show lines + writeFile tmpfile $ unlines $ map show ls renameFile tmpfile file {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> UUID -> IO LogLine -logNow status uuid = do +logNow s u = do now <- getPOSIXTime - return $ LogLine now status uuid + return $ LogLine now s u {- Returns the filename of the log file for a given key. -} logFile :: Git.Repo -> Key -> String @@ -136,28 +132,33 @@ logFile repo key = - the value of a key. -} keyLocations :: Git.Repo -> Key -> IO [UUID] keyLocations thisrepo key = do - lines <- readLog $ logFile thisrepo key - return $ map uuid (filterPresent lines) + ls <- readLog $ logFile thisrepo key + return $ map uuid $ filterPresent ls {- Filters the list of LogLines to find ones where the value - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] -filterPresent lines = filter (\l -> ValuePresent == status l) $ compactLog lines +filterPresent ls = filter (\l -> ValuePresent == status l) $ compactLog ls + +type LogMap = Map.Map UUID LogLine {- Compacts a set of logs, returning a subset that contains the current - status. -} compactLog :: [LogLine] -> [LogLine] -compactLog lines = compactLog' Map.empty lines -compactLog' map [] = Map.elems map -compactLog' map (l:ls) = compactLog' (mapLog map l) ls +compactLog ls = compactLog' Map.empty ls +compactLog' :: LogMap -> [LogLine] -> [LogLine] +compactLog' m [] = Map.elems m +compactLog' m (l:ls) = compactLog' (mapLog m l) ls {- Inserts a log into a map of logs, if the log has better (ie, newer) - information about a repo than the other logs in the map -} -mapLog map log = +mapLog :: LogMap -> LogLine -> LogMap +mapLog m l = if (better) - then Map.insert (uuid log) log map - else map + then Map.insert u l m + else m where - better = case Map.lookup (uuid log) map of - Just l -> (date l <= date log) + better = case Map.lookup u m of + Just l' -> (date l' <= date l) Nothing -> True + u = uuid l From cf4c926f2e78bd315988c1aae063f6b4148a2fae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 16:00:32 -0400 Subject: [PATCH 0368/2835] more Wall cleaning --- Backend/File.hs | 15 +++++---------- Backend/SHA1.hs | 1 + Backend/URL.hs | 10 ++++------ Backend/WORM.hs | 3 +-- Core.hs | 2 +- Locations.hs | 8 ++++---- TypeInternals.hs | 4 ++-- Utility.hs | 12 +++++------- 8 files changed, 23 insertions(+), 32 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index b45354752f..5b93d8227e 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,25 +15,18 @@ module Backend.File (backend) where import Control.Monad.State -import System.IO -import System.Cmd -import System.Cmd.Utils -import Control.Exception import System.Directory -import List -import Maybe import TypeInternals import LocationLog import Locations import qualified Remotes import qualified GitRepo as Git -import Utility import Core import qualified Annex import UUID -import qualified Backend +backend :: Backend backend = Backend { name = mustProvide, getKey = mustProvide, @@ -43,11 +36,12 @@ backend = Backend { hasKey = checkKeyFile } +mustProvide :: a mustProvide = error "must provide this field" {- Storing a key is a no-op. -} dummyStore :: FilePath -> Key -> Annex (Bool) -dummyStore file key = return True +dummyStore _ _ = return True {- Just check if the .git/annex/ file for the key exists. -} checkKeyFile :: Key -> Annex Bool @@ -146,7 +140,8 @@ showLocations key = do if (null uuidsf) then showLongNote $ "No other repository is known to contain the file." else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids - + +showTriedRemotes :: [Git.Repo] -> Annex () showTriedRemotes [] = return () showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index f6daeffec4..76c368f84e 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -15,6 +15,7 @@ import System.IO import qualified Backend.File import TypeInternals +backend :: Backend backend = Backend.File.backend { name = "SHA1", getKey = keyValue diff --git a/Backend/URL.hs b/Backend/URL.hs index 384f933ebf..e6d3eb1ae5 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -9,14 +9,12 @@ module Backend.URL (backend) where import Control.Monad.State (liftIO) import Data.String.Utils -import System.Cmd -import System.Cmd.Utils -import System.Exit import TypeInternals import Core import Utility +backend :: Backend backend = Backend { name = "URL", getKey = keyValue, @@ -28,15 +26,15 @@ backend = Backend { -- cannot generate url from filename keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return Nothing +keyValue _ = return Nothing -- cannot change url contents dummyStore :: FilePath -> Key -> Annex Bool -dummyStore file url = return False +dummyStore _ _ = return False -- allow keys to be removed; presumably they can always be downloaded again dummyOk :: Key -> Annex Bool -dummyOk url = return True +dummyOk _ = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do diff --git a/Backend/WORM.hs b/Backend/WORM.hs index b5ae11807e..848386ecd1 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -10,12 +10,11 @@ module Backend.WORM (backend) where import Control.Monad.State import System.FilePath import System.Posix.Files -import qualified Data.ByteString.Lazy.Char8 as B import qualified Backend.File import TypeInternals -import Utility +backend :: Backend backend = Backend.File.backend { name = "WORM", getKey = keyValue diff --git a/Core.hs b/Core.hs index c5026c6a5f..ebe5d2966c 100644 --- a/Core.hs +++ b/Core.hs @@ -109,7 +109,7 @@ calcGitLink file key = do Just f -> f Nothing -> error $ "unable to normalize " ++ file return $ (relPathDirToDir (parentDir absfile) (Git.workTree g)) ++ - annexLocationRelative g key + annexLocationRelative key {- Updates the LocationLog when a key's presence changes. -} logStatus :: Key -> LogStatus -> Annex () diff --git a/Locations.hs b/Locations.hs index 49ee878c85..a296f8d876 100644 --- a/Locations.hs +++ b/Locations.hs @@ -18,11 +18,11 @@ module Locations ( import Data.String.Utils import Types -import qualified TypeInternals as Internals import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex - directory, in the git repository's working tree. -} +stateLoc :: String stateLoc = ".git-annex/" gitStateDir :: Git.Repo -> FilePath gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc @@ -35,13 +35,13 @@ gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = - (Git.workTree r) ++ "/" ++ (annexLocationRelative r key) + (Git.workTree r) ++ "/" ++ (annexLocationRelative key) {- Annexed file's location relative to git's working tree. - - Note: Assumes repo is NOT bare.-} -annexLocationRelative :: Git.Repo -> Key -> FilePath -annexLocationRelative r key = ".git/annex/" ++ (keyFile key) +annexLocationRelative :: Key -> FilePath +annexLocationRelative key = ".git/annex/" ++ (keyFile key) {- .git-annex/tmp is used for temp files - diff --git a/TypeInternals.hs b/TypeInternals.hs index f45be4760c..46c92cb59b 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -58,9 +58,9 @@ instance Read Key where k = join ":" $ drop 1 l backendName :: Key -> BackendName -backendName (Key (b,k)) = b +backendName (Key (b,_)) = b keyName :: Key -> KeyName -keyName (Key (b,k)) = k +keyName (Key (_,k)) = k -- this structure represents a key-value backend data Backend = Backend { diff --git a/Utility.hs b/Utility.hs index e7b4b510b8..6867f473a7 100644 --- a/Utility.hs +++ b/Utility.hs @@ -15,20 +15,17 @@ module Utility ( ) where import System.IO -import System.Cmd.Utils import System.Exit import System.Posix.Process -import System.Posix.Process.Internals import System.Posix.Signals -import System.Posix.IO import Data.String.Utils import System.Path -import System.IO.HVFS import System.FilePath import System.Directory {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} +hGetContentsStrict :: Handle -> IO String hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s {- Returns the parent directory of a path. Parent of / is "" -} @@ -53,11 +50,11 @@ parentDir dir = relPathCwdToDir :: FilePath -> IO FilePath relPathCwdToDir dir = do cwd <- getCurrentDirectory - let absdir = abs cwd dir + let absdir = absnorm cwd return $ relPathDirToDir cwd absdir where -- absolute, normalized form of the directory - abs cwd dir = + absnorm cwd = case (absNormPath cwd dir) of Just d -> d Nothing -> error $ "unable to normalize " ++ dir @@ -106,13 +103,14 @@ boolSystem command params = do _ -> return False where restoresignals oldint oldset = do - installHandler sigINT oldint Nothing + _ <- installHandler sigINT oldint Nothing setSignalMask oldset childaction oldint oldset = do restoresignals oldint oldset executeFile command True params Nothing {- Escapes a filename to be safely able to be exposed to the shell. -} +shellEscape :: FilePath -> FilePath shellEscape f = "'" ++ quote ++ "'" where -- replace ' with '"'"' From dda0679290aabdb7fe61e52e7a0c84f8baf3d547 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 16:04:19 -0400 Subject: [PATCH 0369/2835] all Walls are clean! --- UUID.hs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/UUID.hs b/UUID.hs index 20f85bae12..d683e0b98e 100644 --- a/UUID.hs +++ b/UUID.hs @@ -36,6 +36,7 @@ import Utility type UUID = String +configkey :: String configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, @@ -53,19 +54,19 @@ getUUID :: Git.Repo -> Annex UUID getUUID r = do g <- Annex.gitRepo - let c = cached r g - let u = uncached r + let c = cached g + let u = uncached if (c /= u && u /= "") then do - updatecache g r u + updatecache g u return u else return c where - uncached r = Git.configGet r "annex.uuid" "" - cached r g = Git.configGet g (cachekey r) "" - updatecache g r u = when (g /= r) $ setConfig (cachekey r) u - cachekey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" + uncached = Git.configGet r "annex.uuid" "" + cached g = Git.configGet g cachekey "" + updatecache g u = when (g /= r) $ setConfig cachekey u + cachekey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -111,26 +112,27 @@ describeUUID :: UUID -> String -> Annex () describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m - log <- uuidLog + logfile <- uuidLog pid <- liftIO $ getProcessID - let tmplog = log ++ ".tmp" ++ show pid - liftIO $ createDirectoryIfMissing True (parentDir log) - liftIO $ writeFile tmplog $ serialize m' - liftIO $ renameFile tmplog log + let tmplogfile = logfile ++ ".tmp" ++ show pid + liftIO $ createDirectoryIfMissing True (parentDir logfile) + liftIO $ writeFile tmplogfile $ serialize m' + liftIO $ renameFile tmplogfile logfile where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m {- Read and parse the uuidLog into a Map -} uuidMap :: Annex (M.Map UUID String) uuidMap = do - log <- uuidLog - s <- liftIO $ catch (readFile log) (\error -> return "") + logfile <- uuidLog + s <- liftIO $ catch (readFile logfile) ignoreerror return $ M.fromList $ map (\l -> pair l) $ lines s where pair l = if (1 < (length $ words l)) then ((words l) !! 0, unwords $ drop 1 $ words l) else ("", "") + ignoreerror _ = return "" {- Filename of uuid.log. -} uuidLog :: Annex String From 9429cdb9dab1321cb69d40996c31572f4657538c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 17:02:49 -0400 Subject: [PATCH 0370/2835] idea --- doc/todo/immutable_annexed_files.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/todo/immutable_annexed_files.mdwn diff --git a/doc/todo/immutable_annexed_files.mdwn b/doc/todo/immutable_annexed_files.mdwn new file mode 100644 index 0000000000..8f19f4f744 --- /dev/null +++ b/doc/todo/immutable_annexed_files.mdwn @@ -0,0 +1,3 @@ +> josh: Do you do anything in git-annex to try to make the files immutable? +> For instance, removing write permission, or even chattr? +> joey: I don't, but that's a very good idea From 763882925daee55584e68f9b0915ecd72877b14f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 17:04:28 -0400 Subject: [PATCH 0371/2835] idea2 --- doc/todo/immutable_annexed_files.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/todo/immutable_annexed_files.mdwn b/doc/todo/immutable_annexed_files.mdwn index 8f19f4f744..e5207dc163 100644 --- a/doc/todo/immutable_annexed_files.mdwn +++ b/doc/todo/immutable_annexed_files.mdwn @@ -1,3 +1,6 @@ > josh: Do you do anything in git-annex to try to make the files immutable? > For instance, removing write permission, or even chattr? > joey: I don't, but that's a very good idea +> josh: Oh, I just thought of another slightly crazy but handy idea. +> josh: I'd hate to run into a program which somehow followed the symlink and then did an unlink to replace the file. +> josh: To break that, you could create a new directory under annex's internal directory for each file, and make the directory have no write permission. From b220e117f2e06133919ad31e9cc225470f501566 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 17:20:05 -0400 Subject: [PATCH 0372/2835] idea --- doc/todo/checkout.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/todo/checkout.mdwn diff --git a/doc/todo/checkout.mdwn b/doc/todo/checkout.mdwn new file mode 100644 index 0000000000..70d31a5ff5 --- /dev/null +++ b/doc/todo/checkout.mdwn @@ -0,0 +1,5 @@ +The checkout subcommand replaces the symlink that normally points at a +file's content, with a copy of the file. Once you've checked a file out, +you can edit it, and `git commit` it. On commit, git-annex will detect +if the file has been changed, and if it has, `add` its content to the +annex. From fd6611f9554e43f2bc365f7ef70f90877c9403d0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 18:04:34 -0400 Subject: [PATCH 0373/2835] Fix crash on unknown symlinks. --- Backend.hs | 41 ++++++++++++++++++++++++++++------------- Core.hs | 7 +++++++ debian/changelog | 1 + git-annex.hs | 7 +------ 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/Backend.hs b/Backend.hs index d75c2a761e..f1b4c28979 100644 --- a/Backend.hs +++ b/Backend.hs @@ -27,9 +27,10 @@ module Backend ( ) where import Control.Monad.State -import Control.Exception.Extensible +import IO (try) import System.FilePath import System.Posix.Files +import Core import Locations import qualified GitRepo as Git @@ -59,12 +60,17 @@ list = do then bs else map (lookupBackendName bs) $ words s -{- Looks up a backend in a list -} +{- Looks up a backend in a list. May fail if unknown. -} lookupBackendName :: [Backend] -> String -> Backend lookupBackendName bs s = + case maybeLookupBackendName bs s of + Just b -> b + Nothing -> error $ "unknown backend " ++ s +maybeLookupBackendName :: [Backend] -> String -> Maybe Backend +maybeLookupBackendName bs s = if ((length matches) /= 1) - then error $ "unknown backend " ++ s - else matches !! 0 + then Nothing + else Just $ matches !! 0 where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} @@ -109,15 +115,24 @@ hasKey key = do lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) lookupFile file = do bs <- Annex.supportedBackends - result <- liftIO $ (try (find bs)::IO (Either SomeException (Maybe (Key, Backend)))) - case (result) of + tl <- liftIO $ try getsymlink + case tl of Left _ -> return Nothing - Right val -> return val - where - find bs = do + Right l -> makekey bs l + where + getsymlink = do l <- readSymbolicLink file - return $ Just $ pair bs $ takeFileName l - pair bs f = (k, b) + return $ takeFileName l + makekey bs l = do + case maybeLookupBackendName bs $ bname of + Nothing -> do + unless (null kname || null bname) $ + warning skip + return Nothing + Just backend -> return $ Just (k, backend) where - k = fileKey f - b = lookupBackendName bs $ backendName k + k = fileKey l + bname = backendName k + kname = keyName k + skip = "skipping " ++ file ++ + " (unknown backend " ++ bname ++ ")" diff --git a/Core.hs b/Core.hs index ebe5d2966c..e2e6eaa0cf 100644 --- a/Core.hs +++ b/Core.hs @@ -165,3 +165,10 @@ showEndOk = verbose $ do showEndFail :: Annex () showEndFail = verbose $ do liftIO $ putStrLn "\nfailed" + +{- Exception pretty-printing. -} +showErr :: (Show a) => a -> Annex () +showErr e = warning $ show e + +warning :: String -> Annex () +warning s = liftIO $ hPutStrLn stderr $ "git-annex: " ++ s diff --git a/debian/changelog b/debian/changelog index 70e9052231..83e76b6ec7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Support building with Debian stable's ghc. * Fixed memory leak; git-annex no longer reads the whole file list from git before starting, and will be much faster with large repos. + * Fix crash on unknown symlinks. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/git-annex.hs b/git-annex.hs index e9e0d6f027..d798d417b5 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,7 +6,6 @@ -} import IO (try) -import System.IO import System.Environment import Monad @@ -41,13 +40,9 @@ tryRun' state errnum (a:as) = do result <- try $ Annex.run state a case (result) of Left err -> do - showErr err + _ <- Annex.run state $ showErr err tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as tryRun' _ errnum [] = when (errnum > 0) $ error $ (show errnum) ++ " failed" - -{- Exception pretty-printing. -} -showErr :: (Show a) => a -> IO () -showErr e = hPutStrLn stderr $ "git-annex: " ++ (show e) From f4383532e69630266c1188527714138dfff8bea0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 19:52:11 -0400 Subject: [PATCH 0374/2835] add toggles --- doc/use_case/Alice.mdwn | 8 ++++++-- doc/use_case/Bob.mdwn | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index c42eb3a74c..5d0360a2b8 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -3,9 +3,12 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server out there on the net. All these things can have different files on them, -but Alice no longer has to deal with the tedious process of keeping them -manually in sync. +but with git-annex Alice no longer has to deal with the tedious process +of keeping them manually in sync. +[[!toggle id=alice text="more..."]] + +[[!toggleable id=alice text=""" When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on @@ -16,3 +19,4 @@ When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knowns that next time she syncs up to the net, her changes will be synced back to her server. +"""]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index a5dc01b373..3aad78eef9 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -4,10 +4,13 @@ Bob has many drives to archive his data, most of them kept offline, in a safe place. With git-annex, Bob has a single directory tree that includes all -his files, even if their content is being stored offline. He can +his files, even those whose content is stored offline. He can reorganize his files using that tree, committing new versions to git, without worry about accidentially deleting anything. +[[!toggle id=bob text="more..."]] + +[[!toggleable id=bob text=""" When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. @@ -16,3 +19,4 @@ Run in a cron job, git-annex adds new files to achival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. +"""]] From 371906efb010c3b441d958adecb2a6fddd498237 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 19:53:17 -0400 Subject: [PATCH 0375/2835] foo --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 5d0360a2b8..590f7b6527 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -6,9 +6,9 @@ out there on the net. All these things can have different files on them, but with git-annex Alice no longer has to deal with the tedious process of keeping them manually in sync. -[[!toggle id=alice text="more..."]] +[[!toggle id=use text="more..."]] -[[!toggleable id=alice text=""" +[[!toggleable id=use text=""" When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 3aad78eef9..0f1247ab22 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -8,9 +8,9 @@ his files, even those whose content is stored offline. He can reorganize his files using that tree, committing new versions to git, without worry about accidentially deleting anything. -[[!toggle id=bob text="more..."]] +[[!toggle id=use text="more..."]] -[[!toggleable id=bob text=""" +[[!toggleable id=use text=""" When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. From b5411926abc5724f7f156ceaea8fa7e37a745cd1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 19:54:12 -0400 Subject: [PATCH 0376/2835] Revert "foo" This reverts commit 371906efb010c3b441d958adecb2a6fddd498237. --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 590f7b6527..5d0360a2b8 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -6,9 +6,9 @@ out there on the net. All these things can have different files on them, but with git-annex Alice no longer has to deal with the tedious process of keeping them manually in sync. -[[!toggle id=use text="more..."]] +[[!toggle id=alice text="more..."]] -[[!toggleable id=use text=""" +[[!toggleable id=alice text=""" When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 0f1247ab22..3aad78eef9 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -8,9 +8,9 @@ his files, even those whose content is stored offline. He can reorganize his files using that tree, committing new versions to git, without worry about accidentially deleting anything. -[[!toggle id=use text="more..."]] +[[!toggle id=bob text="more..."]] -[[!toggleable id=use text=""" +[[!toggleable id=bob text=""" When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. From 4e742d62f45fad486d8772a7b9649b42315e1be7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 19:57:05 -0400 Subject: [PATCH 0377/2835] Revert "add toggles" This reverts commit f4383532e69630266c1188527714138dfff8bea0. --- doc/use_case/Alice.mdwn | 8 ++------ doc/use_case/Bob.mdwn | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 5d0360a2b8..c42eb3a74c 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -3,12 +3,9 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server out there on the net. All these things can have different files on them, -but with git-annex Alice no longer has to deal with the tedious process -of keeping them manually in sync. +but Alice no longer has to deal with the tedious process of keeping them +manually in sync. -[[!toggle id=alice text="more..."]] - -[[!toggleable id=alice text=""" When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on @@ -19,4 +16,3 @@ When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knowns that next time she syncs up to the net, her changes will be synced back to her server. -"""]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 3aad78eef9..a5dc01b373 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -4,13 +4,10 @@ Bob has many drives to archive his data, most of them kept offline, in a safe place. With git-annex, Bob has a single directory tree that includes all -his files, even those whose content is stored offline. He can +his files, even if their content is being stored offline. He can reorganize his files using that tree, committing new versions to git, without worry about accidentially deleting anything. -[[!toggle id=bob text="more..."]] - -[[!toggleable id=bob text=""" When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. @@ -19,4 +16,3 @@ Run in a cron job, git-annex adds new files to achival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. -"""]] From 228b32d473abb72bb26ff8eca008387190428c44 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:13:43 -0400 Subject: [PATCH 0378/2835] bugfix: shell escape for scp --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index b9c1b48f3d..b36856b543 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -217,7 +217,7 @@ copyToRemote r key file = do liftIO $ boolSystem "scp" [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath -sshLocation r file = (Git.urlHost r) ++ ":" ++ file +sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file {- Runs a command in a remote. -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool From 11efa7ef609a99de2841772e415cd2808ee438b8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:19:25 -0400 Subject: [PATCH 0379/2835] more escaping for ssh --- Remotes.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index b36856b543..135a701ffb 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -232,6 +232,6 @@ runCmd r command params = do then do liftIO $ boolSystem "ssh" [Git.urlHost r, "cd " ++ (shellEscape $ Git.workTree r) ++ - " && " ++ command ++ " " ++ - unwords params] + " && " ++ (shellEscape command) ++ " " ++ + (unwords $ map shellEscape params)] else error "running command in non-ssh repo not supported" From d04bfceaad3e96c91c7effa9cebdc0811d336278 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:20:07 -0400 Subject: [PATCH 0380/2835] more accurate type --- Utility.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index 6867f473a7..47c059ded7 100644 --- a/Utility.hs +++ b/Utility.hs @@ -110,7 +110,7 @@ boolSystem command params = do executeFile command True params Nothing {- Escapes a filename to be safely able to be exposed to the shell. -} -shellEscape :: FilePath -> FilePath +shellEscape :: FilePath -> String shellEscape f = "'" ++ quote ++ "'" where -- replace ' with '"'"' From c6206c4560adf35427569a9d9a637f9ee23cb2b8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:22:01 -0400 Subject: [PATCH 0381/2835] tweak --- Utility.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Utility.hs b/Utility.hs index 47c059ded7..4e56289e22 100644 --- a/Utility.hs +++ b/Utility.hs @@ -111,7 +111,7 @@ boolSystem command params = do {- Escapes a filename to be safely able to be exposed to the shell. -} shellEscape :: FilePath -> String -shellEscape f = "'" ++ quote ++ "'" +shellEscape f = "'" ++ escaped ++ "'" where -- replace ' with '"'"' - quote = join "'\"'\"'" $ split "'" f + escaped = join "'\"'\"'" $ split "'" f From e70812eca9748ee53468e68563366ebd856b7e82 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:29:11 -0400 Subject: [PATCH 0382/2835] use ssh -p to preserve perms and refactor --- Remotes.hs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 135a701ffb..a5c4597eba 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -196,8 +196,7 @@ copyFromRemote r key file = do where getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] getssh = do - Core.showProgress -- make way for scp progress bar - liftIO $ boolSystem "scp" [sshLocation r keyloc, file] + scp [sshLocation r keyloc, file] keyloc = annexLocation r key {- Tries to copy a key's content to a file on a remote. -} @@ -213,12 +212,16 @@ copyToRemote r key file = do where putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] putssh src = do - Core.showProgress -- make way for scp progress bar - liftIO $ boolSystem "scp" [src, sshLocation r file] + scp [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file +scp :: [String] -> Annex Bool +scp params = do + Core.showProgress -- make way for scp progress bar + liftIO $ boolSystem "scp" ("-p":params) + {- Runs a command in a remote. -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool runCmd r command params = do From 0194394be61dba0324a5a99f462aa2206066771c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:56:56 -0400 Subject: [PATCH 0383/2835] Added remote.annex-scp-options and remote.annex-ssh-options. --- Remotes.hs | 49 ++++++++++++++++++++++++++-------------------- debian/changelog | 1 + doc/git-annex.mdwn | 5 +++++ 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index a5c4597eba..778ad89e25 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -115,33 +115,28 @@ reposByCost l = do -} repoCost :: Git.Repo -> Annex Int repoCost r = do - g <- Annex.gitRepo - if (not $ null $ config g) - then return $ read $ config g + cost <- repoConfig r "annex-cost" "" + if (not $ null cost) + then return $ read cost else if (Git.repoIsUrl r) then return 200 else return 100 - where - config g = Git.configGet g configkey "" - configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" {- Checks if a repo should be ignored, based either on annex-ignore - setting, or on command-line options. Allows command-line to override - annex-ignore. -} repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do - g <- Annex.gitRepo + ignored <- repoConfig r "annex-ignore" "false" fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" let name = if (not $ null fromName) then fromName else toName if (not $ null name) then return $ match name - else return $ not $ ignored g + else return $ not $ isIgnored ignored where match name = name == Git.repoRemoteName r - ignored g = Git.configTrue $ config g - config g = Git.configGet g configkey "" - configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" + isIgnored ignored = Git.configTrue ignored {- Returns the remote specified by --from or --to, may fail with error. -} commandLineRemote :: Annex Git.Repo @@ -195,8 +190,7 @@ copyFromRemote r key file = do else error "copying from non-ssh repo not supported" where getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] - getssh = do - scp [sshLocation r keyloc, file] + getssh = scp r [sshLocation r keyloc, file] keyloc = annexLocation r key {- Tries to copy a key's content to a file on a remote. -} @@ -211,20 +205,23 @@ copyToRemote r key file = do else error "copying to non-ssh repo not supported" where putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] - putssh src = do - scp [src, sshLocation r file] + putssh src = scp r [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file -scp :: [String] -> Annex Bool -scp params = do +{- Runs scp against a specified remote. (Honors annex-scp-options.) -} +scp :: Git.Repo -> [String] -> Annex Bool +scp r params = do + scpoptions <- repoConfig r "annex-scp-options" "" Core.showProgress -- make way for scp progress bar - liftIO $ boolSystem "scp" ("-p":params) + liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params -{- Runs a command in a remote. -} +{- Runs a command in a remote, using ssh if necessary. + - (Honors annex-ssh-options.) -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool runCmd r command params = do + sshoptions <- repoConfig r "annex-ssh-options" "" if (not $ Git.repoIsUrl r) then do cwd <- liftIO $ getCurrentDirectory @@ -233,8 +230,18 @@ runCmd r command params = do boolSystem command params else if (Git.repoIsSsh r) then do - liftIO $ boolSystem "ssh" [Git.urlHost r, - "cd " ++ (shellEscape $ Git.workTree r) ++ + liftIO $ boolSystem "ssh" $ + (words sshoptions) ++ + [Git.urlHost r, "cd " ++ + (shellEscape $ Git.workTree r) ++ " && " ++ (shellEscape command) ++ " " ++ (unwords $ map shellEscape params)] else error "running command in non-ssh repo not supported" + +{- Looks up a per-remote config option in git config. -} +repoConfig :: Git.Repo -> String -> String -> Annex String +repoConfig r key def = do + g <- Annex.gitRepo + return $ Git.configGet g fullkey def + where + fullkey = "remote." ++ (Git.repoRemoteName r) ++ "." ++ key diff --git a/debian/changelog b/debian/changelog index 83e76b6ec7..cb6a0ef868 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Fixed memory leak; git-annex no longer reads the whole file list from git before starting, and will be much faster with large repos. * Fix crash on unknown symlinks. + * Added remote.annex-scp-options and remote.annex-ssh-options. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 47fbb37601..d176a04c04 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -196,6 +196,11 @@ Like other git commands, git-annex is configured via `.git/config`. from ever using this remote. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. +* `remote..annex-scp-options` -- Options to use when using scp + to or from this repository. For example, to force ipv6, and limit + the bandwidth to 100Kbit/s, set it to "-6 -l 100" +* `remote..annex-ssh-options` -- Options to use when using ssh + to talk to this repository. # FILES From f3e4633e359d0a4afc7a0f90a89e4d276b597d84 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 23:21:16 -0400 Subject: [PATCH 0384/2835] refactor inAnnex remote checking to Remotes --- Core.hs | 13 +++---------- Remotes.hs | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Core.hs b/Core.hs index e2e6eaa0cf..ab7291affa 100644 --- a/Core.hs +++ b/Core.hs @@ -86,19 +86,12 @@ gitPreCommitHook repo = do p <- getPermissions hook setPermissions hook $ p {executable = True} -{- Checks if a given key is currently present in the annexLocation. - - - - This can be run against a remote repository to check the key there. -} +{- Checks if a given key is currently present in the annexLocation. -} inAnnex :: Key -> Annex Bool inAnnex key = do g <- Annex.gitRepo - if (not $ Git.repoIsUrl g) - then liftIO $ doesFileExist $ annexLocation g key - else do - showNote ("checking " ++ Git.repoDescribe g ++ "...") - liftIO $ boolSystem "ssh" [Git.urlHost g, - "test -e " ++ - (shellEscape $ annexLocation g key)] + when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo" + liftIO $ doesFileExist $ annexLocation g key {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath diff --git a/Remotes.hs b/Remotes.hs index 778ad89e25..a01da7d488 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -81,14 +81,24 @@ keyPossibilities key = do - If the remote cannot be accessed, returns a Left error. -} inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) -inAnnex remote key = do - -- the check needs to run in an Annex monad using the remote - liftIO $ ((try $ check)::IO (Either IOException Bool)) +inAnnex r key = do + if (not $ Git.repoIsUrl r) + then check local + else do + Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") + check remote where - check = do - a <- Annex.new remote [] + check a = liftIO $ ((try a)::IO (Either IOException Bool)) + local = do + -- run a local check by making an Annex monad + -- using the remote + a <- Annex.new r [] (result, _) <- Annex.run a (Core.inAnnex key) return result + remote = do + -- remote check via ssh in and test + boolSystem "ssh" [Git.urlHost r, "test -e " ++ + (shellEscape $ annexLocation r key)] {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] From 00d4c7cd01b6f6e863a22483b9ea20ca5260da43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 23:24:16 -0400 Subject: [PATCH 0385/2835] simplify evals --- Annex.hs | 3 +++ Remotes.hs | 3 +-- git-annex.hs | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Annex.hs b/Annex.hs index 63eef51589..8f60a0bf49 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,6 +8,7 @@ module Annex ( new, run, + eval, gitRepo, gitRepoChange, backends, @@ -50,6 +51,8 @@ new gitrepo allbackends = do {- performs an action in the Annex monad -} run :: AnnexState -> StateT AnnexState IO a -> IO (a, AnnexState) run state action = runStateT (action) state +eval :: AnnexState -> StateT AnnexState IO a -> IO a +eval state action = evalStateT (action) state {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo diff --git a/Remotes.hs b/Remotes.hs index a01da7d488..2879516fe1 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -93,8 +93,7 @@ inAnnex r key = do -- run a local check by making an Annex monad -- using the remote a <- Annex.new r [] - (result, _) <- Annex.run a (Core.inAnnex key) - return result + Annex.eval a (Core.inAnnex key) remote = do -- remote check via ssh in and test boolSystem "ssh" [Git.urlHost r, "test -e " ++ diff --git a/git-annex.hs b/git-annex.hs index d798d417b5..e958ac2f96 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -40,7 +40,7 @@ tryRun' state errnum (a:as) = do result <- try $ Annex.run state a case (result) of Left err -> do - _ <- Annex.run state $ showErr err + Annex.eval state $ showErr err tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as From cec25153ecd4f824cf150afbd294ad0c5ed1413e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 23:38:07 -0400 Subject: [PATCH 0386/2835] bugfix: git annex move --from The data structure comparison didn't work because for a file remote, the config gets read for one structure but not the other. --- Commands.hs | 2 +- Remotes.hs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Commands.hs b/Commands.hs index fb76e5502f..507c82cccc 100644 --- a/Commands.hs +++ b/Commands.hs @@ -479,7 +479,7 @@ moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key - if (elem remote l) + if (not $ null $ filter (\r -> Remotes.same r remote) l) then return $ Just $ moveFromPerform key else return Nothing moveFromPerform :: Key -> Annex (Maybe SubCmdCleanup) diff --git a/Remotes.hs b/Remotes.hs index 2879516fe1..dffbe8f502 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -10,6 +10,7 @@ module Remotes ( keyPossibilities, tryGitConfigRead, inAnnex, + same, commandLineRemote, copyFromRemote, copyToRemote, @@ -47,7 +48,7 @@ keyPossibilities key = do uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost -- To determine if a remote has a key, its UUID needs to be known. - -- The locally cached UIIDs of remotes can fall out of date if + -- The locally cached UUIDs of remotes can fall out of date if -- eg, a different drive is mounted at the same location. -- But, reading the config of remotes can be expensive, so make -- sure we only do it once per git-annex run. @@ -147,6 +148,10 @@ repoNotIgnored r = do match name = name == Git.repoRemoteName r isIgnored ignored = Git.configTrue ignored +{- Checks if two repos are the same, by comparing their remote names. -} +same :: Git.Repo -> Git.Repo -> Bool +same a b = Git.repoRemoteName a == Git.repoRemoteName b + {- Returns the remote specified by --from or --to, may fail with error. -} commandLineRemote :: Annex Git.Repo commandLineRemote = do From 0bd7ebbf356c2753fbd6e29b925efef0b4eb3f74 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 23:50:58 -0400 Subject: [PATCH 0387/2835] make a ssh call honor annex-ssh-options --- Remotes.hs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index dffbe8f502..c28ba5afed 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -84,21 +84,21 @@ keyPossibilities key = do inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) inAnnex r key = do if (not $ Git.repoIsUrl r) - then check local - else do - Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") - check remote + then liftIO $ ((try checklocal)::IO (Either IOException Bool)) + else checkremote where - check a = liftIO $ ((try a)::IO (Either IOException Bool)) - local = do + checklocal = do -- run a local check by making an Annex monad -- using the remote a <- Annex.new r [] Annex.eval a (Core.inAnnex key) - remote = do - -- remote check via ssh in and test - boolSystem "ssh" [Git.urlHost r, "test -e " ++ - (shellEscape $ annexLocation r key)] + checkremote = do + Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") + inannex <- runCmd r ("test -e " ++ + (shellEscape $ annexLocation r key)) [] + -- XXX Note that ssh failing and the file not existing + -- are not currently differentiated. + return $ Right inannex {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] From 99c522edeff6add3ece41922f862c59b5afdddbe Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:04:53 -0400 Subject: [PATCH 0388/2835] finished adding support for annex.ssh-options --- Annex.hs | 2 +- GitRepo.hs | 17 +++++++++++------ Remotes.hs | 7 ++++--- UUID.hs | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Annex.hs b/Annex.hs index 8f60a0bf49..303881fa09 100644 --- a/Annex.hs +++ b/Annex.hs @@ -45,7 +45,7 @@ new gitrepo allbackends = do where prep = do -- read git config and update state - gitrepo' <- liftIO $ Git.configRead gitrepo + gitrepo' <- liftIO $ Git.configRead gitrepo Nothing Annex.gitRepoChange gitrepo' {- performs an action in the Annex monad -} diff --git a/GitRepo.hs b/GitRepo.hs index 874b5c3c9b..505dd06eb2 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -246,9 +246,11 @@ pipeNullSplit repo params = do where split0 s = filter (not . null) $ split "\0" s -{- Runs git config and populates a repo with its config. -} -configRead :: Repo -> IO Repo -configRead repo@(Repo { location = Dir d }) = do +{- Runs git config and populates a repo with its config. + - + - For a ssh repository, a list of ssh options may optionally be specified. -} +configRead :: Repo -> Maybe [String] -> IO Repo +configRead repo@(Repo { location = Dir d }) _ = do {- Cannot use pipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory @@ -256,10 +258,13 @@ configRead repo@(Repo { location = Dir d }) = do (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] $ hConfigRead repo -configRead repo = assertSsh repo $ do - pOpen ReadFromPipe "ssh" [urlHost repo, sshcommand] $ hConfigRead repo +configRead repo sshopts = assertSsh repo $ do + pOpen ReadFromPipe "ssh" params $ hConfigRead repo where - sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ + params = case sshopts of + Nothing -> [urlHost repo, command] + Just l -> l ++ [urlHost repo, command] + command = "cd " ++ (shellEscape $ urlPath repo) ++ " && git config --list" hConfigRead :: Repo -> Handle -> IO Repo hConfigRead repo h = do diff --git a/Remotes.hs b/Remotes.hs index c28ba5afed..bb77b00b31 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -94,8 +94,8 @@ inAnnex r key = do Annex.eval a (Core.inAnnex key) checkremote = do Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- runCmd r ("test -e " ++ - (shellEscape $ annexLocation r key)) [] + inannex <- runCmd r "test" + [ "-e", (shellEscape $ annexLocation r key)] -- XXX Note that ssh failing and the file not existing -- are not currently differentiated. return $ Right inannex @@ -172,11 +172,12 @@ commandLineRemote = do - returns the updated git repo. -} tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r = do + sshoptions <- repoConfig r "annex-ssh-options" "" if (Map.null $ Git.configMap r) then do -- configRead can fail due to IO error or -- for other reasons; catch all possible exceptions - result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) + result <- liftIO $ (try (Git.configRead r $ Just $ words sshoptions)::IO (Either SomeException (Git.Repo))) case (result) of Left _ -> return $ Left r Right r' -> do diff --git a/UUID.hs b/UUID.hs index d683e0b98e..ffd2cd46dc 100644 --- a/UUID.hs +++ b/UUID.hs @@ -83,7 +83,7 @@ setConfig key value = do g <- Annex.gitRepo liftIO $ Git.run g ["config", key, value] -- re-read git config and update the repo's state - g' <- liftIO $ Git.configRead g + g' <- liftIO $ Git.configRead g Nothing Annex.gitRepoChange g' {- Filters a list of repos to ones that have listed UUIDs. -} From e638f9647d10ca0cc63799698456a387ca9b8382 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:17:26 -0400 Subject: [PATCH 0389/2835] add global fallback for per-repo options --- Remotes.hs | 19 +++++++++++-------- doc/git-annex.mdwn | 5 ++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index bb77b00b31..7bb1bcd222 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -125,7 +125,7 @@ reposByCost l = do -} repoCost :: Git.Repo -> Annex Int repoCost r = do - cost <- repoConfig r "annex-cost" "" + cost <- repoConfig r "cost" "" if (not $ null cost) then return $ read cost else if (Git.repoIsUrl r) @@ -137,7 +137,7 @@ repoCost r = do - annex-ignore. -} repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do - ignored <- repoConfig r "annex-ignore" "false" + ignored <- repoConfig r "ignore" "false" fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" let name = if (not $ null fromName) then fromName else toName @@ -172,7 +172,7 @@ commandLineRemote = do - returns the updated git repo. -} tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r = do - sshoptions <- repoConfig r "annex-ssh-options" "" + sshoptions <- repoConfig r "ssh-options" "" if (Map.null $ Git.configMap r) then do -- configRead can fail due to IO error or @@ -228,7 +228,7 @@ sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file {- Runs scp against a specified remote. (Honors annex-scp-options.) -} scp :: Git.Repo -> [String] -> Annex Bool scp r params = do - scpoptions <- repoConfig r "annex-scp-options" "" + scpoptions <- repoConfig r "scp-options" "" Core.showProgress -- make way for scp progress bar liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params @@ -236,7 +236,7 @@ scp r params = do - (Honors annex-ssh-options.) -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool runCmd r command params = do - sshoptions <- repoConfig r "annex-ssh-options" "" + sshoptions <- repoConfig r "ssh-options" "" if (not $ Git.repoIsUrl r) then do cwd <- liftIO $ getCurrentDirectory @@ -253,10 +253,13 @@ runCmd r command params = do (unwords $ map shellEscape params)] else error "running command in non-ssh repo not supported" -{- Looks up a per-remote config option in git config. -} +{- Looks up a per-remote config option in git config. + - Failing that, tries looking for a global config option. -} repoConfig :: Git.Repo -> String -> String -> Annex String repoConfig r key def = do g <- Annex.gitRepo - return $ Git.configGet g fullkey def + let def' = Git.configGet g global def + return $ Git.configGet g local def' where - fullkey = "remote." ++ (Git.repoRemoteName r) ++ "." ++ key + local = "remote." ++ (Git.repoRemoteName r) ++ ".annex-" ++ key + global = "annex." ++ key diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d176a04c04..fa584d360c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -178,7 +178,8 @@ Many git-annex subcommands will stage changes for later `git commit` by you. # CONFIGURATION -Like other git commands, git-annex is configured via `.git/config`. +Like other git commands, git-annex is configured via `git-config`. +Here are all the supported configuration settings. * `annex.uuid` -- a unique UUID for this repository (automatically set) * `annex.numcopies` -- number of copies of files to keep across all @@ -201,6 +202,8 @@ Like other git commands, git-annex is configured via `.git/config`. the bandwidth to 100Kbit/s, set it to "-6 -l 100" * `remote..annex-ssh-options` -- Options to use when using ssh to talk to this repository. +* `annex.scp-options` and `annex.ssh-options` -- Default scp and ssh + options to use if a remote does not have specific options. # FILES From 11215b5b118d296dc0d0ac3328802b87964915be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:26:47 -0400 Subject: [PATCH 0390/2835] cleanup --- doc/git-annex.mdwn | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index fa584d360c..c60810424b 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -190,9 +190,7 @@ Here are all the supported configuration settings. * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 100 for local repositories, and 200 for remote - repositories. Note that other factors may be configured when pushing - files to repositories, in particular, whether the repository is on - a filesystem with sufficient free space. + repositories. * `remote..annex-ignore` -- If set to `true`, prevents git-annex from ever using this remote. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories From 4b9990194c239a42fe2916ddf5db83a4be06bf80 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:27:27 -0400 Subject: [PATCH 0391/2835] expand --- doc/git-annex.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index c60810424b..1f2f57fba7 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -192,7 +192,8 @@ Here are all the supported configuration settings. The default cost is 100 for local repositories, and 200 for remote repositories. * `remote..annex-ignore` -- If set to `true`, prevents git-annex - from ever using this remote. + from ever using this remote. This is, for example, useful if the + remote is a bare repository, which git-annex does not currently support. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. * `remote..annex-scp-options` -- Options to use when using scp From 524125e52e8d2a392a2662ac09f8658307513a25 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:28:52 -0400 Subject: [PATCH 0392/2835] fix --- doc/git-annex.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 1f2f57fba7..52a8c712f6 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -230,7 +230,7 @@ should be included, in, for example, `/usr/share/doc/git-annex/` # AUTHOR -Joey Hess +Joey Hess From 0655ae4b8a4c3b55351a37ddad03bb33a7cc9353 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 03:01:58 -0400 Subject: [PATCH 0393/2835] move --- Core.hs | 24 ++++++++++++++++++++++++ git-annex.hs | 26 -------------------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/Core.hs b/Core.hs index ab7291affa..d3d0d7899c 100644 --- a/Core.hs +++ b/Core.hs @@ -7,6 +7,7 @@ module Core where +import IO (try) import System.IO import System.Directory import Control.Monad.State (liftIO) @@ -22,6 +23,29 @@ import qualified GitRepo as Git import qualified GitQueue import qualified Annex import Utility + +{- Runs a list of Annex actions. Catches IO errors and continues + - (but explicitly thrown errors terminate the whole command). + - Propigates an overall error status at the end. + - + - This runs in the IO monad, not in the Annex monad. It seems that + - exceptions can only be caught in the IO monad, not in a stacked monad; + - or more likely I missed an easy way to do it. So, I have to laboriously + - thread AnnexState through this function. + -} +tryRun :: AnnexState -> [Annex Bool] -> IO () +tryRun state actions = tryRun' state 0 actions +tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () +tryRun' state errnum (a:as) = do + result <- try $ Annex.run state a + case (result) of + Left err -> do + Annex.eval state $ showErr err + tryRun' state (errnum + 1) as + Right (True,state') -> tryRun' state' errnum as + Right (False,state') -> tryRun' state' (errnum + 1) as +tryRun' _ errnum [] = + when (errnum > 0) $ error $ (show errnum) ++ " failed" {- Sets up a git repo for git-annex. -} startup :: Annex Bool diff --git a/git-annex.hs b/git-annex.hs index e958ac2f96..370c22a1ef 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -5,12 +5,9 @@ - Licensed under the GNU GPL version 3 or higher. -} -import IO (try) import System.Environment -import Monad import qualified Annex -import Types import Core import Commands import qualified GitRepo as Git @@ -23,26 +20,3 @@ main = do state <- Annex.new gitrepo allBackends (configure, actions) <- parseCmd args state tryRun state $ [startup] ++ configure ++ actions ++ [shutdown] - -{- Runs a list of Annex actions. Catches IO errors and continues - - (but explicitly thrown errors terminate the whole command). - - Propigates an overall error status at the end. - - - - This runs in the IO monad, not in the Annex monad. It seems that - - exceptions can only be caught in the IO monad, not in a stacked monad; - - or more likely I missed an easy way to do it. So, I have to laboriously - - thread AnnexState through this function. - -} -tryRun :: AnnexState -> [Annex Bool] -> IO () -tryRun state actions = tryRun' state 0 actions -tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () -tryRun' state errnum (a:as) = do - result <- try $ Annex.run state a - case (result) of - Left err -> do - Annex.eval state $ showErr err - tryRun' state (errnum + 1) as - Right (True,state') -> tryRun' state' errnum as - Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' _ errnum [] = - when (errnum > 0) $ error $ (show errnum) ++ " failed" From 4da551827fd0e5a437037b316f60b5a000e447e1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 03:12:28 -0400 Subject: [PATCH 0394/2835] trim --- Core.hs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Core.hs b/Core.hs index d3d0d7899c..327a9af483 100644 --- a/Core.hs +++ b/Core.hs @@ -27,11 +27,6 @@ import Utility {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). - Propigates an overall error status at the end. - - - - This runs in the IO monad, not in the Annex monad. It seems that - - exceptions can only be caught in the IO monad, not in a stacked monad; - - or more likely I missed an easy way to do it. So, I have to laboriously - - thread AnnexState through this function. -} tryRun :: AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions From 59e49ae083eb9e6211eec10c901264abcf3e5676 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 14:49:05 -0400 Subject: [PATCH 0395/2835] rework subcommand invocation logic --- Commands.hs | 132 ++++++++++++++++++++++++--------------------- debian/changelog | 2 + doc/git-annex.mdwn | 14 +++-- 3 files changed, 85 insertions(+), 63 deletions(-) diff --git a/Commands.hs b/Commands.hs index 507c82cccc..0f3c6ac347 100644 --- a/Commands.hs +++ b/Commands.hs @@ -27,12 +27,18 @@ import Core import qualified Remotes import qualified TypeInternals +{- A subcommand can take one of several kinds of input parameters. -} +data SubCmdInput = FilesInGit FilePath | FilesNotInGit FilePath | + FilesMissing FilePath | Description String | Keys String | + Tempfile FilePath | FilesToBeCommitted FilePath + {- A subcommand runs in three stages. Each stage can return the next stage - to run. - - 1. The start stage is run before anything is printed about the - - subcommand, and can early abort it if the input does not make sense. - - It should run quickly and should not modify Annex state. + - subcommand, is passed some input, and can early abort it + - if the input does not make sense. It should run quickly and + - should not modify Annex state. - - 2. The perform stage is run after a message is printed about the subcommand - being run, and it should be where the bulk of the work happens. @@ -40,18 +46,18 @@ import qualified TypeInternals - 3. The cleanup stage is run only if the perform stage succeeds, and it - returns the overall success/fail of the subcommand. -} -type SubCmdStart = String -> Annex (Maybe SubCmdPerform) +type SubCmdStart = Annex (Maybe SubCmdPerform) type SubCmdPerform = Annex (Maybe SubCmdCleanup) type SubCmdCleanup = Annex Bool {- Runs a subcommand through its three stages. -} -doSubCmd :: String -> SubCmdStart -> String -> Annex Bool -doSubCmd cmdname start param = do - startres <- start param :: Annex (Maybe SubCmdPerform) +doSubCmd :: String -> SubCmdStart -> Annex Bool +doSubCmd cmdname start = do + startres <- start :: Annex (Maybe SubCmdPerform) case (startres) of Nothing -> return True Just perform -> do - showStart cmdname param + --showStart cmdname param performres <- perform :: Annex (Maybe SubCmdCleanup) case (performres) of Nothing -> do @@ -68,15 +74,10 @@ doSubCmd cmdname start param = do return False -{- A subcommand can broadly want one of several kinds of input parameters. - - This allows a first stage of filtering before starting a subcommand. -} -data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing - | Description | Keys | Tempfile | FilesToBeCommitted - data SubCommand = SubCommand { subcmdname :: String, - subcmdaction :: SubCmdStart, - subcmdwants :: SubCmdWants, + subcmdaction :: (SubCmdInput -> SubCmdStart), + subcmdinput :: (String -> SubCmdInput), subcmddesc :: String } subCmds :: [SubCommand] @@ -139,40 +140,53 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs showcmd c = (subcmdname c) ++ (pad 11 (subcmdname c)) ++ - (descWanted (subcmdwants c)) ++ - (pad 13 (descWanted (subcmdwants c))) ++ + (descSubCmdInput (subcmdinput c)) ++ + (pad 13 (descSubCmdInput (subcmdinput c))) ++ (subcmddesc c) indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' {- Generate descriptions of wanted parameters for subcommands. -} -descWanted :: SubCmdWants -> String -descWanted Description = "DESCRIPTION" -descWanted Keys = "KEY ..." -descWanted _ = "PATH ..." +descSubCmdInput :: (String -> SubCmdInput) -> String +descSubCmdInput Description = "DESCRIPTION" +descSubCmdInput Keys = "KEY ..." +descSubCmdInput _ = "PATH ..." + +{- Prepares a set of actions to run to handle a subcommand, based on + - the parameters passed to it. -} +prepSubCmd :: SubCommand -> Git.Repo -> [String] -> IO [Annex Bool] +prepSubCmd SubCommand { subcmdname = name, subcmdaction = action, + subcmdinput = input, subcmddesc = _ } repo params = do + input <- findInput input params repo + return $ map (doSubCmd name action) input {- Finds the type of parameters a subcommand wants, from among the passed - parameter list. -} -findWanted :: SubCmdWants -> [String] -> Git.Repo -> IO [String] -findWanted FilesNotInGit params repo = do +findInput :: (String -> SubCmdInput) -> [String] -> Git.Repo -> IO [SubCmdInput] +findInput FilesNotInGit params repo = do files <- mapM (Git.notInRepo repo) params - return $ foldl (++) [] files -findWanted FilesInGit params repo = do + return $ map FilesNotInGit $ notState $ foldl (++) [] files +findInput FilesInGit params repo = do files <- mapM (Git.inRepo repo) params - return $ foldl (++) [] files -findWanted FilesMissing params _ = do + return $ map FilesInGit $ notState $ foldl (++) [] files +findInput FilesMissing params _ = do files <- liftIO $ filterM missing params - return $ files + return $ map FilesMissing $ notState $ files where missing f = do e <- doesFileExist f return $ not e -findWanted Description params _ = do - return $ [unwords params] -findWanted FilesToBeCommitted params repo = do +findInput Description params _ = do + return $ map Description $ [unwords params] +findInput FilesToBeCommitted params repo = do files <- mapM (Git.stagedFiles repo) params - return $ foldl (++) [] files -findWanted _ params _ = return params + return $ map FilesToBeCommitted $ notState $ foldl (++) [] files +findInput Keys params _ = return $ map Keys params +findInput Tempfile params _ = return $ map Tempfile params + +{- filter out files from the state directory -} +notState :: [FilePath] -> [FilePath] +notState fs = filter (\f -> stateLoc /= take (length stateLoc) f) fs {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -184,20 +198,15 @@ parseCmd argv state = do when (null params) $ error usage case lookupCmd (params !! 0) of [] -> error usage - [SubCommand { subcmdname = name, subcmdaction = action, - subcmdwants = want, subcmddesc = _ }] -> do - files <- findWanted want (drop 1 params) - (TypeInternals.repo state) - let actions = map (doSubCmd name action) $ - filter notstate files + [subcommand] -> do + let repo = TypeInternals.repo state + actions <- prepSubCmd subcommand repo (drop 1 params) let configactions = map (\flag -> do flag return True) flags return (configactions, actions) _ -> error "internal error: multiple matching subcommands" where - -- never include files from the state directory - notstate f = stateLoc /= take (length stateLoc) f getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) @@ -206,8 +215,8 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: FilePath -> Annex (Maybe SubCmdPerform) -addStart file = notAnnexed file $ do +addStart :: SubCmdInput -> SubCmdStart +addStart (FilesNotInGit file) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return Nothing @@ -231,8 +240,8 @@ addCleanup file key = do return True {- The unannex subcommand undoes an add. -} -unannexStart :: FilePath -> Annex (Maybe SubCmdPerform) -unannexStart file = isAnnexed file $ \(key, backend) -> do +unannexStart :: SubCmdInput -> SubCmdStart +unannexStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do return $ Just $ unannexPerform file key backend unannexPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) unannexPerform file key backend = do @@ -255,8 +264,8 @@ unannexCleanup file key = do return True {- Gets an annexed file from one of the backends. -} -getStart :: FilePath -> Annex (Maybe SubCmdPerform) -getStart file = isAnnexed file $ \(key, backend) -> do +getStart :: SubCmdInput -> Annex (Maybe SubCmdPerform) +getStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return Nothing @@ -270,8 +279,8 @@ getPerform key backend = do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropStart :: FilePath -> Annex (Maybe SubCmdPerform) -dropStart file = isAnnexed file $ \(key, backend) -> do +dropStart :: SubCmdInput -> SubCmdStart +dropStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return Nothing @@ -295,8 +304,8 @@ dropCleanup key = do else return True {- Drops cached content for a key. -} -dropKeyStart :: String -> Annex (Maybe SubCmdPerform) -dropKeyStart keyname = do +dropKeyStart :: SubCmdInput -> SubCmdStart +dropKeyStart (Keys keyname) = do backends <- Backend.list let key = genKey (backends !! 0) keyname present <- inAnnex key @@ -318,8 +327,8 @@ dropKeyCleanup key = do return True {- Sets cached content for a key. -} -setKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) -setKeyStart tmpfile = do +setKeyStart :: SubCmdInput -> SubCmdStart +setKeyStart (Tempfile tmpfile) = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list @@ -339,8 +348,11 @@ setKeyCleanup key = do return True {- Fixes the symlink to an annexed file. -} -fixStart :: FilePath -> Annex (Maybe SubCmdPerform) -fixStart file = isAnnexed file $ \(key, _) -> do +fixStart :: SubCmdInput -> SubCmdStart +fixStart (FilesInGit file) = fixStart' file +fixStart (FilesToBeCommitted file) = fixStart' file +fixStart' :: FilePath -> SubCmdStart +fixStart' file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) @@ -358,8 +370,8 @@ fixCleanup file = do return True {- Stores description for the repository etc. -} -initStart :: String -> Annex (Maybe SubCmdPerform) -initStart description = do +initStart :: SubCmdInput -> SubCmdStart +initStart (Description description) = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage return $ Just $ initPerform description @@ -380,8 +392,8 @@ initCleanup = do return True {- Adds a file pointing at a manually-specified key -} -fromKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) -fromKeyStart file = do +fromKeyStart :: SubCmdInput -> SubCmdStart +fromKeyStart (FilesMissing file) = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list @@ -406,8 +418,8 @@ fromKeyCleanup file = do - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -moveStart :: FilePath -> Annex (Maybe SubCmdPerform) -moveStart file = do +moveStart :: SubCmdInput -> SubCmdStart +moveStart (FilesInGit file) = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" case (fromName, toName) of diff --git a/debian/changelog b/debian/changelog index cb6a0ef868..1709f63465 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,8 @@ git-annex (0.03) UNRELEASED; urgency=low from git before starting, and will be much faster with large repos. * Fix crash on unknown symlinks. * Added remote.annex-scp-options and remote.annex-ssh-options. + * The backends to use when adding different sets of files can be configured + via gitattributes. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 52a8c712f6..bbd7e8cab1 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -159,8 +159,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * --backend=name - Specify the default key-value backend to use, adding it to the front - of the list normally configured by `annex.backends`. + Specifies the key-value backend to use when adding a file. * --key=name @@ -186,7 +185,7 @@ Here are all the supported configuration settings. repositories (default: 1) * `annex.backends` -- space-separated list of names of the key-value backends to use. The first listed is used to store - new files. (default: "WORM SHA1 URL") + new files by default. (default: "WORM SHA1 URL") * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 100 for local repositories, and 200 for remote @@ -204,6 +203,15 @@ Here are all the supported configuration settings. * `annex.scp-options` and `annex.ssh-options` -- Default scp and ssh options to use if a remote does not have specific options. +The backend used when adding a new file to the annex can be configured +on a per-file-type basis via the `.gitattributes` file. In the file, +the `git-annex-backend` attribute can be set to the name of the backend to +use. For example, this here's how to use the WORM backend by default, +but the SHA1 backend for ogg files: + + * git-annex-backend=WORM + *.ogg git-annex-backend=SHA1 + # FILES These files are used, in your git repository: From fefaa5cc486cc435aa720a1fea29974c1ae17c4a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 17:01:27 -0400 Subject: [PATCH 0396/2835] big subcommand dispatch rework not quite done.. head hurts --- Commands.hs | 242 +++++++++++++++++++++++++--------------------------- 1 file changed, 114 insertions(+), 128 deletions(-) diff --git a/Commands.hs b/Commands.hs index 0f3c6ac347..9a41e21b73 100644 --- a/Commands.hs +++ b/Commands.hs @@ -27,82 +27,54 @@ import Core import qualified Remotes import qualified TypeInternals -{- A subcommand can take one of several kinds of input parameters. -} -data SubCmdInput = FilesInGit FilePath | FilesNotInGit FilePath | - FilesMissing FilePath | Description String | Keys String | - Tempfile FilePath | FilesToBeCommitted FilePath - -{- A subcommand runs in three stages. Each stage can return the next stage +{- A subcommand runs in four stages. Each stage can return the next stage - to run. - - - 1. The start stage is run before anything is printed about the - - subcommand, is passed some input, and can early abort it - - if the input does not make sense. It should run quickly and - - should not modify Annex state. - - - - 2. The perform stage is run after a message is printed about the subcommand - - being run, and it should be where the bulk of the work happens. - - - - 3. The cleanup stage is run only if the perform stage succeeds, and it - - returns the overall success/fail of the subcommand. - -} + - 0. The parse stage takes the parameters passed to the subcommand, + - looks through the repo to find the ones that are relevant + - to that subcommand (ie, new files to add), and returns a list of + - start stage actions to run. -} +type SubCmdParse = [String] -> Git.Repo -> IO [SubCmdStart] +{- 1. The start stage is run before anything is printed about the + - subcommand, is passed some input, and can early abort it + - if the input does not make sense. It should run quickly and + - should not modify Annex state. -} type SubCmdStart = Annex (Maybe SubCmdPerform) +{- 2. The perform stage is run after a message is printed about the subcommand + - being run, and it should be where the bulk of the work happens. -} type SubCmdPerform = Annex (Maybe SubCmdCleanup) +{- 3. The cleanup stage is run only if the perform stage succeeds, and it + - returns the overall success/fail of the subcommand. -} type SubCmdCleanup = Annex Bool -{- Runs a subcommand through its three stages. -} -doSubCmd :: String -> SubCmdStart -> Annex Bool -doSubCmd cmdname start = do - startres <- start :: Annex (Maybe SubCmdPerform) - case (startres) of - Nothing -> return True - Just perform -> do - --showStart cmdname param - performres <- perform :: Annex (Maybe SubCmdCleanup) - case (performres) of - Nothing -> do - showEndFail - return False - Just cleanup -> do - cleanupres <- cleanup - if (cleanupres) - then do - showEndOk - return True - else do - showEndFail - return False - - data SubCommand = SubCommand { subcmdname :: String, - subcmdaction :: (SubCmdInput -> SubCmdStart), - subcmdinput :: (String -> SubCmdInput), + subcmdparse :: SubCmdParse, subcmddesc :: String } subCmds :: [SubCommand] subCmds = [ - (SubCommand "add" addStart FilesNotInGit + (SubCommand "add" (withFilesNotInGit addStart) "add files to annex") - , (SubCommand "get" getStart FilesInGit + , (SubCommand "get" (withFilesInGit getStart) "make content of annexed files available") - , (SubCommand "drop" dropStart FilesInGit + , (SubCommand "drop" (withFilesInGit dropStart) "indicate content of files not currently wanted") - , (SubCommand "move" moveStart FilesInGit + , (SubCommand "move" (withFilesInGit moveStart) "transfer content of files to/from another repository") - , (SubCommand "init" initStart Description + , (SubCommand "init" (withDescription initStart) "initialize git-annex with repository description") - , (SubCommand "unannex" unannexStart FilesInGit + , (SubCommand "unannex" (withFilesInGit unannexStart) "undo accidential add command") - , (SubCommand "fix" fixStart FilesInGit + , (SubCommand "fix" (withFilesInGit fixStart) "fix up symlinks to point to annexed content") - , (SubCommand "pre-commit" fixStart FilesToBeCommitted + , (SubCommand "pre-commit" (withFilesToBeCommitted fixStart) "fix up symlinks before they are committed") - , (SubCommand "fromkey" fromKeyStart FilesMissing + , (SubCommand "fromkey" (withFilesMissing fromKeyStart) "adds a file using a specific key") - , (SubCommand "dropkey" dropKeyStart Keys + , (SubCommand "dropkey" (withKeys dropKeyStart) "drops annexed content for specified keys") - , (SubCommand "setkey" setKeyStart Tempfile + , (SubCommand "setkey" (withTempFile setKeyStart) "sets annexed content for a key using a temp file") ] @@ -140,49 +112,66 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs showcmd c = (subcmdname c) ++ (pad 11 (subcmdname c)) ++ - (descSubCmdInput (subcmdinput c)) ++ - (pad 13 (descSubCmdInput (subcmdinput c))) ++ (subcmddesc c) indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' -{- Generate descriptions of wanted parameters for subcommands. -} -descSubCmdInput :: (String -> SubCmdInput) -> String -descSubCmdInput Description = "DESCRIPTION" -descSubCmdInput Keys = "KEY ..." -descSubCmdInput _ = "PATH ..." - -{- Prepares a set of actions to run to handle a subcommand, based on +{- Prepares a set of actions to run to perform a subcommand, based on - the parameters passed to it. -} prepSubCmd :: SubCommand -> Git.Repo -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdname = name, subcmdaction = action, - subcmdinput = input, subcmddesc = _ } repo params = do - input <- findInput input params repo - return $ map (doSubCmd name action) input +prepSubCmd SubCommand { subcmdname = name, subcmdparse = parse, + subcmddesc = _ } repo params = do + list <- parse params repo :: IO [SubCmdStart] + return map (\a -> doSubCmd name a) list -{- Finds the type of parameters a subcommand wants, from among the passed - - parameter list. -} -findInput :: (String -> SubCmdInput) -> [String] -> Git.Repo -> IO [SubCmdInput] -findInput FilesNotInGit params repo = do +{- Runs a subcommand through the perform and cleanup stages -} +doSubCmd :: String -> SubCmdPerform -> SubCmdCleanup +doSubCmd cmdname perform = do + p <- perform + case (p) of + Nothing -> do + showEndFail + return False + Just cleanup -> do + c <- cleanup + if (c) + then do + showEndOk + return True + else do + showEndFail + return False + +{- These functions parse a user's parameters into a list of SubCmdStart + actions to perform. -} +type ParseStrings = (String -> SubCmdStart) -> SubCmdParse +withFilesNotInGit :: ParseStrings +withFilesNotInGit a params repo = do files <- mapM (Git.notInRepo repo) params - return $ map FilesNotInGit $ notState $ foldl (++) [] files -findInput FilesInGit params repo = do + return $ map a $ notState $ foldl (++) [] files +withFilesInGit :: ParseStrings +withFilesInGit a params repo = do files <- mapM (Git.inRepo repo) params - return $ map FilesInGit $ notState $ foldl (++) [] files -findInput FilesMissing params _ = do + return $ map a $ notState $ foldl (++) [] files +withFilesMissing :: ParseStrings +withFilesMissing a params _ = do files <- liftIO $ filterM missing params - return $ map FilesMissing $ notState $ files + return $ map a $ notState files where missing f = do e <- doesFileExist f return $ not e -findInput Description params _ = do - return $ map Description $ [unwords params] -findInput FilesToBeCommitted params repo = do +withDescription :: ParseStrings +withDescription a params _ = do + return $ [a $ unwords params] +withFilesToBeCommitted :: ParseStrings +withFilesToBeCommitted a params repo = do files <- mapM (Git.stagedFiles repo) params - return $ map FilesToBeCommitted $ notState $ foldl (++) [] files -findInput Keys params _ = return $ map Keys params -findInput Tempfile params _ = return $ map Tempfile params + return $ map a $ notState $ foldl (++) [] files +withKeys :: ParseStrings +withKeys a params _ = return $ map a params +withTempFile :: ParseStrings +withTempFile a params _ = return $ map a params {- filter out files from the state directory -} notState :: [FilePath] -> [FilePath] @@ -215,19 +204,19 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: SubCmdInput -> SubCmdStart -addStart (FilesNotInGit file) = notAnnexed file $ do +addStart :: FilePath -> SubCmdStart +addStart file = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return Nothing else return $ Just $ addPerform file -addPerform :: FilePath -> Annex (Maybe SubCmdCleanup) +addPerform :: FilePath -> SubCmdPerform addPerform file = do stored <- Backend.storeFileKey file case (stored) of Nothing -> return Nothing Just (key, _) -> return $ Just $ addCleanup file key -addCleanup :: FilePath -> Key -> Annex Bool +addCleanup :: FilePath -> Key -> SubCmdCleanup addCleanup file key = do logStatus key ValuePresent g <- Annex.gitRepo @@ -240,10 +229,10 @@ addCleanup file key = do return True {- The unannex subcommand undoes an add. -} -unannexStart :: SubCmdInput -> SubCmdStart -unannexStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do +unannexStart :: FilePath -> SubCmdStart +unannexStart file = isAnnexed file $ \(key, backend) -> do return $ Just $ unannexPerform file key backend -unannexPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) +unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform unannexPerform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True @@ -251,7 +240,7 @@ unannexPerform file key backend = do if (ok) then return $ Just $ unannexCleanup file key else return Nothing -unannexCleanup :: FilePath -> Key -> Annex Bool +unannexCleanup :: FilePath -> Key -> SubCmdCleanup unannexCleanup file key = do logStatus key ValueMissing g <- Annex.gitRepo @@ -264,13 +253,13 @@ unannexCleanup file key = do return True {- Gets an annexed file from one of the backends. -} -getStart :: SubCmdInput -> Annex (Maybe SubCmdPerform) -getStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do +getStart :: FilePath -> SubCmdStart +getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return Nothing else return $ Just $ getPerform key backend -getPerform :: Key -> Backend -> Annex (Maybe SubCmdCleanup) +getPerform :: Key -> Backend -> SubCmdPerform getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) @@ -279,19 +268,19 @@ getPerform key backend = do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropStart :: SubCmdInput -> SubCmdStart -dropStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do +dropStart :: FilePath -> SubCmdStart +dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return Nothing else return $ Just $ dropPerform key backend -dropPerform :: Key -> Backend -> Annex (Maybe SubCmdCleanup) +dropPerform :: Key -> Backend -> SubCmdPerform dropPerform key backend = do success <- Backend.removeKey backend key if (success) then return $ Just $ dropCleanup key else return Nothing -dropCleanup :: Key -> Annex Bool +dropCleanup :: Key -> SubCmdCleanup dropCleanup key = do logStatus key ValueMissing inannex <- inAnnex key @@ -304,8 +293,8 @@ dropCleanup key = do else return True {- Drops cached content for a key. -} -dropKeyStart :: SubCmdInput -> SubCmdStart -dropKeyStart (Keys keyname) = do +dropKeyStart :: String -> SubCmdStart +dropKeyStart keyname = do backends <- Backend.list let key = genKey (backends !! 0) keyname present <- inAnnex key @@ -315,26 +304,26 @@ dropKeyStart (Keys keyname) = do else if (not force) then error "dropkey is can cause data loss; use --force if you're sure you want to do this" else return $ Just $ dropKeyPerform key -dropKeyPerform :: Key -> Annex (Maybe SubCmdCleanup) +dropKeyPerform :: Key -> SubCmdPerform dropKeyPerform key = do g <- Annex.gitRepo let loc = annexLocation g key liftIO $ removeFile loc return $ Just $ dropKeyCleanup key -dropKeyCleanup :: Key -> Annex Bool +dropKeyCleanup :: Key -> SubCmdCleanup dropKeyCleanup key = do logStatus key ValueMissing return True {- Sets cached content for a key. -} -setKeyStart :: SubCmdInput -> SubCmdStart -setKeyStart (Tempfile tmpfile) = do +setKeyStart :: FilePath -> SubCmdStart +setKeyStart tmpfile = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname return $ Just $ setKeyPerform tmpfile key -setKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +setKeyPerform :: FilePath -> Key -> SubCmdPerform setKeyPerform tmpfile key = do g <- Annex.gitRepo let loc = annexLocation g key @@ -342,40 +331,37 @@ setKeyPerform tmpfile key = do if (not ok) then error "mv failed!" else return $ Just $ setKeyCleanup key -setKeyCleanup :: Key -> Annex Bool +setKeyCleanup :: Key -> SubCmdCleanup setKeyCleanup key = do logStatus key ValuePresent return True {- Fixes the symlink to an annexed file. -} -fixStart :: SubCmdInput -> SubCmdStart -fixStart (FilesInGit file) = fixStart' file -fixStart (FilesToBeCommitted file) = fixStart' file -fixStart' :: FilePath -> SubCmdStart -fixStart' file = isAnnexed file $ \(key, _) -> do +fixStart :: FilePath -> SubCmdStart +fixStart file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) then return Nothing else return $ Just $ fixPerform file link -fixPerform :: FilePath -> FilePath -> Annex (Maybe SubCmdCleanup) +fixPerform :: FilePath -> FilePath -> SubCmdPerform fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file return $ Just $ fixCleanup file -fixCleanup :: FilePath -> Annex Bool +fixCleanup :: FilePath -> SubCmdCleanup fixCleanup file = do Annex.queue "add" [] file return True {- Stores description for the repository etc. -} -initStart :: SubCmdInput -> SubCmdStart -initStart (Description description) = do +initStart :: String -> SubCmdStart +initStart description = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage return $ Just $ initPerform description -initPerform :: String -> Annex (Maybe SubCmdCleanup) +initPerform :: String -> SubCmdPerform initPerform description = do g <- Annex.gitRepo u <- getUUID g @@ -383,7 +369,7 @@ initPerform description = do liftIO $ gitAttributes g liftIO $ gitPreCommitHook g return $ Just $ initCleanup -initCleanup :: Annex Bool +initCleanup :: SubCmdCleanup initCleanup = do g <- Annex.gitRepo logfile <- uuidLog @@ -392,8 +378,8 @@ initCleanup = do return True {- Adds a file pointing at a manually-specified key -} -fromKeyStart :: SubCmdInput -> SubCmdStart -fromKeyStart (FilesMissing file) = do +fromKeyStart :: FilePath -> SubCmdStart +fromKeyStart file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list @@ -403,13 +389,13 @@ fromKeyStart (FilesMissing file) = do unless (inbackend) $ error $ "key ("++keyname++") is not present in backend" return $ Just $ fromKeyPerform file key -fromKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +fromKeyPerform :: FilePath -> Key -> SubCmdPerform fromKeyPerform file key = do link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ fromKeyCleanup file -fromKeyCleanup :: FilePath -> Annex Bool +fromKeyCleanup :: FilePath -> SubCmdCleanup fromKeyCleanup file = do Annex.queue "add" [] file return True @@ -418,8 +404,8 @@ fromKeyCleanup file = do - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -moveStart :: SubCmdInput -> SubCmdStart -moveStart (FilesInGit file) = do +moveStart :: FilePath -> SubCmdStart +moveStart file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" case (fromName, toName) of @@ -439,13 +425,13 @@ moveStart (FilesInGit file) = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -moveToStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveToStart :: FilePath -> SubCmdStart moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) then return Nothing -- not here, so nothing to do else return $ Just $ moveToPerform key -moveToPerform :: Key -> Annex (Maybe SubCmdCleanup) +moveToPerform :: Key -> SubCmdPerform moveToPerform key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote @@ -462,7 +448,7 @@ moveToPerform key = do then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ dropCleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> Annex Bool +moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", @@ -487,14 +473,14 @@ moveToCleanup remote key tmpfile = do - If the current repository already has the content, it is still removed - from the other repository. -} -moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveFromStart :: FilePath -> SubCmdStart moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key if (not $ null $ filter (\r -> Remotes.same r remote) l) then return $ Just $ moveFromPerform key else return Nothing -moveFromPerform :: Key -> Annex (Maybe SubCmdCleanup) +moveFromPerform :: Key -> SubCmdPerform moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key @@ -506,7 +492,7 @@ moveFromPerform key = do if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> Annex Bool +moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup moveFromCleanup remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), From 15e7d5913757ca8a76cbd83b3626a463fdea1767 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 17:12:58 -0400 Subject: [PATCH 0397/2835] rework complete --- Commands.hs | 75 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/Commands.hs b/Commands.hs index 9a41e21b73..99de9bbc86 100644 --- a/Commands.hs +++ b/Commands.hs @@ -119,28 +119,31 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Prepares a set of actions to run to perform a subcommand, based on - the parameters passed to it. -} prepSubCmd :: SubCommand -> Git.Repo -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdname = name, subcmdparse = parse, - subcmddesc = _ } repo params = do +prepSubCmd SubCommand { subcmdparse = parse } repo params = do list <- parse params repo :: IO [SubCmdStart] - return map (\a -> doSubCmd name a) list + return $ map (\a -> doSubCmd a) list -{- Runs a subcommand through the perform and cleanup stages -} -doSubCmd :: String -> SubCmdPerform -> SubCmdCleanup -doSubCmd cmdname perform = do - p <- perform - case (p) of - Nothing -> do - showEndFail - return False - Just cleanup -> do - c <- cleanup - if (c) - then do - showEndOk - return True - else do +{- Runs a subcommand through the start, perform and cleanup stages -} +doSubCmd :: SubCmdStart -> SubCmdCleanup +doSubCmd start = do + s <- start + case (s) of + Nothing -> return True + Just perform -> do + p <- perform + case (p) of + Nothing -> do showEndFail return False + Just cleanup -> do + c <- cleanup + if (c) + then do + showEndOk + return True + else do + showEndFail + return False {- These functions parse a user's parameters into a list of SubCmdStart actions to perform. -} @@ -209,7 +212,9 @@ addStart file = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return Nothing - else return $ Just $ addPerform file + else do + showStart "add" file + return $ Just $ addPerform file addPerform :: FilePath -> SubCmdPerform addPerform file = do stored <- Backend.storeFileKey file @@ -231,6 +236,7 @@ addCleanup file key = do {- The unannex subcommand undoes an add. -} unannexStart :: FilePath -> SubCmdStart unannexStart file = isAnnexed file $ \(key, backend) -> do + showStart "unannex" file return $ Just $ unannexPerform file key backend unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform unannexPerform file key backend = do @@ -258,7 +264,9 @@ getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return Nothing - else return $ Just $ getPerform key backend + else do + showStart "get" file + return $ Just $ getPerform key backend getPerform :: Key -> Backend -> SubCmdPerform getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) @@ -273,7 +281,9 @@ dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return Nothing - else return $ Just $ dropPerform key backend + else do + showStart "drop" file + return $ Just $ dropPerform key backend dropPerform :: Key -> Backend -> SubCmdPerform dropPerform key backend = do success <- Backend.removeKey backend key @@ -303,7 +313,9 @@ dropKeyStart keyname = do then return Nothing else if (not force) then error "dropkey is can cause data loss; use --force if you're sure you want to do this" - else return $ Just $ dropKeyPerform key + else do + showStart "dropkey" keyname + return $ Just $ dropKeyPerform key dropKeyPerform :: Key -> SubCmdPerform dropKeyPerform key = do g <- Annex.gitRepo @@ -322,6 +334,7 @@ setKeyStart tmpfile = do when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname + showStart "setkey" tmpfile return $ Just $ setKeyPerform tmpfile key setKeyPerform :: FilePath -> Key -> SubCmdPerform setKeyPerform tmpfile key = do @@ -343,7 +356,9 @@ fixStart file = isAnnexed file $ \(key, _) -> do l <- liftIO $ readSymbolicLink file if (link == l) then return Nothing - else return $ Just $ fixPerform file link + else do + showStart "fix" file + return $ Just $ fixPerform file link fixPerform :: FilePath -> FilePath -> SubCmdPerform fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) @@ -360,6 +375,7 @@ initStart :: String -> SubCmdStart initStart description = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage + showStart "init" description return $ Just $ initPerform description initPerform :: String -> SubCmdPerform initPerform description = do @@ -388,6 +404,7 @@ fromKeyStart file = do inbackend <- Backend.hasKey key unless (inbackend) $ error $ "key ("++keyname++") is not present in backend" + showStart "fromkey" file return $ Just $ fromKeyPerform file key fromKeyPerform :: FilePath -> Key -> SubCmdPerform fromKeyPerform file key = do @@ -430,7 +447,9 @@ moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) then return Nothing -- not here, so nothing to do - else return $ Just $ moveToPerform key + else do + showStart "move" file + return $ Just $ moveToPerform key moveToPerform :: Key -> SubCmdPerform moveToPerform key = do -- checking the remote is expensive, so not done in the start step @@ -477,9 +496,11 @@ moveFromStart :: FilePath -> SubCmdStart moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key - if (not $ null $ filter (\r -> Remotes.same r remote) l) - then return $ Just $ moveFromPerform key - else return Nothing + if (null $ filter (\r -> Remotes.same r remote) l) + then return Nothing + else do + showStart "move" file + return $ Just $ moveFromPerform key moveFromPerform :: Key -> SubCmdPerform moveFromPerform key = do remote <- Remotes.commandLineRemote From 287e6e5c1328071ec9b934f75d5250b37a066afe Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 17:18:32 -0400 Subject: [PATCH 0398/2835] bring back param descriptions in usage --- Commands.hs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/Commands.hs b/Commands.hs index 99de9bbc86..80b355f792 100644 --- a/Commands.hs +++ b/Commands.hs @@ -49,34 +49,39 @@ type SubCmdCleanup = Annex Bool data SubCommand = SubCommand { subcmdname :: String, + subcmdparams :: String, subcmdparse :: SubCmdParse, subcmddesc :: String } subCmds :: [SubCommand] subCmds = [ - (SubCommand "add" (withFilesNotInGit addStart) + (SubCommand "add" path (withFilesNotInGit addStart) "add files to annex") - , (SubCommand "get" (withFilesInGit getStart) + , (SubCommand "get" path (withFilesInGit getStart) "make content of annexed files available") - , (SubCommand "drop" (withFilesInGit dropStart) + , (SubCommand "drop" path (withFilesInGit dropStart) "indicate content of files not currently wanted") - , (SubCommand "move" (withFilesInGit moveStart) + , (SubCommand "move" path (withFilesInGit moveStart) "transfer content of files to/from another repository") - , (SubCommand "init" (withDescription initStart) + , (SubCommand "init" desc (withDescription initStart) "initialize git-annex with repository description") - , (SubCommand "unannex" (withFilesInGit unannexStart) + , (SubCommand "unannex" path (withFilesInGit unannexStart) "undo accidential add command") - , (SubCommand "fix" (withFilesInGit fixStart) + , (SubCommand "fix" path (withFilesInGit fixStart) "fix up symlinks to point to annexed content") - , (SubCommand "pre-commit" (withFilesToBeCommitted fixStart) + , (SubCommand "pre-commit" path (withFilesToBeCommitted fixStart) "fix up symlinks before they are committed") - , (SubCommand "fromkey" (withFilesMissing fromKeyStart) + , (SubCommand "fromkey" key (withFilesMissing fromKeyStart) "adds a file using a specific key") - , (SubCommand "dropkey" (withKeys dropKeyStart) + , (SubCommand "dropkey" key (withKeys dropKeyStart) "drops annexed content for specified keys") - , (SubCommand "setkey" (withTempFile setKeyStart) + , (SubCommand "setkey" key (withTempFile setKeyStart) "sets annexed content for a key using a temp file") ] + where + path = "PATH ..." + key = "KEY ..." + desc = "DESCRIPTION" -- Each dashed command-line option results in generation of an action -- in the Annex monad that performs the necessary setting. @@ -112,6 +117,8 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs showcmd c = (subcmdname c) ++ (pad 11 (subcmdname c)) ++ + (subcmdparams c) ++ + (pad 13 (subcmdparams c)) ++ (subcmddesc c) indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' From 899a86f8f9601e359a894fe2839dea9cf7f47def Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 17:50:37 -0400 Subject: [PATCH 0399/2835] now only need to add gitattributes lookup --- Backend.hs | 22 +++++++++++++----- Commands.hs | 64 ++++++++++++++++++++++++++++------------------------- 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/Backend.hs b/Backend.hs index f1b4c28979..693e1371b5 100644 --- a/Backend.hs +++ b/Backend.hs @@ -23,7 +23,8 @@ module Backend ( retrieveKeyFile, removeKey, hasKey, - lookupFile + lookupFile, + chooseBackends ) where import Control.Monad.State @@ -74,12 +75,15 @@ maybeLookupBackendName bs s = where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} -storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) -storeFileKey file = do +storeFileKey :: FilePath -> Maybe Backend -> Annex (Maybe (Key, Backend)) +storeFileKey file trybackend = do g <- Annex.gitRepo let relfile = Git.relative g file - b <- list - storeFileKey' b file relfile + bs <- list + let bs' = case trybackend of + Nothing -> bs + Just backend -> backend:bs + storeFileKey' bs' file relfile storeFileKey' :: [Backend] -> FilePath -> FilePath -> Annex (Maybe (Key, Backend)) storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do @@ -136,3 +140,11 @@ lookupFile file = do kname = keyName k skip = "skipping " ++ file ++ " (unknown backend " ++ bname ++ ")" + +{- Looks up the backends that should be used for each file in a list. + - That can be configured on a per-file basis in the gitattributes file. + -} +chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe Backend)] +chooseBackends fs = do + -- TODO + return $ map (\f -> (f, Nothing)) fs diff --git a/Commands.hs b/Commands.hs index 80b355f792..c012cdca03 100644 --- a/Commands.hs +++ b/Commands.hs @@ -25,7 +25,6 @@ import LocationLog import Types import Core import qualified Remotes -import qualified TypeInternals {- A subcommand runs in four stages. Each stage can return the next stage - to run. @@ -34,7 +33,7 @@ import qualified TypeInternals - looks through the repo to find the ones that are relevant - to that subcommand (ie, new files to add), and returns a list of - start stage actions to run. -} -type SubCmdParse = [String] -> Git.Repo -> IO [SubCmdStart] +type SubCmdParse = [String] -> Annex [SubCmdStart] {- 1. The start stage is run before anything is printed about the - subcommand, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and @@ -125,9 +124,9 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Prepares a set of actions to run to perform a subcommand, based on - the parameters passed to it. -} -prepSubCmd :: SubCommand -> Git.Repo -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdparse = parse } repo params = do - list <- parse params repo :: IO [SubCmdStart] +prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] +prepSubCmd SubCommand { subcmdparse = parse } state params = do + list <- Annex.eval state $ parse params return $ map (\a -> doSubCmd a) list {- Runs a subcommand through the start, perform and cleanup stages -} @@ -155,37 +154,43 @@ doSubCmd start = do {- These functions parse a user's parameters into a list of SubCmdStart actions to perform. -} type ParseStrings = (String -> SubCmdStart) -> SubCmdParse -withFilesNotInGit :: ParseStrings -withFilesNotInGit a params repo = do - files <- mapM (Git.notInRepo repo) params - return $ map a $ notState $ foldl (++) [] files +type ParseBackendFiles = ((FilePath, Maybe Backend) -> SubCmdStart) -> SubCmdParse +withFilesNotInGit :: ParseBackendFiles +withFilesNotInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.notInRepo repo) params + let files' = foldl (++) [] files + pairs <- Backend.chooseBackends files' + return $ map a $ filter (\(f,_) -> notState f) pairs withFilesInGit :: ParseStrings -withFilesInGit a params repo = do - files <- mapM (Git.inRepo repo) params - return $ map a $ notState $ foldl (++) [] files +withFilesInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.inRepo repo) params + return $ map a $ filter notState $ foldl (++) [] files withFilesMissing :: ParseStrings -withFilesMissing a params _ = do +withFilesMissing a params = do files <- liftIO $ filterM missing params - return $ map a $ notState files + return $ map a $ filter notState files where missing f = do e <- doesFileExist f return $ not e withDescription :: ParseStrings -withDescription a params _ = do +withDescription a params = do return $ [a $ unwords params] withFilesToBeCommitted :: ParseStrings -withFilesToBeCommitted a params repo = do - files <- mapM (Git.stagedFiles repo) params - return $ map a $ notState $ foldl (++) [] files +withFilesToBeCommitted a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.stagedFiles repo) params + return $ map a $ filter notState $ foldl (++) [] files withKeys :: ParseStrings -withKeys a params _ = return $ map a params +withKeys a params = return $ map a params withTempFile :: ParseStrings -withTempFile a params _ = return $ map a params +withTempFile a params = return $ map a params {- filter out files from the state directory -} -notState :: [FilePath] -> [FilePath] -notState fs = filter (\f -> stateLoc /= take (length stateLoc) f) fs +notState :: FilePath -> Bool +notState f = stateLoc /= take (length stateLoc) f {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -198,8 +203,7 @@ parseCmd argv state = do case lookupCmd (params !! 0) of [] -> error usage [subcommand] -> do - let repo = TypeInternals.repo state - actions <- prepSubCmd subcommand repo (drop 1 params) + actions <- prepSubCmd subcommand state (drop 1 params) let configactions = map (\flag -> do flag return True) flags @@ -214,17 +218,17 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: FilePath -> SubCmdStart -addStart file = notAnnexed file $ do +addStart :: (FilePath, Maybe Backend) -> SubCmdStart +addStart pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return Nothing else do showStart "add" file - return $ Just $ addPerform file -addPerform :: FilePath -> SubCmdPerform -addPerform file = do - stored <- Backend.storeFileKey file + return $ Just $ addPerform pair +addPerform :: (FilePath, Maybe Backend) -> SubCmdPerform +addPerform (file, backend) = do + stored <- Backend.storeFileKey file backend case (stored) of Nothing -> return Nothing Just (key, _) -> return $ Just $ addCleanup file key From ab3294f1dd734e7573c859e123158eda8e3551ac Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 18:24:19 -0400 Subject: [PATCH 0400/2835] wrote checkAttr --- GitRepo.hs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 505dd06eb2..e8dd0a5dc3 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -35,7 +35,8 @@ module GitRepo ( repoRemoteName, inRepo, notInRepo, - stagedFiles + stagedFiles, + checkAttr ) where import Monad (unless) @@ -145,6 +146,26 @@ attributes repo | bare repo = (workTree repo) ++ "/info/.gitattributes" | otherwise = (workTree repo) ++ "/.gitattributes" +{- Looks up a gitattributes value for each file in a list. -} +checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] +checkAttr repo attr files = do + (pid, fromh, toh) <- hPipeBoth "git" $ + gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] + -- git-check-attr reads all its stdin before outputting anything, + -- so we don't need to worry about deadlock + hPutStr toh files0 + hClose toh + c <- hGetContentsStrict fromh + hClose fromh + forceSuccess pid + return $ map topair $ lines c + where + files0 = join "\0" files + topair l = (bits !! 0, join sep $ drop 1 $ bits) + where + bits = split sep l + sep = ": " ++ attr ++ ": " + {- Path to a repository's .git directory, relative to its workTree. -} gitDir :: Repo -> String gitDir repo From 2926cc64fb838d1ddaa4b9e123a2eaec6c19cc71 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 19:05:38 -0400 Subject: [PATCH 0401/2835] In .gitattributes, the git-annex-backend attribute can be set to the names of backends to use when adding different types of files. --- Backend.hs | 6 ++++-- debian/changelog | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Backend.hs b/Backend.hs index 693e1371b5..e1d0e0a686 100644 --- a/Backend.hs +++ b/Backend.hs @@ -146,5 +146,7 @@ lookupFile file = do -} chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe Backend)] chooseBackends fs = do - -- TODO - return $ map (\f -> (f, Nothing)) fs + g <- Annex.gitRepo + bs <- Annex.supportedBackends + pairs <- liftIO $ Git.checkAttr g "git-annex-backend" fs + return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs diff --git a/debian/changelog b/debian/changelog index 1709f63465..b433ec62fd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,8 @@ git-annex (0.03) UNRELEASED; urgency=low * Added remote.annex-scp-options and remote.annex-ssh-options. * The backends to use when adding different sets of files can be configured via gitattributes. + * In .gitattributes, the git-annex-backend attribute can be set to the + names of backends to use when adding different types of files. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 From f1f4bdcd6054303a5711944e146093d2f4f8069b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 19:11:06 -0400 Subject: [PATCH 0402/2835] expand docs --- doc/backends.mdwn | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/doc/backends.mdwn b/doc/backends.mdwn index cd587726c0..dc359174ae 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -6,8 +6,8 @@ When a file is annexed, a key is generated from its content and/or metadata. The file checked into git symlinks to the key. This key can later be used to retrieve the file's content (its value). -Multiple pluggable backends are supported, and more than one can be used -to store different files' contents in a given repository. +Multiple pluggable backends are supported, and a single repository +can use different backends for different files. * `WORM` ("Write Once, Read Many") This backend stores the file's content only in `.git/annex/`, and assumes that any file with the same basename, @@ -20,3 +20,18 @@ to store different files' contents in a given repository. can make it slower for large files. **Warning** this backend is not ready for use. * `URL` -- This backend downloads the file's content from an external URL. + +The `annex.backends` git-config setting can be used to list the backends +git-annex should use. The first one listed will be used by default when +new files are added. + +For finer control of what backend is used when adding different types of +files, the `.gitattributes` file can be used. The `git-annex-backend` +attribute can be set to the name of the backend to use for matching files. + +For example, to use the SHA1 backend for sound files, which tend to be +smallish and might be modified over time, you could set in +`.gitattributes`: + + *.mp3 git-annex-backend=SHA1 + *.ogg git-annex-backend=SHA1 From 1f9996f7424a6581ed92b56e3ad3f298348c1daf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 19:18:47 -0400 Subject: [PATCH 0403/2835] less confusing names for the subcommand stage types --- Commands.hs | 96 ++++++++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/Commands.hs b/Commands.hs index c012cdca03..f4f675d025 100644 --- a/Commands.hs +++ b/Commands.hs @@ -33,23 +33,25 @@ import qualified Remotes - looks through the repo to find the ones that are relevant - to that subcommand (ie, new files to add), and returns a list of - start stage actions to run. -} -type SubCmdParse = [String] -> Annex [SubCmdStart] +type SubCmdParseStrings = (String -> SubCmdPerform) -> SubCmdStart +type SubCmdParseBackendFiles = ((FilePath, Maybe Backend) -> SubCmdPerform) -> SubCmdStart {- 1. The start stage is run before anything is printed about the - subcommand, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and - should not modify Annex state. -} -type SubCmdStart = Annex (Maybe SubCmdPerform) +type SubCmdStart = [String] -> Annex [SubCmdPerform] {- 2. The perform stage is run after a message is printed about the subcommand - being run, and it should be where the bulk of the work happens. -} type SubCmdPerform = Annex (Maybe SubCmdCleanup) {- 3. The cleanup stage is run only if the perform stage succeeds, and it - returns the overall success/fail of the subcommand. -} -type SubCmdCleanup = Annex Bool +type SubCmdCleanup = Annex (Maybe SubCmdStatus) +type SubCmdStatus = Annex Bool data SubCommand = SubCommand { subcmdname :: String, subcmdparams :: String, - subcmdparse :: SubCmdParse, + subcmdparse :: SubCmdStart, subcmddesc :: String } subCmds :: [SubCommand] @@ -130,7 +132,7 @@ prepSubCmd SubCommand { subcmdparse = parse } state params = do return $ map (\a -> doSubCmd a) list {- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdStart -> SubCmdCleanup +doSubCmd :: SubCmdPerform -> SubCmdStatus doSubCmd start = do s <- start case (s) of @@ -151,23 +153,21 @@ doSubCmd start = do showEndFail return False -{- These functions parse a user's parameters into a list of SubCmdStart +{- These functions parse a user's parameters into a list of SubCmdPerform actions to perform. -} -type ParseStrings = (String -> SubCmdStart) -> SubCmdParse -type ParseBackendFiles = ((FilePath, Maybe Backend) -> SubCmdStart) -> SubCmdParse -withFilesNotInGit :: ParseBackendFiles +withFilesNotInGit :: SubCmdParseBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.notInRepo repo) params let files' = foldl (++) [] files pairs <- Backend.chooseBackends files' return $ map a $ filter (\(f,_) -> notState f) pairs -withFilesInGit :: ParseStrings +withFilesInGit :: SubCmdParseStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params return $ map a $ filter notState $ foldl (++) [] files -withFilesMissing :: ParseStrings +withFilesMissing :: SubCmdParseStrings withFilesMissing a params = do files <- liftIO $ filterM missing params return $ map a $ filter notState files @@ -175,17 +175,17 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e -withDescription :: ParseStrings +withDescription :: SubCmdParseStrings withDescription a params = do return $ [a $ unwords params] -withFilesToBeCommitted :: ParseStrings +withFilesToBeCommitted :: SubCmdParseStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.stagedFiles repo) params return $ map a $ filter notState $ foldl (++) [] files -withKeys :: ParseStrings +withKeys :: SubCmdParseStrings withKeys a params = return $ map a params -withTempFile :: ParseStrings +withTempFile :: SubCmdParseStrings withTempFile a params = return $ map a params {- filter out files from the state directory -} @@ -218,7 +218,7 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: (FilePath, Maybe Backend) -> SubCmdStart +addStart :: (FilePath, Maybe Backend) -> SubCmdPerform addStart pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -226,13 +226,13 @@ addStart pair@(file, _) = notAnnexed file $ do else do showStart "add" file return $ Just $ addPerform pair -addPerform :: (FilePath, Maybe Backend) -> SubCmdPerform +addPerform :: (FilePath, Maybe Backend) -> SubCmdCleanup addPerform (file, backend) = do stored <- Backend.storeFileKey file backend case (stored) of Nothing -> return Nothing Just (key, _) -> return $ Just $ addCleanup file key -addCleanup :: FilePath -> Key -> SubCmdCleanup +addCleanup :: FilePath -> Key -> SubCmdStatus addCleanup file key = do logStatus key ValuePresent g <- Annex.gitRepo @@ -245,11 +245,11 @@ addCleanup file key = do return True {- The unannex subcommand undoes an add. -} -unannexStart :: FilePath -> SubCmdStart +unannexStart :: FilePath -> SubCmdPerform unannexStart file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file return $ Just $ unannexPerform file key backend -unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform +unannexPerform :: FilePath -> Key -> Backend -> SubCmdCleanup unannexPerform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True @@ -257,7 +257,7 @@ unannexPerform file key backend = do if (ok) then return $ Just $ unannexCleanup file key else return Nothing -unannexCleanup :: FilePath -> Key -> SubCmdCleanup +unannexCleanup :: FilePath -> Key -> SubCmdStatus unannexCleanup file key = do logStatus key ValueMissing g <- Annex.gitRepo @@ -270,7 +270,7 @@ unannexCleanup file key = do return True {- Gets an annexed file from one of the backends. -} -getStart :: FilePath -> SubCmdStart +getStart :: FilePath -> SubCmdPerform getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) @@ -278,7 +278,7 @@ getStart file = isAnnexed file $ \(key, backend) -> do else do showStart "get" file return $ Just $ getPerform key backend -getPerform :: Key -> Backend -> SubCmdPerform +getPerform :: Key -> Backend -> SubCmdCleanup getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) @@ -287,7 +287,7 @@ getPerform key backend = do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropStart :: FilePath -> SubCmdStart +dropStart :: FilePath -> SubCmdPerform dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) @@ -295,13 +295,13 @@ dropStart file = isAnnexed file $ \(key, backend) -> do else do showStart "drop" file return $ Just $ dropPerform key backend -dropPerform :: Key -> Backend -> SubCmdPerform +dropPerform :: Key -> Backend -> SubCmdCleanup dropPerform key backend = do success <- Backend.removeKey backend key if (success) then return $ Just $ dropCleanup key else return Nothing -dropCleanup :: Key -> SubCmdCleanup +dropCleanup :: Key -> SubCmdStatus dropCleanup key = do logStatus key ValueMissing inannex <- inAnnex key @@ -314,7 +314,7 @@ dropCleanup key = do else return True {- Drops cached content for a key. -} -dropKeyStart :: String -> SubCmdStart +dropKeyStart :: String -> SubCmdPerform dropKeyStart keyname = do backends <- Backend.list let key = genKey (backends !! 0) keyname @@ -327,19 +327,19 @@ dropKeyStart keyname = do else do showStart "dropkey" keyname return $ Just $ dropKeyPerform key -dropKeyPerform :: Key -> SubCmdPerform +dropKeyPerform :: Key -> SubCmdCleanup dropKeyPerform key = do g <- Annex.gitRepo let loc = annexLocation g key liftIO $ removeFile loc return $ Just $ dropKeyCleanup key -dropKeyCleanup :: Key -> SubCmdCleanup +dropKeyCleanup :: Key -> SubCmdStatus dropKeyCleanup key = do logStatus key ValueMissing return True {- Sets cached content for a key. -} -setKeyStart :: FilePath -> SubCmdStart +setKeyStart :: FilePath -> SubCmdPerform setKeyStart tmpfile = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -347,7 +347,7 @@ setKeyStart tmpfile = do let key = genKey (backends !! 0) keyname showStart "setkey" tmpfile return $ Just $ setKeyPerform tmpfile key -setKeyPerform :: FilePath -> Key -> SubCmdPerform +setKeyPerform :: FilePath -> Key -> SubCmdCleanup setKeyPerform tmpfile key = do g <- Annex.gitRepo let loc = annexLocation g key @@ -355,13 +355,13 @@ setKeyPerform tmpfile key = do if (not ok) then error "mv failed!" else return $ Just $ setKeyCleanup key -setKeyCleanup :: Key -> SubCmdCleanup +setKeyCleanup :: Key -> SubCmdStatus setKeyCleanup key = do logStatus key ValuePresent return True {- Fixes the symlink to an annexed file. -} -fixStart :: FilePath -> SubCmdStart +fixStart :: FilePath -> SubCmdPerform fixStart file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file @@ -370,25 +370,25 @@ fixStart file = isAnnexed file $ \(key, _) -> do else do showStart "fix" file return $ Just $ fixPerform file link -fixPerform :: FilePath -> FilePath -> SubCmdPerform +fixPerform :: FilePath -> FilePath -> SubCmdCleanup fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file return $ Just $ fixCleanup file -fixCleanup :: FilePath -> SubCmdCleanup +fixCleanup :: FilePath -> SubCmdStatus fixCleanup file = do Annex.queue "add" [] file return True {- Stores description for the repository etc. -} -initStart :: String -> SubCmdStart +initStart :: String -> SubCmdPerform initStart description = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage showStart "init" description return $ Just $ initPerform description -initPerform :: String -> SubCmdPerform +initPerform :: String -> SubCmdCleanup initPerform description = do g <- Annex.gitRepo u <- getUUID g @@ -396,7 +396,7 @@ initPerform description = do liftIO $ gitAttributes g liftIO $ gitPreCommitHook g return $ Just $ initCleanup -initCleanup :: SubCmdCleanup +initCleanup :: SubCmdStatus initCleanup = do g <- Annex.gitRepo logfile <- uuidLog @@ -405,7 +405,7 @@ initCleanup = do return True {- Adds a file pointing at a manually-specified key -} -fromKeyStart :: FilePath -> SubCmdStart +fromKeyStart :: FilePath -> SubCmdPerform fromKeyStart file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -417,13 +417,13 @@ fromKeyStart file = do "key ("++keyname++") is not present in backend" showStart "fromkey" file return $ Just $ fromKeyPerform file key -fromKeyPerform :: FilePath -> Key -> SubCmdPerform +fromKeyPerform :: FilePath -> Key -> SubCmdCleanup fromKeyPerform file key = do link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ fromKeyCleanup file -fromKeyCleanup :: FilePath -> SubCmdCleanup +fromKeyCleanup :: FilePath -> SubCmdStatus fromKeyCleanup file = do Annex.queue "add" [] file return True @@ -432,7 +432,7 @@ fromKeyCleanup file = do - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -moveStart :: FilePath -> SubCmdStart +moveStart :: FilePath -> SubCmdPerform moveStart file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" @@ -453,7 +453,7 @@ moveStart file = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -moveToStart :: FilePath -> SubCmdStart +moveToStart :: FilePath -> SubCmdPerform moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) @@ -461,7 +461,7 @@ moveToStart file = isAnnexed file $ \(key, _) -> do else do showStart "move" file return $ Just $ moveToPerform key -moveToPerform :: Key -> SubCmdPerform +moveToPerform :: Key -> SubCmdCleanup moveToPerform key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote @@ -478,7 +478,7 @@ moveToPerform key = do then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ dropCleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup +moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdStatus moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", @@ -503,7 +503,7 @@ moveToCleanup remote key tmpfile = do - If the current repository already has the content, it is still removed - from the other repository. -} -moveFromStart :: FilePath -> SubCmdStart +moveFromStart :: FilePath -> SubCmdPerform moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key @@ -512,7 +512,7 @@ moveFromStart file = isAnnexed file $ \(key, _) -> do else do showStart "move" file return $ Just $ moveFromPerform key -moveFromPerform :: Key -> SubCmdPerform +moveFromPerform :: Key -> SubCmdCleanup moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key @@ -524,7 +524,7 @@ moveFromPerform key = do if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup +moveFromCleanup :: Git.Repo -> Key -> SubCmdStatus moveFromCleanup remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), From f0bf94f76021952f0a4803ab13c8dfdc3c78b148 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 20:03:21 -0400 Subject: [PATCH 0404/2835] better types --- Commands.hs | 65 +++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/Commands.hs b/Commands.hs index f4f675d025..01761d0309 100644 --- a/Commands.hs +++ b/Commands.hs @@ -26,20 +26,21 @@ import Types import Core import qualified Remotes -{- A subcommand runs in four stages. Each stage can return the next stage - - to run. +{- A subcommand runs in four stages. - - - 0. The parse stage takes the parameters passed to the subcommand, + - 0. The seek stage takes the parameters passed to the subcommand, - looks through the repo to find the ones that are relevant - - to that subcommand (ie, new files to add), and returns a list of - - start stage actions to run. -} -type SubCmdParseStrings = (String -> SubCmdPerform) -> SubCmdStart -type SubCmdParseBackendFiles = ((FilePath, Maybe Backend) -> SubCmdPerform) -> SubCmdStart + - to that subcommand (ie, new files to add), and generates + - a start stage action. -} +type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek +type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek +type SubCmdSeek = [String] -> Annex [SubCmdPerform] {- 1. The start stage is run before anything is printed about the - subcommand, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and - should not modify Annex state. -} -type SubCmdStart = [String] -> Annex [SubCmdPerform] +type SubCmdStartString = String -> SubCmdPerform +type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdPerform {- 2. The perform stage is run after a message is printed about the subcommand - being run, and it should be where the bulk of the work happens. -} type SubCmdPerform = Annex (Maybe SubCmdCleanup) @@ -51,7 +52,7 @@ type SubCmdStatus = Annex Bool data SubCommand = SubCommand { subcmdname :: String, subcmdparams :: String, - subcmdparse :: SubCmdStart, + subcmdseek :: SubCmdSeek, subcmddesc :: String } subCmds :: [SubCommand] @@ -127,8 +128,8 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Prepares a set of actions to run to perform a subcommand, based on - the parameters passed to it. -} prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdparse = parse } state params = do - list <- Annex.eval state $ parse params +prepSubCmd SubCommand { subcmdseek = seek } state params = do + list <- Annex.eval state $ seek params return $ map (\a -> doSubCmd a) list {- Runs a subcommand through the start, perform and cleanup stages -} @@ -153,21 +154,21 @@ doSubCmd start = do showEndFail return False -{- These functions parse a user's parameters into a list of SubCmdPerform - actions to perform. -} -withFilesNotInGit :: SubCmdParseBackendFiles +{- These functions find appropriate files or other things based on a + user's parameters. -} +withFilesNotInGit :: SubCmdSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.notInRepo repo) params let files' = foldl (++) [] files pairs <- Backend.chooseBackends files' return $ map a $ filter (\(f,_) -> notState f) pairs -withFilesInGit :: SubCmdParseStrings +withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params return $ map a $ filter notState $ foldl (++) [] files -withFilesMissing :: SubCmdParseStrings +withFilesMissing :: SubCmdSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params return $ map a $ filter notState files @@ -175,17 +176,17 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e -withDescription :: SubCmdParseStrings +withDescription :: SubCmdSeekStrings withDescription a params = do return $ [a $ unwords params] -withFilesToBeCommitted :: SubCmdParseStrings +withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.stagedFiles repo) params return $ map a $ filter notState $ foldl (++) [] files -withKeys :: SubCmdParseStrings +withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params -withTempFile :: SubCmdParseStrings +withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params {- filter out files from the state directory -} @@ -218,7 +219,7 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: (FilePath, Maybe Backend) -> SubCmdPerform +addStart :: SubCmdStartBackendFile addStart pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -245,7 +246,7 @@ addCleanup file key = do return True {- The unannex subcommand undoes an add. -} -unannexStart :: FilePath -> SubCmdPerform +unannexStart :: SubCmdStartString unannexStart file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file return $ Just $ unannexPerform file key backend @@ -270,7 +271,7 @@ unannexCleanup file key = do return True {- Gets an annexed file from one of the backends. -} -getStart :: FilePath -> SubCmdPerform +getStart :: SubCmdStartString getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) @@ -287,7 +288,7 @@ getPerform key backend = do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropStart :: FilePath -> SubCmdPerform +dropStart :: SubCmdStartString dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) @@ -314,7 +315,7 @@ dropCleanup key = do else return True {- Drops cached content for a key. -} -dropKeyStart :: String -> SubCmdPerform +dropKeyStart :: SubCmdStartString dropKeyStart keyname = do backends <- Backend.list let key = genKey (backends !! 0) keyname @@ -339,7 +340,7 @@ dropKeyCleanup key = do return True {- Sets cached content for a key. -} -setKeyStart :: FilePath -> SubCmdPerform +setKeyStart :: SubCmdStartString setKeyStart tmpfile = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -361,7 +362,7 @@ setKeyCleanup key = do return True {- Fixes the symlink to an annexed file. -} -fixStart :: FilePath -> SubCmdPerform +fixStart :: SubCmdStartString fixStart file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file @@ -382,7 +383,7 @@ fixCleanup file = do return True {- Stores description for the repository etc. -} -initStart :: String -> SubCmdPerform +initStart :: SubCmdStartString initStart description = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage @@ -405,7 +406,7 @@ initCleanup = do return True {- Adds a file pointing at a manually-specified key -} -fromKeyStart :: FilePath -> SubCmdPerform +fromKeyStart :: SubCmdStartString fromKeyStart file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -432,7 +433,7 @@ fromKeyCleanup file = do - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -moveStart :: FilePath -> SubCmdPerform +moveStart :: SubCmdStartString moveStart file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" @@ -453,7 +454,7 @@ moveStart file = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -moveToStart :: FilePath -> SubCmdPerform +moveToStart :: SubCmdStartString moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) @@ -503,7 +504,7 @@ moveToCleanup remote key tmpfile = do - If the current repository already has the content, it is still removed - from the other repository. -} -moveFromStart :: FilePath -> SubCmdPerform +moveFromStart :: SubCmdStartString moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key From 82d5a46c5618c6d35ef7b85c5cc257875de4a34b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 20:13:10 -0400 Subject: [PATCH 0405/2835] finally got the types clear enough --- Commands.hs | 64 +++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/Commands.hs b/Commands.hs index 01761d0309..199b83abf6 100644 --- a/Commands.hs +++ b/Commands.hs @@ -31,23 +31,25 @@ import qualified Remotes - 0. The seek stage takes the parameters passed to the subcommand, - looks through the repo to find the ones that are relevant - to that subcommand (ie, new files to add), and generates - - a start stage action. -} -type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek -type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek -type SubCmdSeek = [String] -> Annex [SubCmdPerform] + - a list of start stage actions. -} +type SubCmdSeek = [String] -> Annex [SubCmdStart] {- 1. The start stage is run before anything is printed about the - subcommand, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and - should not modify Annex state. -} -type SubCmdStartString = String -> SubCmdPerform -type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdPerform +type SubCmdStart = Annex (Maybe SubCmdPerform) {- 2. The perform stage is run after a message is printed about the subcommand - being run, and it should be where the bulk of the work happens. -} type SubCmdPerform = Annex (Maybe SubCmdCleanup) {- 3. The cleanup stage is run only if the perform stage succeeds, and it - returns the overall success/fail of the subcommand. -} -type SubCmdCleanup = Annex (Maybe SubCmdStatus) -type SubCmdStatus = Annex Bool +type SubCmdCleanup = Annex Bool +{- Some helper functions are used to build up SubCmdSeek and SubCmdStart + - functions. -} +type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek +type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek +type SubCmdStartString = String -> SubCmdStart +type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart data SubCommand = SubCommand { subcmdname :: String, @@ -125,7 +127,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' -{- Prepares a set of actions to run to perform a subcommand, based on +{- Prepares a list of actions to run to perform a subcommand, based on - the parameters passed to it. -} prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] prepSubCmd SubCommand { subcmdseek = seek } state params = do @@ -133,7 +135,7 @@ prepSubCmd SubCommand { subcmdseek = seek } state params = do return $ map (\a -> doSubCmd a) list {- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdPerform -> SubCmdStatus +doSubCmd :: SubCmdStart -> SubCmdCleanup doSubCmd start = do s <- start case (s) of @@ -227,13 +229,13 @@ addStart pair@(file, _) = notAnnexed file $ do else do showStart "add" file return $ Just $ addPerform pair -addPerform :: (FilePath, Maybe Backend) -> SubCmdCleanup +addPerform :: (FilePath, Maybe Backend) -> SubCmdPerform addPerform (file, backend) = do stored <- Backend.storeFileKey file backend case (stored) of Nothing -> return Nothing Just (key, _) -> return $ Just $ addCleanup file key -addCleanup :: FilePath -> Key -> SubCmdStatus +addCleanup :: FilePath -> Key -> SubCmdCleanup addCleanup file key = do logStatus key ValuePresent g <- Annex.gitRepo @@ -250,7 +252,7 @@ unannexStart :: SubCmdStartString unannexStart file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file return $ Just $ unannexPerform file key backend -unannexPerform :: FilePath -> Key -> Backend -> SubCmdCleanup +unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform unannexPerform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True @@ -258,7 +260,7 @@ unannexPerform file key backend = do if (ok) then return $ Just $ unannexCleanup file key else return Nothing -unannexCleanup :: FilePath -> Key -> SubCmdStatus +unannexCleanup :: FilePath -> Key -> SubCmdCleanup unannexCleanup file key = do logStatus key ValueMissing g <- Annex.gitRepo @@ -279,7 +281,7 @@ getStart file = isAnnexed file $ \(key, backend) -> do else do showStart "get" file return $ Just $ getPerform key backend -getPerform :: Key -> Backend -> SubCmdCleanup +getPerform :: Key -> Backend -> SubCmdPerform getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) @@ -296,13 +298,13 @@ dropStart file = isAnnexed file $ \(key, backend) -> do else do showStart "drop" file return $ Just $ dropPerform key backend -dropPerform :: Key -> Backend -> SubCmdCleanup +dropPerform :: Key -> Backend -> SubCmdPerform dropPerform key backend = do success <- Backend.removeKey backend key if (success) then return $ Just $ dropCleanup key else return Nothing -dropCleanup :: Key -> SubCmdStatus +dropCleanup :: Key -> SubCmdCleanup dropCleanup key = do logStatus key ValueMissing inannex <- inAnnex key @@ -328,13 +330,13 @@ dropKeyStart keyname = do else do showStart "dropkey" keyname return $ Just $ dropKeyPerform key -dropKeyPerform :: Key -> SubCmdCleanup +dropKeyPerform :: Key -> SubCmdPerform dropKeyPerform key = do g <- Annex.gitRepo let loc = annexLocation g key liftIO $ removeFile loc return $ Just $ dropKeyCleanup key -dropKeyCleanup :: Key -> SubCmdStatus +dropKeyCleanup :: Key -> SubCmdCleanup dropKeyCleanup key = do logStatus key ValueMissing return True @@ -348,7 +350,7 @@ setKeyStart tmpfile = do let key = genKey (backends !! 0) keyname showStart "setkey" tmpfile return $ Just $ setKeyPerform tmpfile key -setKeyPerform :: FilePath -> Key -> SubCmdCleanup +setKeyPerform :: FilePath -> Key -> SubCmdPerform setKeyPerform tmpfile key = do g <- Annex.gitRepo let loc = annexLocation g key @@ -356,7 +358,7 @@ setKeyPerform tmpfile key = do if (not ok) then error "mv failed!" else return $ Just $ setKeyCleanup key -setKeyCleanup :: Key -> SubCmdStatus +setKeyCleanup :: Key -> SubCmdCleanup setKeyCleanup key = do logStatus key ValuePresent return True @@ -371,13 +373,13 @@ fixStart file = isAnnexed file $ \(key, _) -> do else do showStart "fix" file return $ Just $ fixPerform file link -fixPerform :: FilePath -> FilePath -> SubCmdCleanup +fixPerform :: FilePath -> FilePath -> SubCmdPerform fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file return $ Just $ fixCleanup file -fixCleanup :: FilePath -> SubCmdStatus +fixCleanup :: FilePath -> SubCmdCleanup fixCleanup file = do Annex.queue "add" [] file return True @@ -389,7 +391,7 @@ initStart description = do "please specify a description of this repository\n" ++ usage showStart "init" description return $ Just $ initPerform description -initPerform :: String -> SubCmdCleanup +initPerform :: String -> SubCmdPerform initPerform description = do g <- Annex.gitRepo u <- getUUID g @@ -397,7 +399,7 @@ initPerform description = do liftIO $ gitAttributes g liftIO $ gitPreCommitHook g return $ Just $ initCleanup -initCleanup :: SubCmdStatus +initCleanup :: SubCmdCleanup initCleanup = do g <- Annex.gitRepo logfile <- uuidLog @@ -418,13 +420,13 @@ fromKeyStart file = do "key ("++keyname++") is not present in backend" showStart "fromkey" file return $ Just $ fromKeyPerform file key -fromKeyPerform :: FilePath -> Key -> SubCmdCleanup +fromKeyPerform :: FilePath -> Key -> SubCmdPerform fromKeyPerform file key = do link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ fromKeyCleanup file -fromKeyCleanup :: FilePath -> SubCmdStatus +fromKeyCleanup :: FilePath -> SubCmdCleanup fromKeyCleanup file = do Annex.queue "add" [] file return True @@ -462,7 +464,7 @@ moveToStart file = isAnnexed file $ \(key, _) -> do else do showStart "move" file return $ Just $ moveToPerform key -moveToPerform :: Key -> SubCmdCleanup +moveToPerform :: Key -> SubCmdPerform moveToPerform key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote @@ -479,7 +481,7 @@ moveToPerform key = do then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ dropCleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdStatus +moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", @@ -513,7 +515,7 @@ moveFromStart file = isAnnexed file $ \(key, _) -> do else do showStart "move" file return $ Just $ moveFromPerform key -moveFromPerform :: Key -> SubCmdCleanup +moveFromPerform :: Key -> SubCmdPerform moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key @@ -525,7 +527,7 @@ moveFromPerform key = do if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> SubCmdStatus +moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup moveFromCleanup remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), From e379307900ba571a5104f3bb2e7ad6c6900d3062 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 20:27:30 -0400 Subject: [PATCH 0406/2835] better message when ikiwiki is not available --- Commands.hs | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Commands.hs b/Commands.hs index 199b83abf6..330b71ed27 100644 --- a/Commands.hs +++ b/Commands.hs @@ -47,8 +47,8 @@ type SubCmdCleanup = Annex Bool {- Some helper functions are used to build up SubCmdSeek and SubCmdStart - functions. -} type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek -type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartString = String -> SubCmdStart +type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart data SubCommand = SubCommand { diff --git a/Makefile b/Makefile index ed262333a3..5e27c1e904 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ install: # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. ifeq ($(shell which ikiwiki),) -IKIWIKI=echo "** ikiwiki not found, skipping building docs" >&2 +IKIWIKI=@echo "** ikiwiki not found, skipping building docs" >&2; true else IKIWIKI=ikiwiki endif From 13514b6afc6cb043dfa742dd0e69efca7a4374df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 21:21:13 -0400 Subject: [PATCH 0407/2835] fix checkAttr to not deadlock and not need strictness Yeah, "will never deadlock" is never a good sign in code comments. ;) --- GitRepo.hs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index e8dd0a5dc3..bc6d450664 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -149,17 +149,12 @@ attributes repo {- Looks up a gitattributes value for each file in a list. -} checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] checkAttr repo attr files = do - (pid, fromh, toh) <- hPipeBoth "git" $ - gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] - -- git-check-attr reads all its stdin before outputting anything, - -- so we don't need to worry about deadlock - hPutStr toh files0 - hClose toh - c <- hGetContentsStrict fromh - hClose fromh - forceSuccess pid - return $ map topair $ lines c + (handle, s) <- pipeBoth "git" params files0 + return $ map topair $ lines s + -- XXX handle is left open, this is ok for git-annex, but may need + -- to be cleaned up for other uses. where + params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] files0 = join "\0" files topair l = (bits !! 0, join sep $ drop 1 $ bits) where From 58c89565e9f9b3a05b0b4ff33944b543d226c398 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 22:50:53 -0400 Subject: [PATCH 0408/2835] updat --- Backend/SHA1.hs | 5 ++++- GitRepo.hs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 76c368f84e..76869b071d 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -14,6 +14,7 @@ import System.IO import qualified Backend.File import TypeInternals +import Core backend :: Backend backend = Backend.File.backend { @@ -23,7 +24,9 @@ backend = Backend.File.backend { -- checksum the file to get its key keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do +keyValue file = do + showNote "checksum" + liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do line <- hGetLine h let bits = split " " line if (null bits) diff --git a/GitRepo.hs b/GitRepo.hs index bc6d450664..244acda862 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -149,7 +149,7 @@ attributes repo {- Looks up a gitattributes value for each file in a list. -} checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] checkAttr repo attr files = do - (handle, s) <- pipeBoth "git" params files0 + (_, s) <- pipeBoth "git" params files0 return $ map topair $ lines s -- XXX handle is left open, this is ok for git-annex, but may need -- to be cleaned up for other uses. From 790613333c2d946d28fe62ef7da8ed3a23428f99 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 01:07:34 -0400 Subject: [PATCH 0409/2835] deal with git's insane octal filename encoding --- GitRepo.hs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 244acda862..389aa35fe1 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -156,11 +156,40 @@ checkAttr repo attr files = do where params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] files0 = join "\0" files - topair l = (bits !! 0, join sep $ drop 1 $ bits) + topair l = (file, value) where + file = decodeGitFile $ join sep $ take end bits + value = bits !! end + end = length bits - 1 bits = split sep l sep = ": " ++ attr ++ ": " +{- Some git commands output encoded filenames. Such a filename + - will always be double-quoted, and then \nnn (in octal) is used + - to escape high characters. -} +decodeGitFile :: String -> FilePath +decodeGitFile [] = [] +decodeGitFile f@(c:s) + | c == '"' = unescape middle + | otherwise = f + where + e = "\\" + middle = take (length s - 1) s + unescape v = foldl (++) beginning $ map decode $ split e rest + where + pair = span (/= '\\') v + beginning = fst pair + rest = snd pair + decode [] = "" + decode n + | length num == 3 = (chr $ readoctal num):rest + | otherwise = e++n + where + pair = span isOctDigit n + num = fst pair + rest = snd pair + readoctal o = read $ "0o" ++ o :: Int + {- Path to a repository's .git directory, relative to its workTree. -} gitDir :: Repo -> String gitDir repo From bdb7ed76cf233a5c7415b2235123fe448f130ab9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 02:24:19 -0400 Subject: [PATCH 0410/2835] tweak --- Backend/SHA1.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 76869b071d..4858922585 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -25,7 +25,7 @@ backend = Backend.File.backend { -- checksum the file to get its key keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do - showNote "checksum" + showNote "checksum..." liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do line <- hGetLine h let bits = split " " line From 82056a79213268c4d7d93c64c013ca44056f70ba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 12:43:34 -0400 Subject: [PATCH 0411/2835] reorg --- GitRepo.hs | 88 +++++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 389aa35fe1..9fda0a23fe 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -146,50 +146,6 @@ attributes repo | bare repo = (workTree repo) ++ "/info/.gitattributes" | otherwise = (workTree repo) ++ "/.gitattributes" -{- Looks up a gitattributes value for each file in a list. -} -checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] -checkAttr repo attr files = do - (_, s) <- pipeBoth "git" params files0 - return $ map topair $ lines s - -- XXX handle is left open, this is ok for git-annex, but may need - -- to be cleaned up for other uses. - where - params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] - files0 = join "\0" files - topair l = (file, value) - where - file = decodeGitFile $ join sep $ take end bits - value = bits !! end - end = length bits - 1 - bits = split sep l - sep = ": " ++ attr ++ ": " - -{- Some git commands output encoded filenames. Such a filename - - will always be double-quoted, and then \nnn (in octal) is used - - to escape high characters. -} -decodeGitFile :: String -> FilePath -decodeGitFile [] = [] -decodeGitFile f@(c:s) - | c == '"' = unescape middle - | otherwise = f - where - e = "\\" - middle = take (length s - 1) s - unescape v = foldl (++) beginning $ map decode $ split e rest - where - pair = span (/= '\\') v - beginning = fst pair - rest = snd pair - decode [] = "" - decode n - | length num == 3 = (chr $ readoctal num):rest - | otherwise = e++n - where - pair = span isOctDigit n - num = fst pair - rest = snd pair - readoctal o = read $ "0o" ++ o :: Int - {- Path to a repository's .git directory, relative to its workTree. -} gitDir :: Repo -> String gitDir repo @@ -352,6 +308,50 @@ configGet repo key defaultValue = configMap :: Repo -> Map.Map String String configMap repo = config repo +{- Looks up a gitattributes value for each file in a list. -} +checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] +checkAttr repo attr files = do + (_, s) <- pipeBoth "git" params files0 + return $ map topair $ lines s + -- XXX handle is left open, this is ok for git-annex, but may need + -- to be cleaned up for other uses. + where + params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] + files0 = join "\0" files + topair l = (file, value) + where + file = decodeGitFile $ join sep $ take end bits + value = bits !! end + end = length bits - 1 + bits = split sep l + sep = ": " ++ attr ++ ": " + +{- Some git commands output encoded filenames. Such a filename + - will always be double-quoted, and then \nnn (in octal) is used + - to escape high characters. -} +decodeGitFile :: String -> FilePath +decodeGitFile [] = [] +decodeGitFile f@(c:s) + | c == '"' = unescape middle + | otherwise = f + where + e = "\\" + middle = take (length s - 1) s + unescape v = foldl (++) beginning $ map decode $ split e rest + where + pair = span (/= '\\') v + beginning = fst pair + rest = snd pair + decode [] = "" + decode n + | length num == 3 = (chr $ readoctal num):rest + | otherwise = e++n + where + pair = span isOctDigit n + num = fst pair + rest = snd pair + readoctal o = read $ "0o" ++ o :: Int + {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo repoFromCwd = do From 9e5985ff983c4b489d545e99946cf62e34fadcd9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 15:54:43 -0400 Subject: [PATCH 0412/2835] blew several hours on getting the decodeGitFile 100% right with quickcheck --- GitRepo.hs | 76 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 9fda0a23fe..c62a820d8e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -36,7 +36,11 @@ module GitRepo ( inRepo, notInRepo, stagedFiles, - checkAttr + checkAttr, + decodeGitFile, + encodeGitFile, + + prop_idempotent_encode ) where import Monad (unless) @@ -51,6 +55,9 @@ import qualified Data.Map as Map hiding (map, split) import Network.URI import Maybe import Char +import Text.Printf +import Data.Word (Word8) +import Codec.Binary.UTF8.String (encode) import Utility @@ -332,25 +339,70 @@ checkAttr repo attr files = do decodeGitFile :: String -> FilePath decodeGitFile [] = [] decodeGitFile f@(c:s) - | c == '"' = unescape middle + | c == '"' = unescape ("", middle) | otherwise = f where - e = "\\" middle = take (length s - 1) s - unescape v = foldl (++) beginning $ map decode $ split e rest + unescape (b, []) = b + unescape (b, v) = b ++ beginning ++ unescape (decode rest) where pair = span (/= '\\') v beginning = fst pair rest = snd pair - decode [] = "" - decode n - | length num == 3 = (chr $ readoctal num):rest - | otherwise = e++n + isescape c = c == '\\' + decode (e:n1:n2:n3:rest) + | isescape e && alloctal = (fromoctal, rest) + where + alloctal = isOctDigit n1 && + isOctDigit n2 && + isOctDigit n3 + fromoctal = [chr $ readoctal (n1:n2:n3:[])] + readoctal o = read $ "0o" ++ o :: Int + decode (e:nc:rest) + | isescape e = ([echar nc], rest) where - pair = span isOctDigit n - num = fst pair - rest = snd pair - readoctal o = read $ "0o" ++ o :: Int + -- special character escapes + echar 'a' = '\a' + echar 'b' = '\b' + echar 'f' = '\f' + echar 'n' = '\n' + echar 'r' = '\r' + echar 't' = '\t' + echar 'v' = '\v' + echar x = x + decode n = ("", n) + +{- Should not need to use this, except for testing decodeGitFile. -} +encodeGitFile :: FilePath -> String +encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" + where + e c = "\\" ++ [c] + echar '\a' = e 'a' + echar '\b' = e 'b' + echar '\f' = e 'f' + echar '\n' = e 'n' + echar '\r' = e 'r' + echar '\t' = e 't' + echar '\v' = e 'v' + echar '\\' = e '\\' + echar '"' = e '"' + echar x + | ord x < 0x20 = e_num x -- low ascii + | ord x >= 256 = e_utf x + | ord x > 0x7E = e_num x -- high ascii + | otherwise = [x] -- printable ascii + where + showoctal i = "\\" ++ (printf "%03o" i) + e_num c = showoctal $ ord c + -- unicode character is decomposed to Word8 + -- and each is shown with e_num + e_utf c = foldl (++) "" $ map showoctal $ + (encode [c] :: [Word8]) + + +{- for quickcheck -} +prop_idempotent_encode :: String -> Bool +prop_idempotent_encode s = s == (decodeGitFile $ encodeGitFile s) {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo From cecb1cbeb2017e9e3be252c253ee7f2b49235c63 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 16:00:55 -0400 Subject: [PATCH 0413/2835] clean up --- GitRepo.hs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index c62a820d8e..752253b416 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -333,23 +333,26 @@ checkAttr repo attr files = do bits = split sep l sep = ": " ++ attr ++ ": " -{- Some git commands output encoded filenames. Such a filename - - will always be double-quoted, and then \nnn (in octal) is used - - to escape high characters. -} +{- Some git commands output encoded filenames. Decode that (annoyingly + - complex) encoding. -} decodeGitFile :: String -> FilePath decodeGitFile [] = [] decodeGitFile f@(c:s) + -- encoded strings will be inside double quotes | c == '"' = unescape ("", middle) | otherwise = f where + e = '\\' middle = take (length s - 1) s unescape (b, []) = b + -- look for escapes starting with '\' unescape (b, v) = b ++ beginning ++ unescape (decode rest) where - pair = span (/= '\\') v + pair = span (/= e) v beginning = fst pair rest = snd pair - isescape c = c == '\\' + isescape c = c == e + -- \NNN is an octal encoded character decode (e:n1:n2:n3:rest) | isescape e && alloctal = (fromoctal, rest) where @@ -358,10 +361,10 @@ decodeGitFile f@(c:s) isOctDigit n3 fromoctal = [chr $ readoctal (n1:n2:n3:[])] readoctal o = read $ "0o" ++ o :: Int + -- \C is used for a few special characters decode (e:nc:rest) | isescape e = ([echar nc], rest) where - -- special character escapes echar 'a' = '\a' echar 'b' = '\b' echar 'f' = '\f' From c7b0f60fba9f53ed97c43f3ad9a48e7698b97760 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 16:02:43 -0400 Subject: [PATCH 0414/2835] clean up --- GitRepo.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 752253b416..d9dd086f2a 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -397,8 +397,8 @@ encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" where showoctal i = "\\" ++ (printf "%03o" i) e_num c = showoctal $ ord c - -- unicode character is decomposed to Word8 - -- and each is shown with e_num + -- unicode character is decomposed to + -- Word8s and each is shown in octal e_utf c = foldl (++) "" $ map showoctal $ (encode [c] :: [Word8]) From d93a37289406fbeb0cedf05f1c8009f68cdb2570 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 16:49:35 -0400 Subject: [PATCH 0415/2835] add a stupid test harness --- GitRepo.hs | 18 +++++++++--------- Makefile | 3 +++ test.hs | 8 ++++++++ 3 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 test.hs diff --git a/GitRepo.hs b/GitRepo.hs index d9dd086f2a..7d5291ff1e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -40,7 +40,7 @@ module GitRepo ( decodeGitFile, encodeGitFile, - prop_idempotent_encode + prop_idempotent_deencode ) where import Monad (unless) @@ -351,10 +351,10 @@ decodeGitFile f@(c:s) pair = span (/= e) v beginning = fst pair rest = snd pair - isescape c = c == e + isescape x = x == e -- \NNN is an octal encoded character - decode (e:n1:n2:n3:rest) - | isescape e && alloctal = (fromoctal, rest) + decode (x:n1:n2:n3:rest) + | isescape x && alloctal = (fromoctal, rest) where alloctal = isOctDigit n1 && isOctDigit n2 && @@ -362,8 +362,8 @@ decodeGitFile f@(c:s) fromoctal = [chr $ readoctal (n1:n2:n3:[])] readoctal o = read $ "0o" ++ o :: Int -- \C is used for a few special characters - decode (e:nc:rest) - | isescape e = ([echar nc], rest) + decode (x:nc:rest) + | isescape x = ([echar nc], rest) where echar 'a' = '\a' echar 'b' = '\b' @@ -372,7 +372,7 @@ decodeGitFile f@(c:s) echar 'r' = '\r' echar 't' = '\t' echar 'v' = '\v' - echar x = x + echar a = a decode n = ("", n) {- Should not need to use this, except for testing decodeGitFile. -} @@ -404,8 +404,8 @@ encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" {- for quickcheck -} -prop_idempotent_encode :: String -> Bool -prop_idempotent_encode s = s == (decodeGitFile $ encodeGitFile s) +prop_idempotent_deencode :: String -> Bool +prop_idempotent_deencode s = s == (decodeGitFile $ encodeGitFile s) {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo diff --git a/Makefile b/Makefile index 5e27c1e904..eb74bb7273 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ else IKIWIKI=ikiwiki endif +test: + runghc test.hs + docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ diff --git a/test.hs b/test.hs new file mode 100644 index 0000000000..e9ea684593 --- /dev/null +++ b/test.hs @@ -0,0 +1,8 @@ +-- TODO find a test harness that is actually in Debian and use it. + +import Test.QuickCheck +import GitRepo + +main = do + putStr "prop_idempotent_deencode " + quickCheck prop_idempotent_deencode From 606ed6bb3566fa86c1783e3f1c7d799a6f1be8d1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 17:08:16 -0400 Subject: [PATCH 0416/2835] better test framework --- test.hs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test.hs b/test.hs index e9ea684593..9897236176 100644 --- a/test.hs +++ b/test.hs @@ -1,8 +1,13 @@ -- TODO find a test harness that is actually in Debian and use it. import Test.QuickCheck +import Test.HUnit +import Test.HUnit.Tools + import GitRepo -main = do - putStr "prop_idempotent_deencode " - quickCheck prop_idempotent_deencode +alltests = [ + qctest "prop_idempotent_deencode" prop_idempotent_deencode + ] + +main = runVerboseTests (TestList alltests) From 0eae5b806c76b0fa3e21fbae6e5f2d9a39a04cce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 19:04:24 -0400 Subject: [PATCH 0417/2835] broke subcommands out into separate modules --- CmdLine.hs | 201 ++++++++++++++++ Command.hs | 50 ++++ Command/Add.hs | 52 +++++ Command/Drop.hs | 50 ++++ Command/DropKey.hs | 47 ++++ Command/Fix.hs | 40 ++++ Command/FromKey.hs | 44 ++++ Command/Get.hs | 31 +++ Command/Init.hs | 42 ++++ Command/Move.hs | 131 +++++++++++ Command/SetKey.hs | 43 ++++ Command/Unannex.hs | 48 ++++ Commands.hs | 555 --------------------------------------------- git-annex.hs | 2 +- 14 files changed, 780 insertions(+), 556 deletions(-) create mode 100644 CmdLine.hs create mode 100644 Command.hs create mode 100644 Command/Add.hs create mode 100644 Command/Drop.hs create mode 100644 Command/DropKey.hs create mode 100644 Command/Fix.hs create mode 100644 Command/FromKey.hs create mode 100644 Command/Get.hs create mode 100644 Command/Init.hs create mode 100644 Command/Move.hs create mode 100644 Command/SetKey.hs create mode 100644 Command/Unannex.hs delete mode 100644 Commands.hs diff --git a/CmdLine.hs b/CmdLine.hs new file mode 100644 index 0000000000..494da2873c --- /dev/null +++ b/CmdLine.hs @@ -0,0 +1,201 @@ +{- git-annex command line + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine (parseCmd) where + +import System.Console.GetOpt +import Control.Monad.State (liftIO) +import System.Directory +import Data.String.Utils +import Control.Monad (filterM) +import Monad (when) + +import qualified GitRepo as Git +import qualified Annex +import Locations +import qualified Backend +import Types +import Core + +import Command +import qualified Command.Add +import qualified Command.Unannex +import qualified Command.Drop +import qualified Command.Move +import qualified Command.Get +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.SetKey +import qualified Command.Fix +import qualified Command.Init + +data SubCommand = SubCommand { + subcmdname :: String, + subcmdparams :: String, + subcmdseek :: SubCmdSeek, + subcmddesc :: String +} +subCmds :: [SubCommand] +subCmds = [ + (SubCommand "add" path (withFilesNotInGit Command.Add.start) + "add files to annex") + , (SubCommand "get" path (withFilesInGit Command.Get.start) + "make content of annexed files available") + , (SubCommand "drop" path (withFilesInGit Command.Drop.start) + "indicate content of files not currently wanted") + , (SubCommand "move" path (withFilesInGit Command.Move.start) + "transfer content of files to/from another repository") + , (SubCommand "init" desc (withDescription Command.Init.start) + "initialize git-annex with repository description") + , (SubCommand "unannex" path (withFilesInGit Command.Unannex.start) + "undo accidential add command") + , (SubCommand "fix" path (withFilesInGit Command.Fix.start) + "fix up symlinks to point to annexed content") + , (SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) + "fix up symlinks before they are committed") + , (SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) + "adds a file using a specific key") + , (SubCommand "dropkey" key (withKeys Command.DropKey.start) + "drops annexed content for specified keys") + , (SubCommand "setkey" key (withTempFile Command.SetKey.start) + "sets annexed content for a key using a temp file") + ] + where + path = "PATH ..." + key = "KEY ..." + desc = "DESCRIPTION" + +-- Each dashed command-line option results in generation of an action +-- in the Annex monad that performs the necessary setting. +options :: [OptDescr (Annex ())] +options = [ + Option ['f'] ["force"] (NoArg (storebool "force" True)) + "allow actions that may lose annexed data" + , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) + "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (storebool "quiet" False)) + "allow verbose output" + , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") + "specify default key-value backend to use" + , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") + "specify from where to transfer content" + ] + where + storebool n b = Annex.flagChange n $ FlagBool b + storestring n s = Annex.flagChange n $ FlagString s + +header :: String +header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) + +{- Usage message with lists of options and subcommands. -} +usage :: String +usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs + where + cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds + showcmd c = + (subcmdname c) ++ + (pad 11 (subcmdname c)) ++ + (subcmdparams c) ++ + (pad 13 (subcmdparams c)) ++ + (subcmddesc c) + indent l = " " ++ l + pad n s = take (n - (length s)) $ repeat ' ' + +{- Prepares a list of actions to run to perform a subcommand, based on + - the parameters passed to it. -} +prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] +prepSubCmd SubCommand { subcmdseek = seek } state params = do + list <- Annex.eval state $ seek params + return $ map (\a -> doSubCmd a) list + +{- Runs a subcommand through the start, perform and cleanup stages -} +doSubCmd :: SubCmdStart -> SubCmdCleanup +doSubCmd start = do + s <- start + case (s) of + Nothing -> return True + Just perform -> do + p <- perform + case (p) of + Nothing -> do + showEndFail + return False + Just cleanup -> do + c <- cleanup + if (c) + then do + showEndOk + return True + else do + showEndFail + return False + +{- These functions find appropriate files or other things based on a + user's parameters. -} +withFilesNotInGit :: SubCmdSeekBackendFiles +withFilesNotInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.notInRepo repo) params + let files' = foldl (++) [] files + pairs <- Backend.chooseBackends files' + return $ map a $ filter (\(f,_) -> notState f) pairs +withFilesInGit :: SubCmdSeekStrings +withFilesInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.inRepo repo) params + return $ map a $ filter notState $ foldl (++) [] files +withFilesMissing :: SubCmdSeekStrings +withFilesMissing a params = do + files <- liftIO $ filterM missing params + return $ map a $ filter notState files + where + missing f = do + e <- doesFileExist f + return $ not e +withDescription :: SubCmdSeekStrings +withDescription a params = do + return $ [a $ unwords params] +withFilesToBeCommitted :: SubCmdSeekStrings +withFilesToBeCommitted a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.stagedFiles repo) params + return $ map a $ filter notState $ foldl (++) [] files +withKeys :: SubCmdSeekStrings +withKeys a params = return $ map a params +withTempFile :: SubCmdSeekStrings +withTempFile a params = return $ map a params + +{- filter out files from the state directory -} +notState :: FilePath -> Bool +notState f = stateLoc /= take (length stateLoc) f + +{- Parses command line and returns two lists of actions to be + - run in the Annex monad. The first actions configure it + - according to command line options, while the second actions + - handle subcommands. -} +parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) +parseCmd argv state = do + (flags, params) <- getopt + when (null params) $ error usage + case lookupCmd (params !! 0) of + [] -> error usage + [subcommand] -> do + actions <- prepSubCmd subcommand state (drop 1 params) + let configactions = map (\flag -> do + flag + return True) flags + return (configactions, actions) + _ -> error "internal error: multiple matching subcommands" + where + getopt = case getOpt Permute options argv of + (flags, params, []) -> return (flags, params) + (_, _, errs) -> ioError (userError (concat errs ++ usage)) + lookupCmd cmd = filter (\c -> cmd == subcmdname c) subCmds diff --git a/Command.hs b/Command.hs new file mode 100644 index 0000000000..3d1e75e5f5 --- /dev/null +++ b/Command.hs @@ -0,0 +1,50 @@ +{- git-annex command types + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command where + +import Types +import Backend + +{- A subcommand runs in four stages. + - + - 0. The seek stage takes the parameters passed to the subcommand, + - looks through the repo to find the ones that are relevant + - to that subcommand (ie, new files to add), and generates + - a list of start stage actions. -} +type SubCmdSeek = [String] -> Annex [SubCmdStart] +{- 1. The start stage is run before anything is printed about the + - subcommand, is passed some input, and can early abort it + - if the input does not make sense. It should run quickly and + - should not modify Annex state. -} +type SubCmdStart = Annex (Maybe SubCmdPerform) +{- 2. The perform stage is run after a message is printed about the subcommand + - being run, and it should be where the bulk of the work happens. -} +type SubCmdPerform = Annex (Maybe SubCmdCleanup) +{- 3. The cleanup stage is run only if the perform stage succeeds, and it + - returns the overall success/fail of the subcommand. -} +type SubCmdCleanup = Annex Bool +{- Some helper functions are used to build up SubCmdSeek and SubCmdStart + - functions. -} +type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek +type SubCmdStartString = String -> SubCmdStart +type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek +type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart + +notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) +notAnnexed file a = do + r <- Backend.lookupFile file + case (r) of + Just _ -> return Nothing + Nothing -> a + +isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) +isAnnexed file a = do + r <- Backend.lookupFile file + case (r) of + Just v -> a v + Nothing -> return Nothing diff --git a/Command/Add.hs b/Command/Add.hs new file mode 100644 index 0000000000..825c1d8c1e --- /dev/null +++ b/Command/Add.hs @@ -0,0 +1,52 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Add where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory + +import Command +import qualified Annex +import Utility +import Locations +import qualified Backend +import LocationLog +import Types +import Core + +{- The add subcommand annexes a file, storing it in a backend, and then + - moving it into the annex directory and setting up the symlink pointing + - to its content. -} +start :: SubCmdStartBackendFile +start pair@(file, _) = notAnnexed file $ do + s <- liftIO $ getSymbolicLinkStatus file + if ((isSymbolicLink s) || (not $ isRegularFile s)) + then return Nothing + else do + showStart "add" file + return $ Just $ perform pair + +perform :: (FilePath, Maybe Backend) -> SubCmdPerform +perform (file, backend) = do + stored <- Backend.storeFileKey file backend + case (stored) of + Nothing -> return Nothing + Just (key, _) -> return $ Just $ cleanup file key + +cleanup :: FilePath -> Key -> SubCmdCleanup +cleanup file key = do + logStatus key ValuePresent + g <- Annex.gitRepo + let dest = annexLocation g key + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ renameFile file dest + link <- calcGitLink file key + liftIO $ createSymbolicLink link file + Annex.queue "add" [] file + return True diff --git a/Command/Drop.hs b/Command/Drop.hs new file mode 100644 index 0000000000..6cdf216f41 --- /dev/null +++ b/Command/Drop.hs @@ -0,0 +1,50 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Drop where + +import Control.Monad.State (liftIO) +import System.Directory + +import Command +import qualified Annex +import Locations +import qualified Backend +import LocationLog +import Types +import Core + +{- Indicates a file's content is not wanted anymore, and should be removed + - if it's safe to do so. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + inbackend <- Backend.hasKey key + if (not inbackend) + then return Nothing + else do + showStart "drop" file + return $ Just $ perform key backend + +perform :: Key -> Backend -> SubCmdPerform +perform key backend = do + success <- Backend.removeKey backend key + if (success) + then return $ Just $ cleanup key + else return Nothing + +cleanup :: Key -> SubCmdCleanup +cleanup key = do + logStatus key ValueMissing + inannex <- inAnnex key + if (inannex) + then do + g <- Annex.gitRepo + let loc = annexLocation g key + liftIO $ removeFile loc + return True + else return True + diff --git a/Command/DropKey.hs b/Command/DropKey.hs new file mode 100644 index 0000000000..bdd9b55b12 --- /dev/null +++ b/Command/DropKey.hs @@ -0,0 +1,47 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DropKey where + +import Control.Monad.State (liftIO) +import System.Directory + +import Command +import qualified Annex +import Locations +import qualified Backend +import LocationLog +import Types +import Core + +{- Drops cached content for a key. -} +start :: SubCmdStartString +start keyname = do + backends <- Backend.list + let key = genKey (backends !! 0) keyname + present <- inAnnex key + force <- Annex.flagIsSet "force" + if (not present) + then return Nothing + else if (not force) + then error "dropkey is can cause data loss; use --force if you're sure you want to do this" + else do + showStart "dropkey" keyname + return $ Just $ perform key + +perform :: Key -> SubCmdPerform +perform key = do + g <- Annex.gitRepo + let loc = annexLocation g key + liftIO $ removeFile loc + return $ Just $ cleanup key + +cleanup :: Key -> SubCmdCleanup +cleanup key = do + logStatus key ValueMissing + return True + diff --git a/Command/Fix.hs b/Command/Fix.hs new file mode 100644 index 0000000000..90257a8a53 --- /dev/null +++ b/Command/Fix.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Fix where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory + +import Command +import qualified Annex +import Utility +import Core + +{- Fixes the symlink to an annexed file. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, _) -> do + link <- calcGitLink file key + l <- liftIO $ readSymbolicLink file + if (link == l) + then return Nothing + else do + showStart "fix" file + return $ Just $ perform file link + +perform :: FilePath -> FilePath -> SubCmdPerform +perform file link = do + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + return $ Just $ cleanup file + +cleanup :: FilePath -> SubCmdCleanup +cleanup file = do + Annex.queue "add" [] file + return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs new file mode 100644 index 0000000000..3071f218f4 --- /dev/null +++ b/Command/FromKey.hs @@ -0,0 +1,44 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.FromKey where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory +import Control.Monad (when, unless) + +import Command +import qualified Annex +import Utility +import qualified Backend +import Types +import Core + +{- Adds a file pointing at a manually-specified key -} +start :: SubCmdStartString +start file = do + keyname <- Annex.flagGet "key" + when (null keyname) $ error "please specify the key with --key" + backends <- Backend.list + let key = genKey (backends !! 0) keyname + + inbackend <- Backend.hasKey key + unless (inbackend) $ error $ + "key ("++keyname++") is not present in backend" + showStart "fromkey" file + return $ Just $ perform file key +perform :: FilePath -> Key -> SubCmdPerform +perform file key = do + link <- calcGitLink file key + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ createSymbolicLink link file + return $ Just $ cleanup file +cleanup :: FilePath -> SubCmdCleanup +cleanup file = do + Annex.queue "add" [] file + return True diff --git a/Command/Get.hs b/Command/Get.hs new file mode 100644 index 0000000000..1433bc8d00 --- /dev/null +++ b/Command/Get.hs @@ -0,0 +1,31 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Get where + +import Command +import qualified Backend +import Types +import Core + +{- Gets an annexed file from one of the backends. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + inannex <- inAnnex key + if (inannex) + then return Nothing + else do + showStart "get" file + return $ Just $ perform key backend + +perform :: Key -> Backend -> SubCmdPerform +perform key backend = do + ok <- getViaTmp key (Backend.retrieveKeyFile backend key) + if (ok) + then return $ Just $ return True -- no cleanup needed + else return Nothing + diff --git a/Command/Init.hs b/Command/Init.hs new file mode 100644 index 0000000000..b1e4e0e066 --- /dev/null +++ b/Command/Init.hs @@ -0,0 +1,42 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Init where + +import Control.Monad.State (liftIO) +import Control.Monad (when) + +import Command +import qualified Annex +import Core +import qualified GitRepo as Git +import UUID + +{- Stores description for the repository etc. -} +start :: SubCmdStartString +start description = do + when (null description) $ error $ + "please specify a description of this repository\n" + showStart "init" description + return $ Just $ perform description + +perform :: String -> SubCmdPerform +perform description = do + g <- Annex.gitRepo + u <- getUUID g + describeUUID u description + liftIO $ gitAttributes g + liftIO $ gitPreCommitHook g + return $ Just $ cleanup + +cleanup :: SubCmdCleanup +cleanup = do + g <- Annex.gitRepo + logfile <- uuidLog + liftIO $ Git.run g ["add", logfile] + liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] + return True diff --git a/Command/Move.hs b/Command/Move.hs new file mode 100644 index 0000000000..cee9416222 --- /dev/null +++ b/Command/Move.hs @@ -0,0 +1,131 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Move where + +import Control.Monad.State (liftIO) +import Monad (when) + +import Command +import Command.Drop +import qualified Annex +import Locations +import LocationLog +import Types +import Core +import qualified GitRepo as Git +import qualified Remotes +import UUID + +{- Move a file either --to or --from a repository. + - + - This only operates on the cached file content; it does not involve + - moving data in the key-value backend. -} +start :: SubCmdStartString +start file = do + fromName <- Annex.flagGet "fromrepository" + toName <- Annex.flagGet "torepository" + case (fromName, toName) of + ("", "") -> error "specify either --from or --to" + ("", _) -> moveToStart file + (_ , "") -> moveFromStart file + (_ , _) -> error "only one of --from or --to can be specified" + +{- Moves the content of an annexed file to another repository, + - removing it from the current repository, and updates locationlog + - information on both. + - + - If the destination already has the content, it is still removed + - from the current repository. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. + -} +moveToStart :: SubCmdStartString +moveToStart file = isAnnexed file $ \(key, _) -> do + ishere <- inAnnex key + if (not ishere) + then return Nothing -- not here, so nothing to do + else do + showStart "move" file + return $ Just $ moveToPerform key +moveToPerform :: Key -> SubCmdPerform +moveToPerform key = do + -- checking the remote is expensive, so not done in the start step + remote <- Remotes.commandLineRemote + isthere <- Remotes.inAnnex remote key + case isthere of + Left err -> do + showNote $ show err + return Nothing + Right False -> do + Core.showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." + let tmpfile = (annexTmpLocation remote) ++ (keyFile key) + ok <- Remotes.copyToRemote remote key tmpfile + if (ok) + then return $ Just $ moveToCleanup remote key tmpfile + else return Nothing -- failed + Right True -> return $ Just $ Command.Drop.cleanup key +moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup +moveToCleanup remote key tmpfile = do + -- Tell remote to use the transferred content. + ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", + "--backend=" ++ (backendName key), + "--key=" ++ keyName key, + tmpfile] + if ok + then do + -- Record that the key is present on the remote. + g <- Annex.gitRepo + remoteuuid <- getUUID remote + logfile <- liftIO $ logChange g key remoteuuid ValuePresent + Annex.queue "add" [] logfile + -- Cleanup on the local side is the same as done for the + -- drop subcommand. + Command.Drop.cleanup key + else return False + +{- Moves the content of an annexed file from another repository to the current + - repository and updates locationlog information on both. + - + - If the current repository already has the content, it is still removed + - from the other repository. + -} +moveFromStart :: SubCmdStartString +moveFromStart file = isAnnexed file $ \(key, _) -> do + remote <- Remotes.commandLineRemote + l <- Remotes.keyPossibilities key + if (null $ filter (\r -> Remotes.same r remote) l) + then return Nothing + else do + showStart "move" file + return $ Just $ moveFromPerform key +moveFromPerform :: Key -> SubCmdPerform +moveFromPerform key = do + remote <- Remotes.commandLineRemote + ishere <- inAnnex key + if (ishere) + then return $ Just $ moveFromCleanup remote key + else do + Core.showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." + ok <- getViaTmp key (Remotes.copyFromRemote remote key) + if (ok) + then return $ Just $ moveFromCleanup remote key + else return Nothing -- fail +moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup +moveFromCleanup remote key = do + ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", + "--backend=" ++ (backendName key), + keyName key] + when ok $ do + -- Record locally that the key is not on the remote. + remoteuuid <- getUUID remote + g <- Annex.gitRepo + logfile <- liftIO $ logChange g key remoteuuid ValueMissing + Annex.queue "add" [] logfile + return ok diff --git a/Command/SetKey.hs b/Command/SetKey.hs new file mode 100644 index 0000000000..a5710643ec --- /dev/null +++ b/Command/SetKey.hs @@ -0,0 +1,43 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.SetKey where + +import Control.Monad.State (liftIO) +import Control.Monad (when) + +import Command +import qualified Annex +import Utility +import Locations +import qualified Backend +import LocationLog +import Types +import Core + +{- Sets cached content for a key. -} +start :: SubCmdStartString +start tmpfile = do + keyname <- Annex.flagGet "key" + when (null keyname) $ error "please specify the key with --key" + backends <- Backend.list + let key = genKey (backends !! 0) keyname + showStart "setkey" tmpfile + return $ Just $ perform tmpfile key +perform :: FilePath -> Key -> SubCmdPerform +perform tmpfile key = do + g <- Annex.gitRepo + let loc = annexLocation g key + ok <- liftIO $ boolSystem "mv" [tmpfile, loc] + if (not ok) + then error "mv failed!" + else return $ Just $ cleanup key +cleanup :: Key -> SubCmdCleanup +cleanup key = do + logStatus key ValuePresent + return True + diff --git a/Command/Unannex.hs b/Command/Unannex.hs new file mode 100644 index 0000000000..5cffb2d894 --- /dev/null +++ b/Command/Unannex.hs @@ -0,0 +1,48 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unannex where + +import Control.Monad.State (liftIO) +import System.Directory + +import Command +import qualified Annex +import Utility +import Locations +import qualified Backend +import LocationLog +import Types +import Core +import qualified GitRepo as Git + +{- The unannex subcommand undoes an add. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + showStart "unannex" file + return $ Just $ perform file key backend + +perform :: FilePath -> Key -> Backend -> SubCmdPerform +perform file key backend = do + -- force backend to always remove + Annex.flagChange "force" $ FlagBool True + ok <- Backend.removeKey backend key + if (ok) + then return $ Just $ cleanup file key + else return Nothing + +cleanup :: FilePath -> Key -> SubCmdCleanup +cleanup file key = do + logStatus key ValueMissing + g <- Annex.gitRepo + let src = annexLocation g key + liftIO $ removeFile file + liftIO $ Git.run g ["rm", "--quiet", file] + -- git rm deletes empty directories; put them back + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ renameFile src file + return True diff --git a/Commands.hs b/Commands.hs deleted file mode 100644 index 330b71ed27..0000000000 --- a/Commands.hs +++ /dev/null @@ -1,555 +0,0 @@ -{- git-annex command line - - - - Copyright 2010 Joey Hess - - - - Licensed under the GNU GPL version 3 or higher. - -} - -module Commands (parseCmd) where - -import System.Console.GetOpt -import Control.Monad.State (liftIO) -import System.Posix.Files -import System.Directory -import Data.String.Utils -import Control.Monad (filterM) -import Monad (when, unless) - -import qualified GitRepo as Git -import qualified Annex -import Utility -import Locations -import qualified Backend -import UUID -import LocationLog -import Types -import Core -import qualified Remotes - -{- A subcommand runs in four stages. - - - - 0. The seek stage takes the parameters passed to the subcommand, - - looks through the repo to find the ones that are relevant - - to that subcommand (ie, new files to add), and generates - - a list of start stage actions. -} -type SubCmdSeek = [String] -> Annex [SubCmdStart] -{- 1. The start stage is run before anything is printed about the - - subcommand, is passed some input, and can early abort it - - if the input does not make sense. It should run quickly and - - should not modify Annex state. -} -type SubCmdStart = Annex (Maybe SubCmdPerform) -{- 2. The perform stage is run after a message is printed about the subcommand - - being run, and it should be where the bulk of the work happens. -} -type SubCmdPerform = Annex (Maybe SubCmdCleanup) -{- 3. The cleanup stage is run only if the perform stage succeeds, and it - - returns the overall success/fail of the subcommand. -} -type SubCmdCleanup = Annex Bool -{- Some helper functions are used to build up SubCmdSeek and SubCmdStart - - functions. -} -type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek -type SubCmdStartString = String -> SubCmdStart -type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek -type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart - -data SubCommand = SubCommand { - subcmdname :: String, - subcmdparams :: String, - subcmdseek :: SubCmdSeek, - subcmddesc :: String -} -subCmds :: [SubCommand] -subCmds = [ - (SubCommand "add" path (withFilesNotInGit addStart) - "add files to annex") - , (SubCommand "get" path (withFilesInGit getStart) - "make content of annexed files available") - , (SubCommand "drop" path (withFilesInGit dropStart) - "indicate content of files not currently wanted") - , (SubCommand "move" path (withFilesInGit moveStart) - "transfer content of files to/from another repository") - , (SubCommand "init" desc (withDescription initStart) - "initialize git-annex with repository description") - , (SubCommand "unannex" path (withFilesInGit unannexStart) - "undo accidential add command") - , (SubCommand "fix" path (withFilesInGit fixStart) - "fix up symlinks to point to annexed content") - , (SubCommand "pre-commit" path (withFilesToBeCommitted fixStart) - "fix up symlinks before they are committed") - , (SubCommand "fromkey" key (withFilesMissing fromKeyStart) - "adds a file using a specific key") - , (SubCommand "dropkey" key (withKeys dropKeyStart) - "drops annexed content for specified keys") - , (SubCommand "setkey" key (withTempFile setKeyStart) - "sets annexed content for a key using a temp file") - ] - where - path = "PATH ..." - key = "KEY ..." - desc = "DESCRIPTION" - --- Each dashed command-line option results in generation of an action --- in the Annex monad that performs the necessary setting. -options :: [OptDescr (Annex ())] -options = [ - Option ['f'] ["force"] (NoArg (storebool "force" True)) - "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) - "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storebool "quiet" False)) - "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") - "specify default key-value backend to use" - , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") - "specify from where to transfer content" - ] - where - storebool n b = Annex.flagChange n $ FlagBool b - storestring n s = Annex.flagChange n $ FlagString s - -header :: String -header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) - -{- Usage message with lists of options and subcommands. -} -usage :: String -usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs - where - cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds - showcmd c = - (subcmdname c) ++ - (pad 11 (subcmdname c)) ++ - (subcmdparams c) ++ - (pad 13 (subcmdparams c)) ++ - (subcmddesc c) - indent l = " " ++ l - pad n s = take (n - (length s)) $ repeat ' ' - -{- Prepares a list of actions to run to perform a subcommand, based on - - the parameters passed to it. -} -prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdseek = seek } state params = do - list <- Annex.eval state $ seek params - return $ map (\a -> doSubCmd a) list - -{- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdStart -> SubCmdCleanup -doSubCmd start = do - s <- start - case (s) of - Nothing -> return True - Just perform -> do - p <- perform - case (p) of - Nothing -> do - showEndFail - return False - Just cleanup -> do - c <- cleanup - if (c) - then do - showEndOk - return True - else do - showEndFail - return False - -{- These functions find appropriate files or other things based on a - user's parameters. -} -withFilesNotInGit :: SubCmdSeekBackendFiles -withFilesNotInGit a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.notInRepo repo) params - let files' = foldl (++) [] files - pairs <- Backend.chooseBackends files' - return $ map a $ filter (\(f,_) -> notState f) pairs -withFilesInGit :: SubCmdSeekStrings -withFilesInGit a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.inRepo repo) params - return $ map a $ filter notState $ foldl (++) [] files -withFilesMissing :: SubCmdSeekStrings -withFilesMissing a params = do - files <- liftIO $ filterM missing params - return $ map a $ filter notState files - where - missing f = do - e <- doesFileExist f - return $ not e -withDescription :: SubCmdSeekStrings -withDescription a params = do - return $ [a $ unwords params] -withFilesToBeCommitted :: SubCmdSeekStrings -withFilesToBeCommitted a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.stagedFiles repo) params - return $ map a $ filter notState $ foldl (++) [] files -withKeys :: SubCmdSeekStrings -withKeys a params = return $ map a params -withTempFile :: SubCmdSeekStrings -withTempFile a params = return $ map a params - -{- filter out files from the state directory -} -notState :: FilePath -> Bool -notState f = stateLoc /= take (length stateLoc) f - -{- Parses command line and returns two lists of actions to be - - run in the Annex monad. The first actions configure it - - according to command line options, while the second actions - - handle subcommands. -} -parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) -parseCmd argv state = do - (flags, params) <- getopt - when (null params) $ error usage - case lookupCmd (params !! 0) of - [] -> error usage - [subcommand] -> do - actions <- prepSubCmd subcommand state (drop 1 params) - let configactions = map (\flag -> do - flag - return True) flags - return (configactions, actions) - _ -> error "internal error: multiple matching subcommands" - where - getopt = case getOpt Permute options argv of - (flags, params, []) -> return (flags, params) - (_, _, errs) -> ioError (userError (concat errs ++ usage)) - lookupCmd cmd = filter (\c -> cmd == subcmdname c) subCmds - -{- The add subcommand annexes a file, storing it in a backend, and then - - moving it into the annex directory and setting up the symlink pointing - - to its content. -} -addStart :: SubCmdStartBackendFile -addStart pair@(file, _) = notAnnexed file $ do - s <- liftIO $ getSymbolicLinkStatus file - if ((isSymbolicLink s) || (not $ isRegularFile s)) - then return Nothing - else do - showStart "add" file - return $ Just $ addPerform pair -addPerform :: (FilePath, Maybe Backend) -> SubCmdPerform -addPerform (file, backend) = do - stored <- Backend.storeFileKey file backend - case (stored) of - Nothing -> return Nothing - Just (key, _) -> return $ Just $ addCleanup file key -addCleanup :: FilePath -> Key -> SubCmdCleanup -addCleanup file key = do - logStatus key ValuePresent - g <- Annex.gitRepo - let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - liftIO $ renameFile file dest - link <- calcGitLink file key - liftIO $ createSymbolicLink link file - Annex.queue "add" [] file - return True - -{- The unannex subcommand undoes an add. -} -unannexStart :: SubCmdStartString -unannexStart file = isAnnexed file $ \(key, backend) -> do - showStart "unannex" file - return $ Just $ unannexPerform file key backend -unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform -unannexPerform file key backend = do - -- force backend to always remove - Annex.flagChange "force" $ FlagBool True - ok <- Backend.removeKey backend key - if (ok) - then return $ Just $ unannexCleanup file key - else return Nothing -unannexCleanup :: FilePath -> Key -> SubCmdCleanup -unannexCleanup file key = do - logStatus key ValueMissing - g <- Annex.gitRepo - let src = annexLocation g key - liftIO $ removeFile file - liftIO $ Git.run g ["rm", "--quiet", file] - -- git rm deletes empty directories; put them back - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ renameFile src file - return True - -{- Gets an annexed file from one of the backends. -} -getStart :: SubCmdStartString -getStart file = isAnnexed file $ \(key, backend) -> do - inannex <- inAnnex key - if (inannex) - then return Nothing - else do - showStart "get" file - return $ Just $ getPerform key backend -getPerform :: Key -> Backend -> SubCmdPerform -getPerform key backend = do - ok <- getViaTmp key (Backend.retrieveKeyFile backend key) - if (ok) - then return $ Just $ return True -- no cleanup needed - else return Nothing - -{- Indicates a file's content is not wanted anymore, and should be removed - - if it's safe to do so. -} -dropStart :: SubCmdStartString -dropStart file = isAnnexed file $ \(key, backend) -> do - inbackend <- Backend.hasKey key - if (not inbackend) - then return Nothing - else do - showStart "drop" file - return $ Just $ dropPerform key backend -dropPerform :: Key -> Backend -> SubCmdPerform -dropPerform key backend = do - success <- Backend.removeKey backend key - if (success) - then return $ Just $ dropCleanup key - else return Nothing -dropCleanup :: Key -> SubCmdCleanup -dropCleanup key = do - logStatus key ValueMissing - inannex <- inAnnex key - if (inannex) - then do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc - return True - else return True - -{- Drops cached content for a key. -} -dropKeyStart :: SubCmdStartString -dropKeyStart keyname = do - backends <- Backend.list - let key = genKey (backends !! 0) keyname - present <- inAnnex key - force <- Annex.flagIsSet "force" - if (not present) - then return Nothing - else if (not force) - then error "dropkey is can cause data loss; use --force if you're sure you want to do this" - else do - showStart "dropkey" keyname - return $ Just $ dropKeyPerform key -dropKeyPerform :: Key -> SubCmdPerform -dropKeyPerform key = do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc - return $ Just $ dropKeyCleanup key -dropKeyCleanup :: Key -> SubCmdCleanup -dropKeyCleanup key = do - logStatus key ValueMissing - return True - -{- Sets cached content for a key. -} -setKeyStart :: SubCmdStartString -setKeyStart tmpfile = do - keyname <- Annex.flagGet "key" - when (null keyname) $ error "please specify the key with --key" - backends <- Backend.list - let key = genKey (backends !! 0) keyname - showStart "setkey" tmpfile - return $ Just $ setKeyPerform tmpfile key -setKeyPerform :: FilePath -> Key -> SubCmdPerform -setKeyPerform tmpfile key = do - g <- Annex.gitRepo - let loc = annexLocation g key - ok <- liftIO $ boolSystem "mv" [tmpfile, loc] - if (not ok) - then error "mv failed!" - else return $ Just $ setKeyCleanup key -setKeyCleanup :: Key -> SubCmdCleanup -setKeyCleanup key = do - logStatus key ValuePresent - return True - -{- Fixes the symlink to an annexed file. -} -fixStart :: SubCmdStartString -fixStart file = isAnnexed file $ \(key, _) -> do - link <- calcGitLink file key - l <- liftIO $ readSymbolicLink file - if (link == l) - then return Nothing - else do - showStart "fix" file - return $ Just $ fixPerform file link -fixPerform :: FilePath -> FilePath -> SubCmdPerform -fixPerform file link = do - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ removeFile file - liftIO $ createSymbolicLink link file - return $ Just $ fixCleanup file -fixCleanup :: FilePath -> SubCmdCleanup -fixCleanup file = do - Annex.queue "add" [] file - return True - -{- Stores description for the repository etc. -} -initStart :: SubCmdStartString -initStart description = do - when (null description) $ error $ - "please specify a description of this repository\n" ++ usage - showStart "init" description - return $ Just $ initPerform description -initPerform :: String -> SubCmdPerform -initPerform description = do - g <- Annex.gitRepo - u <- getUUID g - describeUUID u description - liftIO $ gitAttributes g - liftIO $ gitPreCommitHook g - return $ Just $ initCleanup -initCleanup :: SubCmdCleanup -initCleanup = do - g <- Annex.gitRepo - logfile <- uuidLog - liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] - return True - -{- Adds a file pointing at a manually-specified key -} -fromKeyStart :: SubCmdStartString -fromKeyStart file = do - keyname <- Annex.flagGet "key" - when (null keyname) $ error "please specify the key with --key" - backends <- Backend.list - let key = genKey (backends !! 0) keyname - - inbackend <- Backend.hasKey key - unless (inbackend) $ error $ - "key ("++keyname++") is not present in backend" - showStart "fromkey" file - return $ Just $ fromKeyPerform file key -fromKeyPerform :: FilePath -> Key -> SubCmdPerform -fromKeyPerform file key = do - link <- calcGitLink file key - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ createSymbolicLink link file - return $ Just $ fromKeyCleanup file -fromKeyCleanup :: FilePath -> SubCmdCleanup -fromKeyCleanup file = do - Annex.queue "add" [] file - return True - -{- Move a file either --to or --from a repository. - - - - This only operates on the cached file content; it does not involve - - moving data in the key-value backend. -} -moveStart :: SubCmdStartString -moveStart file = do - fromName <- Annex.flagGet "fromrepository" - toName <- Annex.flagGet "torepository" - case (fromName, toName) of - ("", "") -> error "specify either --from or --to" - ("", _) -> moveToStart file - (_ , "") -> moveFromStart file - (_ , _) -> error "only one of --from or --to can be specified" - -{- Moves the content of an annexed file to another repository, - - removing it from the current repository, and updates locationlog - - information on both. - - - - If the destination already has the content, it is still removed - - from the current repository. - - - - Note that unlike drop, this does not honor annex.numcopies. - - A file's content can be moved even if there are insufficient copies to - - allow it to be dropped. - -} -moveToStart :: SubCmdStartString -moveToStart file = isAnnexed file $ \(key, _) -> do - ishere <- inAnnex key - if (not ishere) - then return Nothing -- not here, so nothing to do - else do - showStart "move" file - return $ Just $ moveToPerform key -moveToPerform :: Key -> SubCmdPerform -moveToPerform key = do - -- checking the remote is expensive, so not done in the start step - remote <- Remotes.commandLineRemote - isthere <- Remotes.inAnnex remote key - case isthere of - Left err -> do - showNote $ show err - return Nothing - Right False -> do - Core.showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." - let tmpfile = (annexTmpLocation remote) ++ (keyFile key) - ok <- Remotes.copyToRemote remote key tmpfile - if (ok) - then return $ Just $ moveToCleanup remote key tmpfile - else return Nothing -- failed - Right True -> return $ Just $ dropCleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup -moveToCleanup remote key tmpfile = do - -- Tell remote to use the transferred content. - ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", - "--backend=" ++ (backendName key), - "--key=" ++ keyName key, - tmpfile] - if ok - then do - -- Record that the key is present on the remote. - g <- Annex.gitRepo - remoteuuid <- getUUID remote - logfile <- liftIO $ logChange g key remoteuuid ValuePresent - Annex.queue "add" [] logfile - -- Cleanup on the local side is the same as done for the - -- drop subcommand. - dropCleanup key - else return False - -{- Moves the content of an annexed file from another repository to the current - - repository and updates locationlog information on both. - - - - If the current repository already has the content, it is still removed - - from the other repository. - -} -moveFromStart :: SubCmdStartString -moveFromStart file = isAnnexed file $ \(key, _) -> do - remote <- Remotes.commandLineRemote - l <- Remotes.keyPossibilities key - if (null $ filter (\r -> Remotes.same r remote) l) - then return Nothing - else do - showStart "move" file - return $ Just $ moveFromPerform key -moveFromPerform :: Key -> SubCmdPerform -moveFromPerform key = do - remote <- Remotes.commandLineRemote - ishere <- inAnnex key - if (ishere) - then return $ Just $ moveFromCleanup remote key - else do - Core.showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." - ok <- getViaTmp key (Remotes.copyFromRemote remote key) - if (ok) - then return $ Just $ moveFromCleanup remote key - else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup -moveFromCleanup remote key = do - ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", - "--backend=" ++ (backendName key), - keyName key] - when ok $ do - -- Record locally that the key is not on the remote. - remoteuuid <- getUUID remote - g <- Annex.gitRepo - logfile <- liftIO $ logChange g key remoteuuid ValueMissing - Annex.queue "add" [] logfile - return ok - --- helpers -notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) -notAnnexed file a = do - r <- Backend.lookupFile file - case (r) of - Just _ -> return Nothing - Nothing -> a -isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) -isAnnexed file a = do - r <- Backend.lookupFile file - case (r) of - Just v -> a v - Nothing -> return Nothing diff --git a/git-annex.hs b/git-annex.hs index 370c22a1ef..098ccac2d4 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -9,7 +9,7 @@ import System.Environment import qualified Annex import Core -import Commands +import CmdLine import qualified GitRepo as Git import BackendList From df4b461df1bf70016ffa02d08e822067f8544a63 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 4 Nov 2010 13:28:49 -0400 Subject: [PATCH 0418/2835] refactor --- CmdLine.hs | 35 ----------------------------------- Command.hs | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 494da2873c..98bdab12fb 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -33,12 +33,6 @@ import qualified Command.SetKey import qualified Command.Fix import qualified Command.Init -data SubCommand = SubCommand { - subcmdname :: String, - subcmdparams :: String, - subcmdseek :: SubCmdSeek, - subcmddesc :: String -} subCmds :: [SubCommand] subCmds = [ (SubCommand "add" path (withFilesNotInGit Command.Add.start) @@ -109,35 +103,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' -{- Prepares a list of actions to run to perform a subcommand, based on - - the parameters passed to it. -} -prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdseek = seek } state params = do - list <- Annex.eval state $ seek params - return $ map (\a -> doSubCmd a) list - -{- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdStart -> SubCmdCleanup -doSubCmd start = do - s <- start - case (s) of - Nothing -> return True - Just perform -> do - p <- perform - case (p) of - Nothing -> do - showEndFail - return False - Just cleanup -> do - c <- cleanup - if (c) - then do - showEndOk - return True - else do - showEndFail - return False - {- These functions find appropriate files or other things based on a user's parameters. -} withFilesNotInGit :: SubCmdSeekBackendFiles diff --git a/Command.hs b/Command.hs index 3d1e75e5f5..47c73370f3 100644 --- a/Command.hs +++ b/Command.hs @@ -8,7 +8,9 @@ module Command where import Types -import Backend +import qualified Backend +import Core +import qualified Annex {- A subcommand runs in four stages. - @@ -35,6 +37,42 @@ type SubCmdStartString = String -> SubCmdStart type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart +data SubCommand = SubCommand { + subcmdname :: String, + subcmdparams :: String, + subcmdseek :: SubCmdSeek, + subcmddesc :: String +} + +{- Prepares a list of actions to run to perform a subcommand, based on + - the parameters passed to it. -} +prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] +prepSubCmd SubCommand { subcmdseek = seek } state params = do + list <- Annex.eval state $ seek params + return $ map (\a -> doSubCmd a) list + +{- Runs a subcommand through the start, perform and cleanup stages -} +doSubCmd :: SubCmdStart -> SubCmdCleanup +doSubCmd start = do + s <- start + case (s) of + Nothing -> return True + Just perform -> do + p <- perform + case (p) of + Nothing -> do + showEndFail + return False + Just cleanup -> do + c <- cleanup + if (c) + then do + showEndOk + return True + else do + showEndFail + return False + notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) notAnnexed file a = do r <- Backend.lookupFile file From cc4794ce85f8e8e511a4aadb62db53bfff35ca8d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 4 Nov 2010 13:37:06 -0400 Subject: [PATCH 0419/2835] support subcommands that take no params --- CmdLine.hs | 10 ++++++++-- Command.hs | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 98bdab12fb..e03c4d4c6d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -32,6 +32,7 @@ import qualified Command.DropKey import qualified Command.SetKey import qualified Command.Fix import qualified Command.Init +import qualified Command.Fsck subCmds :: [SubCommand] subCmds = [ @@ -47,8 +48,6 @@ subCmds = [ "initialize git-annex with repository description") , (SubCommand "unannex" path (withFilesInGit Command.Unannex.start) "undo accidential add command") - , (SubCommand "fix" path (withFilesInGit Command.Fix.start) - "fix up symlinks to point to annexed content") , (SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) "fix up symlinks before they are committed") , (SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) @@ -57,11 +56,16 @@ subCmds = [ "drops annexed content for specified keys") , (SubCommand "setkey" key (withTempFile Command.SetKey.start) "sets annexed content for a key using a temp file") + , (SubCommand "fix" path (withFilesInGit Command.Fix.start) + "fix up symlinks to point to annexed content") + , (SubCommand "fsck" nothing (withNothing Command.Fsck.start) + "check annex for problems") ] where path = "PATH ..." key = "KEY ..." desc = "DESCRIPTION" + nothing = "" -- Each dashed command-line option results in generation of an action -- in the Annex monad that performs the necessary setting. @@ -137,6 +141,8 @@ withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params +withNothing :: SubCmdSeekNothing +withNothing a params = return [a] {- filter out files from the state directory -} notState :: FilePath -> Bool diff --git a/Command.hs b/Command.hs index 47c73370f3..d557651aa3 100644 --- a/Command.hs +++ b/Command.hs @@ -36,6 +36,7 @@ type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek type SubCmdStartString = String -> SubCmdStart type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart +type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek data SubCommand = SubCommand { subcmdname :: String, From 6b80356f6de05efef1f14fd2af9835cf5abe69a0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 4 Nov 2010 13:40:00 -0400 Subject: [PATCH 0420/2835] fixes --- CmdLine.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index e03c4d4c6d..7aaa1c842e 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,7 +10,6 @@ module CmdLine (parseCmd) where import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Directory -import Data.String.Utils import Control.Monad (filterM) import Monad (when) @@ -19,7 +18,6 @@ import qualified Annex import Locations import qualified Backend import Types -import Core import Command import qualified Command.Add @@ -91,7 +89,7 @@ options = [ storestring n s = Annex.flagChange n $ FlagString s header :: String -header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) +header = "Usage: git-annex subcommand [option ..]" {- Usage message with lists of options and subcommands. -} usage :: String @@ -142,7 +140,7 @@ withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params withNothing :: SubCmdSeekNothing -withNothing a params = return [a] +withNothing a _ = return [a] {- filter out files from the state directory -} notState :: FilePath -> Bool From 016b6a59e7187ead0ed630699c85d0fec729a30d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 6 Nov 2010 17:06:19 -0400 Subject: [PATCH 0421/2835] add fsck subcommand (stub) --- CmdLine.hs | 72 ++++++++++++++++++++++------------------------ Command.hs | 2 +- Command/Fsck.hs | 39 +++++++++++++++++++++++++ debian/changelog | 1 + doc/git-annex.mdwn | 5 ++++ 5 files changed, 81 insertions(+), 38 deletions(-) create mode 100644 Command/Fsck.hs diff --git a/CmdLine.hs b/CmdLine.hs index 7aaa1c842e..3823c72476 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,8 +10,7 @@ module CmdLine (parseCmd) where import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Directory -import Control.Monad (filterM) -import Monad (when) +import Control.Monad (filterM, when) import qualified GitRepo as Git import qualified Annex @@ -33,31 +32,31 @@ import qualified Command.Init import qualified Command.Fsck subCmds :: [SubCommand] -subCmds = [ - (SubCommand "add" path (withFilesNotInGit Command.Add.start) - "add files to annex") - , (SubCommand "get" path (withFilesInGit Command.Get.start) - "make content of annexed files available") - , (SubCommand "drop" path (withFilesInGit Command.Drop.start) - "indicate content of files not currently wanted") - , (SubCommand "move" path (withFilesInGit Command.Move.start) - "transfer content of files to/from another repository") - , (SubCommand "init" desc (withDescription Command.Init.start) - "initialize git-annex with repository description") - , (SubCommand "unannex" path (withFilesInGit Command.Unannex.start) - "undo accidential add command") - , (SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) - "fix up symlinks before they are committed") - , (SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) - "adds a file using a specific key") - , (SubCommand "dropkey" key (withKeys Command.DropKey.start) - "drops annexed content for specified keys") - , (SubCommand "setkey" key (withTempFile Command.SetKey.start) - "sets annexed content for a key using a temp file") - , (SubCommand "fix" path (withFilesInGit Command.Fix.start) - "fix up symlinks to point to annexed content") - , (SubCommand "fsck" nothing (withNothing Command.Fsck.start) - "check annex for problems") +subCmds = + [ SubCommand "add" path (withFilesNotInGit Command.Add.start) + "add files to annex" + , SubCommand "get" path (withFilesInGit Command.Get.start) + "make content of annexed files available" + , SubCommand "drop" path (withFilesInGit Command.Drop.start) + "indicate content of files not currently wanted" + , SubCommand "move" path (withFilesInGit Command.Move.start) + "transfer content of files to/from another repository" + , SubCommand "init" desc (withDescription Command.Init.start) + "initialize git-annex with repository description" + , SubCommand "unannex" path (withFilesInGit Command.Unannex.start) + "undo accidential add command" + , SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) + "fix up symlinks before they are committed" + , SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) + "adds a file using a specific key" + , SubCommand "dropkey" key (withKeys Command.DropKey.start) + "drops annexed content for specified keys" + , SubCommand "setkey" key (withTempFile Command.SetKey.start) + "sets annexed content for a key using a temp file" + , SubCommand "fix" path (withFilesInGit Command.Fix.start) + "fix up symlinks to point to annexed content" + , SubCommand "fsck" nothing (withNothing Command.Fsck.start) + "check annex for problems" ] where path = "PATH ..." @@ -95,15 +94,15 @@ header = "Usage: git-annex subcommand [option ..]" usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where - cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds + cmddescs = unlines $ map (indent . showcmd) subCmds showcmd c = - (subcmdname c) ++ - (pad 11 (subcmdname c)) ++ - (subcmdparams c) ++ - (pad 13 (subcmdparams c)) ++ - (subcmddesc c) + subcmdname c ++ + pad 11 (subcmdname c) ++ + subcmdparams c ++ + pad 13 (subcmdparams c) ++ + subcmddesc c indent l = " " ++ l - pad n s = take (n - (length s)) $ repeat ' ' + pad n s = replicate (n - length s) ' ' {- These functions find appropriate files or other things based on a user's parameters. -} @@ -128,8 +127,7 @@ withFilesMissing a params = do e <- doesFileExist f return $ not e withDescription :: SubCmdSeekStrings -withDescription a params = do - return $ [a $ unwords params] +withDescription a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo @@ -154,7 +152,7 @@ parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) parseCmd argv state = do (flags, params) <- getopt when (null params) $ error usage - case lookupCmd (params !! 0) of + case lookupCmd (head params) of [] -> error usage [subcommand] -> do actions <- prepSubCmd subcommand state (drop 1 params) diff --git a/Command.hs b/Command.hs index d557651aa3..a0e3280d6b 100644 --- a/Command.hs +++ b/Command.hs @@ -50,7 +50,7 @@ data SubCommand = SubCommand { prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] prepSubCmd SubCommand { subcmdseek = seek } state params = do list <- Annex.eval state $ seek params - return $ map (\a -> doSubCmd a) list + return $ map doSubCmd list {- Runs a subcommand through the start, perform and cleanup stages -} doSubCmd :: SubCmdStart -> SubCmdCleanup diff --git a/Command/Fsck.hs b/Command/Fsck.hs new file mode 100644 index 0000000000..bd5a9ad7f0 --- /dev/null +++ b/Command/Fsck.hs @@ -0,0 +1,39 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Fsck where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory + +import Command +import qualified Annex +import Types +import Utility +import Core + +{- Checks the whole annex for problems. -} +start :: SubCmdStart +start = do + showStart "fsck" "" + return $ Just perform + +perform :: SubCmdPerform +perform = do + ok <- checkUnused + if (ok) + then return $ Just $ return True + else do + showLongNote "Possible problems detected." + return Nothing + +checkUnused :: Annex Bool +checkUnused = do + showNote "checking for unused data..." + -- TODO + return False diff --git a/debian/changelog b/debian/changelog index b433ec62fd..ae68f657b9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -13,6 +13,7 @@ git-annex (0.03) UNRELEASED; urgency=low via gitattributes. * In .gitattributes, the git-annex-backend attribute can be set to the names of backends to use when adding different types of files. + * Add fsck subcommand. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index bbd7e8cab1..856b474e05 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -141,6 +141,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git annex setkey --key=1287765018:3 /tmp/file +* fsck + + This subcommand checks the whole annex for consistency, and warns + about any problems found. + # OPTIONS * --force From a3519c365feec45b5ab1236c7610863678b868a0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 6 Nov 2010 17:07:11 -0400 Subject: [PATCH 0422/2835] hlinted a few files --- Annex.hs | 4 ++-- Backend.hs | 13 ++++++------- Core.hs | 14 +++++++------- GitQueue.hs | 4 ++-- GitRepo.hs | 31 +++++++++++++++---------------- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/Annex.hs b/Annex.hs index 303881fa09..e86e1967e3 100644 --- a/Annex.hs +++ b/Annex.hs @@ -50,9 +50,9 @@ new gitrepo allbackends = do {- performs an action in the Annex monad -} run :: AnnexState -> StateT AnnexState IO a -> IO (a, AnnexState) -run state action = runStateT (action) state +run state action = runStateT action state eval :: AnnexState -> StateT AnnexState IO a -> IO a -eval state action = evalStateT (action) state +eval state action = evalStateT action state {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo diff --git a/Backend.hs b/Backend.hs index e1d0e0a686..e2c8a43b6a 100644 --- a/Backend.hs +++ b/Backend.hs @@ -53,7 +53,7 @@ list = do let l' = if (not $ null backendflag) then (lookupBackendName bs backendflag):defaults else defaults - Annex.backendsChange $ l' + Annex.backendsChange l' return l' where parseBackendList bs s = @@ -71,7 +71,7 @@ maybeLookupBackendName :: [Backend] -> String -> Maybe Backend maybeLookupBackendName bs s = if ((length matches) /= 1) then Nothing - else Just $ matches !! 0 + else Just $ head matches where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} @@ -88,14 +88,13 @@ storeFileKey' :: [Backend] -> FilePath -> FilePath -> Annex (Maybe (Key, Backend storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do result <- (Internals.getKey b) relfile - case (result) of + case result of Nothing -> nextbackend Just key -> do stored <- (Internals.storeFileKey b) file key if (not stored) then nextbackend - else do - return $ Just (key, b) + else return $ Just (key, b) where nextbackend = storeFileKey' bs file relfile @@ -127,8 +126,8 @@ lookupFile file = do getsymlink = do l <- readSymbolicLink file return $ takeFileName l - makekey bs l = do - case maybeLookupBackendName bs $ bname of + makekey bs l = + case maybeLookupBackendName bs bname of Nothing -> do unless (null kname || null bname) $ warning skip diff --git a/Core.hs b/Core.hs index 327a9af483..f34b2ebbeb 100644 --- a/Core.hs +++ b/Core.hs @@ -13,7 +13,7 @@ import System.Directory import Control.Monad.State (liftIO) import System.Path import Data.String.Utils -import Monad (when, unless) +import Control.Monad (when, unless) import Types import Locations @@ -40,7 +40,7 @@ tryRun' state errnum (a:as) = do Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as tryRun' _ errnum [] = - when (errnum > 0) $ error $ (show errnum) ++ " failed" + when (errnum > 0) $ error $ show errnum ++ " failed" {- Sets up a git repo for git-annex. -} startup :: Annex Bool @@ -63,7 +63,7 @@ shutdown = do -- the tmp directory itself let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp - when (exists) $ liftIO $ removeDirectoryRecursive $ tmp + when (exists) $ liftIO $ removeDirectoryRecursive tmp liftIO $ createDirectoryIfMissing True tmp return True @@ -93,7 +93,7 @@ gitAttributes repo = do {- set up a git pre-commit hook, if one is not already present -} gitPreCommitHook :: Git.Repo -> IO () gitPreCommitHook repo = do - let hook = (Git.workTree repo) ++ "/" ++ (Git.gitDir repo) ++ + let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit" exists <- doesFileExist hook if (exists) @@ -120,7 +120,7 @@ calcGitLink file key = do let absfile = case (absNormPath cwd file) of Just f -> f Nothing -> error $ "unable to normalize " ++ file - return $ (relPathDirToDir (parentDir absfile) (Git.workTree g)) ++ + return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++ annexLocationRelative key {- Updates the LocationLog when a key's presence changes. -} @@ -138,7 +138,7 @@ getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool getViaTmp key action = do g <- Annex.gitRepo let dest = annexLocation g key - let tmp = (annexTmpLocation g) ++ (keyFile key) + let tmp = annexTmpLocation g ++ keyFile key liftIO $ createDirectoryIfMissing True (parentDir tmp) success <- action tmp if (success) @@ -165,7 +165,7 @@ showNote s = verbose $ do liftIO $ putStr $ "(" ++ s ++ ") " liftIO $ hFlush stdout showProgress :: Annex () -showProgress = verbose $ liftIO $ putStr $ "\n" +showProgress = verbose $ liftIO $ putStr "\n" showLongNote :: String -> Annex () showLongNote s = verbose $ do liftIO $ putStr $ "\n" ++ indented diff --git a/GitQueue.hs b/GitQueue.hs index 632d1d3910..2d44a8f108 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -16,7 +16,7 @@ import qualified Data.Map as M import System.IO import System.Cmd.Utils import Data.String.Utils -import Monad (unless) +import Control.Monad (unless) import qualified GitRepo as Git @@ -57,5 +57,5 @@ runAction repo action files = do where runxargs = pOpen WriteToPipe "xargs" ("-0":gitcmd) feedxargs gitcmd = ["git"] ++ Git.gitCommandLine repo - ((getSubcommand action):(getParams action)) + (getSubcommand action:getParams action) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index 7d5291ff1e..da86c225e2 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -43,8 +43,8 @@ module GitRepo ( prop_idempotent_deencode ) where -import Monad (unless) -import Directory +import Control.Monad (unless) +import System.Directory import System.Posix.Directory import System.Path import System.Cmd.Utils @@ -53,11 +53,11 @@ import Data.String.Utils import System.IO import qualified Data.Map as Map hiding (map, split) import Network.URI -import Maybe -import Char -import Text.Printf +import Data.Maybe +import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) +import Text.Printf import Utility @@ -127,31 +127,31 @@ assertLocal :: Repo -> a -> a assertLocal repo action = if (not $ repoIsUrl repo) then action - else error $ "acting on URL git repo " ++ (repoDescribe repo) ++ + else error $ "acting on URL git repo " ++ repoDescribe repo ++ " not supported" assertUrl :: Repo -> a -> a assertUrl repo action = if (repoIsUrl repo) then action - else error $ "acting on local git repo " ++ (repoDescribe repo) ++ + else error $ "acting on local git repo " ++ repoDescribe repo ++ " not supported" assertSsh :: Repo -> a -> a assertSsh repo action = if (repoIsSsh repo) then action - else error $ "unsupported url in repo " ++ (repoDescribe repo) + else error $ "unsupported url in repo " ++ repoDescribe repo bare :: Repo -> Bool bare repo = case Map.lookup "core.bare" $ config repo of Just v -> configTrue v Nothing -> error $ "it is not known if git repo " ++ - (repoDescribe repo) ++ + repoDescribe repo ++ " is a bare repository; config not read" {- Path to a repository's gitattributes file. -} attributes :: Repo -> String attributes repo - | bare repo = (workTree repo) ++ "/info/.gitattributes" - | otherwise = (workTree repo) ++ "/.gitattributes" + | bare repo = workTree repo ++ "/info/.gitattributes" + | otherwise = workTree repo ++ "/.gitattributes" {- Path to a repository's .git directory, relative to its workTree. -} gitDir :: Repo -> String @@ -176,7 +176,7 @@ relative repo@(Repo { location = Dir d }) file = drop (length absrepo) absfile -- will be substring of file absrepo = case (absNormPath "/" d) of Just f -> f ++ "/" - Nothing -> error $ "bad repo" ++ (repoDescribe repo) + Nothing -> error $ "bad repo" ++ repoDescribe repo absfile = case (secureAbsNormPath absrepo file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo @@ -185,7 +185,7 @@ relative repo _ = assertLocal repo $ error "internal" {- Hostname of an URL repo. (May include a username and/or port too.) -} urlHost :: Repo -> String urlHost Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a - where a = fromJust $ uriAuthority $ u + where a = fromJust $ uriAuthority u urlHost repo = assertUrl repo $ error "internal" {- Path of an URL repo. -} @@ -204,14 +204,13 @@ gitCommandLine repo _ = assertLocal repo $ error "internal" run :: Repo -> [String] -> IO () run repo params = assertLocal repo $ do ok <- boolSystem "git" (gitCommandLine repo params) - unless (ok) $ error $ "git " ++ (show params) ++ " failed" + unless (ok) $ error $ "git " ++ show params ++ " failed" {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String pipeRead repo params = assertLocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - ret <- hGetContentsStrict h - return ret + hGetContentsStrict h {- Like pipeRead, but does not read output strictly; recommended - for git commands that produce a lot of output that will be processed From b1e26b19c6055b8c635a158068a2ad0188f17d17 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 00:26:44 +0000 Subject: [PATCH 0423/2835] idea --- debian/changelog | 1 - doc/todo/auto_remotes.mdwn | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 doc/todo/auto_remotes.mdwn diff --git a/debian/changelog b/debian/changelog index ae68f657b9..1d3c6427f6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,6 @@ git-annex (0.03) UNRELEASED; urgency=low * Add --verbose * Fix SIGINT handling. * Fix handling of files with unusual characters in their name. - * Support building with Debian stable's ghc. * Fixed memory leak; git-annex no longer reads the whole file list from git before starting, and will be much faster with large repos. * Fix crash on unknown symlinks. diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn new file mode 100644 index 0000000000..4976a2a778 --- /dev/null +++ b/doc/todo/auto_remotes.mdwn @@ -0,0 +1,20 @@ +It should be possible for clones to learn about how to contact +each other without remotes needing to always be explicitly set +up. Say that `.git-annex/remote.log` is maintained by git-annex +to contain: + + UUID hostname URI + +The URI comes from configured remotes and maybe from +`file://$(pwd)` for the current repo. This format will merge +without conflicts or data loss. + +Then when content is belived to be in a UUID, and no +configured remote has it, the remote.log can be consulted and +URIs that look likely tried. (file:// ones if the hostname +is the same (or maybe always -- a removable drive might tend +to be mounted at the same location on different hosts), +otherwise ssh:// ones.) + +Question: When should git-annex update the remote.log? +(Not just on init.) Whenever it reads in a repo's remotes? From 8156af90edc27c041bd1c22d8401aff21f4b8947 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 00:28:17 +0000 Subject: [PATCH 0424/2835] update --- doc/todo/auto_remotes.mdwn | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn index 4976a2a778..2ac4790af9 100644 --- a/doc/todo/auto_remotes.mdwn +++ b/doc/todo/auto_remotes.mdwn @@ -6,8 +6,9 @@ to contain: UUID hostname URI The URI comes from configured remotes and maybe from -`file://$(pwd)` for the current repo. This format will merge -without conflicts or data loss. +`file://$(pwd)`, or even `ssh://$(hostname -f)` +for the current repo. This format will merge without +conflicts or data loss. Then when content is belived to be in a UUID, and no configured remote has it, the remote.log can be consulted and @@ -17,4 +18,4 @@ to be mounted at the same location on different hosts), otherwise ssh:// ones.) Question: When should git-annex update the remote.log? -(Not just on init.) Whenever it reads in a repo's remotes? +(If not just on init.) Whenever it reads in a repo's remotes? From 55b92860ceb099614ac9ebe4c37e92b57ad6a430 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 6 Nov 2010 21:12:45 -0400 Subject: [PATCH 0425/2835] bigfix: doubled shell escape --- Remotes.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 7bb1bcd222..280543968c 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -94,8 +94,7 @@ inAnnex r key = do Annex.eval a (Core.inAnnex key) checkremote = do Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- runCmd r "test" - [ "-e", (shellEscape $ annexLocation r key)] + inannex <- runCmd r "test" ["-e", annexLocation r key] -- XXX Note that ssh failing and the file not existing -- are not currently differentiated. return $ Right inannex From ea8ccaa3d5416044ca69e4a3dcb7b879aec0ff4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 17:26:21 -0400 Subject: [PATCH 0426/2835] rough in fsck --- Command/Fsck.hs | 46 ++++++++++++++++++++++++++++++++++++++-------- TypeInternals.hs | 2 +- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index bd5a9ad7f0..c86f30ff80 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -7,15 +7,10 @@ module Command.Fsck where -import Control.Monad.State (liftIO) -import System.Posix.Files -import System.Directory - import Command -import qualified Annex import Types -import Utility import Core +import qualified Data.Map as M {- Checks the whole annex for problems. -} start :: SubCmdStart @@ -35,5 +30,40 @@ perform = do checkUnused :: Annex Bool checkUnused = do showNote "checking for unused data..." - -- TODO - return False + unused <- unusedKeys + if (null unused) + then return True + else do + showLongNote $ w unused + return False + where + w u = unlines $ [ + "Some annexed data is no longer pointed to by any file.", + "If this data is no longer needed, it can be removed using git-annex dropkey:" + ] ++ map show u + +unusedKeys :: Annex [Key] +unusedKeys = do + present <- getKeysPresent + referenced <- getKeysReferenced + + -- Constructing a single map, of the set that tends to be smaller, + -- appears more efficient in both memory and CPU than constructing + -- and taking the M.difference of two maps. + let present_m = existsMap present + let unused_m = remove referenced present_m + return $ M.keys unused_m + where + remove [] m = m + remove (x:xs) m = remove xs $ M.delete x m + +existsMap :: Ord k => [k] -> M.Map k Int +existsMap l = M.fromList $ map (\k -> (k, 1)) l + +getKeysPresent :: Annex [Key] +getKeysPresent = do + return [] + +getKeysReferenced :: Annex [Key] +getKeysReferenced = do + return [] diff --git a/TypeInternals.hs b/TypeInternals.hs index 46c92cb59b..4b5cff9d9f 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -39,7 +39,7 @@ type Annex = StateT AnnexState IO -- annexed filenames are mapped through a backend into keys type KeyName = String type BackendName = String -data Key = Key (BackendName, KeyName) deriving (Eq) +data Key = Key (BackendName, KeyName) deriving (Eq, Ord) -- constructs a key in a backend genKey :: Backend -> KeyName -> Key From 316264f3e8f6dbcbd2c3752566d7d754dbfe9994 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 17:36:24 -0400 Subject: [PATCH 0427/2835] add annexDir --- Locations.hs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Locations.hs b/Locations.hs index a296f8d876..951924c407 100644 --- a/Locations.hs +++ b/Locations.hs @@ -12,7 +12,8 @@ module Locations ( fileKey, annexLocation, annexLocationRelative, - annexTmpLocation + annexTmpLocation, + annexDir ) where import Data.String.Utils @@ -43,11 +44,15 @@ annexLocation r key = annexLocationRelative :: Key -> FilePath annexLocationRelative key = ".git/annex/" ++ (keyFile key) -{- .git-annex/tmp is used for temp files +{- The annex directory of a repository. - - Note: Assumes repo is NOT bare. -} +annexDir :: Git.Repo -> FilePath +annexDir r = Git.workTree r ++ "/.git/annex" + +{- .git-annex/tmp is used for temp files -} annexTmpLocation :: Git.Repo -> FilePath -annexTmpLocation r = (Git.workTree r) ++ "/.git/annex/tmp/" +annexTmpLocation r = annexDir r ++ "/tmp/" {- Converts a key into a filename fragment. - From 009873e0eb296b6f373f9f2d847659038a1d2bde Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:22:25 -0400 Subject: [PATCH 0428/2835] fsck works --- Backend.hs | 2 +- Command/Fsck.hs | 33 ++++++++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/Backend.hs b/Backend.hs index e2c8a43b6a..456a98bd41 100644 --- a/Backend.hs +++ b/Backend.hs @@ -107,7 +107,7 @@ retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest removeKey :: Backend -> Key -> Annex Bool removeKey backend key = (Internals.removeKey backend) key -{- Checks if a backend has its key. -} +{- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool hasKey key = do bs <- Annex.supportedBackends diff --git a/Command/Fsck.hs b/Command/Fsck.hs index c86f30ff80..785aecd8af 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -7,10 +7,20 @@ module Command.Fsck where +import qualified Data.Map as M +import System.Directory +import System.Posix.Files +import Monad (filterM) +import Control.Monad.State (liftIO) +import Data.Maybe + import Command import Types import Core -import qualified Data.Map as M +import Locations +import qualified Annex +import qualified GitRepo as Git +import qualified Backend {- Checks the whole annex for problems. -} start :: SubCmdStart @@ -38,10 +48,12 @@ checkUnused = do return False where w u = unlines $ [ - "Some annexed data is no longer pointed to by any file.", + "Some annexed data is no longer pointed to by any files in the repository.", "If this data is no longer needed, it can be removed using git-annex dropkey:" - ] ++ map show u + ] ++ map (\k -> " " ++ show k) u +{- Finds keys whose content is present, but that do not seem to be used + - by any files in the git repo. -} unusedKeys :: Annex [Key] unusedKeys = do present <- getKeysPresent @@ -62,8 +74,19 @@ existsMap l = M.fromList $ map (\k -> (k, 1)) l getKeysPresent :: Annex [Key] getKeysPresent = do - return [] + g <- Annex.gitRepo + let top = annexDir g + contents <- liftIO $ getDirectoryContents top + files <- liftIO $ filterM (isreg top) contents + return $ map fileKey files + where + isreg top f = do + s <- getFileStatus $ top ++ "/" ++ f + return $ isRegularFile s getKeysReferenced :: Annex [Key] getKeysReferenced = do - return [] + g <- Annex.gitRepo + files <- liftIO $ Git.inRepo g $ Git.workTree g + keypairs <- mapM Backend.lookupFile files + return $ map fst $ catMaybes keypairs From bbee90cd557f0e1acd3744a120bf5251928b7056 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:23:25 -0400 Subject: [PATCH 0429/2835] update --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 1d3c6427f6..ccfaa0d074 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,7 +12,8 @@ git-annex (0.03) UNRELEASED; urgency=low via gitattributes. * In .gitattributes, the git-annex-backend attribute can be set to the names of backends to use when adding different types of files. - * Add fsck subcommand. + * Add fsck subcommand. (For now it only finds unused key contents in the + annex.) -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 From 05d321ace65291322e0393b1d931f29c5058f5c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:24:44 -0400 Subject: [PATCH 0430/2835] done --- doc/todo/fsck.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/fsck.mdwn b/doc/todo/fsck.mdwn index 308a1cb63a..6126154e9f 100644 --- a/doc/todo/fsck.mdwn +++ b/doc/todo/fsck.mdwn @@ -1 +1,3 @@ add a git annex fsck that finds keys that have no referring file + +[[done]] From 8f949b164404e70be56d283280045ba4c72426d6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:30:21 -0400 Subject: [PATCH 0431/2835] releasing version 0.03 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index ccfaa0d074..f9cec02c44 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.03) UNRELEASED; urgency=low +git-annex (0.03) unstable; urgency=low * Fix support for file:// remotes. * Add --verbose @@ -15,7 +15,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Add fsck subcommand. (For now it only finds unused key contents in the annex.) - -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 + -- Joey Hess Sun, 07 Nov 2010 18:26:04 -0400 git-annex (0.02) unstable; urgency=low From 714619d9e834a3a04340b164255ff2c92f654228 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:30:38 -0400 Subject: [PATCH 0432/2835] add news item for git-annex 0.03 --- doc/news/version_0.03.mdwn | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/news/version_0.03.mdwn diff --git a/doc/news/version_0.03.mdwn b/doc/news/version_0.03.mdwn new file mode 100644 index 0000000000..b22aad91af --- /dev/null +++ b/doc/news/version_0.03.mdwn @@ -0,0 +1,16 @@ +git-annex 0.03 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Fix support for file:// remotes. + * Add --verbose + * Fix SIGINT handling. + * Fix handling of files with unusual characters in their name. + * Fixed memory leak; git-annex no longer reads the whole file list + from git before starting, and will be much faster with large repos. + * Fix crash on unknown symlinks. + * Added remote.annex-scp-options and remote.annex-ssh-options. + * The backends to use when adding different sets of files can be configured + via gitattributes. + * In .gitattributes, the git-annex-backend attribute can be set to the + names of backends to use when adding different types of files. + * Add fsck subcommand. (For now it only finds unused key contents in the + annex.)"""]] \ No newline at end of file From 377bf24d9a951186b374cd7a3f920b6bc9deb8f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 21:02:25 -0400 Subject: [PATCH 0433/2835] documentation for checkout --- debian/changelog | 8 ++++++++ doc/backends.mdwn | 3 +-- doc/git-annex.mdwn | 14 +++++++++++++- doc/todo/backendSHA1.mdwn | 2 ++ doc/todo/checkout.mdwn | 18 ++++++++++++++++++ doc/walkthrough.mdwn | 34 ++++++++++++++++++++++++++++++++++ 6 files changed, 76 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index f9cec02c44..9205dbe06f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +git-annex (0.04) UNRELEASED; urgency=low + + * Add checkout subcommand, which allows checking out file content + in preparation of changing it. + * Add uncheckout subcommand. + + -- Joey Hess Sun, 07 Nov 2010 21:01:29 -0400 + git-annex (0.03) unstable; urgency=low * Fix support for file:// remotes. diff --git a/doc/backends.mdwn b/doc/backends.mdwn index dc359174ae..69c17649a4 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -17,8 +17,7 @@ can use different backends for different files. * `SHA1` -- This backend stores the file's content in `.git/annex/`, with a name based on its sha1 checksum. This backend allows modifications of files to be tracked. Its need to generate checksums - can make it slower for large files. **Warning** this backend is not ready - for use. + can make it slower for large files. * `URL` -- This backend downloads the file's content from an external URL. The `annex.backends` git-config setting can be used to list the backends diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 856b474e05..80680820fa 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -81,6 +81,13 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git-annex may refuse to drop content if the backend does not think it is safe to do so, typically because of the setting of annex.numcopies. +* checkout [path ...] + + Checks out annexed files so they can be modified. This replaces the + symlink for each specified file with a copy of the file content. + When you `git commit`, the file, the new content is injected back into + the annex. + * move [path ...] When used with the --to option, moves the content of annexed files from @@ -95,6 +102,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. and sets up `.gitattributes` and the pre-commit hook. This is an optional, but recommended step. +* uncheckout [path ...] + + Use this to undo a checkout command if you don't want to modify + the checked out files, or have made modifications you want to discard. + * unannex [path ...] Use this to undo an accidental add command. This is not the command you @@ -110,7 +122,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * pre-commit [path ...] Fixes up symlinks that are staged as part of a commit, to ensure they - point to annexed content. + point to annexed content. Also handles committing checked-out files. This is meant to be called from git's pre-commit hook. `git annex init` automatically creates a pre-commit hook using this. diff --git a/doc/todo/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn index fa9728af64..44df406dea 100644 --- a/doc/todo/backendSHA1.mdwn +++ b/doc/todo/backendSHA1.mdwn @@ -3,3 +3,5 @@ This backend is not finished. In particular, while files can be added using it, git-annex will not notice when their content changes, and will not create a new key for the new sha1 of the net content. + +[[done]]; use checkout subcommand diff --git a/doc/todo/checkout.mdwn b/doc/todo/checkout.mdwn index 70d31a5ff5..50da2d62e1 100644 --- a/doc/todo/checkout.mdwn +++ b/doc/todo/checkout.mdwn @@ -3,3 +3,21 @@ file's content, with a copy of the file. Once you've checked a file out, you can edit it, and `git commit` it. On commit, git-annex will detect if the file has been changed, and if it has, `add` its content to the annex. + +> Internally, this will need to store the original symlink to the file, in +> `.git/annex/checkedout/$filename`. +> +> * git-annex uncheckout moves that back +> * git-annex pre-commit hook checks each file being committed to see if +> it has a symlink there, and if so, removes the symlink and adds the new +> content to the annex. +> +> And it seems the file content should be copied, not moved or hard linked: +> +> * Makes sure other annexes can find it if transferring it from +> this annex. +> * Ensures it's always available for uncheckout. +> * Avoids the last copy of a file's content being lost when +> the checked out file is modified. + +[[done]] diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index ab0067470d..cb564fa979 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -190,6 +190,26 @@ makes it very easy. WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 ok +## modifying annexed files + +Normally, the content of files in the annex cannot be modified. +In order to modify a file, it should first be checked out: + + # git annex checkout my_cool_big_file + checkout my_cool_big_file (copying...) ok + +Checking a file out replaces the symlink that normally points at its content +with a copy of the content. You can then modify the file like any regular +file. Because it is a regular file. + +When you `git commit`, git-annex's pre-commit hook will automatically +notice that you are committing a checked-out file, and add its new content +to the annex. The file will be replaced with a symlink to the new content, +and this symlink is what gets committed to git. + +If you decide you don't need to modify the file after all, or want to discard +modifications, just use the uncheckout subcommand to undo the checkout. + ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has @@ -216,3 +236,17 @@ that the URL is stable; no local backup is kept. # git annex drop somefile drop somefile (ok) + +## using the SHA1 backend + +Another handy alternative to the default [[backend|backends]] is the +SHA1 backend. This backend provides more git-style assurance that your data +has not been damanged. And the checksum means that when you add the same +content to the annex twice, only one copy need be stored in the backend. + +The only reason it's not the default is that it needs to checksum +files when they're added to the annex, and this can slow things down +significantly for really big files. To make SHA1 the detault, just +add something like this to `.gitattributes`: + + * git-annex-backend=SHA1 From f03adec793d378cc4807392400d09e70e293a991 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 12:36:55 -0400 Subject: [PATCH 0434/2835] Add build dep on libghc6-testpack-dev. --- debian/changelog | 6 ++++++ debian/control | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index f9cec02c44..a363d442c9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.04) UNRELEASED; urgency=low + + * Add build dep on libghc6-testpack-dev. + + -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 + git-annex (0.03) unstable; urgency=low * Fix support for file:// remotes. diff --git a/debian/control b/debian/control index d8abc487cb..3fba367427 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, ikiwiki +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From ab4de454914954676aa1e05ef26dc8a85bd8f6f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 14:39:12 -0400 Subject: [PATCH 0435/2835] Add annex.version, which will be used to automate upgrades. --- Annex.hs | 12 +++++++++++- Core.hs | 16 ++++++++++++++++ UUID.hs | 13 ++----------- debian/changelog | 1 + doc/git-annex.mdwn | 2 ++ 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/Annex.hs b/Annex.hs index e86e1967e3..7c046b141d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -19,7 +19,8 @@ module Annex ( flagGet, Flag(..), queue, - queueGet + queueGet, + setConfig ) where import Control.Monad.State @@ -118,3 +119,12 @@ queueGet :: Annex GitQueue.Queue queueGet = do state <- get return (Internals.repoqueue state) + +{- Changes a git config setting in both internal state and .git/config -} +setConfig :: String -> String -> Annex () +setConfig key value = do + g <- Annex.gitRepo + liftIO $ Git.run g ["config", key, value] + -- re-read git config and update the repo's state + g' <- liftIO $ Git.configRead g Nothing + Annex.gitRepoChange g' diff --git a/Core.hs b/Core.hs index f34b2ebbeb..347e635939 100644 --- a/Core.hs +++ b/Core.hs @@ -46,6 +46,7 @@ tryRun' _ errnum [] = startup :: Annex Bool startup = do prepUUID + autoUpgrade return True {- When git-annex is done, it runs this. -} @@ -151,6 +152,21 @@ getViaTmp key action = do -- to resume its transfer return False +{- Uses the annex.version git config setting to automate upgrades. -} +autoUpgrade :: Annex () +autoUpgrade = do + g <- Annex.gitRepo + + case Git.configGet g field "0" of + "0" -> do -- before there was repo versioning + setVersion + v | v == currentVersion -> return () + _ -> error "this version of git-annex is too old for this git repository!" + where + currentVersion = "1" + setVersion = Annex.setConfig field currentVersion + field = "annex.version" + {- Output logging -} verbose :: Annex () -> Annex () verbose a = do diff --git a/UUID.hs b/UUID.hs index ffd2cd46dc..0f8a2173ef 100644 --- a/UUID.hs +++ b/UUID.hs @@ -65,7 +65,7 @@ getUUID r = do where uncached = Git.configGet r "annex.uuid" "" cached g = Git.configGet g cachekey "" - updatecache g u = when (g /= r) $ setConfig cachekey u + updatecache g u = when (g /= r) $ Annex.setConfig cachekey u cachekey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} @@ -75,16 +75,7 @@ prepUUID = do u <- getUUID g when ("" == u) $ do uuid <- liftIO $ genUUID - setConfig configkey uuid - -{- Changes a git config setting in both internal state and .git/config -} -setConfig :: String -> String -> Annex () -setConfig key value = do - g <- Annex.gitRepo - liftIO $ Git.run g ["config", key, value] - -- re-read git config and update the repo's state - g' <- liftIO $ Git.configRead g Nothing - Annex.gitRepoChange g' + Annex.setConfig configkey uuid {- Filters a list of repos to ones that have listed UUIDs. -} reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] diff --git a/debian/changelog b/debian/changelog index a363d442c9..98b814eb94 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.04) UNRELEASED; urgency=low * Add build dep on libghc6-testpack-dev. + * Add annex.version, which will be used to automate upgrades. -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 856b474e05..6f2c85d573 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -207,6 +207,8 @@ Here are all the supported configuration settings. to talk to this repository. * `annex.scp-options` and `annex.ssh-options` -- Default scp and ssh options to use if a remote does not have specific options. +* `annex.version` -- Automatically maintained, and used to automate upgrades + between versions. The backend used when adding a new file to the annex can be configured on a per-file-type basis via the `.gitattributes` file. In the file, From 02a21d7f274568a2e2f94498607955aab8713a24 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 15:14:54 -0400 Subject: [PATCH 0436/2835] reorg .git/annex --- Locations.hs | 13 ++++++++++--- debian/changelog | 2 ++ doc/backends.mdwn | 12 ++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/Locations.hs b/Locations.hs index 951924c407..78c0bff4b3 100644 --- a/Locations.hs +++ b/Locations.hs @@ -13,7 +13,8 @@ module Locations ( annexLocation, annexLocationRelative, annexTmpLocation, - annexDir + annexDir, + annexObjectDir ) where import Data.String.Utils @@ -29,7 +30,7 @@ gitStateDir :: Git.Repo -> FilePath gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc {- An annexed file's content is stored in - - /path/to/repo/.git/annex/, where is of the form + - /path/to/repo/.git/annex/objects//, where is of the form - - - That allows deriving the key and backend by looking at the symlink to it. @@ -42,7 +43,8 @@ annexLocation r key = - - Note: Assumes repo is NOT bare.-} annexLocationRelative :: Key -> FilePath -annexLocationRelative key = ".git/annex/" ++ (keyFile key) +annexLocationRelative key = ".git/annex/objects/" ++ f ++ f + where f = keyFile key {- The annex directory of a repository. - @@ -50,6 +52,11 @@ annexLocationRelative key = ".git/annex/" ++ (keyFile key) annexDir :: Git.Repo -> FilePath annexDir r = Git.workTree r ++ "/.git/annex" +{- The part of the annex directory where file contents are stored. + -} +annexObjectDir :: Git.Repo -> FilePath +annexObjectDir r = annexDir r ++ "/objects" + {- .git-annex/tmp is used for temp files -} annexTmpLocation :: Git.Repo -> FilePath annexTmpLocation r = annexDir r ++ "/tmp/" diff --git a/debian/changelog b/debian/changelog index 98b814eb94..49aa9829a0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,8 @@ git-annex (0.04) UNRELEASED; urgency=low * Add build dep on libghc6-testpack-dev. * Add annex.version, which will be used to automate upgrades. + * Reorganised the layout of .git/annex/ , moving cached file contents + to .git/annex/objects// -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/backends.mdwn b/doc/backends.mdwn index dc359174ae..fde23df5ed 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -10,13 +10,13 @@ Multiple pluggable backends are supported, and a single repository can use different backends for different files. * `WORM` ("Write Once, Read Many") This backend stores the file's content - only in `.git/annex/`, and assumes that any file with the same basename, - size, and modification time has the same content. So with this backend, - files can be moved around, but should never be added to or changed. - This is the default, and the least expensive backend. + only in `.git/annex/objects/`, and assumes that any file with the same + basename, size, and modification time has the same content. So with + this backend, files can be moved around, but should never be added to + or changed. This is the default, and the least expensive backend. * `SHA1` -- This backend stores the file's content in - `.git/annex/`, with a name based on its sha1 checksum. This backend allows - modifications of files to be tracked. Its need to generate checksums + `.git/annex/objects/`, with a name based on its sha1 checksum. This backend + allows modifications of files to be tracked. Its need to generate checksums can make it slower for large files. **Warning** this backend is not ready for use. * `URL` -- This backend downloads the file's content from an external URL. From 070e8530c1151dc96dec099eac8b967277751b10 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 15:15:21 -0400 Subject: [PATCH 0437/2835] refactoring, no code changes really --- Backend.hs | 2 +- Backend/File.hs | 1 + Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Command.hs | 2 +- Command/Add.hs | 1 + Command/Drop.hs | 1 + Command/DropKey.hs | 1 + Command/Fix.hs | 1 + Command/FromKey.hs | 1 + Command/Fsck.hs | 29 +------------------- Command/Get.hs | 1 + Command/Init.hs | 1 + Command/Move.hs | 5 ++-- Command/SetKey.hs | 1 + Command/Unannex.hs | 1 + Core.hs | 66 ++++++++++++++++++++-------------------------- Messages.hs | 54 +++++++++++++++++++++++++++++++++++++ Remotes.hs | 7 ++--- 19 files changed, 105 insertions(+), 74 deletions(-) create mode 100644 Messages.hs diff --git a/Backend.hs b/Backend.hs index 456a98bd41..43b450736d 100644 --- a/Backend.hs +++ b/Backend.hs @@ -31,13 +31,13 @@ import Control.Monad.State import IO (try) import System.FilePath import System.Posix.Files -import Core import Locations import qualified GitRepo as Git import qualified Annex import Types import qualified TypeInternals as Internals +import Messages {- List of backends in the order to try them when storing a new key. -} list :: Annex [Backend] diff --git a/Backend/File.hs b/Backend/File.hs index 5b93d8227e..9178b830a5 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -25,6 +25,7 @@ import qualified GitRepo as Git import Core import qualified Annex import UUID +import Messages backend :: Backend backend = Backend { diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 4858922585..5a232ec1db 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -14,7 +14,7 @@ import System.IO import qualified Backend.File import TypeInternals -import Core +import Messages backend :: Backend backend = Backend.File.backend { diff --git a/Backend/URL.hs b/Backend/URL.hs index e6d3eb1ae5..830d343c53 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -11,8 +11,8 @@ import Control.Monad.State (liftIO) import Data.String.Utils import TypeInternals -import Core import Utility +import Messages backend :: Backend backend = Backend { diff --git a/Command.hs b/Command.hs index a0e3280d6b..f896a53f6f 100644 --- a/Command.hs +++ b/Command.hs @@ -9,7 +9,7 @@ module Command where import Types import qualified Backend -import Core +import Messages import qualified Annex {- A subcommand runs in four stages. diff --git a/Command/Add.hs b/Command/Add.hs index 825c1d8c1e..3cc681f69a 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -19,6 +19,7 @@ import qualified Backend import LocationLog import Types import Core +import Messages {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing diff --git a/Command/Drop.hs b/Command/Drop.hs index 6cdf216f41..d1ebd7f64d 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -17,6 +17,7 @@ import qualified Backend import LocationLog import Types import Core +import Messages {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} diff --git a/Command/DropKey.hs b/Command/DropKey.hs index bdd9b55b12..8076e6fd3f 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -17,6 +17,7 @@ import qualified Backend import LocationLog import Types import Core +import Messages {- Drops cached content for a key. -} start :: SubCmdStartString diff --git a/Command/Fix.hs b/Command/Fix.hs index 90257a8a53..7963a1d2ea 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -15,6 +15,7 @@ import Command import qualified Annex import Utility import Core +import Messages {- Fixes the symlink to an annexed file. -} start :: SubCmdStartString diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 3071f218f4..de555475c1 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -18,6 +18,7 @@ import Utility import qualified Backend import Types import Core +import Messages {- Adds a file pointing at a manually-specified key -} start :: SubCmdStartString diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 785aecd8af..5405ce1201 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -8,19 +8,11 @@ module Command.Fsck where import qualified Data.Map as M -import System.Directory -import System.Posix.Files -import Monad (filterM) -import Control.Monad.State (liftIO) -import Data.Maybe import Command import Types import Core -import Locations -import qualified Annex -import qualified GitRepo as Git -import qualified Backend +import Messages {- Checks the whole annex for problems. -} start :: SubCmdStart @@ -71,22 +63,3 @@ unusedKeys = do existsMap :: Ord k => [k] -> M.Map k Int existsMap l = M.fromList $ map (\k -> (k, 1)) l - -getKeysPresent :: Annex [Key] -getKeysPresent = do - g <- Annex.gitRepo - let top = annexDir g - contents <- liftIO $ getDirectoryContents top - files <- liftIO $ filterM (isreg top) contents - return $ map fileKey files - where - isreg top f = do - s <- getFileStatus $ top ++ "/" ++ f - return $ isRegularFile s - -getKeysReferenced :: Annex [Key] -getKeysReferenced = do - g <- Annex.gitRepo - files <- liftIO $ Git.inRepo g $ Git.workTree g - keypairs <- mapM Backend.lookupFile files - return $ map fst $ catMaybes keypairs diff --git a/Command/Get.hs b/Command/Get.hs index 1433bc8d00..c50b5a3775 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -11,6 +11,7 @@ import Command import qualified Backend import Types import Core +import Messages {- Gets an annexed file from one of the backends. -} start :: SubCmdStartString diff --git a/Command/Init.hs b/Command/Init.hs index b1e4e0e066..fd55242a46 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -15,6 +15,7 @@ import qualified Annex import Core import qualified GitRepo as Git import UUID +import Messages {- Stores description for the repository etc. -} start :: SubCmdStartString diff --git a/Command/Move.hs b/Command/Move.hs index cee9416222..6ca923a310 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -20,6 +20,7 @@ import Core import qualified GitRepo as Git import qualified Remotes import UUID +import Messages {- Move a file either --to or --from a repository. - @@ -64,7 +65,7 @@ moveToPerform key = do showNote $ show err return Nothing Right False -> do - Core.showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." + showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." let tmpfile = (annexTmpLocation remote) ++ (keyFile key) ok <- Remotes.copyToRemote remote key tmpfile if (ok) @@ -112,7 +113,7 @@ moveFromPerform key = do if (ishere) then return $ Just $ moveFromCleanup remote key else do - Core.showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." + showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) then return $ Just $ moveFromCleanup remote key diff --git a/Command/SetKey.hs b/Command/SetKey.hs index a5710643ec..9286e740b6 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -18,6 +18,7 @@ import qualified Backend import LocationLog import Types import Core +import Messages {- Sets cached content for a key. -} start :: SubCmdStartString diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 5cffb2d894..e0848cd4a0 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -19,6 +19,7 @@ import LocationLog import Types import Core import qualified GitRepo as Git +import Messages {- The unannex subcommand undoes an add. -} start :: SubCmdStartString diff --git a/Core.hs b/Core.hs index 347e635939..7aadfb5fbf 100644 --- a/Core.hs +++ b/Core.hs @@ -8,12 +8,12 @@ module Core where import IO (try) -import System.IO import System.Directory import Control.Monad.State (liftIO) import System.Path -import Data.String.Utils -import Control.Monad (when, unless) +import Control.Monad (when, unless, filterM) +import System.Posix.Files +import Data.Maybe import Types import Locations @@ -22,7 +22,9 @@ import UUID import qualified GitRepo as Git import qualified GitQueue import qualified Annex +import qualified Backend import Utility +import Messages {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). @@ -152,6 +154,27 @@ getViaTmp key action = do -- to resume its transfer return False +{- List of keys whose content exists in .git/annex/objects/ -} +getKeysPresent :: Annex [Key] +getKeysPresent = do + g <- Annex.gitRepo + let top = annexObjectDir g + contents <- liftIO $ getDirectoryContents top + files <- liftIO $ filterM (isreg top) contents + return $ map fileKey files + where + isreg top f = do + s <- getFileStatus $ top ++ "/" ++ f + return $ isRegularFile s + +{- List of keys referenced by symlinks in the git repo. -} +getKeysReferenced :: Annex [Key] +getKeysReferenced = do + g <- Annex.gitRepo + files <- liftIO $ Git.inRepo g $ Git.workTree g + keypairs <- mapM Backend.lookupFile files + return $ map fst $ catMaybes keypairs + {- Uses the annex.version git config setting to automate upgrades. -} autoUpgrade :: Annex () autoUpgrade = do @@ -159,6 +182,8 @@ autoUpgrade = do case Git.configGet g field "0" of "0" -> do -- before there was repo versioning + upgradeNote "Upgrading object directory layout..." + setVersion v | v == currentVersion -> return () _ -> error "this version of git-annex is too old for this git repository!" @@ -166,37 +191,4 @@ autoUpgrade = do currentVersion = "1" setVersion = Annex.setConfig field currentVersion field = "annex.version" - -{- Output logging -} -verbose :: Annex () -> Annex () -verbose a = do - q <- Annex.flagIsSet "quiet" - unless q a -showStart :: String -> String -> Annex () -showStart command file = verbose $ do - liftIO $ putStr $ command ++ " " ++ file ++ " " - liftIO $ hFlush stdout -showNote :: String -> Annex () -showNote s = verbose $ do - liftIO $ putStr $ "(" ++ s ++ ") " - liftIO $ hFlush stdout -showProgress :: Annex () -showProgress = verbose $ liftIO $ putStr "\n" -showLongNote :: String -> Annex () -showLongNote s = verbose $ do - liftIO $ putStr $ "\n" ++ indented - where - indented = join "\n" $ map (\l -> " " ++ l) $ lines s -showEndOk :: Annex () -showEndOk = verbose $ do - liftIO $ putStrLn "ok" -showEndFail :: Annex () -showEndFail = verbose $ do - liftIO $ putStrLn "\nfailed" - -{- Exception pretty-printing. -} -showErr :: (Show a) => a -> Annex () -showErr e = warning $ show e - -warning :: String -> Annex () -warning s = liftIO $ hPutStrLn stderr $ "git-annex: " ++ s + upgradeNote s = verbose $ liftIO $ putStrLn $ "("++s++")" diff --git a/Messages.hs b/Messages.hs new file mode 100644 index 0000000000..89f78e2441 --- /dev/null +++ b/Messages.hs @@ -0,0 +1,54 @@ +{- git-annex output messages + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Messages where + +import Control.Monad.State (liftIO) +import System.IO +import Control.Monad (unless) +import Data.String.Utils + +import Types +import qualified Annex + +verbose :: Annex () -> Annex () +verbose a = do + q <- Annex.flagIsSet "quiet" + unless q a + +showStart :: String -> String -> Annex () +showStart command file = verbose $ do + liftIO $ putStr $ command ++ " " ++ file ++ " " + liftIO $ hFlush stdout + +showNote :: String -> Annex () +showNote s = verbose $ do + liftIO $ putStr $ "(" ++ s ++ ") " + liftIO $ hFlush stdout + +showProgress :: Annex () +showProgress = verbose $ liftIO $ putStr "\n" + +showLongNote :: String -> Annex () +showLongNote s = verbose $ do + liftIO $ putStr $ "\n" ++ indented + where + indented = join "\n" $ map (\l -> " " ++ l) $ lines s +showEndOk :: Annex () +showEndOk = verbose $ do + liftIO $ putStrLn "ok" + +showEndFail :: Annex () +showEndFail = verbose $ do + liftIO $ putStrLn "\nfailed" + +{- Exception pretty-printing. -} +showErr :: (Show a) => a -> Annex () +showErr e = warning $ show e + +warning :: String -> Annex () +warning s = liftIO $ hPutStrLn stderr $ "git-annex: " ++ s diff --git a/Remotes.hs b/Remotes.hs index 280543968c..7aad6c2a06 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -36,6 +36,7 @@ import Locations import UUID import Utility import qualified Core +import Messages {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -64,7 +65,7 @@ keyPossibilities key = do let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive unless (null doexpensive) $ do - Core.showNote $ "getting UUID for " ++ + showNote $ "getting UUID for " ++ (list doexpensive) ++ "..." let todo = cheap ++ doexpensive if (not $ null todo) @@ -93,7 +94,7 @@ inAnnex r key = do a <- Annex.new r [] Annex.eval a (Core.inAnnex key) checkremote = do - Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") + showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- runCmd r "test" ["-e", annexLocation r key] -- XXX Note that ssh failing and the file not existing -- are not currently differentiated. @@ -228,7 +229,7 @@ sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file scp :: Git.Repo -> [String] -> Annex Bool scp r params = do scpoptions <- repoConfig r "scp-options" "" - Core.showProgress -- make way for scp progress bar + showProgress -- make way for scp progress bar liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params {- Runs a command in a remote, using ssh if necessary. From c281747b0eb39c10eb7bae0ea3202dca6077b74f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:40:02 -0400 Subject: [PATCH 0438/2835] add queueRun --- Annex.hs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Annex.hs b/Annex.hs index 7c046b141d..af761051dc 100644 --- a/Annex.hs +++ b/Annex.hs @@ -20,6 +20,7 @@ module Annex ( Flag(..), queue, queueGet, + queueRun, setConfig ) where @@ -120,6 +121,15 @@ queueGet = do state <- get return (Internals.repoqueue state) +{- Runs (and empties) the queue. -} +queueRun :: Annex () +queueRun = do + state <- get + let q = Internals.repoqueue state + g <- gitRepo + liftIO $ GitQueue.run g q + put state { Internals.repoqueue = GitQueue.empty } + {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () setConfig key value = do From 50ec22e322ecc0538a0629e32313c0d8ec4ffd45 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:40:28 -0400 Subject: [PATCH 0439/2835] set version on init --- Command/Init.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Command/Init.hs b/Command/Init.hs index fd55242a46..fa5725c48f 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -15,6 +15,7 @@ import qualified Annex import Core import qualified GitRepo as Git import UUID +import Version import Messages {- Stores description for the repository etc. -} @@ -30,6 +31,7 @@ perform description = do g <- Annex.gitRepo u <- getUUID g describeUUID u description + setVersion liftIO $ gitAttributes g liftIO $ gitPreCommitHook g return $ Just $ cleanup From 98a77ab7256e484d29d67fcffc1f173fcb830f60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:40:42 -0400 Subject: [PATCH 0440/2835] add --- Version.hs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 Version.hs diff --git a/Version.hs b/Version.hs new file mode 100644 index 0000000000..ce39c0c1b1 --- /dev/null +++ b/Version.hs @@ -0,0 +1,39 @@ +{- git-annex repository versioning + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Version where + +import Control.Monad.State (liftIO) +import System.Directory + +import Types +import qualified Annex +import qualified GitRepo as Git +import Locations + +currentVersion :: String +currentVersion = "1" + +versionField :: String +versionField = "annex.version" + +getVersion :: Annex (Maybe String) +getVersion = do + g <- Annex.gitRepo + let v = Git.configGet g versionField "" + if (not $ null v) + then return $ Just v + else do + -- version 0 was not recorded in .git/config; + -- such a repo should have an annexDir + d <- liftIO $ doesDirectoryExist $ annexDir g + if (d) + then return $ Just "0" + else return Nothing -- no version yet + +setVersion :: Annex () +setVersion = Annex.setConfig versionField currentVersion From ba59ac13b25d5be671e38cb7b4c40257f3fdac4f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:45:41 -0400 Subject: [PATCH 0441/2835] add showSideAction --- Messages.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Messages.hs b/Messages.hs index 89f78e2441..ed4f3b90a1 100644 --- a/Messages.hs +++ b/Messages.hs @@ -20,6 +20,9 @@ verbose a = do q <- Annex.flagIsSet "quiet" unless q a +showSideAction :: String -> Annex () +showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")" + showStart :: String -> String -> Annex () showStart command file = verbose $ do liftIO $ putStr $ command ++ " " ++ file ++ " " From 6395b790ce3d2f97803f0c642af71d1a9eb169c6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:47:36 -0400 Subject: [PATCH 0442/2835] Reorganised the layout of .git/annex/ --- Core.hs | 77 ++++++++++++++++++++++++++++++++-------------- Locations.hs | 23 ++++++++------ debian/changelog | 10 ++++-- doc/git-annex.mdwn | 2 +- test.hs | 4 ++- 5 files changed, 78 insertions(+), 38 deletions(-) diff --git a/Core.hs b/Core.hs index 7aadfb5fbf..90af62eb67 100644 --- a/Core.hs +++ b/Core.hs @@ -25,6 +25,7 @@ import qualified Annex import qualified Backend import Utility import Messages +import Version {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). @@ -54,16 +55,14 @@ startup = do {- When git-annex is done, it runs this. -} shutdown :: Annex Bool shutdown = do - g <- Annex.gitRepo - - -- Runs all queued git commands. q <- Annex.queueGet unless (q == GitQueue.empty) $ do - verbose $ liftIO $ putStrLn "Recording state in git..." - liftIO $ GitQueue.run g q + showSideAction "Recording state in git..." + Annex.queueRun -- clean up any files left in the temp directory, but leave -- the tmp directory itself + g <- Annex.gitRepo let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp when (exists) $ liftIO $ removeDirectoryRecursive tmp @@ -140,13 +139,12 @@ logStatus key status = do getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool getViaTmp key action = do g <- Annex.gitRepo - let dest = annexLocation g key let tmp = annexTmpLocation g ++ keyFile key liftIO $ createDirectoryIfMissing True (parentDir tmp) success <- action tmp if (success) then do - liftIO $ renameFile tmp dest + moveToObjectDir key tmp logStatus key ValuePresent return True else do @@ -154,17 +152,28 @@ getViaTmp key action = do -- to resume its transfer return False +{- Moves a file into .git/annex/objects/ -} +moveToObjectDir :: Key -> FilePath -> Annex () +moveToObjectDir key src = do + g <- Annex.gitRepo + let dest = annexLocation g key + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ renameFile src dest + -- TODO directory and file mode tweaks + {- List of keys whose content exists in .git/annex/objects/ -} getKeysPresent :: Annex [Key] getKeysPresent = do g <- Annex.gitRepo - let top = annexObjectDir g - contents <- liftIO $ getDirectoryContents top - files <- liftIO $ filterM (isreg top) contents + getKeysPresent' $ annexObjectDir g +getKeysPresent' :: FilePath -> Annex [Key] +getKeysPresent' dir = do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM isreg contents return $ map fileKey files where - isreg top f = do - s <- getFileStatus $ top ++ "/" ++ f + isreg f = do + s <- getFileStatus $ dir ++ "/" ++ f return $ isRegularFile s {- List of keys referenced by symlinks in the git repo. -} @@ -178,17 +187,39 @@ getKeysReferenced = do {- Uses the annex.version git config setting to automate upgrades. -} autoUpgrade :: Annex () autoUpgrade = do + version <- getVersion + case version of + Just "0" -> upgradeFrom0 + Nothing -> return () -- repo not initted yet, no version + Just v | v == currentVersion -> return () + Just _ -> error "this version of git-annex is too old for this git repository!" + +upgradeFrom0 :: Annex () +upgradeFrom0 = do + showSideAction "Upgrading object directory layout for git-annex 0.04..." g <- Annex.gitRepo - case Git.configGet g field "0" of - "0" -> do -- before there was repo versioning - upgradeNote "Upgrading object directory layout..." - - setVersion - v | v == currentVersion -> return () - _ -> error "this version of git-annex is too old for this git repository!" + -- do the reorganisation of the files + let olddir = annexDir g + keys <- getKeysPresent' olddir + _ <- mapM (\k -> moveToObjectDir k $ olddir ++ "/" ++ keyFile k) keys + + -- update the symlinks to the files + files <- liftIO $ Git.inRepo g $ Git.workTree g + fixlinks files + Annex.queueRun + + setVersion + where - currentVersion = "1" - setVersion = Annex.setConfig field currentVersion - field = "annex.version" - upgradeNote s = verbose $ liftIO $ putStrLn $ "("++s++")" + fixlinks [] = return () + fixlinks (f:fs) = do + r <- Backend.lookupFile f + case r of + Nothing -> return () + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" [] f + fixlinks fs diff --git a/Locations.hs b/Locations.hs index 78c0bff4b3..e5f78a31ce 100644 --- a/Locations.hs +++ b/Locations.hs @@ -14,7 +14,9 @@ module Locations ( annexLocationRelative, annexTmpLocation, annexDir, - annexObjectDir + annexObjectDir, + + prop_idempotent_fileKey ) where import Data.String.Utils @@ -29,12 +31,7 @@ stateLoc = ".git-annex/" gitStateDir :: Git.Repo -> FilePath gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc -{- An annexed file's content is stored in - - /path/to/repo/.git/annex/objects//, where is of the form - - - - - - That allows deriving the key and backend by looking at the symlink to it. - -} +{- Annexed file's absolute location. -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = (Git.workTree r) ++ "/" ++ (annexLocationRelative key) @@ -43,8 +40,9 @@ annexLocation r key = - - Note: Assumes repo is NOT bare.-} annexLocationRelative :: Key -> FilePath -annexLocationRelative key = ".git/annex/objects/" ++ f ++ f - where f = keyFile key +annexLocationRelative key = ".git/annex/objects/" ++ f ++ "/" ++ f + where + f = keyFile key {- The annex directory of a repository. - @@ -72,10 +70,15 @@ annexTmpLocation r = annexDir r ++ "/tmp/" - is one to one. - -} keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key +keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Key fileKey file = read $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + +{- for quickcheck -} +prop_idempotent_fileKey :: String -> Bool +prop_idempotent_fileKey s = k == (fileKey $ keyFile k) + where k = read "test:s" diff --git a/debian/changelog b/debian/changelog index 49aa9829a0..dc9dcedc2b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,13 @@ git-annex (0.04) UNRELEASED; urgency=low * Add build dep on libghc6-testpack-dev. - * Add annex.version, which will be used to automate upgrades. - * Reorganised the layout of .git/annex/ , moving cached file contents - to .git/annex/objects// + * Add annex.version, which will be used to automate upgrades + between incompatable versions. + * Reorganised the layout of .git/annex/ + * The new layout will be automatically upgraded to the first time + git-annex is used in a repository with the old layout. + * Note that git-annex 0.04 cannot transfer content from old repositories + that have not yet been upgraded. -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 6f2c85d573..6a580f0050 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -223,7 +223,7 @@ but the SHA1 backend for ogg files: These files are used, in your git repository: -`.git/annex/` contains the annexed file contents that are currently +`.git/annex/objects/` contains the annexed file contents that are currently available. Annexed files in your git repository symlink to that content. `.git-annex/uuid.log` is used to map between repository UUID and diff --git a/test.hs b/test.hs index 9897236176..288532d7be 100644 --- a/test.hs +++ b/test.hs @@ -5,9 +5,11 @@ import Test.HUnit import Test.HUnit.Tools import GitRepo +import Locations alltests = [ - qctest "prop_idempotent_deencode" prop_idempotent_deencode + qctest "prop_idempotent_deencode" prop_idempotent_deencode, + qctest "prop_idempotent_fileKey" prop_idempotent_fileKey ] main = runVerboseTests (TestList alltests) From 40a815d873a828fbccee453f45fc519feffe15fd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 17:44:08 -0400 Subject: [PATCH 0443/2835] add unsetFileMode --- Utility.hs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index 4e56289e22..0053c687bb 100644 --- a/Utility.hs +++ b/Utility.hs @@ -11,17 +11,21 @@ module Utility ( relPathCwdToDir, relPathDirToDir, boolSystem, - shellEscape + shellEscape, + unsetFileMode ) where import System.IO import System.Exit import System.Posix.Process import System.Posix.Signals +import System.Posix.Files +import System.Posix.Types import Data.String.Utils import System.Path import System.FilePath import System.Directory +import Foreign (complement) {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} @@ -115,3 +119,10 @@ shellEscape f = "'" ++ escaped ++ "'" where -- replace ' with '"'"' escaped = join "'\"'\"'" $ split "'" f + +{- Removes a FileMode from a file. + - For example, call with otherWriteMode to chmod o-w -} +unsetFileMode :: FilePath -> FileMode -> IO () +unsetFileMode f m = do + s <- getFileStatus f + setFileMode f $ (fileMode s) `intersectFileModes` (complement m) From 8dd9f8e49eae081e7503facff6d5a53285194c09 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 17:44:30 -0400 Subject: [PATCH 0444/2835] typo --- Locations.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Locations.hs b/Locations.hs index e5f78a31ce..58244cef0e 100644 --- a/Locations.hs +++ b/Locations.hs @@ -81,4 +81,4 @@ fileKey file = read $ {- for quickcheck -} prop_idempotent_fileKey :: String -> Bool prop_idempotent_fileKey s = k == (fileKey $ keyFile k) - where k = read "test:s" + where k = read $ "test:" ++ s From 1d32d902c95a49c53c46951641852c209476cb3d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 19:26:37 -0400 Subject: [PATCH 0445/2835] Annexed file contents are now made unwritable and put in unwriteable directories, to avoid them accidentially being removed or modified. (Thanks Josh Triplett for the idea.) --- Command/Add.hs | 9 +---- Command/Drop.hs | 17 ++------- Command/DropKey.hs | 8 +--- Command/SetKey.hs | 22 +++++------ Command/Unannex.hs | 9 +++-- Core.hs | 53 +++++++++++++++++++++++---- debian/changelog | 3 ++ doc/todo/immutable_annexed_files.mdwn | 2 + 8 files changed, 74 insertions(+), 49 deletions(-) diff --git a/Command/Add.hs b/Command/Add.hs index 3cc681f69a..6c5d24f842 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -9,12 +9,9 @@ module Command.Add where import Control.Monad.State (liftIO) import System.Posix.Files -import System.Directory import Command import qualified Annex -import Utility -import Locations import qualified Backend import LocationLog import Types @@ -42,11 +39,9 @@ perform (file, backend) = do cleanup :: FilePath -> Key -> SubCmdCleanup cleanup file key = do + moveAnnex key file logStatus key ValuePresent - g <- Annex.gitRepo - let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - liftIO $ renameFile file dest + link <- calcGitLink file key liftIO $ createSymbolicLink link file Annex.queue "add" [] file diff --git a/Command/Drop.hs b/Command/Drop.hs index d1ebd7f64d..48433b14cf 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -7,12 +7,9 @@ module Command.Drop where -import Control.Monad.State (liftIO) -import System.Directory +import Control.Monad (when) import Command -import qualified Annex -import Locations import qualified Backend import LocationLog import Types @@ -39,13 +36,7 @@ perform key backend = do cleanup :: Key -> SubCmdCleanup cleanup key = do - logStatus key ValueMissing inannex <- inAnnex key - if (inannex) - then do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc - return True - else return True - + when (inannex) $ removeAnnex key + logStatus key ValueMissing + return True diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 8076e6fd3f..e0b20918cb 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -7,12 +7,8 @@ module Command.DropKey where -import Control.Monad.State (liftIO) -import System.Directory - import Command import qualified Annex -import Locations import qualified Backend import LocationLog import Types @@ -36,9 +32,7 @@ start keyname = do perform :: Key -> SubCmdPerform perform key = do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc + removeAnnex key return $ Just $ cleanup key cleanup :: Key -> SubCmdCleanup diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 9286e740b6..50e9a590b1 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -13,7 +13,6 @@ import Control.Monad (when) import Command import qualified Annex import Utility -import Locations import qualified Backend import LocationLog import Types @@ -22,21 +21,22 @@ import Messages {- Sets cached content for a key. -} start :: SubCmdStartString -start tmpfile = do +start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname - showStart "setkey" tmpfile - return $ Just $ perform tmpfile key + showStart "setkey" file + return $ Just $ perform file key perform :: FilePath -> Key -> SubCmdPerform -perform tmpfile key = do - g <- Annex.gitRepo - let loc = annexLocation g key - ok <- liftIO $ boolSystem "mv" [tmpfile, loc] - if (not ok) - then error "mv failed!" - else return $ Just $ cleanup key +perform file key = do + -- the file might be on a different filesystem, so mv is used + -- rather than simply calling moveToObjectDir key file + ok <- getViaTmp key $ \dest -> liftIO $ boolSystem "mv" [file, dest] + if ok + then return $ Just $ cleanup key + else error "mv failed!" + cleanup :: Key -> SubCmdCleanup cleanup key = do logStatus key ValuePresent diff --git a/Command/Unannex.hs b/Command/Unannex.hs index e0848cd4a0..a9c18f765e 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -13,7 +13,6 @@ import System.Directory import Command import qualified Annex import Utility -import Locations import qualified Backend import LocationLog import Types @@ -38,12 +37,14 @@ perform file key backend = do cleanup :: FilePath -> Key -> SubCmdCleanup cleanup file key = do - logStatus key ValueMissing g <- Annex.gitRepo - let src = annexLocation g key + liftIO $ removeFile file liftIO $ Git.run g ["rm", "--quiet", file] -- git rm deletes empty directories; put them back liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ renameFile src file + + fromAnnex key file + logStatus key ValueMissing + return True diff --git a/Core.hs b/Core.hs index 90af62eb67..f04a3dfac8 100644 --- a/Core.hs +++ b/Core.hs @@ -144,7 +144,7 @@ getViaTmp key action = do success <- action tmp if (success) then do - moveToObjectDir key tmp + moveAnnex key tmp logStatus key ValuePresent return True else do @@ -152,14 +152,53 @@ getViaTmp key action = do -- to resume its transfer return False +{- Removes the write bits from a file. -} +preventWrite :: FilePath -> IO () +preventWrite f = unsetFileMode f writebits + where + writebits = foldl unionFileModes ownerWriteMode + [groupWriteMode, otherWriteMode] + +{- Turns a file's write bit back on. -} +allowWrite :: FilePath -> IO () +allowWrite f = do + s <- getFileStatus f + setFileMode f $ (fileMode s) `unionFileModes` ownerWriteMode + {- Moves a file into .git/annex/objects/ -} -moveToObjectDir :: Key -> FilePath -> Annex () -moveToObjectDir key src = do +moveAnnex :: Key -> FilePath -> Annex () +moveAnnex key src = do g <- Annex.gitRepo let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - liftIO $ renameFile src dest - -- TODO directory and file mode tweaks + let dir = parentDir dest + liftIO $ do + createDirectoryIfMissing True dir + renameFile src dest + preventWrite dest + preventWrite dir + +{- Removes a key's file from .git/annex/objects/ -} +removeAnnex :: Key -> Annex () +removeAnnex key = do + g <- Annex.gitRepo + let file = annexLocation g key + let dir = parentDir file + liftIO $ do + allowWrite dir + removeFile file + removeDirectory dir + +{- Moves a key's file out of .git/annex/objects/ -} +fromAnnex :: Key -> FilePath -> Annex () +fromAnnex key dest = do + g <- Annex.gitRepo + let file = annexLocation g key + let dir = parentDir file + liftIO $ do + allowWrite dir + allowWrite file + renameFile file dest + removeDirectory dir {- List of keys whose content exists in .git/annex/objects/ -} getKeysPresent :: Annex [Key] @@ -202,7 +241,7 @@ upgradeFrom0 = do -- do the reorganisation of the files let olddir = annexDir g keys <- getKeysPresent' olddir - _ <- mapM (\k -> moveToObjectDir k $ olddir ++ "/" ++ keyFile k) keys + _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys -- update the symlinks to the files files <- liftIO $ Git.inRepo g $ Git.workTree g diff --git a/debian/changelog b/debian/changelog index dc9dcedc2b..1ce6a2debe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,9 @@ git-annex (0.04) UNRELEASED; urgency=low git-annex is used in a repository with the old layout. * Note that git-annex 0.04 cannot transfer content from old repositories that have not yet been upgraded. + * Annexed file contents are now made unwritable and put in unwriteable + directories, to avoid them accidentially being removed or modified. + (Thanks Josh Triplett for the idea.) -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/todo/immutable_annexed_files.mdwn b/doc/todo/immutable_annexed_files.mdwn index e5207dc163..b26838e95e 100644 --- a/doc/todo/immutable_annexed_files.mdwn +++ b/doc/todo/immutable_annexed_files.mdwn @@ -4,3 +4,5 @@ > josh: Oh, I just thought of another slightly crazy but handy idea. > josh: I'd hate to run into a program which somehow followed the symlink and then did an unlink to replace the file. > josh: To break that, you could create a new directory under annex's internal directory for each file, and make the directory have no write permission. + +[[done]] and done --[[Joey]] From 8d5374f4a33f398baa166035e5fafb716a78fd1d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 9 Nov 2010 15:04:13 -0400 Subject: [PATCH 0446/2835] tweak --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index f04a3dfac8..304c8a9232 100644 --- a/Core.hs +++ b/Core.hs @@ -235,7 +235,7 @@ autoUpgrade = do upgradeFrom0 :: Annex () upgradeFrom0 = do - showSideAction "Upgrading object directory layout for git-annex 0.04..." + showSideAction "Upgrading object directory layout..." g <- Annex.gitRepo -- do the reorganisation of the files From d56feda25dd82ffa34fe5e3f28eff3ecf9eac5b5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 9 Nov 2010 15:11:45 -0400 Subject: [PATCH 0447/2835] maybe call it unlock and not checkout It'd be confusing to have a git-annex subcommand with the same name as a git subcommand. --- debian/changelog | 6 +++--- doc/git-annex.mdwn | 11 ++++++----- doc/todo/backendSHA1.mdwn | 2 +- doc/walkthrough.mdwn | 18 +++++++++--------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/debian/changelog b/debian/changelog index fa8bb0f938..9fd96de95b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,8 @@ git-annex (0.04) UNRELEASED; urgency=low - * Add checkout subcommand, which allows checking out file content - in preparation of changing it. - * Add uncheckout subcommand. + * Add unlock subcommand, which replaces the symlink with a copy of + the file's content in preparation of changing it. + * Add lock subcommand. * Add build dep on libghc6-testpack-dev. * Add annex.version, which will be used to automate upgrades between incompatable versions. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3bb3b08357..61f0f8fca6 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -81,9 +81,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git-annex may refuse to drop content if the backend does not think it is safe to do so, typically because of the setting of annex.numcopies. -* checkout [path ...] +* unlock [path ...] - Checks out annexed files so they can be modified. This replaces the + Normally, the content of annexed files is protected from being changed. + Unlocking a annexed file allows it to be modified. This replaces the symlink for each specified file with a copy of the file content. When you `git commit`, the file, the new content is injected back into the annex. @@ -102,10 +103,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. and sets up `.gitattributes` and the pre-commit hook. This is an optional, but recommended step. -* uncheckout [path ...] +* lock [path ...] - Use this to undo a checkout command if you don't want to modify - the checked out files, or have made modifications you want to discard. + Use this to undo an unlock command if you don't want to modify + the files, or have made modifications you want to discard. * unannex [path ...] diff --git a/doc/todo/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn index 44df406dea..8c16b75ad0 100644 --- a/doc/todo/backendSHA1.mdwn +++ b/doc/todo/backendSHA1.mdwn @@ -4,4 +4,4 @@ In particular, while files can be added using it, git-annex will not notice when their content changes, and will not create a new key for the new sha1 of the net content. -[[done]]; use checkout subcommand +[[done]]; use unlock subcommand and commit changes with git diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index cb564fa979..29922bd75e 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -192,23 +192,23 @@ makes it very easy. ## modifying annexed files -Normally, the content of files in the annex cannot be modified. -In order to modify a file, it should first be checked out: +Normally, the content of files in the annex is prevented from being modified. +In order to modify a file, it should first be unlocked: - # git annex checkout my_cool_big_file - checkout my_cool_big_file (copying...) ok + # git annex unlock my_cool_big_file + unlock my_cool_big_file (copying...) ok -Checking a file out replaces the symlink that normally points at its content -with a copy of the content. You can then modify the file like any regular -file. Because it is a regular file. +They replaces the symlink that normally points at its content with a copy +of the content. You can then modify the file like any regular file. Because +it is a regular file. When you `git commit`, git-annex's pre-commit hook will automatically -notice that you are committing a checked-out file, and add its new content +notice that you are committing an unlocked file, and add its new content to the annex. The file will be replaced with a symlink to the new content, and this symlink is what gets committed to git. If you decide you don't need to modify the file after all, or want to discard -modifications, just use the uncheckout subcommand to undo the checkout. +modifications, just use the lock subcommand. ## using the URL backend From 536bc97d25479ac969273b49442c2fd8c31358c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 9 Nov 2010 15:59:49 -0400 Subject: [PATCH 0448/2835] lock and unlock subcommands --- CmdLine.hs | 6 ++++++ Command/Lock.hs | 41 +++++++++++++++++++++++++++++++++++++++++ Command/Unlock.hs | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 Command/Lock.hs create mode 100644 Command/Unlock.hs diff --git a/CmdLine.hs b/CmdLine.hs index 3823c72476..adcf25e9a9 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -30,6 +30,8 @@ import qualified Command.SetKey import qualified Command.Fix import qualified Command.Init import qualified Command.Fsck +import qualified Command.Unlock +import qualified Command.Lock subCmds :: [SubCommand] subCmds = @@ -41,6 +43,10 @@ subCmds = "indicate content of files not currently wanted" , SubCommand "move" path (withFilesInGit Command.Move.start) "transfer content of files to/from another repository" + , SubCommand "unlock" path (withFilesInGit Command.Unlock.start) + "unlock files for modification" + , SubCommand "lock" path (withFilesInGit Command.Lock.start) + "undo unlock command" , SubCommand "init" desc (withDescription Command.Init.start) "initialize git-annex with repository description" , SubCommand "unannex" path (withFilesInGit Command.Unannex.start) diff --git a/Command/Lock.hs b/Command/Lock.hs new file mode 100644 index 0000000000..c28e643276 --- /dev/null +++ b/Command/Lock.hs @@ -0,0 +1,41 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Lock where + +import Control.Monad.State (liftIO) +import System.Directory +import System.Posix.Files + +import Command +import Messages +import qualified Annex +import qualified GitRepo as Git + +{- Undo unlock -} +start :: SubCmdStartString +start file = do + -- Want to avoid calling git checkout on files that are not + -- annexed -- but without the symlink to the annex, cannot tell + -- for sure if the file was annexed. So, check if git thinks the + -- file's type has changed (from a symlink to a regular file). + g <- Annex.gitRepo + test <- liftIO $ + Git.pipeRead g ["diff", "--name-only", "--diff-filter=T", file] + s <- liftIO $ getSymbolicLinkStatus file + if (null test || isSymbolicLink s) + then return Nothing + else do + showStart "lock" file + return $ Just $ perform file + +perform :: FilePath -> SubCmdPerform +perform file = do + liftIO $ removeFile file + g <- Annex.gitRepo + liftIO $ Git.run g ["checkout", file] + return $ Just $ return True -- no cleanup needed diff --git a/Command/Unlock.hs b/Command/Unlock.hs new file mode 100644 index 0000000000..57d4ad87af --- /dev/null +++ b/Command/Unlock.hs @@ -0,0 +1,36 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unlock where + +import Control.Monad.State (liftIO) +import System.Directory + +import Command +import qualified Annex +import Types +import Messages +import Locations +import Utility + +{- The unlock subcommand replaces the symlink with a copy of the file's + - content. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, _) -> do + showStart "unlock" file + return $ Just $ perform file key + +perform :: FilePath -> Key -> SubCmdPerform +perform dest key = do + g <- Annex.gitRepo + let src = annexLocation g key + liftIO $ removeFile dest + showNote "copying..." + ok <- liftIO $ boolSystem "cp" ["-p", src, dest] + if ok + then return $ Just $ return True -- no cleanup needed + else error "cp failed!" From 515d6b6c7d90e1ff44e791421066450cf4322b47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:49:35 -0400 Subject: [PATCH 0449/2835] Avoid using runghc to run test suite as it is not available on all architectures. Closes: #603006 --- CmdLine.hs | 5 +++-- Makefile | 11 +++++++---- debian/changelog | 2 ++ doc/git-annex.mdwn | 4 ++-- test.hs | 3 ++- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index adcf25e9a9..1c73533df2 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -32,6 +32,7 @@ import qualified Command.Init import qualified Command.Fsck import qualified Command.Unlock import qualified Command.Lock +import qualified Command.PreCommit subCmds :: [SubCommand] subCmds = @@ -51,8 +52,8 @@ subCmds = "initialize git-annex with repository description" , SubCommand "unannex" path (withFilesInGit Command.Unannex.start) "undo accidential add command" - , SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) - "fix up symlinks before they are committed" + , SubCommand "pre-commit" path (withFilesToBeCommitted Command.PreCommit.start) + "run by git pre-commit hook" , SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) "adds a file using a specific key" , SubCommand "dropkey" key (withKeys Command.DropKey.start) diff --git a/Makefile b/Makefile index eb74bb7273..ee4d50f9fc 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ all: git-annex docs +ghcmake=ghc -Wall -odir build -hidir build --make + git-annex: mkdir -p build - ghc -Wall -odir build -hidir build --make git-annex + $(ghcmake) git-annex install: install -d $(DESTDIR)/usr/bin @@ -17,7 +19,8 @@ IKIWIKI=ikiwiki endif test: - runghc test.hs + $(ghcmake) test + ./test docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 @@ -27,7 +30,7 @@ docs: --disable-plugin=smiley clean: - rm -rf build git-annex git-annex.1 + rm -rf build git-annex git-annex.1 test rm -rf doc/.ikiwiki html -.PHONY: git-annex +.PHONY: git-annex test diff --git a/debian/changelog b/debian/changelog index 9fd96de95b..97939af661 100644 --- a/debian/changelog +++ b/debian/changelog @@ -14,6 +14,8 @@ git-annex (0.04) UNRELEASED; urgency=low * Annexed file contents are now made unwritable and put in unwriteable directories, to avoid them accidentially being removed or modified. (Thanks Josh Triplett for the idea.) + * Avoid using runghc to run test suite as it is not available on all + architectures. Closes: #603006 -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 61f0f8fca6..d3cf594aa8 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -101,7 +101,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Initializes git-annex with a description of the git repository, and sets up `.gitattributes` and the pre-commit hook. - This is an optional, but recommended step. * lock [path ...] @@ -123,7 +122,8 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * pre-commit [path ...] Fixes up symlinks that are staged as part of a commit, to ensure they - point to annexed content. Also handles committing checked-out files. + point to annexed content. Also handles injecting changes to unlocked + files into the annex. This is meant to be called from git's pre-commit hook. `git annex init` automatically creates a pre-commit hook using this. diff --git a/test.hs b/test.hs index 288532d7be..2faf579a20 100644 --- a/test.hs +++ b/test.hs @@ -1,15 +1,16 @@ -- TODO find a test harness that is actually in Debian and use it. -import Test.QuickCheck import Test.HUnit import Test.HUnit.Tools import GitRepo import Locations +alltests :: [Test] alltests = [ qctest "prop_idempotent_deencode" prop_idempotent_deencode, qctest "prop_idempotent_fileKey" prop_idempotent_fileKey ] +main :: IO (Counts, Int) main = runVerboseTests (TestList alltests) From b52aa2d12f064310d193955b15ef7dccc281eec8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:51:17 -0400 Subject: [PATCH 0450/2835] in debian unstable! --- doc/download.mdwn | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/download.mdwn b/doc/download.mdwn index 664f46ed95..ccf21c091b 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,7 +1,6 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] -There are no binary packages yet, but you can build Debian packages from -the source tree with `dpkg-buildpackage`. +Users of Debian unstable can `apt-get install git-annex` Next: [[install]] From 05ca2bebff521b1fa9b79014b1856b828d897b6d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:51:17 -0400 Subject: [PATCH 0451/2835] in debian unstable! --- doc/download.mdwn | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/download.mdwn b/doc/download.mdwn index 664f46ed95..ccf21c091b 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,7 +1,6 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] -There are no binary packages yet, but you can build Debian packages from -the source tree with `dpkg-buildpackage`. +Users of Debian unstable can `apt-get install git-annex` Next: [[install]] From 91c5fe71af0f43461d933abc9a9b9a8b94594897 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:52:43 -0400 Subject: [PATCH 0452/2835] add --- Command/PreCommit.hs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Command/PreCommit.hs diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs new file mode 100644 index 0000000000..2128f207da --- /dev/null +++ b/Command/PreCommit.hs @@ -0,0 +1,22 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.PreCommit where + +import Command +import qualified Command.Fix + +{- Run by git pre-commit hook. -} +start :: SubCmdStartString +start file = do + -- If a file is unlocked for edit, inject its content into the + -- annex, and replace it with a symlink to the content. Git will + -- then commit that. + error "TODO" + + -- fix symlinks + Command.Fix.start file From 25b014ec26650c549bcb6dc900bf1c0646df57e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:52:59 -0400 Subject: [PATCH 0453/2835] update --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 13deb526a9..6956c49dd2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build/* +test git-annex git-annex.1 doc/.ikiwiki From f1c4a5a8dc222f19245f26e3deb7b25237bfc712 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:55:00 -0400 Subject: [PATCH 0454/2835] close --- debian/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/changelog b/debian/changelog index 97939af661..0e121fda21 100644 --- a/debian/changelog +++ b/debian/changelog @@ -16,6 +16,7 @@ git-annex (0.04) UNRELEASED; urgency=low (Thanks Josh Triplett for the idea.) * Avoid using runghc to run test suite as it is not available on all architectures. Closes: #603006 + * Missing build dep. Closes: #603016 -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 From 55720885aeb90e9b8d3e4153145b6a13cac1c0c7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 12:50:00 -0400 Subject: [PATCH 0455/2835] set write bit on unlocked file --- Command/Unlock.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 57d4ad87af..de21988de5 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -16,6 +16,7 @@ import Types import Messages import Locations import Utility +import Core {- The unlock subcommand replaces the symlink with a copy of the file's - content. -} @@ -32,5 +33,7 @@ perform dest key = do showNote "copying..." ok <- liftIO $ boolSystem "cp" ["-p", src, dest] if ok - then return $ Just $ return True -- no cleanup needed + then do + liftIO $ allowWrite dest + return $ Just $ return True else error "cp failed!" From 2ab448276c90cfb588b8608c2e84ebbe88c87e57 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:01:01 -0400 Subject: [PATCH 0456/2835] fix handling of staged unlocked files --- Command/Lock.hs | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/Command/Lock.hs b/Command/Lock.hs index c28e643276..1be35d6126 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -11,6 +11,7 @@ import Control.Monad.State (liftIO) import System.Directory import System.Posix.Files +import Types import Command import Messages import qualified Annex @@ -19,15 +20,8 @@ import qualified GitRepo as Git {- Undo unlock -} start :: SubCmdStartString start file = do - -- Want to avoid calling git checkout on files that are not - -- annexed -- but without the symlink to the annex, cannot tell - -- for sure if the file was annexed. So, check if git thinks the - -- file's type has changed (from a symlink to a regular file). - g <- Annex.gitRepo - test <- liftIO $ - Git.pipeRead g ["diff", "--name-only", "--diff-filter=T", file] - s <- liftIO $ getSymbolicLinkStatus file - if (null test || isSymbolicLink s) + locked <- isLocked file + if locked then return Nothing else do showStart "lock" file @@ -37,5 +31,26 @@ perform :: FilePath -> SubCmdPerform perform file = do liftIO $ removeFile file g <- Annex.gitRepo - liftIO $ Git.run g ["checkout", file] + -- first reset the file to drop any changes checked into the index + liftIO $ Git.run g ["reset", "-q", "--", file] + -- checkout the symlink + liftIO $ Git.run g ["checkout", "--", file] return $ Just $ return True -- no cleanup needed + +{- Checks if a file is unlocked for edit. + - + - But, without the symlink to the annex, cannot tell for sure if the + - file was annexed before. So, check if git thinks the file's type has + - changed (from a symlink to a regular file). -} +isLocked :: FilePath -> Annex Bool +isLocked file = do + g <- Annex.gitRepo + changed <- typechanged g Nothing + changedCached <- typechanged g $ Just "--cached" + s <- liftIO $ getSymbolicLinkStatus file + return $ null (changed++changedCached) || isSymbolicLink s + where + typechanged g Nothing = typechanged' g params + typechanged g (Just param) = typechanged' g $ params++[param] + typechanged' g p = liftIO $ Git.pipeRead g $ p++[file] + params = ["diff", "--name-only", "--diff-filter=T"] From 361d28e1389e218c1d0baf378a740570a139f730 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:01:17 -0400 Subject: [PATCH 0457/2835] Unlocked files will now automatically be added back into the annex when committed (and the updated symlink committed), by some magic in the pre-commit hook. --- Command/PreCommit.hs | 28 +++++++++++++++++++++++----- debian/changelog | 3 +++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 2128f207da..117546359d 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -7,16 +7,34 @@ module Command.PreCommit where +import Control.Monad.State (liftIO) +import Control.Monad (when, unless) + import Command +import qualified Annex +import qualified Backend +import qualified GitRepo as Git import qualified Command.Fix +import qualified Command.Lock +import qualified Command.Add {- Run by git pre-commit hook. -} start :: SubCmdStartString start file = do - -- If a file is unlocked for edit, inject its content into the - -- annex, and replace it with a symlink to the content. Git will - -- then commit that. - error "TODO" + -- If a file is unlocked for edit, add its new content to the + -- annex, -} + locked <- Command.Lock.isLocked file + when (not locked) $ do + pairs <- Backend.chooseBackends [file] + ok <- doSubCmd $ Command.Add.start $ pairs !! 0 + unless (ok) $ do + error $ "failed to add " ++ file ++ "; canceling commit" + -- git commit will have staged the file's content; + -- drop that and stage the symlink + g <- Annex.gitRepo + liftIO $ Git.run g ["reset", "-q", "--", file] + liftIO $ Git.run g ["add", "--", file] - -- fix symlinks + -- Fix symlinks as they are committed, this ensures the + -- relative links are not broken when moved around. Command.Fix.start file diff --git a/debian/changelog b/debian/changelog index 0e121fda21..02c4ec295a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,9 @@ git-annex (0.04) UNRELEASED; urgency=low * Add unlock subcommand, which replaces the symlink with a copy of the file's content in preparation of changing it. * Add lock subcommand. + * Unlocked files will now automatically be added back into the annex when + committed (and the updated symlink committed), by some magic in the + pre-commit hook. * Add build dep on libghc6-testpack-dev. * Add annex.version, which will be used to automate upgrades between incompatable versions. From 43412419ea8c2f26620a0bc837acf6f14f0afb12 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:08:29 -0400 Subject: [PATCH 0458/2835] bugfix The object's directory might exist if the backend picked the same key as was already present. That could happen, for example, if the sha1 is the same. Note that I chose to go ahead and replace the old content with the new. We don't know if they are the same (even with sha1, their timestamp or perms could differ), so have to assume the newer one is preffered. Although it won't propigate to other annexes, so it had better not be significantly different! --- Core.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Core.hs b/Core.hs index 304c8a9232..f9c9417bd8 100644 --- a/Core.hs +++ b/Core.hs @@ -173,6 +173,7 @@ moveAnnex key src = do let dir = parentDir dest liftIO $ do createDirectoryIfMissing True dir + allowWrite dir -- in case the directory already exists renameFile src dest preventWrite dest preventWrite dir From ee6052cbf8337f60d4c1ee590f21b24f77f3b210 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:27:52 -0400 Subject: [PATCH 0459/2835] update for modifying files --- doc/walkthrough.mdwn | 65 ++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 29922bd75e..cfdf0ddabb 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -139,6 +139,51 @@ But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. +## modifying annexed files + +Normally, the content of files in the annex is prevented from being modified. + + # echo oops > my_cool_big_file + bash: my_cool_big_file: Permission deined + +In order to modify a file, it should first be unlocked. + + # git annex unlock my_cool_big_file + unlock my_cool_big_file (copying...) ok + +They replaces the symlink that normally points at its content with a copy +of the content. You can then modify the file like any regular file. Because +it is a regular file. + +If you decide you don't need to modify the file after all, or want to discard +modifications, just use `git annex lock`. + +When you `git commit`, git-annex's pre-commit hook will automatically +notice that you are committing an unlocked file, and add its new content +to the annex. The file will be replaced with a symlink to the new content, +and this symlink is what gets committed to git in the end. + + # echo "now smaller, but even cooler" > my_cool_big_file + # git commit my_cool_big_file -m "changed an annexed file" + add my_cool_big_file ok + (Recording state in git...) + [master 64cda67] changed an annexed file + 2 files changed, 2 insertions(+), 1 deletions(-) + create mode 100644 .git-annex/SHA1:0b1d8616d0238cb9418a0e0a649bdad2e9e7faae.log + +There is one problem with using `git commit` like this: Git wants to first +stage the entire contents of the file in its index. That can be slow for +big files (sorta why git-annex exists in the first place). So, the +automatic handling on commit is a nice safety feature, since it prevents +the file content being accidentially commited into git. But when working with +big files, it's faster to explicitly add them to the annex yourself +before committing. + + # echo "now smaller, but even cooler yet" > my_cool_big_file + # git annex add my_cool_big_file + add my_cool_big_file ok + # git commit my_cool_big_file -m "changed an annexed file" + ## using ssh remotes So far in this walkthrough, git-annex has been used with a remote @@ -190,26 +235,6 @@ makes it very easy. WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 ok -## modifying annexed files - -Normally, the content of files in the annex is prevented from being modified. -In order to modify a file, it should first be unlocked: - - # git annex unlock my_cool_big_file - unlock my_cool_big_file (copying...) ok - -They replaces the symlink that normally points at its content with a copy -of the content. You can then modify the file like any regular file. Because -it is a regular file. - -When you `git commit`, git-annex's pre-commit hook will automatically -notice that you are committing an unlocked file, and add its new content -to the annex. The file will be replaced with a symlink to the new content, -and this symlink is what gets committed to git. - -If you decide you don't need to modify the file after all, or want to discard -modifications, just use the lock subcommand. - ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has From 99d9c1cf893f1ad6bf400219a47593cf70993768 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:28:04 -0400 Subject: [PATCH 0460/2835] edit an alias for unlock --- CmdLine.hs | 2 ++ debian/changelog | 3 ++- doc/git-annex.mdwn | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CmdLine.hs b/CmdLine.hs index 1c73533df2..3f2b1e94eb 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -46,6 +46,8 @@ subCmds = "transfer content of files to/from another repository" , SubCommand "unlock" path (withFilesInGit Command.Unlock.start) "unlock files for modification" + , SubCommand "edit" path (withFilesInGit Command.Unlock.start) + "same as unlock" , SubCommand "lock" path (withFilesInGit Command.Lock.start) "undo unlock command" , SubCommand "init" desc (withDescription Command.Init.start) diff --git a/debian/changelog b/debian/changelog index 02c4ec295a..d3733aeb8e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,8 @@ git-annex (0.04) UNRELEASED; urgency=low * Add unlock subcommand, which replaces the symlink with a copy of - the file's content in preparation of changing it. + the file's content in preparation of changing it. The "edit" subcommand + is an alias for unlock. * Add lock subcommand. * Unlocked files will now automatically be added back into the annex when committed (and the updated symlink committed), by some magic in the diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d3cf594aa8..cb3cde0a73 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -89,6 +89,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. When you `git commit`, the file, the new content is injected back into the annex. +* edit [path ...] + + This is an alias for the unlock subcommand. May be easier to remember, + if you think of this as allowing you to edit an annexed file. + * move [path ...] When used with the --to option, moves the content of annexed files from From d0886a9ac7b662b2cdfe386af1d81af74925d0b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:32:46 -0400 Subject: [PATCH 0461/2835] explicity run queue to git add files --- Command/PreCommit.hs | 2 +- doc/walkthrough.mdwn | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 117546359d..cd6ce6f080 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -33,7 +33,7 @@ start file = do -- drop that and stage the symlink g <- Annex.gitRepo liftIO $ Git.run g ["reset", "-q", "--", file] - liftIO $ Git.run g ["add", "--", file] + Annex.queueRun -- Fix symlinks as they are committed, this ensures the -- relative links are not broken when moved around. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index cfdf0ddabb..61cf29b89f 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -166,7 +166,6 @@ and this symlink is what gets committed to git in the end. # echo "now smaller, but even cooler" > my_cool_big_file # git commit my_cool_big_file -m "changed an annexed file" add my_cool_big_file ok - (Recording state in git...) [master 64cda67] changed an annexed file 2 files changed, 2 insertions(+), 1 deletions(-) create mode 100644 .git-annex/SHA1:0b1d8616d0238cb9418a0e0a649bdad2e9e7faae.log From 81524d19a7e5bc5f7b573260babe2def279187f7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:01:41 -0400 Subject: [PATCH 0462/2835] add typeChangedFiles --- GitRepo.hs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index da86c225e2..5fc077c449 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -39,6 +39,7 @@ module GitRepo ( checkAttr, decodeGitFile, encodeGitFile, + typeChangedFiles, prop_idempotent_deencode ) where @@ -58,6 +59,7 @@ import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf +import Data.List import Utility @@ -227,20 +229,31 @@ hPipeRead repo params = assertLocal repo $ do - are checked into git at that location. -} inRepo :: Repo -> FilePath -> IO [FilePath] inRepo repo l = pipeNullSplit repo - ["ls-files", "--cached", "--exclude-standard", "-z", l] + ["ls-files", "--cached", "--exclude-standard", "-z", "--", l] {- Passed a location, recursively scans for all files that are not checked - into git, and not gitignored. -} notInRepo :: Repo -> FilePath -> IO [FilePath] notInRepo repo l = pipeNullSplit repo - ["ls-files", "--others", "--exclude-standard", "-z", l] + ["ls-files", "--others", "--exclude-standard", "-z", "--", l] {- Passed a location, returns a list of the files, staged for - commit, that are being added, moved, or changed (but not deleted). -} stagedFiles :: Repo -> FilePath -> IO [FilePath] stagedFiles repo l = pipeNullSplit repo - ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", - "HEAD", l] + ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", + "--", l] + +{- Passed a location, returns a list of the files whose type has changed. -} +typeChangedFiles :: Repo -> FilePath -> IO [FilePath] +typeChangedFiles repo l = do + changed <- pipeNullSplit repo $ start ++ end + changedCached <- pipeNullSplit repo $ start ++ ["--cached"] ++ end + -- a file can be found twice by the above, so nub + return $ nub $ changed ++ changedCached + where + start = ["diff", "--name-only", "--diff-filter=T", "-z"] + end = ["--", l] {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} From e826368cecb2e515cc3c4f5f8d0385c025b069a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:02:08 -0400 Subject: [PATCH 0463/2835] allow adding unlocked files --- CmdLine.hs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 3f2b1e94eb..7e6626573a 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -36,7 +36,7 @@ import qualified Command.PreCommit subCmds :: [SubCommand] subCmds = - [ SubCommand "add" path (withFilesNotInGit Command.Add.start) + [ SubCommand "add" path (withFilesToAdd Command.Add.start) "add files to annex" , SubCommand "get" path (withFilesInGit Command.Get.start) "make content of annexed files available" @@ -115,13 +115,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- These functions find appropriate files or other things based on a user's parameters. -} -withFilesNotInGit :: SubCmdSeekBackendFiles -withFilesNotInGit a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.notInRepo repo) params - let files' = foldl (++) [] files - pairs <- Backend.chooseBackends files' - return $ map a $ filter (\(f,_) -> notState f) pairs withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo @@ -135,6 +128,14 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e +withFilesToAdd :: SubCmdSeekBackendFiles +withFilesToAdd a params = do + repo <- Annex.gitRepo + newfiles <- liftIO $ mapM (Git.notInRepo repo) params + unlockedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params + let files = foldl (++) [] $ newfiles ++ unlockedfiles + pairs <- Backend.chooseBackends files + return $ map a $ filter (\(f,_) -> notState f) pairs withDescription :: SubCmdSeekStrings withDescription a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings From 31101a8b278a1b875defdea627307ef90ac3df21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:08:28 -0400 Subject: [PATCH 0464/2835] use new git function --- Command/Lock.hs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Command/Lock.hs b/Command/Lock.hs index 1be35d6126..955749e93d 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -45,12 +45,6 @@ perform file = do isLocked :: FilePath -> Annex Bool isLocked file = do g <- Annex.gitRepo - changed <- typechanged g Nothing - changedCached <- typechanged g $ Just "--cached" + typechanged <- liftIO $ Git.typeChangedFiles g file s <- liftIO $ getSymbolicLinkStatus file - return $ null (changed++changedCached) || isSymbolicLink s - where - typechanged g Nothing = typechanged' g params - typechanged g (Just param) = typechanged' g $ params++[param] - typechanged' g p = liftIO $ Git.pipeRead g $ p++[file] - params = ["diff", "--name-only", "--diff-filter=T"] + return $ (not $ elem file typechanged) || isSymbolicLink s From 16ba23d48de00daccbfb481dd1baca91047f8b3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:10:48 -0400 Subject: [PATCH 0465/2835] tweak --- doc/git-annex.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index cb3cde0a73..d0bd3a754e 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -85,9 +85,9 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Normally, the content of annexed files is protected from being changed. Unlocking a annexed file allows it to be modified. This replaces the - symlink for each specified file with a copy of the file content. - When you `git commit`, the file, the new content is injected back into - the annex. + symlink for each specified file with a copy of the file's content. + You can then modify it and `git annex add` (or `git commit`) to inject + it back into the annex. * edit [path ...] From fb824f7eb03c10301ad897d9e1eeb0aa40492a3d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:15:21 -0400 Subject: [PATCH 0466/2835] use -- before filenames when running git add, git rm, etc --- Command/Add.hs | 2 +- Command/Fix.hs | 2 +- Command/FromKey.hs | 2 +- Command/Move.hs | 4 ++-- Command/Unannex.hs | 2 +- Core.hs | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Command/Add.hs b/Command/Add.hs index 6c5d24f842..649b466bb3 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -44,5 +44,5 @@ cleanup file key = do link <- calcGitLink file key liftIO $ createSymbolicLink link file - Annex.queue "add" [] file + Annex.queue "add" ["--"] file return True diff --git a/Command/Fix.hs b/Command/Fix.hs index 7963a1d2ea..9db832cc76 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -37,5 +37,5 @@ perform file link = do cleanup :: FilePath -> SubCmdCleanup cleanup file = do - Annex.queue "add" [] file + Annex.queue "add" ["--"] file return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs index de555475c1..229a93684a 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -41,5 +41,5 @@ perform file key = do return $ Just $ cleanup file cleanup :: FilePath -> SubCmdCleanup cleanup file = do - Annex.queue "add" [] file + Annex.queue "add" ["--"] file return True diff --git a/Command/Move.hs b/Command/Move.hs index 6ca923a310..e0b079193a 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -85,7 +85,7 @@ moveToCleanup remote key tmpfile = do g <- Annex.gitRepo remoteuuid <- getUUID remote logfile <- liftIO $ logChange g key remoteuuid ValuePresent - Annex.queue "add" [] logfile + Annex.queue "add" ["--"] logfile -- Cleanup on the local side is the same as done for the -- drop subcommand. Command.Drop.cleanup key @@ -128,5 +128,5 @@ moveFromCleanup remote key = do remoteuuid <- getUUID remote g <- Annex.gitRepo logfile <- liftIO $ logChange g key remoteuuid ValueMissing - Annex.queue "add" [] logfile + Annex.queue "add" ["--"] logfile return ok diff --git a/Command/Unannex.hs b/Command/Unannex.hs index a9c18f765e..f5e78e55af 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -40,7 +40,7 @@ cleanup file key = do g <- Annex.gitRepo liftIO $ removeFile file - liftIO $ Git.run g ["rm", "--quiet", file] + liftIO $ Git.run g ["rm", "--quiet", "--", file] -- git rm deletes empty directories; put them back liftIO $ createDirectoryIfMissing True (parentDir file) diff --git a/Core.hs b/Core.hs index f9c9417bd8..8497a7f368 100644 --- a/Core.hs +++ b/Core.hs @@ -131,7 +131,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g logfile <- liftIO $ logChange g key u status - Annex.queue "add" [] logfile + Annex.queue "add" ["--"] logfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into @@ -261,5 +261,5 @@ upgradeFrom0 = do link <- calcGitLink f k liftIO $ removeFile f liftIO $ createSymbolicLink link f - Annex.queue "add" [] f + Annex.queue "add" ["--"] f fixlinks fs From d9d6b256fc4ad430dad48374bd64a82913e7b080 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:16:53 -0400 Subject: [PATCH 0467/2835] cleanup --- debian/changelog | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index d3733aeb8e..9020f60855 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,7 +7,6 @@ git-annex (0.04) UNRELEASED; urgency=low * Unlocked files will now automatically be added back into the annex when committed (and the updated symlink committed), by some magic in the pre-commit hook. - * Add build dep on libghc6-testpack-dev. * Add annex.version, which will be used to automate upgrades between incompatable versions. * Reorganised the layout of .git/annex/ @@ -18,9 +17,9 @@ git-annex (0.04) UNRELEASED; urgency=low * Annexed file contents are now made unwritable and put in unwriteable directories, to avoid them accidentially being removed or modified. (Thanks Josh Triplett for the idea.) + * Add build dep on libghc6-testpack-dev. Closes: #603016 * Avoid using runghc to run test suite as it is not available on all architectures. Closes: #603006 - * Missing build dep. Closes: #603016 -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 From 10f35dceb62882d90dd019357ebd257b7e7f64fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:20:50 -0400 Subject: [PATCH 0468/2835] tweak --- doc/walkthrough.mdwn | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 61cf29b89f..d6c0214ffd 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -142,6 +142,8 @@ some other repository before dropping it. ## modifying annexed files Normally, the content of files in the annex is prevented from being modified. +That's a good thing, because it might be the only copy, you wouldn't +want to lose it in a fumblefingered mistake. # echo oops > my_cool_big_file bash: my_cool_big_file: Permission deined @@ -151,12 +153,12 @@ In order to modify a file, it should first be unlocked. # git annex unlock my_cool_big_file unlock my_cool_big_file (copying...) ok -They replaces the symlink that normally points at its content with a copy +That replaces the symlink that normally points at its content with a copy of the content. You can then modify the file like any regular file. Because it is a regular file. -If you decide you don't need to modify the file after all, or want to discard -modifications, just use `git annex lock`. +(If you decide you don't need to modify the file after all, or want to discard +modifications, just use `git annex lock`.) When you `git commit`, git-annex's pre-commit hook will automatically notice that you are committing an unlocked file, and add its new content From b9d7e67f614b0da96130c99206ad82a690bc826e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:29:51 -0400 Subject: [PATCH 0469/2835] releasing version 0.04 --- debian/changelog | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 9020f60855..0c09eb7ea4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.04) UNRELEASED; urgency=low +git-annex (0.04) unstable; urgency=low * Add unlock subcommand, which replaces the symlink with a copy of the file's content in preparation of changing it. The "edit" subcommand @@ -7,8 +7,9 @@ git-annex (0.04) UNRELEASED; urgency=low * Unlocked files will now automatically be added back into the annex when committed (and the updated symlink committed), by some magic in the pre-commit hook. + * The SHA1 backend is now fully usable. * Add annex.version, which will be used to automate upgrades - between incompatable versions. + between incompatible versions. * Reorganised the layout of .git/annex/ * The new layout will be automatically upgraded to the first time git-annex is used in a repository with the old layout. @@ -21,7 +22,7 @@ git-annex (0.04) UNRELEASED; urgency=low * Avoid using runghc to run test suite as it is not available on all architectures. Closes: #603006 - -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 + -- Joey Hess Wed, 10 Nov 2010 14:23:23 -0400 git-annex (0.03) unstable; urgency=low From 8a8987579f8efe290e5084202a416f72eec491a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:30:33 -0400 Subject: [PATCH 0470/2835] add news item for git-annex 0.04 --- doc/news/version_0.04.mdwn | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 doc/news/version_0.04.mdwn diff --git a/doc/news/version_0.04.mdwn b/doc/news/version_0.04.mdwn new file mode 100644 index 0000000000..c5a9fd3ead --- /dev/null +++ b/doc/news/version_0.04.mdwn @@ -0,0 +1,23 @@ +git-annex 0.04 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Add unlock subcommand, which replaces the symlink with a copy of + the file's content in preparation of changing it. The "edit" subcommand + is an alias for unlock. + * Add lock subcommand. + * Unlocked files will now automatically be added back into the annex when + committed (and the updated symlink committed), by some magic in the + pre-commit hook. + * The SHA1 backend is now fully usable. + * Add annex.version, which will be used to automate upgrades + between incompatible versions. + * Reorganised the layout of .git/annex/ + * The new layout will be automatically upgraded to the first time + git-annex is used in a repository with the old layout. + * Note that git-annex 0.04 cannot transfer content from old repositories + that have not yet been upgraded. + * Annexed file contents are now made unwritable and put in unwriteable + directories, to avoid them accidentially being removed or modified. + (Thanks Josh Triplett for the idea.) + * Add build dep on libghc6-testpack-dev. Closes: #[603016](http://bugs.debian.org/603016) + * Avoid using runghc to run test suite as it is not available on all + architectures. Closes: #[603006](http://bugs.debian.org/603006)"""]] \ No newline at end of file From dffe9499630a8dded76b23ef762f03aa264dc45c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 14:54:29 -0400 Subject: [PATCH 0471/2835] Optimize both pre-commit and lock subcommands. isLocked was doing the expensive check before the cheap one. Let's not fork git diff twice per file when committing, especially. git diff is still run more than strictly necessary (ie, more than once) if multiple unlocked files are being committed. But much better now. --- Command/Lock.hs | 18 ++++++++++-------- Command/PreCommit.hs | 5 +++-- debian/changelog | 6 ++++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Command/Lock.hs b/Command/Lock.hs index 955749e93d..6ae59221c8 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -37,14 +37,16 @@ perform file = do liftIO $ Git.run g ["checkout", "--", file] return $ Just $ return True -- no cleanup needed -{- Checks if a file is unlocked for edit. - - - - But, without the symlink to the annex, cannot tell for sure if the - - file was annexed before. So, check if git thinks the file's type has - - changed (from a symlink to a regular file). -} +{- Checks if a file is unlocked for edit. -} isLocked :: FilePath -> Annex Bool isLocked file = do - g <- Annex.gitRepo - typechanged <- liftIO $ Git.typeChangedFiles g file + -- check if it's a symlink first, as that's cheapest s <- liftIO $ getSymbolicLinkStatus file - return $ (not $ elem file typechanged) || isSymbolicLink s + if (isSymbolicLink s) + then return True -- Symlinked files are always locked. + else do + -- Not a symlink, so see if the type has changed, + -- if so it is presumed to have been unlocked. + g <- Annex.gitRepo + typechanged <- liftIO $ Git.typeChangedFiles g file + return $ not $ elem file typechanged diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index cd6ce6f080..72cece8d50 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -22,7 +22,7 @@ import qualified Command.Add start :: SubCmdStartString start file = do -- If a file is unlocked for edit, add its new content to the - -- annex, -} + -- annex. -} locked <- Command.Lock.isLocked file when (not locked) $ do pairs <- Backend.chooseBackends [file] @@ -30,7 +30,8 @@ start file = do unless (ok) $ do error $ "failed to add " ++ file ++ "; canceling commit" -- git commit will have staged the file's content; - -- drop that and stage the symlink + -- drop that and run command queued by Add.state to + -- stage the symlink g <- Annex.gitRepo liftIO $ Git.run g ["reset", "-q", "--", file] Annex.queueRun diff --git a/debian/changelog b/debian/changelog index 0c09eb7ea4..a4c8bceac0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.05) UNRELEASED; urgency=low + + * Optimize both pre-commit and lock subcommands. + + -- Joey Hess Thu, 11 Nov 2010 14:52:05 -0400 + git-annex (0.04) unstable; urgency=low * Add unlock subcommand, which replaces the symlink with a copy of From 58fffdb73be115ad095342eba2f4dd90190e998f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 15:34:28 -0400 Subject: [PATCH 0472/2835] remove unnecessary mkdir --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index ee4d50f9fc..ffefbb2a39 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ all: git-annex docs ghcmake=ghc -Wall -odir build -hidir build --make git-annex: - mkdir -p build $(ghcmake) git-annex install: From b5ce88dd2aa2d6cc5eac6fd014f94d387c38bce0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 17:01:19 -0400 Subject: [PATCH 0473/2835] build with -O2 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ffefbb2a39..c94fbbc8f2 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: git-annex docs -ghcmake=ghc -Wall -odir build -hidir build --make +ghcmake=ghc -Wall -odir build -hidir build -O2 --make git-annex: $(ghcmake) git-annex From ce62f5abf16e578f9f4b86cd140ea2ddfb1e4217 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 17:58:55 -0400 Subject: [PATCH 0474/2835] rework command dispatching for add and pre-commit Both subcommands do two different operations on different sets of files, so allowing a subcommand to perform a list of operations cleans things up. --- CmdLine.hs | 57 ++++++++++++++++++++++++++++---------------- Command.hs | 6 ++--- Command/Lock.hs | 28 ++++------------------ Command/PreCommit.hs | 42 ++++++++++++++++---------------- Core.hs | 14 +++++++++++ debian/changelog | 8 ++++++- 6 files changed, 84 insertions(+), 71 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 7e6626573a..7c9d75c18d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -17,6 +17,7 @@ import qualified Annex import Locations import qualified Backend import Types +import Core import Command import qualified Command.Add @@ -36,35 +37,37 @@ import qualified Command.PreCommit subCmds :: [SubCommand] subCmds = - [ SubCommand "add" path (withFilesToAdd Command.Add.start) + [ SubCommand "add" path [withFilesNotInGit Command.Add.start, + withFilesUnlocked Command.Add.start] "add files to annex" - , SubCommand "get" path (withFilesInGit Command.Get.start) + , SubCommand "get" path [withFilesInGit Command.Get.start] "make content of annexed files available" - , SubCommand "drop" path (withFilesInGit Command.Drop.start) + , SubCommand "drop" path [withFilesInGit Command.Drop.start] "indicate content of files not currently wanted" - , SubCommand "move" path (withFilesInGit Command.Move.start) + , SubCommand "move" path [withFilesInGit Command.Move.start] "transfer content of files to/from another repository" - , SubCommand "unlock" path (withFilesInGit Command.Unlock.start) + , SubCommand "unlock" path [withFilesInGit Command.Unlock.start] "unlock files for modification" - , SubCommand "edit" path (withFilesInGit Command.Unlock.start) + , SubCommand "edit" path [withFilesInGit Command.Unlock.start] "same as unlock" - , SubCommand "lock" path (withFilesInGit Command.Lock.start) + , SubCommand "lock" path [withFilesUnlocked Command.Lock.start] "undo unlock command" - , SubCommand "init" desc (withDescription Command.Init.start) + , SubCommand "init" desc [withDescription Command.Init.start] "initialize git-annex with repository description" - , SubCommand "unannex" path (withFilesInGit Command.Unannex.start) + , SubCommand "unannex" path [withFilesInGit Command.Unannex.start] "undo accidential add command" - , SubCommand "pre-commit" path (withFilesToBeCommitted Command.PreCommit.start) + , SubCommand "pre-commit" path [withFilesToBeCommitted Command.Fix.start, + withUnlockedFilesToBeCommitted Command.PreCommit.start] "run by git pre-commit hook" - , SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) + , SubCommand "fromkey" key [withFilesMissing Command.FromKey.start] "adds a file using a specific key" - , SubCommand "dropkey" key (withKeys Command.DropKey.start) + , SubCommand "dropkey" key [withKeys Command.DropKey.start] "drops annexed content for specified keys" - , SubCommand "setkey" key (withTempFile Command.SetKey.start) + , SubCommand "setkey" key [withTempFile Command.SetKey.start] "sets annexed content for a key using a temp file" - , SubCommand "fix" path (withFilesInGit Command.Fix.start) + , SubCommand "fix" path [withFilesInGit Command.Fix.start] "fix up symlinks to point to annexed content" - , SubCommand "fsck" nothing (withNothing Command.Fsck.start) + , SubCommand "fsck" nothing [withNothing Command.Fsck.start] "check annex for problems" ] where @@ -128,12 +131,17 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e -withFilesToAdd :: SubCmdSeekBackendFiles -withFilesToAdd a params = do +withFilesNotInGit :: SubCmdSeekBackendFiles +withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ mapM (Git.notInRepo repo) params - unlockedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params - let files = foldl (++) [] $ newfiles ++ unlockedfiles + backendPairs a $ foldl (++) [] newfiles +withFilesUnlocked :: SubCmdSeekBackendFiles +withFilesUnlocked a params = do + unlocked <- mapM unlockedFiles params + backendPairs a $ foldl (++) [] unlocked +backendPairs :: SubCmdSeekBackendFiles +backendPairs a files = do pairs <- Backend.chooseBackends files return $ map a $ filter (\(f,_) -> notState f) pairs withDescription :: SubCmdSeekStrings @@ -141,8 +149,15 @@ withDescription a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.stagedFiles repo) params - return $ map a $ filter notState $ foldl (++) [] files + tocommit <- liftIO $ mapM (Git.stagedFiles repo) params + return $ map a $ filter notState $ foldl (++) [] tocommit +withUnlockedFilesToBeCommitted :: SubCmdSeekStrings +withUnlockedFilesToBeCommitted a params = do + repo <- Annex.gitRepo + unlocked <- mapM unlockedFiles params + tocommit <- liftIO $ mapM (Git.stagedFiles repo) $ + filter notState $ foldl (++) [] unlocked + return $ map a $ foldl (++) [] tocommit withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings diff --git a/Command.hs b/Command.hs index f896a53f6f..90c4d53854 100644 --- a/Command.hs +++ b/Command.hs @@ -41,7 +41,7 @@ type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek data SubCommand = SubCommand { subcmdname :: String, subcmdparams :: String, - subcmdseek :: SubCmdSeek, + subcmdseek :: [SubCmdSeek], subcmddesc :: String } @@ -49,8 +49,8 @@ data SubCommand = SubCommand { - the parameters passed to it. -} prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] prepSubCmd SubCommand { subcmdseek = seek } state params = do - list <- Annex.eval state $ seek params - return $ map doSubCmd list + lists <- Annex.eval state $ mapM (\s -> s params) seek + return $ map doSubCmd $ foldl (++) [] lists {- Runs a subcommand through the start, perform and cleanup stages -} doSubCmd :: SubCmdStart -> SubCmdCleanup diff --git a/Command/Lock.hs b/Command/Lock.hs index 6ae59221c8..f03d6b6c88 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -9,23 +9,17 @@ module Command.Lock where import Control.Monad.State (liftIO) import System.Directory -import System.Posix.Files -import Types import Command import Messages import qualified Annex import qualified GitRepo as Git {- Undo unlock -} -start :: SubCmdStartString -start file = do - locked <- isLocked file - if locked - then return Nothing - else do - showStart "lock" file - return $ Just $ perform file +start :: SubCmdStartBackendFile +start (file, _) = do + showStart "lock" file + return $ Just $ perform file perform :: FilePath -> SubCmdPerform perform file = do @@ -36,17 +30,3 @@ perform file = do -- checkout the symlink liftIO $ Git.run g ["checkout", "--", file] return $ Just $ return True -- no cleanup needed - -{- Checks if a file is unlocked for edit. -} -isLocked :: FilePath -> Annex Bool -isLocked file = do - -- check if it's a symlink first, as that's cheapest - s <- liftIO $ getSymbolicLinkStatus file - if (isSymbolicLink s) - then return True -- Symlinked files are always locked. - else do - -- Not a symlink, so see if the type has changed, - -- if so it is presumed to have been unlocked. - g <- Annex.gitRepo - typechanged <- liftIO $ Git.typeChangedFiles g file - return $ not $ elem file typechanged diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 72cece8d50..b3b940cdd1 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -8,34 +8,32 @@ module Command.PreCommit where import Control.Monad.State (liftIO) -import Control.Monad (when, unless) import Command import qualified Annex import qualified Backend import qualified GitRepo as Git -import qualified Command.Fix -import qualified Command.Lock import qualified Command.Add -{- Run by git pre-commit hook. -} +{- Run by git pre-commit hook; passed unlocked files that are being + - committed. -} start :: SubCmdStartString -start file = do - -- If a file is unlocked for edit, add its new content to the - -- annex. -} - locked <- Command.Lock.isLocked file - when (not locked) $ do - pairs <- Backend.chooseBackends [file] - ok <- doSubCmd $ Command.Add.start $ pairs !! 0 - unless (ok) $ do - error $ "failed to add " ++ file ++ "; canceling commit" - -- git commit will have staged the file's content; - -- drop that and run command queued by Add.state to - -- stage the symlink - g <- Annex.gitRepo - liftIO $ Git.run g ["reset", "-q", "--", file] - Annex.queueRun +start file = return $ Just $ perform file - -- Fix symlinks as they are committed, this ensures the - -- relative links are not broken when moved around. - Command.Fix.start file +perform :: FilePath -> SubCmdPerform +perform file = do + pairs <- Backend.chooseBackends [file] + ok <- doSubCmd $ Command.Add.start $ pairs !! 0 + if ok + then return $ Just $ cleanup file + else error $ "failed to add " ++ file ++ "; canceling commit" + +cleanup :: FilePath -> SubCmdCleanup +cleanup file = do + -- git commit will have staged the file's content; + -- drop that and run command queued by Add.state to + -- stage the symlink + g <- Annex.gitRepo + liftIO $ Git.run g ["reset", "-q", "--", file] + Annex.queueRun + return True diff --git a/Core.hs b/Core.hs index 8497a7f368..0c06d23101 100644 --- a/Core.hs +++ b/Core.hs @@ -224,6 +224,20 @@ getKeysReferenced = do keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs +{- Passed a location (a directory or a single file, returns + - files there that are unlocked for editing. -} +unlockedFiles :: FilePath -> Annex [FilePath] +unlockedFiles l = do + -- unlocked files have changed type from a symlink to a regular file + g <- Annex.gitRepo + typechangedfiles <- liftIO $ Git.typeChangedFiles g l + unlockedfiles <- filterM notsymlink typechangedfiles + return unlockedfiles + where + notsymlink f = do + s <- liftIO $ getSymbolicLinkStatus f + return $ not $ isSymbolicLink s + {- Uses the annex.version git config setting to automate upgrades. -} autoUpgrade :: Annex () autoUpgrade = do diff --git a/debian/changelog b/debian/changelog index a4c8bceac0..f705bfaf54 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,12 @@ git-annex (0.05) UNRELEASED; urgency=low - * Optimize both pre-commit and lock subcommands. + * Optimize both pre-commit and lock subcommands to not call git diff + on every file being committed or locked. + (This actually also works around a bug in ghc 6.12.1, that caused + git-annex 0.04 pre-commit to sometimes corrupt filenames and fail. + The excessive number of calls made by pre-commit exposed the ghc bug. + Thanks Josh Triplett for the debugging.) + * Build with -O3. -- Joey Hess Thu, 11 Nov 2010 14:52:05 -0400 From f2c7a6e73d342f4f82be4e2839a2022f97539c65 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:21:54 -0400 Subject: [PATCH 0475/2835] got rid of several more calls to git when finding unlocked files --- CmdLine.hs | 22 +++++++++++++++------- Core.hs | 14 -------------- GitRepo.hs | 20 +++++++++++++------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 7c9d75c18d..5c25b41c3b 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,6 +10,7 @@ module CmdLine (parseCmd) where import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Directory +import System.Posix.Files import Control.Monad (filterM, when) import qualified GitRepo as Git @@ -17,7 +18,6 @@ import qualified Annex import Locations import qualified Backend import Types -import Core import Command import qualified Command.Add @@ -138,8 +138,11 @@ withFilesNotInGit a params = do backendPairs a $ foldl (++) [] newfiles withFilesUnlocked :: SubCmdSeekBackendFiles withFilesUnlocked a params = do - unlocked <- mapM unlockedFiles params - backendPairs a $ foldl (++) [] unlocked + -- unlocked files have changed type from a symlink to a regular file + repo <- Annex.gitRepo + typechangedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params + unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + backendPairs a $ filter notState unlockedfiles backendPairs :: SubCmdSeekBackendFiles backendPairs a files = do pairs <- Backend.chooseBackends files @@ -154,10 +157,9 @@ withFilesToBeCommitted a params = do withUnlockedFilesToBeCommitted :: SubCmdSeekStrings withUnlockedFilesToBeCommitted a params = do repo <- Annex.gitRepo - unlocked <- mapM unlockedFiles params - tocommit <- liftIO $ mapM (Git.stagedFiles repo) $ - filter notState $ foldl (++) [] unlocked - return $ map a $ foldl (++) [] tocommit + typechangedfiles <- liftIO $ mapM (Git.typeChangedStagedFiles repo) params + unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + return $ map a $ filter notState unlockedfiles withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings @@ -168,6 +170,12 @@ withNothing a _ = return [a] {- filter out files from the state directory -} notState :: FilePath -> Bool notState f = stateLoc /= take (length stateLoc) f + +{- filter out symlinks -} +notSymlink :: FilePath -> IO Bool +notSymlink f = do + s <- liftIO $ getSymbolicLinkStatus f + return $ not $ isSymbolicLink s {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it diff --git a/Core.hs b/Core.hs index 0c06d23101..8497a7f368 100644 --- a/Core.hs +++ b/Core.hs @@ -224,20 +224,6 @@ getKeysReferenced = do keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs -{- Passed a location (a directory or a single file, returns - - files there that are unlocked for editing. -} -unlockedFiles :: FilePath -> Annex [FilePath] -unlockedFiles l = do - -- unlocked files have changed type from a symlink to a regular file - g <- Annex.gitRepo - typechangedfiles <- liftIO $ Git.typeChangedFiles g l - unlockedfiles <- filterM notsymlink typechangedfiles - return unlockedfiles - where - notsymlink f = do - s <- liftIO $ getSymbolicLinkStatus f - return $ not $ isSymbolicLink s - {- Uses the annex.version git config setting to automate upgrades. -} autoUpgrade :: Annex () autoUpgrade = do diff --git a/GitRepo.hs b/GitRepo.hs index 5fc077c449..fa78e51222 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -40,6 +40,7 @@ module GitRepo ( decodeGitFile, encodeGitFile, typeChangedFiles, + typeChangedStagedFiles, prop_idempotent_deencode ) where @@ -59,7 +60,6 @@ import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf -import Data.List import Utility @@ -244,16 +244,22 @@ stagedFiles repo l = pipeNullSplit repo ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", "--", l] -{- Passed a location, returns a list of the files whose type has changed. -} +{- Passed a location, returns a list of the files, staged for + - commit, whose type has changed. -} +typeChangedStagedFiles :: Repo -> FilePath -> IO [FilePath] +typeChangedStagedFiles repo l = typeChangedFiles' repo l ["--cached"] + +{- Passed a location, returns a list of the files whose type has changed. + - Files only staged for commit will not be included. -} typeChangedFiles :: Repo -> FilePath -> IO [FilePath] -typeChangedFiles repo l = do - changed <- pipeNullSplit repo $ start ++ end - changedCached <- pipeNullSplit repo $ start ++ ["--cached"] ++ end - -- a file can be found twice by the above, so nub - return $ nub $ changed ++ changedCached +typeChangedFiles repo l = typeChangedFiles' repo l [] + +typeChangedFiles' :: Repo -> FilePath -> [String] -> IO [FilePath] +typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where start = ["diff", "--name-only", "--diff-filter=T", "-z"] end = ["--", l] + {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} From 5357d3a37af9e3d3a0aec207a8ba7fb94bfea953 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:26:27 -0400 Subject: [PATCH 0476/2835] remove dup filter --- CmdLine.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 5c25b41c3b..93404e546d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -117,7 +117,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs pad n s = replicate (n - length s) ' ' {- These functions find appropriate files or other things based on a - user's parameters. -} + user's parameters, and run a specified action on them. -} withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo @@ -135,7 +135,7 @@ withFilesNotInGit :: SubCmdSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ mapM (Git.notInRepo repo) params - backendPairs a $ foldl (++) [] newfiles + backendPairs a $ filter notState $ foldl (++) [] newfiles withFilesUnlocked :: SubCmdSeekBackendFiles withFilesUnlocked a params = do -- unlocked files have changed type from a symlink to a regular file @@ -146,7 +146,7 @@ withFilesUnlocked a params = do backendPairs :: SubCmdSeekBackendFiles backendPairs a files = do pairs <- Backend.chooseBackends files - return $ map a $ filter (\(f,_) -> notState f) pairs + return $ map a pairs withDescription :: SubCmdSeekStrings withDescription a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings From da0de293d16ace6aac574d0cdc37ec41715b7d66 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:54:52 -0400 Subject: [PATCH 0477/2835] refactor param seeking --- CmdLine.hs | 101 +++++++------------------------------------ Command.hs | 70 +++++++++++++++++++++++++++++- Command/Add.hs | 4 ++ Command/Drop.hs | 3 ++ Command/DropKey.hs | 3 ++ Command/Fix.hs | 3 ++ Command/FromKey.hs | 3 ++ Command/Fsck.hs | 3 ++ Command/Get.hs | 3 ++ Command/Init.hs | 3 ++ Command/Lock.hs | 3 ++ Command/Move.hs | 5 ++- Command/PreCommit.hs | 9 +++- Command/SetKey.hs | 3 ++ Command/Unannex.hs | 3 ++ Command/Unlock.hs | 3 ++ debian/changelog | 9 ++-- 17 files changed, 138 insertions(+), 93 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 93404e546d..efa541ebcc 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -8,15 +8,9 @@ module CmdLine (parseCmd) where import System.Console.GetOpt -import Control.Monad.State (liftIO) -import System.Directory -import System.Posix.Files -import Control.Monad (filterM, when) +import Control.Monad (when) -import qualified GitRepo as Git import qualified Annex -import Locations -import qualified Backend import Types import Command @@ -37,37 +31,35 @@ import qualified Command.PreCommit subCmds :: [SubCommand] subCmds = - [ SubCommand "add" path [withFilesNotInGit Command.Add.start, - withFilesUnlocked Command.Add.start] + [ SubCommand "add" path Command.Add.seek "add files to annex" - , SubCommand "get" path [withFilesInGit Command.Get.start] + , SubCommand "get" path Command.Get.seek "make content of annexed files available" - , SubCommand "drop" path [withFilesInGit Command.Drop.start] + , SubCommand "drop" path Command.Drop.seek "indicate content of files not currently wanted" - , SubCommand "move" path [withFilesInGit Command.Move.start] + , SubCommand "move" path Command.Move.seek "transfer content of files to/from another repository" - , SubCommand "unlock" path [withFilesInGit Command.Unlock.start] + , SubCommand "unlock" path Command.Unlock.seek "unlock files for modification" - , SubCommand "edit" path [withFilesInGit Command.Unlock.start] + , SubCommand "edit" path Command.Unlock.seek "same as unlock" - , SubCommand "lock" path [withFilesUnlocked Command.Lock.start] + , SubCommand "lock" path Command.Lock.seek "undo unlock command" - , SubCommand "init" desc [withDescription Command.Init.start] + , SubCommand "init" desc Command.Init.seek "initialize git-annex with repository description" - , SubCommand "unannex" path [withFilesInGit Command.Unannex.start] + , SubCommand "unannex" path Command.Unannex.seek "undo accidential add command" - , SubCommand "pre-commit" path [withFilesToBeCommitted Command.Fix.start, - withUnlockedFilesToBeCommitted Command.PreCommit.start] + , SubCommand "pre-commit" path Command.PreCommit.seek "run by git pre-commit hook" - , SubCommand "fromkey" key [withFilesMissing Command.FromKey.start] + , SubCommand "fromkey" key Command.FromKey.seek "adds a file using a specific key" - , SubCommand "dropkey" key [withKeys Command.DropKey.start] + , SubCommand "dropkey" key Command.DropKey.seek "drops annexed content for specified keys" - , SubCommand "setkey" key [withTempFile Command.SetKey.start] + , SubCommand "setkey" key Command.SetKey.seek "sets annexed content for a key using a temp file" - , SubCommand "fix" path [withFilesInGit Command.Fix.start] + , SubCommand "fix" path Command.Fix.seek "fix up symlinks to point to annexed content" - , SubCommand "fsck" nothing [withNothing Command.Fsck.start] + , SubCommand "fsck" nothing Command.Fsck.seek "check annex for problems" ] where @@ -116,67 +108,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = replicate (n - length s) ' ' -{- These functions find appropriate files or other things based on a - user's parameters, and run a specified action on them. -} -withFilesInGit :: SubCmdSeekStrings -withFilesInGit a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.inRepo repo) params - return $ map a $ filter notState $ foldl (++) [] files -withFilesMissing :: SubCmdSeekStrings -withFilesMissing a params = do - files <- liftIO $ filterM missing params - return $ map a $ filter notState files - where - missing f = do - e <- doesFileExist f - return $ not e -withFilesNotInGit :: SubCmdSeekBackendFiles -withFilesNotInGit a params = do - repo <- Annex.gitRepo - newfiles <- liftIO $ mapM (Git.notInRepo repo) params - backendPairs a $ filter notState $ foldl (++) [] newfiles -withFilesUnlocked :: SubCmdSeekBackendFiles -withFilesUnlocked a params = do - -- unlocked files have changed type from a symlink to a regular file - repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params - unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - backendPairs a $ filter notState unlockedfiles -backendPairs :: SubCmdSeekBackendFiles -backendPairs a files = do - pairs <- Backend.chooseBackends files - return $ map a pairs -withDescription :: SubCmdSeekStrings -withDescription a params = return [a $ unwords params] -withFilesToBeCommitted :: SubCmdSeekStrings -withFilesToBeCommitted a params = do - repo <- Annex.gitRepo - tocommit <- liftIO $ mapM (Git.stagedFiles repo) params - return $ map a $ filter notState $ foldl (++) [] tocommit -withUnlockedFilesToBeCommitted :: SubCmdSeekStrings -withUnlockedFilesToBeCommitted a params = do - repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (Git.typeChangedStagedFiles repo) params - unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - return $ map a $ filter notState unlockedfiles -withKeys :: SubCmdSeekStrings -withKeys a params = return $ map a params -withTempFile :: SubCmdSeekStrings -withTempFile a params = return $ map a params -withNothing :: SubCmdSeekNothing -withNothing a _ = return [a] - -{- filter out files from the state directory -} -notState :: FilePath -> Bool -notState f = stateLoc /= take (length stateLoc) f - -{- filter out symlinks -} -notSymlink :: FilePath -> IO Bool -notSymlink f = do - s <- liftIO $ getSymbolicLinkStatus f - return $ not $ isSymbolicLink s - {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it - according to command line options, while the second actions diff --git a/Command.hs b/Command.hs index 90c4d53854..21d636463e 100644 --- a/Command.hs +++ b/Command.hs @@ -1,4 +1,4 @@ -{- git-annex command types +{- git-annex commands - - Copyright 2010 Joey Hess - @@ -7,10 +7,17 @@ module Command where +import Control.Monad.State (liftIO) +import System.Directory +import System.Posix.Files +import Control.Monad (filterM) + import Types import qualified Backend import Messages import qualified Annex +import qualified GitRepo as Git +import Locations {- A subcommand runs in four stages. - @@ -87,3 +94,64 @@ isAnnexed file a = do case (r) of Just v -> a v Nothing -> return Nothing + +{- These functions find appropriate files or other things based on a + user's parameters, and run a specified action on them. -} +withFilesInGit :: SubCmdSeekStrings +withFilesInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.inRepo repo) params + return $ map a $ filter notState $ foldl (++) [] files +withFilesMissing :: SubCmdSeekStrings +withFilesMissing a params = do + files <- liftIO $ filterM missing params + return $ map a $ filter notState files + where + missing f = do + e <- doesFileExist f + return $ not e +withFilesNotInGit :: SubCmdSeekBackendFiles +withFilesNotInGit a params = do + repo <- Annex.gitRepo + newfiles <- liftIO $ mapM (Git.notInRepo repo) params + backendPairs a $ filter notState $ foldl (++) [] newfiles +withFilesUnlocked :: SubCmdSeekBackendFiles +withFilesUnlocked a params = do + -- unlocked files have changed type from a symlink to a regular file + repo <- Annex.gitRepo + typechangedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params + unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + backendPairs a $ filter notState unlockedfiles +backendPairs :: SubCmdSeekBackendFiles +backendPairs a files = do + pairs <- Backend.chooseBackends files + return $ map a pairs +withDescription :: SubCmdSeekStrings +withDescription a params = return [a $ unwords params] +withFilesToBeCommitted :: SubCmdSeekStrings +withFilesToBeCommitted a params = do + repo <- Annex.gitRepo + tocommit <- liftIO $ mapM (Git.stagedFiles repo) params + return $ map a $ filter notState $ foldl (++) [] tocommit +withUnlockedFilesToBeCommitted :: SubCmdSeekStrings +withUnlockedFilesToBeCommitted a params = do + repo <- Annex.gitRepo + typechangedfiles <- liftIO $ mapM (Git.typeChangedStagedFiles repo) params + unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + return $ map a $ filter notState unlockedfiles +withKeys :: SubCmdSeekStrings +withKeys a params = return $ map a params +withTempFile :: SubCmdSeekStrings +withTempFile a params = return $ map a params +withNothing :: SubCmdSeekNothing +withNothing a _ = return [a] + +{- filter out files from the state directory -} +notState :: FilePath -> Bool +notState f = stateLoc /= take (length stateLoc) f + +{- filter out symlinks -} +notSymlink :: FilePath -> IO Bool +notSymlink f = do + s <- liftIO $ getSymbolicLinkStatus f + return $ not $ isSymbolicLink s diff --git a/Command/Add.hs b/Command/Add.hs index 649b466bb3..586807b530 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -18,6 +18,10 @@ import Types import Core import Messages +{- Add acts on both files not checked into git yet, and unlocked files. -} +seek :: [SubCmdSeek] +seek = [withFilesNotInGit start, withFilesUnlocked start] + {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} diff --git a/Command/Drop.hs b/Command/Drop.hs index 48433b14cf..1e73d8b821 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -16,6 +16,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} start :: SubCmdStartString diff --git a/Command/DropKey.hs b/Command/DropKey.hs index e0b20918cb..34010481dd 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -15,6 +15,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withKeys start] + {- Drops cached content for a key. -} start :: SubCmdStartString start keyname = do diff --git a/Command/Fix.hs b/Command/Fix.hs index 9db832cc76..323aca95e3 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -17,6 +17,9 @@ import Utility import Core import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- Fixes the symlink to an annexed file. -} start :: SubCmdStartString start file = isAnnexed file $ \(key, _) -> do diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 229a93684a..f25de23a2c 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -20,6 +20,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withFilesMissing start] + {- Adds a file pointing at a manually-specified key -} start :: SubCmdStartString start file = do diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 5405ce1201..e5f0debe0f 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -14,6 +14,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withNothing start] + {- Checks the whole annex for problems. -} start :: SubCmdStart start = do diff --git a/Command/Get.hs b/Command/Get.hs index c50b5a3775..13d1375377 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -13,6 +13,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- Gets an annexed file from one of the backends. -} start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do diff --git a/Command/Init.hs b/Command/Init.hs index fa5725c48f..e3b05a83fa 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -18,6 +18,9 @@ import UUID import Version import Messages +seek :: [SubCmdSeek] +seek = [withDescription start] + {- Stores description for the repository etc. -} start :: SubCmdStartString start description = do diff --git a/Command/Lock.hs b/Command/Lock.hs index f03d6b6c88..27a030bc22 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -15,6 +15,9 @@ import Messages import qualified Annex import qualified GitRepo as Git +seek :: [SubCmdSeek] +seek = [withFilesUnlocked start] + {- Undo unlock -} start :: SubCmdStartBackendFile start (file, _) = do diff --git a/Command/Move.hs b/Command/Move.hs index e0b079193a..7f8f40737d 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import Monad (when) import Command -import Command.Drop +import qualified Command.Drop import qualified Annex import Locations import LocationLog @@ -22,6 +22,9 @@ import qualified Remotes import UUID import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- Move a file either --to or --from a repository. - - This only operates on the cached file content; it does not involve diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index b3b940cdd1..a15510bd93 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -14,9 +14,14 @@ import qualified Annex import qualified Backend import qualified GitRepo as Git import qualified Command.Add +import qualified Command.Fix + +{- The pre-commit hook needs to fix symlinks to all files being committed. + - And, it needs to inject unlocked files into the annex. -} +seek :: [SubCmdSeek] +seek = [withFilesToBeCommitted Command.Fix.start, + withUnlockedFilesToBeCommitted start] -{- Run by git pre-commit hook; passed unlocked files that are being - - committed. -} start :: SubCmdStartString start file = return $ Just $ perform file diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 50e9a590b1..e8d407b832 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -19,6 +19,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withTempFile start] + {- Sets cached content for a key. -} start :: SubCmdStartString start file = do diff --git a/Command/Unannex.hs b/Command/Unannex.hs index f5e78e55af..e85e8486f5 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -20,6 +20,9 @@ import Core import qualified GitRepo as Git import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- The unannex subcommand undoes an add. -} start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do diff --git a/Command/Unlock.hs b/Command/Unlock.hs index de21988de5..3ff3023b24 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -18,6 +18,9 @@ import Locations import Utility import Core +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- The unlock subcommand replaces the symlink with a copy of the file's - content. -} start :: SubCmdStartString diff --git a/debian/changelog b/debian/changelog index f705bfaf54..b9f9569abc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,14 +1,15 @@ -git-annex (0.05) UNRELEASED; urgency=low +git-annex (0.05) unstable; urgency=low * Optimize both pre-commit and lock subcommands to not call git diff - on every file being committed or locked. + on every file being committed/locked. (This actually also works around a bug in ghc 6.12.1, that caused - git-annex 0.04 pre-commit to sometimes corrupt filenames and fail. + git-annex 0.04 pre-commit to sometimes corrupt filename being read + from git ls-files and fail. The excessive number of calls made by pre-commit exposed the ghc bug. Thanks Josh Triplett for the debugging.) * Build with -O3. - -- Joey Hess Thu, 11 Nov 2010 14:52:05 -0400 + -- Joey Hess Thu, 11 Nov 2010 18:31:09 -0400 git-annex (0.04) unstable; urgency=low From 3e60c3a3f921d7e6e3d4e227aeea3b0125ded93e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:59:19 -0400 Subject: [PATCH 0478/2835] releasing version 0.05 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index b9f9569abc..7b59883bb3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,12 +2,12 @@ git-annex (0.05) unstable; urgency=low * Optimize both pre-commit and lock subcommands to not call git diff on every file being committed/locked. - (This actually also works around a bug in ghc 6.12.1, that caused + (This actually also works around a bug in ghc, that caused git-annex 0.04 pre-commit to sometimes corrupt filename being read from git ls-files and fail. The excessive number of calls made by pre-commit exposed the ghc bug. Thanks Josh Triplett for the debugging.) - * Build with -O3. + * Build with -O2. -- Joey Hess Thu, 11 Nov 2010 18:31:09 -0400 From 728fc4134e8c1f1040356b752a9be2c894080bc9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:59:37 -0400 Subject: [PATCH 0479/2835] add news item for git-annex 0.05 --- doc/news/version_0.05.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/news/version_0.05.mdwn diff --git a/doc/news/version_0.05.mdwn b/doc/news/version_0.05.mdwn new file mode 100644 index 0000000000..88c7391c13 --- /dev/null +++ b/doc/news/version_0.05.mdwn @@ -0,0 +1,10 @@ +git-annex 0.05 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Optimize both pre-commit and lock subcommands to not call git diff + on every file being committed/locked. + (This actually also works around a bug in ghc, that caused + git-annex 0.04 pre-commit to sometimes corrupt filename being read + from git ls-files and fail. + The excessive number of calls made by pre-commit exposed the ghc bug. + Thanks Josh Triplett for the debugging.) + * Build with -O2."""]] \ No newline at end of file From fc6b972da6f35be21429b96f8c50c1771edf1230 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 19:00:38 -0400 Subject: [PATCH 0480/2835] link to ghc bug --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 7b59883bb3..d1c3344ffa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,8 @@ git-annex (0.05) unstable; urgency=low on every file being committed/locked. (This actually also works around a bug in ghc, that caused git-annex 0.04 pre-commit to sometimes corrupt filename being read - from git ls-files and fail. + from git ls-files and fail. + See The excessive number of calls made by pre-commit exposed the ghc bug. Thanks Josh Triplett for the debugging.) * Build with -O2. From c7ec38b829192ce6c29acf37d49a30f28ebd6cb6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 20:00:49 -0400 Subject: [PATCH 0481/2835] bug --- doc/bugs/fat_support.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/fat_support.mdwn diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn new file mode 100644 index 0000000000..05164d4571 --- /dev/null +++ b/doc/bugs/fat_support.mdwn @@ -0,0 +1,9 @@ +Klaus pointed out that there are two problems that keep +git-annex from being used on USB keys, that would typically +be VFAT formatted: + +- Use of symlinks, which VFAT does not support. Very hard to fix. +- Use of ":" in filenames of object files, also not supported. + Could easily be fixed by reorganizing the object directory. + +[[!tag wishlist]] From 09da0da02f683a218c56cbba0fe0d2c2d5909fe9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 12 Nov 2010 00:56:08 -0400 Subject: [PATCH 0482/2835] add --- doc/todo/use_cp_reflink.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/todo/use_cp_reflink.mdwn diff --git a/doc/todo/use_cp_reflink.mdwn b/doc/todo/use_cp_reflink.mdwn new file mode 100644 index 0000000000..d7974928ce --- /dev/null +++ b/doc/todo/use_cp_reflink.mdwn @@ -0,0 +1,5 @@ +The unlock command needs to copy a file, and it would be great to use this: + cp --reflink=auto src dst + +O(1) overhead on BTRFS. Needs coreutils 7.6; and remember that git-annex +may be used on systems without coreutils.. From 3a5efc54d0c4e3cf4bdb54e830a2bb2a9125f68c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 12 Nov 2010 11:00:20 -0400 Subject: [PATCH 0483/2835] fullfledged design for moving location tracking info into branches --- doc/todo/branching.mdwn | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index 21996ecc00..81f230a92a 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -34,3 +34,41 @@ get keys in that branch. Would have to be careful about conflicts when deleting and bringing back files with the same name. And would need to avoid expensive searching thru all history to try to find an old log file. + +## fleshed out proposal + +Let's use one branch per uuid, named git-annex/$UUID. + +- I came to realize this would be a good idea when thinking about how + to upgrade. Each individual annex will be upgraded independantly, + so each will want to make a branch, and if the branches aren't distinct, + they will merge conflict for sure. +- TODO: What will need to be done to git to make it push/pull these new + branches? +- A given repo only ever writes to its UUID branch. So no conflicts. +- (BTW, UUIDs probably don't compress well, and this reduces the bloat of having + them repeated lots of times in the tree.) +- Per UUID branches mean that if it wants to find a file's location + amoung configured remotes, it can examine only their branches, if + desired. + +In the branch, only one file is needed. Call it locationlog. git-annex +can cache location log changes and write them all to locationlog in +a single git operation on shutdown. + +- TODO: what if it's ctrl-c'd with changes pending? Perhaps it should + collect them to ,git/annex/locationlog, and inject that file on shutdown? +- This will be less overhead than the current staging of all the log files. + +The log is not appended to, so in git we have a series of commits each of +which replaces the log's entire contens. + +To find locations of a key, all (or all relevant) branches need to be +examined, looking backward through the history of each until a log +with a indication of the presense/absense of the key is found. + +- This will be less expensive for files that have recently been added + or transfered. +- It could get pretty slow when digging deeper. +- Only 3 places in git-annex will be affected by any slowdown: move --from, + get and drop. From d4d65a3c923de1eece50463145e875326bfe57e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 13:11:41 -0400 Subject: [PATCH 0484/2835] new fsck items --- doc/todo/fsck.mdwn | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/todo/fsck.mdwn b/doc/todo/fsck.mdwn index 6126154e9f..eebe6446f4 100644 --- a/doc/todo/fsck.mdwn +++ b/doc/todo/fsck.mdwn @@ -1,3 +1,9 @@ add a git annex fsck that finds keys that have no referring file -[[done]] +(done) + +* Need per-backend fsck support. sha1 can checksum all files in the annex. + WORM can check filesize. + +* Both can check that annex.numcopies is satisfied. Probably only + querying the locationlog, not doing an online verification. From 5fa25a812a8a03af9f6a5fdb3d06eb4d89ee06f5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 14:59:27 -0400 Subject: [PATCH 0485/2835] fsck improvements * fsck: Check if annex.numcopies is satisfied. * fsck: Verify the sha1 of files when the SHA1 backend is used. * fsck: Verify the size of files when the WORM backend is used. * fsck: Allow specifying individual files to fsk if fscking everything is not desired. * fsck: Fix bug, introduced in 0.04, in detection of unused data. --- Backend.hs | 7 ++++++- Backend/File.hs | 43 ++++++++++++++++++++++++++++++++++++------- Backend/SHA1.hs | 37 ++++++++++++++++++++++++++++++++----- Backend/URL.hs | 8 ++++++-- Backend/WORM.hs | 34 +++++++++++++++++++++++++++++++++- CmdLine.hs | 6 +++--- Command.hs | 10 ++++++++++ Command/Fsck.hs | 9 ++++----- Command/FsckFile.hs | 33 +++++++++++++++++++++++++++++++++ Core.hs | 18 +++++++++++++++--- Locations.hs | 5 +++++ TypeInternals.hs | 4 +++- debian/changelog | 11 +++++++++++ doc/git-annex.mdwn | 8 +++++--- doc/walkthrough.mdwn | 34 ++++++++++++++++++++++++++++++++++ 15 files changed, 236 insertions(+), 31 deletions(-) create mode 100644 Command/FsckFile.hs diff --git a/Backend.hs b/Backend.hs index 43b450736d..14af56bbfa 100644 --- a/Backend.hs +++ b/Backend.hs @@ -23,6 +23,7 @@ module Backend ( retrieveKeyFile, removeKey, hasKey, + fsckKey, lookupFile, chooseBackends ) where @@ -105,7 +106,7 @@ retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} removeKey :: Backend -> Key -> Annex Bool -removeKey backend key = (Internals.removeKey backend) key +removeKey backend key = (Internals.removeKey backend) key {- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool @@ -113,6 +114,10 @@ hasKey key = do bs <- Annex.supportedBackends (Internals.hasKey (lookupBackendName bs $ backendName key)) key +{- Checks a key's backend for problems. -} +fsckKey :: Backend -> Key -> Annex Bool +fsckKey backend key = (Internals.fsckKey backend) key + {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) diff --git a/Backend/File.hs b/Backend/File.hs index 9178b830a5..9bda0d5718 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -4,15 +4,15 @@ - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. - - - This is an abstract backend; getKey has to be implemented to complete - - it. + - This is an abstract backend; name, getKey and fsckKey have to be implemented + - to complete it. - - Copyright 2010 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module Backend.File (backend) where +module Backend.File (backend, checkKey) where import Control.Monad.State import System.Directory @@ -34,7 +34,8 @@ backend = Backend { storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, removeKey = checkRemoveKey, - hasKey = checkKeyFile + hasKey = checkKeyFile, + fsckKey = mustProvide } mustProvide :: a @@ -97,14 +98,12 @@ checkRemoveKey key = do if (force) then return True else do - g <- Annex.gitRepo remotes <- Remotes.keyPossibilities key - let numcopies = read $ Git.configGet g config "1" + numcopies <- getNumCopies if (numcopies > length remotes) then notEnoughCopies numcopies (length remotes) [] else findcopies numcopies 0 remotes [] where - config = "annex.numcopies" findcopies need have [] bad = if (have >= need) then return True @@ -147,3 +146,33 @@ showTriedRemotes [] = return () showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ (Remotes.list remotes) + +getNumCopies :: Annex Int +getNumCopies = do + g <- Annex.gitRepo + return $ read $ Git.configGet g config "1" + where + config = "annex.numcopies" + +{- This is used to check that numcopies is satisfied for the key on fsck. + - This trusts the location log, and so checks all keys, even those with + - data not present in the current annex. + - + - The passed action is first run to allow backends deriving this one + - to do their own checks. + -} +checkKey :: (Key -> Annex Bool) -> Key -> Annex Bool +checkKey a key = do + a_ok <- a key + copies_ok <- checkKeyNumCopies key + return $ a_ok && copies_ok + +checkKeyNumCopies :: Key -> Annex Bool +checkKeyNumCopies key = do + remotes <- Remotes.keyPossibilities key + numcopies <- getNumCopies + if (length remotes < numcopies) + then do + showLongNote $ "only " ++ show (length remotes) ++ " of " ++ show numcopies ++ " copies" + return False + else return True diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 5a232ec1db..8852e72e94 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -11,24 +11,51 @@ import Control.Monad.State import Data.String.Utils import System.Cmd.Utils import System.IO +import System.Directory import qualified Backend.File import TypeInternals import Messages +import qualified Annex +import Locations +import Core backend :: Backend backend = Backend.File.backend { name = "SHA1", - getKey = keyValue + getKey = keyValue, + fsckKey = Backend.File.checkKey checkKeySHA1 } --- checksum the file to get its key -keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = do +sha1 :: FilePath -> Annex String +sha1 file = do showNote "checksum..." liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do line <- hGetLine h let bits = split " " line if (null bits) then error "sha1sum parse error" - else return $ Just $ Key ((name backend), bits !! 0) + else return $ bits !! 0 + +-- A key is a sha1 of its contents. +keyValue :: FilePath -> Annex (Maybe Key) +keyValue file = do + s <- sha1 file + return $ Just $ Key ((name backend), s) + +-- A key's sha1 is checked during fsck. +checkKeySHA1 :: Key -> Annex Bool +checkKeySHA1 key = do + g <- Annex.gitRepo + let file = annexLocation g key + present <- liftIO $ doesFileExist file + if (not present) + then return True + else do + s <- sha1 file + if (s == keyName key) + then return True + else do + dest <- moveBad key + showNote $ "bad file content (moved to "++dest++")" + return False diff --git a/Backend/URL.hs b/Backend/URL.hs index 830d343c53..b38ea71c96 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -20,8 +20,13 @@ backend = Backend { getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = downloadUrl, + -- allow keys to be removed; presumably they can always be + -- downloaded again removeKey = dummyOk, - hasKey = dummyOk + -- similarly, keys are always assumed to be out there on the web + hasKey = dummyOk, + -- and nothing needed to fsck + fsckKey = dummyOk } -- cannot generate url from filename @@ -32,7 +37,6 @@ keyValue _ = return Nothing dummyStore :: FilePath -> Key -> Annex Bool dummyStore _ _ = return False --- allow keys to be removed; presumably they can always be downloaded again dummyOk :: Key -> Annex Bool dummyOk _ = return True diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 848386ecd1..21b3876b90 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -10,14 +10,22 @@ module Backend.WORM (backend) where import Control.Monad.State import System.FilePath import System.Posix.Files +import System.Posix.Types +import System.Directory +import Data.String.Utils import qualified Backend.File import TypeInternals +import Locations +import qualified Annex +import Core +import Messages backend :: Backend backend = Backend.File.backend { name = "WORM", - getKey = keyValue + getKey = keyValue, + fsckKey = Backend.File.checkKey checkKeySize } -- The key is formed from the file size, modification time, and the @@ -36,3 +44,27 @@ keyValue file = do (show $ fileSize stat) base = takeFileName file sep = ":" + +{- Extracts the file size from a key. -} +keySize :: Key -> FileOffset +keySize key = read $ section !! 2 + where + section = split ":" (keyName key) + +{- The size of the data for a key is checked against the size encoded in + - the key. Note that the modification time is not checked. -} +checkKeySize :: Key -> Annex Bool +checkKeySize key = do + g <- Annex.gitRepo + let file = annexLocation g key + present <- liftIO $ doesFileExist file + if (not present) + then return True + else do + s <- liftIO $ getFileStatus file + if (fileSize s == keySize key) + then return True + else do + dest <- moveBad key + showNote $ "bad file size (moved to "++dest++")" + return False diff --git a/CmdLine.hs b/CmdLine.hs index efa541ebcc..a683be5c53 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -59,14 +59,14 @@ subCmds = "sets annexed content for a key using a temp file" , SubCommand "fix" path Command.Fix.seek "fix up symlinks to point to annexed content" - , SubCommand "fsck" nothing Command.Fsck.seek - "check annex for problems" + , SubCommand "fsck" maybepath Command.Fsck.seek + "check for problems" ] where path = "PATH ..." + maybepath = "[PATH ...]" key = "KEY ..." desc = "DESCRIPTION" - nothing = "" -- Each dashed command-line option results in generation of an action -- in the Annex monad that performs the necessary setting. diff --git a/Command.hs b/Command.hs index 21d636463e..4180155faf 100644 --- a/Command.hs +++ b/Command.hs @@ -146,6 +146,16 @@ withTempFile a params = return $ map a params withNothing :: SubCmdSeekNothing withNothing a _ = return [a] +{- Default to acting on all files matching the seek action if + - none are specified. -} +withAll :: SubCmdSeekStrings -> SubCmdSeekStrings +withAll w a params = do + if null params + then do + g <- Annex.gitRepo + w a [Git.workTree g] + else w a params + {- filter out files from the state directory -} notState :: FilePath -> Bool notState f = stateLoc /= take (length stateLoc) f diff --git a/Command/Fsck.hs b/Command/Fsck.hs index e5f0debe0f..b0b9f7bb6b 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -13,9 +13,10 @@ import Command import Types import Core import Messages +import qualified Command.FsckFile seek :: [SubCmdSeek] -seek = [withNothing start] +seek = [withNothing start, withAll withFilesInGit Command.FsckFile.start] {- Checks the whole annex for problems. -} start :: SubCmdStart @@ -26,11 +27,9 @@ start = do perform :: SubCmdPerform perform = do ok <- checkUnused - if (ok) + if ok then return $ Just $ return True - else do - showLongNote "Possible problems detected." - return Nothing + else return Nothing checkUnused :: Annex Bool checkUnused = do diff --git a/Command/FsckFile.hs b/Command/FsckFile.hs new file mode 100644 index 0000000000..2f9efa56ee --- /dev/null +++ b/Command/FsckFile.hs @@ -0,0 +1,33 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.FsckFile where + +import Command +import qualified Backend +import Types +import Messages + +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + +{- Checks a file's backend data for problems. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + inbackend <- Backend.hasKey key + if (not inbackend) + then return Nothing + else do + showStart "fsck" file + return $ Just $ perform key backend + +perform :: Key -> Backend -> SubCmdPerform +perform key backend = do + success <- Backend.fsckKey backend key + if (success) + then return $ Just $ return True + else return Nothing diff --git a/Core.hs b/Core.hs index 8497a7f368..789b369cc8 100644 --- a/Core.hs +++ b/Core.hs @@ -14,6 +14,7 @@ import System.Path import Control.Monad (when, unless, filterM) import System.Posix.Files import Data.Maybe +import System.FilePath import Types import Locations @@ -201,6 +202,16 @@ fromAnnex key dest = do renameFile file dest removeDirectory dir +{- Moves a key out of .git/annex/objects/ into .git/annex/bad, and + - returns the directory it was moved to. -} +moveBad :: Key -> Annex FilePath +moveBad key = do + g <- Annex.gitRepo + let src = parentDir $ annexLocation g key + let dest = annexBadLocation g + liftIO $ renameDirectory src dest + return dest + {- List of keys whose content exists in .git/annex/objects/ -} getKeysPresent :: Annex [Key] getKeysPresent = do @@ -209,11 +220,12 @@ getKeysPresent = do getKeysPresent' :: FilePath -> Annex [Key] getKeysPresent' dir = do contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM isreg contents + files <- liftIO $ filterM present contents return $ map fileKey files where - isreg f = do - s <- getFileStatus $ dir ++ "/" ++ f + present d = do + s <- getFileStatus $ dir ++ "/" ++ d ++ "/" + ++ (takeFileName d) return $ isRegularFile s {- List of keys referenced by symlinks in the git repo. -} diff --git a/Locations.hs b/Locations.hs index 58244cef0e..c3bab285d4 100644 --- a/Locations.hs +++ b/Locations.hs @@ -13,6 +13,7 @@ module Locations ( annexLocation, annexLocationRelative, annexTmpLocation, + annexBadLocation, annexDir, annexObjectDir, @@ -59,6 +60,10 @@ annexObjectDir r = annexDir r ++ "/objects" annexTmpLocation :: Git.Repo -> FilePath annexTmpLocation r = annexDir r ++ "/tmp/" +{- .git-annex/bad is used for bad files found during fsck -} +annexBadLocation :: Git.Repo -> FilePath +annexBadLocation r = annexDir r ++ "/bad/" + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid diff --git a/TypeInternals.hs b/TypeInternals.hs index 4b5cff9d9f..3078224b15 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -75,7 +75,9 @@ data Backend = Backend { -- removes a key removeKey :: Key -> Annex Bool, -- checks if a backend is storing the content of a key - hasKey :: Key -> Annex Bool + hasKey :: Key -> Annex Bool, + -- called during fsck to check a key + fsckKey :: Key -> Annex Bool } instance Show Backend where diff --git a/debian/changelog b/debian/changelog index d1c3344ffa..71d163d116 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +git-annex (0.06) UNRELEASED; urgency=low + + * fsck: Check if annex.numcopies is satisfied. + * fsck: Verify the sha1 of files when the SHA1 backend is used. + * fsck: Verify the size of files when the WORM backend is used. + * fsck: Allow specifying individual files to fsk if fscking everything + is not desired. + * fsck: Fix bug, introduced in 0.04, in detection of unused data. + + -- Joey Hess Sat, 13 Nov 2010 14:08:58 -0400 + git-annex (0.05) unstable; urgency=low * Optimize both pre-commit and lock subcommands to not call git diff diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d0bd3a754e..61a5962f1f 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -159,10 +159,12 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git annex setkey --key=1287765018:3 /tmp/file -* fsck +* fsck [path ...] - This subcommand checks the whole annex for consistency, and warns - about any problems found. + With no parameters, this subcommand checks the whole annex for consistency, + and warns about any problems found. + + With parameters, only the specified files are checked. # OPTIONS diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index d6c0214ffd..7effb53178 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -276,3 +276,37 @@ significantly for really big files. To make SHA1 the detault, just add something like this to `.gitattributes`: * git-annex-backend=SHA1 + +## fsck: verifying your data + +You can use the fsck subcommand to check for problems in your data. +What can be checked depends on the [[backend|backends]] you've used to store +the data. For example, when you use the SHA1 backend, fsck will verify that +the checksums of your files are good. Fsck also checks that the annex.numcopies +setting is satisfied for all files, and it warns about any dangling values +in `.git/annex/objects/`. + + # git annex fsck + fsck (checking for unused data...) (checking files...) ok + +Fsck checks the entire repository for problems by default. But you can +also specify the files to check. +This is particularly useful if you're using sha1 and don't want to spend +a long time checksumming everything. + + # git annex fsck my_cool_big_file + fsck my_cool_big_file (checksum..) ok + +## fsck: When things go wrong + +Fsck never deletes possibly bad data; instead it will be moved to +`.git/annex/bad/` for you to review. Here is a sample of what fsck +might say about a badly messed up annex: + + # git annex fsck + fsck (checking for unused data...) + Some annexed data is no longer pointed to by any files in the repository. + If this data is no longer needed, it can be removed using git-annex dropkey: + WORM:1289672605:3:file + (checking files...) + From abebbcfd544953f3a2c9dab042368069fd2d916a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:24:36 -0400 Subject: [PATCH 0486/2835] fsck improvements --- Backend/File.hs | 14 +++++++++++--- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- doc/walkthrough.mdwn | 26 +++++++++++++++++--------- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 9bda0d5718..8351778568 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -169,10 +169,18 @@ checkKey a key = do checkKeyNumCopies :: Key -> Annex Bool checkKeyNumCopies key = do + needed <- getNumCopies remotes <- Remotes.keyPossibilities key - numcopies <- getNumCopies - if (length remotes < numcopies) + inannex <- inAnnex key + let present = length remotes + if inannex then 1 else 0 + if (present < needed) then do - showLongNote $ "only " ++ show (length remotes) ++ " of " ++ show numcopies ++ " copies" + showLongNote $ note present needed return False else return True + where + note 0 _ = "** No known copies of the file exist!" + note present needed = + "Only " ++ show present ++ " of " ++ show needed ++ + " copies exist. " ++ + "Run git annex get somewhere else to back it up." diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 8852e72e94..9e9000ba93 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -57,5 +57,5 @@ checkKeySHA1 key = do then return True else do dest <- moveBad key - showNote $ "bad file content (moved to "++dest++")" + showLongNote $ "Bad file content; moved to "++dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 21b3876b90..e53ec6de76 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -66,5 +66,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - showNote $ "bad file size (moved to "++dest++")" + showLongNote $ "Bad file size; moved to "++dest return False diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 7effb53178..887fde48cd 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -170,7 +170,7 @@ and this symlink is what gets committed to git in the end. add my_cool_big_file ok [master 64cda67] changed an annexed file 2 files changed, 2 insertions(+), 1 deletions(-) - create mode 100644 .git-annex/SHA1:0b1d8616d0238cb9418a0e0a649bdad2e9e7faae.log + create mode 100644 .git-annex/WORM:1289672605:30:file.log There is one problem with using `git commit` like this: Git wants to first stage the entire contents of the file in its index. That can be slow for @@ -287,20 +287,21 @@ setting is satisfied for all files, and it warns about any dangling values in `.git/annex/objects/`. # git annex fsck - fsck (checking for unused data...) (checking files...) ok + fsck (checking for unused data...) ok + fsck my_cool_big_file (checksum..) ok + ...... -Fsck checks the entire repository for problems by default. But you can -also specify the files to check. -This is particularly useful if you're using sha1 and don't want to spend -a long time checksumming everything. +You can also specifiy the files to check. This is particularly useful if +you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file + fsck (checking for unused data...) ok fsck my_cool_big_file (checksum..) ok ## fsck: When things go wrong Fsck never deletes possibly bad data; instead it will be moved to -`.git/annex/bad/` for you to review. Here is a sample of what fsck +`.git/annex/bad/` for you to recover. Here is a sample of what fsck might say about a badly messed up annex: # git annex fsck @@ -308,5 +309,12 @@ might say about a badly messed up annex: Some annexed data is no longer pointed to by any files in the repository. If this data is no longer needed, it can be removed using git-annex dropkey: WORM:1289672605:3:file - (checking files...) - + failed + fsck my_cool_big_file (checksum..) + Bad file content; moved to .git/annex/bad/ + ** No known copies of the file exist! + failed + fsck important_file + Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. + failed + git-annex: 3 failed From dd573e70105706bd1a769f74af9c3006dab903d0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:40:12 -0400 Subject: [PATCH 0487/2835] fsck bugfixes --- Backend/WORM.hs | 2 +- Core.hs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Backend/WORM.hs b/Backend/WORM.hs index e53ec6de76..3749899968 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -47,7 +47,7 @@ keyValue file = do {- Extracts the file size from a key. -} keySize :: Key -> FileOffset -keySize key = read $ section !! 2 +keySize key = read $ section !! 1 where section = split ":" (keyName key) diff --git a/Core.hs b/Core.hs index 789b369cc8..2a81678aa6 100644 --- a/Core.hs +++ b/Core.hs @@ -207,9 +207,11 @@ fromAnnex key dest = do moveBad :: Key -> Annex FilePath moveBad key = do g <- Annex.gitRepo - let src = parentDir $ annexLocation g key + let src = annexLocation g key let dest = annexBadLocation g - liftIO $ renameDirectory src dest + liftIO $ createDirectoryIfMissing True dest + liftIO $ renameFile src (dest ++ takeFileName src) + liftIO $ removeDirectory (parentDir src) return dest {- List of keys whose content exists in .git/annex/objects/ -} From aec34ee1bda9b8bcdfe54903331eae582ee32792 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:42:56 -0400 Subject: [PATCH 0488/2835] tweak --- Core.hs | 6 +++--- doc/walkthrough.mdwn | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Core.hs b/Core.hs index 2a81678aa6..9faaada56f 100644 --- a/Core.hs +++ b/Core.hs @@ -203,14 +203,14 @@ fromAnnex key dest = do removeDirectory dir {- Moves a key out of .git/annex/objects/ into .git/annex/bad, and - - returns the directory it was moved to. -} + - returns the file it was moved to. -} moveBad :: Key -> Annex FilePath moveBad key = do g <- Annex.gitRepo let src = annexLocation g key - let dest = annexBadLocation g + let dest = annexBadLocation g ++ takeFileName src liftIO $ createDirectoryIfMissing True dest - liftIO $ renameFile src (dest ++ takeFileName src) + liftIO $ renameFile src dest liftIO $ removeDirectory (parentDir src) return dest diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 887fde48cd..049dc40f67 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -311,7 +311,7 @@ might say about a badly messed up annex: WORM:1289672605:3:file failed fsck my_cool_big_file (checksum..) - Bad file content; moved to .git/annex/bad/ + Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 ** No known copies of the file exist! failed fsck important_file From 14d897a33827450799a713dac9d6a5a384c80e40 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:43:08 -0400 Subject: [PATCH 0489/2835] close --- doc/todo/fsck.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/fsck.mdwn b/doc/todo/fsck.mdwn index eebe6446f4..1dcaad9a51 100644 --- a/doc/todo/fsck.mdwn +++ b/doc/todo/fsck.mdwn @@ -7,3 +7,5 @@ add a git annex fsck that finds keys that have no referring file * Both can check that annex.numcopies is satisfied. Probably only querying the locationlog, not doing an online verification. + +[[done]] From 147affa252989b99e876a052433bd5d2ce32c672 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:44:02 -0400 Subject: [PATCH 0490/2835] tweak --- doc/walkthrough.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 049dc40f67..a15f0a9a44 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -288,7 +288,7 @@ in `.git/annex/objects/`. # git annex fsck fsck (checking for unused data...) ok - fsck my_cool_big_file (checksum..) ok + fsck my_cool_big_file (checksum...) ok ...... You can also specifiy the files to check. This is particularly useful if @@ -296,7 +296,7 @@ you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file fsck (checking for unused data...) ok - fsck my_cool_big_file (checksum..) ok + fsck my_cool_big_file (checksum...) ok ## fsck: When things go wrong @@ -310,7 +310,7 @@ might say about a badly messed up annex: If this data is no longer needed, it can be removed using git-annex dropkey: WORM:1289672605:3:file failed - fsck my_cool_big_file (checksum..) + fsck my_cool_big_file (checksum...) Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 ** No known copies of the file exist! failed From d9d79a7980b5c9a84823c0cf38750a0189d03b0a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:46:56 -0400 Subject: [PATCH 0491/2835] idea --- doc/bugs/fat_support.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn index 05164d4571..2a998cf1ad 100644 --- a/doc/bugs/fat_support.mdwn +++ b/doc/bugs/fat_support.mdwn @@ -3,6 +3,8 @@ git-annex from being used on USB keys, that would typically be VFAT formatted: - Use of symlinks, which VFAT does not support. Very hard to fix. + One possibility is to add [[bare_git_repos]] support, then + a git repo on a thumb drive could be used to transfer data. - Use of ":" in filenames of object files, also not supported. Could easily be fixed by reorganizing the object directory. From 498c8e8544f04841d3221ce7a47b2409858e9f38 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:03:25 -0400 Subject: [PATCH 0492/2835] fsck: avoid global checks if files specified --- Command.hs | 6 ++---- Command/Fsck.hs | 19 +++++++++++++------ Command/Init.hs | 2 +- doc/walkthrough.mdwn | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Command.hs b/Command.hs index 4180155faf..3f68cf661f 100644 --- a/Command.hs +++ b/Command.hs @@ -126,8 +126,8 @@ backendPairs :: SubCmdSeekBackendFiles backendPairs a files = do pairs <- Backend.chooseBackends files return $ map a pairs -withDescription :: SubCmdSeekStrings -withDescription a params = return [a $ unwords params] +withString :: SubCmdSeekStrings +withString a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo @@ -143,8 +143,6 @@ withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params -withNothing :: SubCmdSeekNothing -withNothing a _ = return [a] {- Default to acting on all files matching the seek action if - none are specified. -} diff --git a/Command/Fsck.hs b/Command/Fsck.hs index b0b9f7bb6b..85a26a89b9 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -16,13 +16,20 @@ import Messages import qualified Command.FsckFile seek :: [SubCmdSeek] -seek = [withNothing start, withAll withFilesInGit Command.FsckFile.start] +seek = [withString start, withAll withFilesInGit Command.FsckFile.start] -{- Checks the whole annex for problems. -} -start :: SubCmdStart -start = do - showStart "fsck" "" - return $ Just perform +{- Checks the whole annex for problems, only if specific files were not + - specified. -} +start :: SubCmdStartString +start whatspecified = do + if (null whatspecified) + then do + showStart "fsck" "" + return $ Just perform + else do + showStart "fsck" "" + showNote "only checking specified files" + return $ Just $ return $ Just $ return True perform :: SubCmdPerform perform = do diff --git a/Command/Init.hs b/Command/Init.hs index e3b05a83fa..8110948a41 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -19,7 +19,7 @@ import Version import Messages seek :: [SubCmdSeek] -seek = [withDescription start] +seek = [withString start] {- Stores description for the repository etc. -} start :: SubCmdStartString diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index a15f0a9a44..cdb2392e7a 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -295,7 +295,7 @@ You can also specifiy the files to check. This is particularly useful if you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file - fsck (checking for unused data...) ok + fsck (only checking specified files) ok fsck my_cool_big_file (checksum...) ok ## fsck: When things go wrong From 7293ba2940dc7dcefa7376667fa4462b21e05ee0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:12:02 -0400 Subject: [PATCH 0493/2835] fsck even files not in backend --- Command/FsckFile.hs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Command/FsckFile.hs b/Command/FsckFile.hs index 2f9efa56ee..c74e94e62f 100644 --- a/Command/FsckFile.hs +++ b/Command/FsckFile.hs @@ -18,12 +18,8 @@ seek = [withFilesInGit start] {- Checks a file's backend data for problems. -} start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do - inbackend <- Backend.hasKey key - if (not inbackend) - then return Nothing - else do - showStart "fsck" file - return $ Just $ perform key backend + showStart "fsck" file + return $ Just $ perform key backend perform :: Key -> Backend -> SubCmdPerform perform key backend = do From 19ee56559a191e0e794380363a1bd0b0c57ea22a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:15:45 -0400 Subject: [PATCH 0494/2835] better fsck file handling --- Command.hs | 4 ++++ Command/Fsck.hs | 16 +++++----------- doc/walkthrough.mdwn | 1 - 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Command.hs b/Command.hs index 3f68cf661f..40a21cacc8 100644 --- a/Command.hs +++ b/Command.hs @@ -44,6 +44,7 @@ type SubCmdStartString = String -> SubCmdStart type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek +type SubCmdStartNothing = SubCmdStart data SubCommand = SubCommand { subcmdname :: String, @@ -143,6 +144,9 @@ withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params +withNothing :: SubCmdSeekNothing +withNothing a [] = return [a] +withNothing _ _ = return [] {- Default to acting on all files matching the seek action if - none are specified. -} diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 85a26a89b9..5b731a6968 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -16,20 +16,14 @@ import Messages import qualified Command.FsckFile seek :: [SubCmdSeek] -seek = [withString start, withAll withFilesInGit Command.FsckFile.start] +seek = [withNothing start, withAll withFilesInGit Command.FsckFile.start] {- Checks the whole annex for problems, only if specific files were not - specified. -} -start :: SubCmdStartString -start whatspecified = do - if (null whatspecified) - then do - showStart "fsck" "" - return $ Just perform - else do - showStart "fsck" "" - showNote "only checking specified files" - return $ Just $ return $ Just $ return True +start :: SubCmdStartNothing +start = do + showStart "fsck" "" + return $ Just perform perform :: SubCmdPerform perform = do diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index cdb2392e7a..9a6ee220c2 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -295,7 +295,6 @@ You can also specifiy the files to check. This is particularly useful if you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file - fsck (only checking specified files) ok fsck my_cool_big_file (checksum...) ok ## fsck: When things go wrong From 2403fece78b2f25d7369babd0852bc214127e5f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:29:33 -0400 Subject: [PATCH 0495/2835] releasing version 0.06 --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 71d163d116..64973557ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,13 +1,13 @@ -git-annex (0.06) UNRELEASED; urgency=low +git-annex (0.06) unstable; urgency=low * fsck: Check if annex.numcopies is satisfied. * fsck: Verify the sha1 of files when the SHA1 backend is used. * fsck: Verify the size of files when the WORM backend is used. - * fsck: Allow specifying individual files to fsk if fscking everything + * fsck: Allow specifying individual files if fscking everything is not desired. * fsck: Fix bug, introduced in 0.04, in detection of unused data. - -- Joey Hess Sat, 13 Nov 2010 14:08:58 -0400 + -- Joey Hess Sat, 13 Nov 2010 16:24:29 -0400 git-annex (0.05) unstable; urgency=low From dd5aa59cb2242c8c2070fc3faa56bd06d58b736c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:29:40 -0400 Subject: [PATCH 0496/2835] add news item for git-annex 0.06 --- doc/news/version_0.06.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/news/version_0.06.mdwn diff --git a/doc/news/version_0.06.mdwn b/doc/news/version_0.06.mdwn new file mode 100644 index 0000000000..39689232a7 --- /dev/null +++ b/doc/news/version_0.06.mdwn @@ -0,0 +1,8 @@ +git-annex 0.06 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * fsck: Check if annex.numcopies is satisfied. + * fsck: Verify the sha1 of files when the SHA1 backend is used. + * fsck: Verify the size of files when the WORM backend is used. + * fsck: Allow specifying individual files if fscking everything + is not desired. + * fsck: Fix bug, introduced in 0.04, in detection of unused data."""]] \ No newline at end of file From bfacd10b9628c070f190ce642cb82c8bf9514632 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 21:29:30 -0400 Subject: [PATCH 0497/2835] add logo --- doc/index.mdwn | 2 ++ doc/logo.png | Bin 0 -> 15170 bytes 2 files changed, 2 insertions(+) create mode 100644 doc/logo.png diff --git a/doc/index.mdwn b/doc/index.mdwn index ae5dbf85e5..add0505d97 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -14,6 +14,8 @@ revision control. To get a feel for it, see the [[walkthrough]]. [[!sidebar content=""" +[[!img logo.png size=200x]] + * **[[download]]** * [[install]] * [[news]] diff --git a/doc/logo.png b/doc/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..a92ff42f4768ee1f55480dcb5c0f323f2a083577 GIT binary patch literal 15170 zcmZ8|by!s2_w|4vARS7B4Bg!yKxs+o?(URMC8VWGKtPaA>Fx&U?(Xh-58wB{-#o-K z_s*Pq?mm0(v-aBShJ2QjKz&2-1_FVgev%Ybgg{^n!LJh{EI1POHx3*8gK<)n5QdZt z6YYXGNcNH%P7nw(IDw1}14;Rb2M!`Qf07kLm_tT`A*ApMNvMH9-a|f#epGgwJ4|!+ z!kL`w85z7vSn@0U^^6{&N2ywQcGk@2)-b4Ux#34^KWD_> z7-N#h-mwR!_k;I6Kl*;a%}}IP-`_vAZ4Vk(4|XIk3y$!PxTAGdFu*|k&P6)lAdp`0 z%Ld-bfNO)f{hzmxWC-K}mQ@DQs-Ia?twLzECxdEJ&DB&TYc^RKpc1B{sF{Unxmgq!|GZ{sCFMdo;Q%G@rF%%SkQ*)c7{gQXXJh6eWdGPL*#?t8NYGvD&1&Ip4PO# z!h}w2J|zlbcqTS<+LSeqZBy#mFg(OrF-O%#P@vZ{o+T&Oj?h}8MT-NY~U&=9772SxNqD)O>7b#?H zZGB~fi}~o=Rq12_qJ`vwDJRw6u?(zX{rt%rD36DY&`$_3iz6T0*6t_`!h+>~yq znTT)Bt{(lhDSTPoQKy->m77yrFo9JvR;E7Q3_OyAv`kyr(h0xqo?g7Kvvf`2s+_h9 z^uUJvpsIB7eO$E=jBWg#F6UxpCD+uevWBek+L!G#Z7E^^*x%lCPd1{4SSP1)4~dLE zD}fPN@9VEu5PLL1w$gd1?bcidj;)Eu=DQEzqmJ(awcnP8PzJ!!!+Itvhl0;-_vo{e zSn#ADu#Rc%y_ipzEmfY>MzV>^*#8_=i;w8w4qiK&PwBD4LS6&Us<~ukQ{nmRM<$eR zDnI&)!aH=^Qq-F!J$&g>{oZa~$Gd{rKaf@& ziEMegIjKD55279$g4y&Ed@5It2}z|q3nY+m{&(I+W|KfE$c6LKC#2f)+JUloc;^jR zke`WzTjSbBaS~aDi=Mw0GOR$j00TSAN0gUzjY%P5Dj8De(T^$#$G2%`tLuUM^UE`I zPMVjpKFYpr9$hVs(MQdw;F}EOpFrSYOBU0P6o&9Zr&-vw1Jhe$SU=l5t|@y|q2ws! zrRI9?;|eY8>k_E-21gm)%(P^}v`?;585x`GbKhgPiW%rQelagTJ@_w=AB=xa zW~J9@V8FoScAsOjsZE~eT}cZ3Gi~(m2x7ISGdN6XW1>x-B{1>^_=?%h(8v8D)Do9t zl!2d?!&`IOm^$Foiz|z|w-vJsYnIo#dCb-2T*BXLCI%$7P+v>{i({53xpiqwS>@Df z^DKsle;7THjVny$XN6fDx4`xwtK8O=IcNhi`S!7;B7|DM#$^ zY%r~2NoxYpx#q%>s!npT8v7U516VH7r?}epDJ%51RU2UN;jmOu_HOcIPoF)R;SA$T ziJ_&6YDeMk!W4RVUM9>-Fl;c|(Vj+6ymREc^*;hx_h`18m5_~Ql`rt8o4GdVu60#r zTO&6hJnULwHNJ`$_}d~jsSoA#!A?XcFYyu1wO&0&tbJeoJ=OCMCb-~Fd~72wR#Rcg zzkF+Zx)D+*lO*GgP)Yf?9(yN`xj*|O8pHTXyvc=&MmQh{FL^%5zkXPpQp zWj*8BmR9?_ah%P#>wC@NyNm*t2I&Q^oW^~|%dcSFFcnkBKteB^QXV0GO$$2}?#0l? z_~bQ>yDQxuM+UX5Jz6d=cJ7f^X+RP58#t9zKAh|Q=U%u9T{@pn-~Ti{{@JLg~7*yJypIX?$Is@o1^TdSo_7QFHzgGXge=4CL^}SP6E~r z_JX8Jy*MH7FdnI!3b)U*M*8rBq2VAY?$h=D*>Um}X$#1vBoQskN_(S+mlkt_+)m5u zH6`x@I8sstY3Drj2Gcq=pQ8ewW?pK$=4Nn#lv)7f6rjhnx#CJ7Cm3tWiRac@)e6BpRp6xAI z8P2uhAE^A;|CzE|Gu&L<_h-0KL%XaV*2zhBxCCbP*=^}>_k2y39m9wb$sD7-G;U(L zGN=EGc|u84KQ+7}B&2TS?5wOqzU;DM(bZ<7)e2vUCS(BKRiO9*VKb1B*Q1{8dN1{mA-TF)F(B~=D zC(GeGGi*}Q3fD)g%Z$>k#-YO@5ll<>&WIYZC7DYtWyEAiN4%-@fTx+@7JVC*aPHl2 zut;__a-v{cC~my2OmVtlCtnm_z`Y-dwk5w|Cky?Bu&!D>q$`1VTj{bKi-dliMWXFH zZfN06f7~Qw^}K}^5ADK6Lf6^juhJ930XIO(U;t9a`8W2|zX_&C;lZ7c%`^OF_ag@$ z4zU^yoX{fi@R&y@661t2f z3_Sb!lMkOZ{@ii~GMIjNOm#)g*^5lKMLxM>RR=zyAClTc5|F>}khI z!-{cmcCj9I;|aa1IAvnd<0LyG|9H!e!G?Z&MCLr%S$_Db)LHkmo`D+|hyB zbIXodQ08f<`r9pnRF~Vzph!sd=iT|F6b&S!Q|E@ttew=4@4QTW?GuEwg8)TAJ53>EBjRO)YJRW z4A;2l&(qEC12M1)BXx;gxm!%|UpB>0j-+*zn@z~tz-8Jj;A9ofUzMoids{6hDw|bp zSTiokA>NidPB;3^0yM&FucAT?fD}Ky&u#S@wUehN0p&U5%4MCBe8r5xCkmkoSn8yS@$ysBSar-*r0&zGW& z(e~rYf}GW@9ZFNl(!>hoWj&qwk=sbQ$R8IpV^|bHl!1eM{u8mjNG}~2UqwdMMFIz< znv%}tFTsK|Go6fP=4@8e5$+E9&1Rv~HzB&~ksL(J3Qr=IMg-Ib@47weDYu=aC4Bw_ zcP`2wWn<756~#kwIx6@IOGXzu|Jd1|1`ZeKy=xY|c_LphTqeGr!+1ddDpL!+!u`FP zNh>AFTZ_g&P!Zp~w_NZ%r%@L+!axzwD^_9k6?Xy4=bQL~d%kCR^F`W2lp?%XdG-Y6 zD+|8R&%m$cuF}`msj z;0L2aDA3%sf=rt8XK1VVcTCi&J{$@4c{aYn#Z9vvetU;T#HLyQ-h63jb=oye>Yv6X zfVhvDXSlXa*kqkLYc8G}jL6@%#wYZT$Vbg>rEH`IZ2RcP661CT2!>?06E)}bwg`(* z-lzpdUK6oTQUmBg)VlVq<}`;$Hax^-qL~gS`pRBE+yI~6iV|&)!BBJIlfwd~tr~)! zyUoeh65eU9sp?9{>8-30fb16V!PZM>z7(2gB65N-YtHWYV#V!76nsXd?;A&0-01lY zNdyT`{9hw8!$wUD-WpF!TI{}mvL3@>%pv01V1h#FJ-JprQbv9!`;0SRrp+Dx@JZov z5_#$oZUJy<2f^?-^m8EU&@*S9B#E(vAdfLDPf+6&yO$QN;*O1toRgIoixBt988tS^ z(Nzk8(c(RP#zFq#NCuKB+!C?3ggiSf#JD{f6D_?#dhzl#bzH~?_Yc!8DR}4Gl+X7S z7lh5Nal#ZWybnQOqd^O#fbE6D6aHOuY%GCz`lI$dCo2ufPGdW>U&!*xN4YkqTfK)^ zF`$0oc?@Ks3q2nWKN%D?yo68Frf0$1g{T4xO-QR>!@P9bf*zvA;arLTgt)UD*&6uM z0yd@HKjbT@x_lERMDk(3iZ~0?_yVLME0%Fs^dP3LZ)uOq57XPpvJ$|RW*2oT=`!~u z5z@@`Y~VDy-43f1_O(>=okMmdEjh5j&sbU5g|9iy_#p9X;F@@rS%a6ov1ZQ27F6+apFuO1@1gC9t? z(Pw2dZx-Ze3XCX6Q#yMx0Ydm^l)}i^Ue7@EP$9YfL0G!mjRKw)mRx1^hBR2lPq!9} zdSpFgfC#T?zt|>jkrs(zAk;kir;9qTI4fl{+ezraL+*MDAsdVbCARG1K<9agyd}|J z{%;XgkLNd!S9Y2m7*wBP?OiX3BMZ1cDkztYGiC?K`1!A^yDo{ALG3G|I$Yc<*i^0L zv+xoq3CcZw&m;D1iB>R3ll3|BeCU7^359xpYRb>tzcs!Vl@QDjU}o;P+Ec;3q&*P= zOoG6c`Fe=$2(>3gD-U4C5Xm-f+>N25sBFNbei+zB@1~Mg9AeGeRWi%a0c+-1K`T=0 zL?IvJ`Sq2w<|{}tUN;fHd}nmP_gA1&>bdP^#$E(v8W<>_Z7QxR-c^7(vx|ITO`jBa z!-lztWzDwHgaQpUFt;CuaQzobM6$jkaswi8HUI<5A-yW*6J8SJ8|suZ%G<5ig9Yr9 ziGSmDR()^5etWy8@(F-;V_17h^V7rV8Vq4MPP20Y%-@J7F&^zNvmYj_yyaLvY9X@u zHuOG+(nZ(W#M-o529c{2EJdcj{r*4EYoFN@DbbLr2po!me7=dDoQcn~pU32rA{#-_(FN0?#&gMmWc zdO0N31kXq{s4nX#k8~Y%XckUiHl_az8~}6&EA}0%%>sQ^?G|lRe#GmETw3c_BPqT& zL0M^?b-yTLT76sAp3r;YjA3W|%ObBNuW$4UeE0WcF@(+Y<5cNr<}|o2F~exf5ufkz zQXwv6xP7g_@FA}@4`5NW`oEwJ`U09bPIIKGOGfs-`Y0l=B=41)THD>=0woD-zo;A$%b2#5p! z@Pwte-uwrESY2TdQaM~wJ&TVTyblQohTVK(LBy-j&o#2=8I5}arOF!-mHMlR;eH(! zM1X&WadxF|T;elRFvg6ha=CIilL!rn{C>T{aI}rQLaeC;WJLcuI#+TPU}d_I?Cu91 z^`Q{E_8f~!{IS2u534xGA@ZjGjTDpGmH*APl5Z1*u&wU><}`Uzpz}J2R^-3mM-O`V z?X9AakHLjbzfWub#p|ncHBd$gjoYMf4X|}~^T>L?4$VahmHj}~3Xg#%BXgDhc}_LA zX0J|!JA_Jf*2JAt*2;7^`gJg$Y!i}jm@(_jDOjsvk<8{=kRGc{{yN7=D3<=WhKV4%flr*9Iv!2lDlLQZ) zw;bqKFh7t!L~^F<&s3hLDHI_3)*ZKcl}uVzX%;gK(BXuZki#w{Z*6VCRDM&;kCUXp zO9j?GGRf+QKS`b;B_;E^y72JU)z#I|(6F(wF)|__s0e$<7#b0A$S3|II=b9py*oEI z7sk!jhHHMI&dIKRmnvU5lnJl~+pT2>(cKerudqQrACZ2d&dI3>U{*!kGrUqD4-E}% zetx{OEhzMGA`;e>rHK0ZatNDRZZL_J914YPf_;hJ z2Fk?c(E-`4)v-L1Po7)hB|k6}?JV7(tC@m60bPdq1qE}}Hd?l}C8E$wixdpun#4mh zjRb>w`BRS6p^H}4YoTRsV3NeG4;`4zX=|M6xjSRkjxV$^Z}F|SGqitTpymFkVPZl# zB{xTrw%rHhFCRp9D!@;@4>%$C_)=?L$-S@jGn_8b;dedytImKG0tX9HVMh#skxpiF zlGQlQ%eWYh^>s#$&cC(1j(;Ln&OVGvkmls(wts{C2r)R4LaZpjS`@GS#1VjDfY#MZ zVmsJoAu26B&>cZoSy{gsM|^C>_=_W&XLl2EQ@s;w}4PD3_L& zK7RZtl~WPBml+I5mBoonvVe`h&Kug|h2(XySM~=p>)BES==SGHxJP|7$ zsOBKiIkU5{<}~;t@&6IlNZ?GCxxYTH(JXFyV)G`#{AHoZl<2gMEZ|YN*x>4WIjM#h z_2C~(y+jgGTa;evFEFMd`o4q}Ke41sbDz~c~ z>%5Manldvp!Gy$0K{z_O(b&GUL*k%ad$lSH$sjd`<-}VRb{70Ad3)X9L@7fxKAJ5N zDB^e3i3`4cd$tKiC6B~oGuBA()Cv*BA|`&k7?*#(S&tHWx*_ji!Syx^%_>sezh3su zNJ`o{I*N=05i5$J)LBY;A9^D5$3Ns{8 z))f;IQ(-wt7a`uk%;kC$VSmhP&>j#>>N8`_HP0?g8!vTzIIs5kGb94sG;;K0hXq<_ z)?dwHxj&WLqlOw&rj@|2inZ%1p5d~_sk?i6mOYP~u4H^a<4|ISRNE~_+WrqCWL0q- z=P&9}%uIw~fF2lpJ=8$ba6~IUK0aB7ieE^rSatc!w2tvFENq%kLk*XOS_f<@nX>b{ z`uh5_Gsg>z-hjgFn(AU6kLQ*OU0rvtDtneJNj+;@z1LgR=jD~=DzdT>k8G?S7dw7_ ze*Q9wD_#M!m1bdGE)g#HC_#;l|2boUAXuI>I~hI%@J8Lv^u@ zMjAAVr@Fhv`Adn|O;L886kSR`$;qh}PF{I=n3&`YnTn|S%hWRE%UY;j9rTB>m`MJB zo>S2YkSe_i|-EO0CRbGfar^SvDR9JZ@HtAd;yTKya2_Ai(U#N_U%TU5JC&C$xwOTtOkogM+47`4z>NI#+@}Z@L%SVa~kIno*2379Fq#&L{tkxi!SZ2D3YYm ztTc^C!{xVJd38I7$oGN!r0W2*TeAPdx<4*IZt1zq!p)xHZbbS=eH2=+!>G7hbVu0n zE`+xHQ9k!Ey-~IB^5g`s({pfm_(=K_GFToTI#iX?ts*-X?xlAQ=M2yIX-{;xeX632 z`@5b}6j*wCdc~?%Bh2mw2CKCW>#ynEGHD753f_#ueMPe;vla9*1O1<(V#AK)DB_!& zS!*uObb)BexReyB*R)NcTCY?;f7btV{B0}=pa4<3$@4>Tr`R@A=iYg#oUzu{51u4+ zeZIIuC&7N51f=U?COz%Xr= zg=UW8?Crc&m1YzXFB?q>9GUU;(V`0zmI+?>_&B!rYa?u?9{;<@?bvyFbskfP?NfwH z)h4MT(8}cZcYG!h@b~fYF)W`U$tjz)MnEIN6Y%(}%|xo70p;YZDk&L$`c6(mgXsNw zDZr|&Y%n@#{ei575~0WGkM5Q*R#O$DPU`G9_x1UCWy|T96c+?S(EFkVz}U)H`cqL^ zSr`2kCX5H&Ja1e8cN?V;_~^=HukbTH-8rMt$q7b+`MXzd&w>P z&pnEmgv5LuI>yVc4cb4OK~g1*nUt}OBeWz|M*Jklpin-VWPZ5V9k)oSezw@A6K#4p z8_=c-G+KPqZ0hRnep^`DY{^03dGVx}&%wrqrJ?=s}-Z|zd zCdb7E*s^Q42IQ?HP%)h*kKJZMClIig7PM-FwDA)*(*N!EIJSL-f7$31u5iQ@-i2*$ z^u-pcl{TOC)0+(@ktGfUE;S1rQ#A&eXnbq?_wQes`}x+|nqi3On9`qC)5-`!ZaKPm z!}Zct*ua3<=5gaGT~~YF4z{0A>x_z!dBvI4S1=&9zM%_B%F7>j&Cw;8Fun^@%ny%< z*p$)_4hp(5XC47)c5Q8q-E7#{%xrsmTbd$9_rP;dHn2pmW$9_?GfX3r*@~nQ-5-F3 z)Mb@@&EYIEc7L*)$pK$wwp4Utu2E+IXcxTLlEdl6HV+REkB>>;z3UzlM5Ly5a&-Ky zXaNARmX?-=M)Hs;d$Nr7q*t=c6u(5PH5apb*@t}P_IN*;+IRQ+Xs3tyOR;yCmEz%e zm?R|Wv9aS}6=(}aeUDb<>J0C~{&Tg?W<1(m2 zBJgze5i3iR@SQA`G=pVsWcEDT64q??&xMLur!EDK-Uk$E*C|rP&MMiBk*oroLxJ@P zETkwjUdjZvHWnJ)s}OHW6Pf~WYO7iFtKQD{PFb{RdPsZp=OL5MOH$iIjn~yUHut$+ z+e*pLNB2hE6-DCN8ciNmY5acZ#I-hUx$+s2|H}|)Ttshcv1oC4w_Rf>513V2oDQMh zA)}^_-xWnAB=o#5Cjkb;3&ITRx?b|Mi&7K<{1U!ClIM}~&W0SWGJe5rwc_u%5TYOH zgJN$0c94{m+)*ItdT$w^&E$JAD#7ZAlx#2Ns5Ly1I&3OZ*xAoY(wD}J!La##ywmxN z^KPwGI%jOMP%R3N)o{mdhW=exBC{UJ+qdulE7U>}2>7%^0Jll;_xCuwJ8bgGfF*xt z=KFwu9FH?bQIu3vXd2w<(j+<+#(kY3*e{$6?&fj*H{w^=OSC%7y;h7X^W+}QEEidZ z9~CAFR0`MD*MnD%mQXqXq@A~G@!lFt{)gBb5T1&S0oT*T=p zaEB{b-hIUTpY7Y8v&6!@yuDoyW=5x`y4~@=(O?8%*x4ORkhpjD^dJzxB-=YA^9I@f9_-cyb1#Crj-D|;d@m2?1~n=o0u>#7WN`4|;d~@DH5KfNegZk#yn{0{GjHitZ|?7% zZZThOSK29WH7P-P!@ zO@~tA6aM##kIxz31%*WR1F|hSTjAv1(NX%(pZ@`*#cI@jQpHGy0g72fGRi|IqJSS3 zi(*7ya_<1aVl^OkwlvvtkZqQ!O!r4FjnAMR?pqtEPG3NKEVcaEK$FKQi_=~!3ySPG z5)x9S=}>4$hyo4H!aKq5Y=Da(0uq6jwmQEYw$rMA2;LV80k;wKnhU49i`|8a!ok78 z#>U2nhlik`*Cnc`*BjWvMXK~dIZ_HZ*C$RXP^=IzXQ>=1Il0KtQ1Ew4Y8^2L1}qz7 z+urUV4Jy@%b#=5e-;mbMLv18XEbv`VMLi=jD{tfFWi0%C$kYm$uef+}|8jw&K(%mF zsRaPq#6;W<7P@$;VpUIfcMA@J)dQq!+aIrc;dq$A-s(#&;O>unrp5o7>dy0_;*Uq0 zzy>gU_NX~WN_wP6tE)ZdkHA^IpS;x1Y?gW6O-lkBFW( z-SIzTLENxK!hqVz_AfpX8Op#`_165lY^OQUWw5PEogmExVC3915{{vy&3*>O>?^5= z@`O@xF(v1kFXo73uT#sjyyp#4vWzC8S%^UdzX>LcJU=wP28>6PEi?VUw`_-X;Xp!n zTQ`peC`5Zx<*zi^lUe-BCTU}T4Y*(C9kYmC_xN$0W0bJF#ix1TY@}HHRfGR38kMN0 zU(15)c&95pib{0BNr~8B_>U5p%Op^XImid9*K9|hb!I-6RZKzK2+r{+WC5}D3mY|8 z9C^GP8%V)Wxd1(2kLKg3*WBq6I_-uB#{+|NA|+TT0$7;0xje< zleoQJVk-qgGdi(1>gXS+ev*D0ay#_T(T;RUV%unm74(yZ00f67K@jN#$O8lQ*RR3w z|B{=&Al$grOgaFnt(HSPbr&#tPxiUXl(@~!bLt>TFZOF1@U3vXw_Ae}+5Bf~9>w#p z*$hZhCVL0KianeBXZAN4&;rY(&##LRR%C18d?yap=M(Sb8cqa%70L@5{7ptAn`Nwf z8$|aqt=FW*IqpC-#HZvvEi{=*fb85=tg5pNMOz09P2~5EYba*0{ZJlD-y2|!SHZ&t zKz)!lVY*Shm0`hXU5Q_NJ^;1V#;l8Lv33P*;LYX^fjok8J}F|>eu2}36GD&r-22ZR ztl96^@ODXj{lf?cBObW)*fa?WP{glce}wpFcdK7Z94-O&bgMdj*G1~gH)ojvy+uJd zqbbcMhyRB3T(yu%$CD2H=&9YDysgK=jpPX+bP~-c7mY()ME*c^M~rMF z@Y&bRY_lYG&Ux#rg}#3%L>KTkZK17(ZFlsMK=9Wtl2FE1s4F)R$b7I-NAy;dDDhs; zgH>LFdV=$U5t6*47tS?){Z`0ZV0|Fm6AJg;5connc=A@*d$h@9l#&snc}%0F}J zfYCd0;TF>8^7W6e5{$hCD#vxYa!Vj@AQUcHee(%%R?F4T_0w0-#9dVUZ&cu;a2R8F zeC`ft^@%UT%60sZp5kKPBBgWn);$vv96lC}qca4;glOB-PeIuK(+77I43F?A)=DGe*&r-O#Z8ecMh~~b-eIT8 z>-Kf9kwFnScUz`kIW^|xKR^=Nu)ViJKdJ^UX3sJwvwy~QQ1Vn6hQNAXvNJ!dwz^b4 zqdpJXD?*HSS#1kL_W9unJ_0s{nnm?H5fAE@g>eZFttPmWh{C<&$IBb?CianxVIcgP|mQ{%@4J!J8=vl8;?-ifw&n4jC{YjhKANHs6IzKe5se#BdGbIXxDJ? zSEL9(kb>c!&tIAG2j2pt_Q0{R9vW)7L0E|JzgqxRom_Pq57kZnQOZ2L@%}fqI|w z>9e}p7}DnumDon=y&{jNL8asovzm?n%^v-xal_Zk14?EbdM!%LzMRvnL_&B9@iqUO zoIdDA!dV4l+^&1}1wJ*b<}?^|)%S(p@nyh2W4d)cmeU7*oLb)<(RJ&i1q5s5L7_lz zV$EfoMCoPcmbMI`b`YZfFlr$bns(c7za2lx_{pOq|bg1&A* zh=NPW|8E{*{1s@U;^pIBNDdjD1G0ffR#(b;*TG|Jy1T}4s$59zPZ^xOu;&u~PEteY zXW_66X)j@>7>8ihsXMPJD-Si|3b1tvBi_v- z+^xXz(r4XD38t7*{M#9hj880x_C9uN*Ly8DjE&;8U7)RHFpm7scTwc%7GnJtrjNj?doGy$q%53&7(dp2UgTc<==n8-+}&VZUr8+%hoP@ zggF{0NA%epZRGP2eBOHg8p3&$eDLNDUS6S^%IlN@;H6l`7;%iU+avrdt*6wl1yXT{ z9EtPB-;L83w8cUR%s0GIoxVE%i3Ewo%(KT^C4{#ERWW|QRAtzLag6^IfO&} zLv+s`cGa#Yj{YE4@f{smeSL=c2H8HPdUM&;j;Hn2lD?_EUlW|B4k!!o=Jw5AntPoJ z*^UlT^yXcbvkwa<_S_v%Fw8W5(D6m&f*B^qj_aqX z`cWI*S@Sye_3Au$9s|tb8+SRAtB8L|?FY37%V7_^@?$tesR@qcD&Uc?lYyi&9rgEo ze{s=Umjm+lPbTC>0U!O?djm9!*5(-#RA_wZf0qXkSIMLu}~J?eEMD?7-U27-@Ik@1@@(XjqnfQyrJk`d;7sG5qq#WI9+| zCy$;to4yvv=Ota0OZVu95=@&JiLPPv@dm&ZJp^|@C5rE!|zwocb%Pt4Pq_$gT?kRb7So7dWu;5>x>V zqYo+#q*DjPpicOo=SnWTmZQW5efaJ9hQj9nPS2j*1nrlG9?5H%mlO9o{ros$cQF7B zvY0a?#<+%S1R+T!yjnqFTcy2B+Tn~8TF|{?M43b0LKyx8qVq?s5D=GA93z3ivb?CO zxB{{yd9$Zrp$Er)Dr?x+kj4p&y=3n%Fa1i>~wy81xdn8|rqPR_mD|Kb5FN<94RLr`~h`U1$^sQLqVf zzleo=i|y#0Sr?PH!*XNKPOm7IpZjDGL{I)OLH^W?Nd!$$F?7F9AHKpSVvssIK z(dqA`cS5XdZ2&$0JA^cX{V95Mtg6onXwNrC@NByIql%vY0oIHm`bdeZc4w8R4Btucd9Jjpf{k3=da#_LFB7%FB_b{ zCH2xftvoRXqVF_^Qd+S$)YS|f6?U5~*CJia!sz=EtjrX3QKB`kPN;?-A)!7rN9n=q zNAZX*{}}(hCB^uS$4mM;^|Y?2Dve{UYlHJ2A|#oI8Ts3G3-{9(gUH;(i4F}LO@qAp z=gxH?Q(PON5erQe7oaA}hk<;QxCVCuBvIf2GNs0DMzn?mXxuse1@MT(%4N%{J-Dws zbzHZzAT93Lya*fcY;v!W1$Q~#U;~vVza^?7V_TW{f3>7~W(BxPh9j$_em^Y*9~G}s z)80G)neAbIK=1~X_YNPF5i!6GXB7X+N=ZMZT}*5kXu0W{P1-TtJ?x*622hK|nTt(S zq_&qh^|u1}iuWEbLFeS!u(@|~cy~T{@r)(N#8u_F8Ax)eDplPCMKLcfdL=-eI@$!n zNAROkTp7la1iTh_w#5z)|HxRWTOu&S6#43sJm7PHCf&D| zD8A~bkpD)kZ~C_WYl&~u!tq!`mfdyYeHJ?a)-5dazNY>q=AOIUY|$5>3JCh;?^8H? z{*){a37{%6?DjBHt<0!UUATBEt!%18(W!9|7rma^2TV%8wu&aI2>V@@)AVgc5seZzATB}qGJ)`RFDzsDMn1lLi|9EN%h$J$ zD_*@XV^rgkO_M6KYljK)vFG#Z*2(_z@cpmxK2R`3ai>!RNLpAlQc+0}W71rsD+Sd{ zFD&HP(nSO9sRJeBg{pvvdhF1IwFMZZD%Yb}!xPLhF!Ssbvrxj5;`I3T%j~cm;6i|8 z3}<_4Kp{+JhX%6L`D5MOXb(VGQcIqGo{RlaoJWg(&&)&rj)W{5tk#F=}E!dh2Zg zB6y+!nk9OphtQgA0t;a|`NsnDe;+5+-#L5E Date: Sat, 13 Nov 2010 21:30:58 -0400 Subject: [PATCH 0498/2835] update --- doc/index.mdwn | 2 +- doc/logo_small.png | Bin 0 -> 7154 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 doc/logo_small.png diff --git a/doc/index.mdwn b/doc/index.mdwn index add0505d97..c1546dbce8 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -14,7 +14,7 @@ revision control. To get a feel for it, see the [[walkthrough]]. [[!sidebar content=""" -[[!img logo.png size=200x]] +[[!img logo_small.png]] * **[[download]]** * [[install]] diff --git a/doc/logo_small.png b/doc/logo_small.png new file mode 100644 index 0000000000000000000000000000000000000000..38a87e19cd20b6763754cfc89e99ac585880bad2 GIT binary patch literal 7154 zcmVPx#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0GsElL!))Bpe*gh@m}RCwC$ zoq2pz)xE&ax!bIn%#xXeWU`WkY$O;&2(qbB=u@Z^Teo6gedQ@tpY|!OT5Mgg*7vNo zSX!x7eBe?+E4Z)}LCPY8H9#PceVZhcnMr29ckXuHA6KSh62c^zYdZ32QBpey;TsKa zZ)w{qA~8{Eo~ECAn{`Q&GBJ+OhyeAl?#4jNVb{@P-6xCm`63b#Ksdx@Lpt{G0({UB za#~f%1V+5wv{{KOu1%OVNtHP8*V}^aM_k9|nP*#+<}2D&8Q}=>JJD15LF+aVpkEcN z(h!<~{o>aV+E!ORZ=#IP75!38s1QI9Ksh2EWuJ#$tJ|>p>>3PWKgnDk4psi8Is+<} zRh{xo;gb#irdR9M0{{TJ(j{eym58E<5b6%~tg3vfeBXEX+V`^p8~IW6CqW49vmacr z_m;|@vjZP5YhjG_uZwa1^6Ya>!PYmf{^NC)xdbGlQM<{;%DCgK=X?jZrdGx&J0)+i%fHK0^hytgcEqpRtn;G*n zFG?z(rZ0NCd2^5t_Iqc@seQ~!XkhRa>L{hwSO4FYPd-)f^J`4gWrVSp03Y=80kgt{ z!I3g)yvU#IIg_GJN>L@t2xBZF61Um5*ZCVB%vsU51T#8{F8jh^Nl>8w={OWe#*Gla z$jq`i(9%iU`xcfiT0$!$jIpS26_0pc30%Z*Hw4V=*blk*?d`qw` zUpGZY7-PxOW_mcUlXZ<;QX!qT&fj38JFZQbDI<*W;2J4CiIGi#){#ppq|+YnsWedf z6jgHH8%0Jqtie^`s(js)Hrh6#7jlMNd%O1En0UR0Q1`u2k~Fooe`6reR16s&0APLn zhD!I@-<7OUk3e=WT?HW|wE|23{^pmt2W~nV=KUIZ@=n_Rz=_AU*>>ELyeN`iOjJTT z2`h3*mmri{~>`5{?0f)&|aoqIR8e$*6b*?7(ROl{h5C`w-p^7Oj8zaDfR z{&mqW2#gr`gx^erje(Y+Kz|LP$oG0@2>jk{-m;~2TZSg>*`g;3CPwfADwBMe#IN&n~4yESg-h7ghSDc zuvu#iw7hWs<%U4himZEYhVvwLP?U?sLWJGhh7_zT^LjXOT_to0hcX#f+-ZJ+Wd5ugY3b_J z1cfQ`01=6t$Wa&tA*}J$t*%;QAoX{p-?lVmu>z+CTO}YSo_<7wq0gFFgn$x@D$4lM zd30Cjp5xt>rN&}uF1L$uzT2|J&3T?Kd|XRtZ=SSpfZu} zYTlfrw1n5~^%W8${$c+@KtiuFRjHxaqOe@Ho(TcX(~W^368+TKrfYh|u0zX4 zCRl3!YVBX%Z}~fk6L7$~k`#}GRWUrQ1FTpF06~D|>!v)B|Fa~e1&4T}zxnyWH@QH1whEU%!Y2i}qVjUR=Mr^Utw-F&P^0oeY08j^G7m#p8v(|5{c*RSk zYOIgvJrbS*9R5mO8R5T#7MRoSvpB>LIgbPdX5f(}A<-+KfDj5u5HIfw5kS)oC67*d zFp}Sd7?}x!kP(^F?sGWA1AK7f%=LOocex|AxMtbmduV~_RRnn||FM~-=^VOj&!-Yo z#*x~IqI?q%;YDwq)cu|tmu*!it8nFD&k#?=>nVM#st4moCcvz{+;*vB$QyH5uB-~!8>pH2CxoB~cfgg(yy zm-@HdtjD4>I~mtJa~b&8{QI{l^_z$YB@@~dY;lB~Hzr>9LiNk8kZV@LyA-65D2l-# zV(8eqLL`WY4ao>&X+i-2P5$QApzUhow9Ty_nJ5E=Q2+qal4O49QNVLCziE_H3S@+_ z6h>gqdoLzXrrfyf1(vz9E0v!*Jm*UdB_t%DEqVe(c0v$D zSaraqW_W3n4p9^dg1|6LddVJSiT7)yJVzQV;Y zhGFb>`+)-os;jG6mVMxX2Qo4;5JDe+{PEjwzfF>4G?xgu8>N;0PyhV@;i5~X0+gq(&P zk3Skh=;i&D?lXCDIetDE;De=xl3t7vLbbKE8#ZikI-LYT40CyV*pJExV=SLus;F=kozwbx#wX*yEvh9g?7HX$KFHl#6=J@)+;MM8!qeUE)Vg()V- zWdiiI8s>ZNz1Q8{9qH$!=yc$IUkt;ft>waIRun~($+U9i%JlSfAa}6EP%ijttm@=A zBCeC^ni^jK5kL_G+uhxL;J|@!s2PT*P$<&U($wnypW;IhL|Iu`adELip@@Xq+1a^i z(~vRgd||IZw5qB~O5VfyfW=~2xpL)<88a@&PdYivvH*Ztvu3SWu|hh7bV8+4 zSy54O{``3`@_QA0!x2IVJ)HO6V-K#c+W-Kt(H$7X(>1AItJ?JR`2GG!Eo52t_S0*b4<=wY@VN7ZRi5hBU|T?-d691nv`p4!s&HlG zu)18X$mm}bMZI1>b?VeGi6}3$c=6&;C=_XpYin!eq#_2=5@;hKb(LYNoAWjXnseeN zE581igFqk9;$ji&?OAbOv5Jcp*#gaP=8&$1Vo1B~+z8fHf48u4a4mq3|t|PoD z>3Xv=;X2FQI$uL$py{Th>y%%Ara&kZigZ^}JkqaCE-NdeX*wLZDin&0jEwN72qC>* z-`(9EF3La%JsywQY?cuYonJ@@i9je%oDTrdQ}K7EEt_ea-s@3Z1VKc4EGfJ=lVu-UZPY>{Ykc)(2{Q?d2nnI5&%L;> z>kudQZm_UeEJ~#^9HBr6Ns`>OX%o-$5|1lB6&OP3^?G;i+!;BQh!7%4aw3*Vv2mh; z0Dv~Sy&=$et!X9!2}TTgSg%=OA~CWrgY4|=$YLgjVO?EaYuB!oCj9yZ8Ce$%1OjW< zt_>p(f0~(@2_~S77emU5>{-uw4WXG5myJQFz^L0&ZI$K*?{eFKqK@10?>FMdErKJXP?EH%m%Jk|d7f5JEu^c%GLOS4#PN zU7h2&<;#~}@d>c9AstA*bR9KP`kc70)`q=Y^QT>%d;8T1=^c~F^y44@I9%k6JTg29 z%yAsUFf>iGEGvp4yd=F&7z%~H|NZYvj!>Sd7`lAXbue9>I$4_m0Jz|-#Q>J3u@2yw zAcWJWPrvJ~yTTb^WWWM1FOl(!)Niq1!GiMg@?NG@MmRWevE66g=Q1@JNyNolmoHtqRHxH%T(r&Ik;PSlAm-1Xf9IWdN?FPTMK~^e;pg^$Y6x|S zzDRmm<*6nhk)zEXI1iC1E?l@UH8pkr{{1IUo(zRTBuQSe!oqQ!AP6}*ISUpnC@U+& zaeN}W!10A}N5}3sLVJy=RC;;dcagvdsk9ai#e#x@jEsyqbLN~refq?S6J1?hQrRUk z(vfT?&+|&9vZSPB`t<3!xw#gL1x(C-nQ@0ORh^QdNt3Lohx2+^Z>l;4i(**AMK!fr zT~t(*o11&f6(cy6P^z_g)&2gMYqtR$IX0tgpH8ndsTcgn^6pC;KU_$d7 zV}#*JgEdoEhhNtD>v_Z%=<)_t*^z{zD3T=mouKG@-M(L!!tg9;CvEp|-oDekvW&-YMv5tl^+ox*Jo3^OeKw}U z%lT#+XMFQK);CZKA%yk*#;vx`c*KQ)rK(a&4JF@6E|>CYB+C3o4`Jl| zfGgB<%yn`{$8KqVsajwCzOI9HzJ`8Twlkeop7V1O%GCH*3z53n?)O?g_}I2xxco!x zWr<}t6JEU#64;O1x9_zd=wLc&fuRK&1K2#v?6(@;sOFmhGi!#5@_h)k zW~7W$;5I~c)S(CL{(i^lur>oOW zyU+df#G`W)%C1kG-$u88&$?LZl#$M9QWzh}m7kS2!V#%7)?JKaW7FS0Z2LG}lXgq; zO}AT@k~jtW+pD%$cv<7C-_pAEp!0A$-I=4!S~GQZo-SAVloy2)J*Na9USph2KwJhh zf=NY#zv<eC*uxj@!u75fXD-mxQ>0=zAIgAT`~FJ!w0xo&qF7Fo~%w< zoxduM)XHE+(2%~~@P8Gqma#z~-3*+-K2nW0Oa~Ndg)_~GU&bZH!7&%#+DQ_b(B6*?Y|6aWC z`?H?&R&~l$eF41m{!q3L{`z#Qv@PAI?K>?BvssZKBN-8dA%T6b<-@-<|Gg#HHdxJz zM#XW6FEGz>hFlvO-|1o;K%S;!ea3xmsr8oYE%Rbt`jS+6nRUtilkcr?o%mzz z>$E^eztT*WD~wdw3s!q;5|w5nWf+M9v~*5RW;uXWSuE9IhdS5Vp~_-&Vi!NypAe zt2c7LMu{t*E_mFa(7)Zh359gAW9S$Cocp5GzVL?xW=R~%KQdxNFjuN&VSlnWu){AEOWrSlXht9x@@ z-Cr4jkvRfQyiQxWQ0(ozu8C`GBjyp5=uxOF*`16^Xv@~SmmH5 zYG2sH_O#KqVtpY7v9TJpa8kIh>)>Ad0ePpi!D*qF^K~*^rG{zvn6|M;b6LYka2}npS#d0_ktrltt%{0SQ4WSwH-zTYE zZro%>V5He#S)xMN;BWd(^~*N8W6Xb_q;?saG!Ai&kW&U0Ey7Nw>%Eo_T4~!j`IT{m zRzxBrgk)gRB2?i@4X$ysZgG5s4;aJ|00I(ZV9|y%dAi&`T=hbME)N?Y;e(I!VGvUh zN*P#G2oXXWLOmy;j1faI?^hc_I0GSU3ARy?G%L(9u&4~F1gxU_`2Sq^gO~S>Z5%IM zz1CN+!c|F1iwrDUUkCtbr#nAs-*M1+WXu!l=0Hmi=gzpactd6uEyAmfS7m84H#NOK z=0DM-FfFq#`H#%y0LZoO!5~akr;8+DQ?zdQWyb1!%w!up0D2D_(sE~eO(8O^+l@-9G z<*vh6B8(jCk8L{-IS)UayE0joG@MZmf7i`=c65IJVe2+W$aSmr=KsiCt|Q}Qeo@LG zB8BjPQaHJ@^YiSuta~$mh!5#8KjG)Mu>$u=`;LFuzc}Nr%GPGBnYy|xVFm#Sncs*Y z49m5qnR63nH~5=ELP$-hLDcJ>!iP!EZ`)?u@k#p*8{GjAdLVnnttmHa2zB@XnN92s zX{3lq6cHjYyux)NUzZz4XsNZqFhAQ6e2$N>N?$^<*(c%xzC&d$AB zLcM+777yn=-hFC@@hTmu1M(u!(bvK-%dPF79daD0^*7pR8xC=+D%nUGHG~?27%Q>? zK4@ng6h>~H^Ueit?H_CZLSba7vE+y8w?}3z^7O;lAPnmbE3y}Tb*({LJKbSt93IX? z3yg?F5+hZ(DxTD(sZ;JqTLuB_W_xsG9Ep*#&W#r>>}3EELbSkeh!aEs0fa#ehd7Cm oDqL<`8pjCxf*jfbvJ6@NAID37cnRF~@&Et;07*qoM6N<$g4p4e;s5{u literal 0 HcmV?d00001 From a3ddfd8717d5224d2c81e3b6e42d457329c08956 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 21:37:47 -0400 Subject: [PATCH 0499/2835] no link --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index c1546dbce8..439e2da09f 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -14,7 +14,7 @@ revision control. To get a feel for it, see the [[walkthrough]]. [[!sidebar content=""" -[[!img logo_small.png]] +[[!img logo_small.png link=no]] * **[[download]]** * [[install]] From 422cb1e469849b1ce516a6c4e3b2e9edb1caf4b8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 21:45:37 -0400 Subject: [PATCH 0500/2835] change --- doc/logo.png | Bin 15170 -> 16760 bytes doc/logo_small.png | Bin 7154 -> 8163 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index a92ff42f4768ee1f55480dcb5c0f323f2a083577..9667bbd8b034f9dc9f37b881379770281f1a7e44 100644 GIT binary patch literal 16760 zcmX9`1zc3!(_VV%rMnT8mhM=(K~R+D1?dzJkZx&dX=#v@?hfhh2I&-#j_>&W_eWmt z-h0nEb7r1-<{3iORpqcT$T1)g2)2T}^lJzNffanlLJ`4{nN{oO;2VO|YdJWiY=m+T z{DSTv|K14#!6f+ii2zASBLxT1oE21L&=xRp5Ll=`bh%$bAhZw#X-O@&g`;#=PlCTs zed4b3IVy3-Em?7ik+f10WN#h5)6(M*5^3obilfM&XtnwiXux1(+4KYIFv&y|WDMJ^ z)Dh5?QHltGFatdwEu-m z1VW5>tuAVk_e&^Myp1&-lgsoh)t4{SO!X|Qb$`<;+;hJFSS^6j$60JhgnAz6>ojRI zV`w2Stm#Upl3Zp{)Ra(;L4;baVzAe99+(1y*hR%~roYamE*5qFNw+LP+9(nd;MDA% zVIZQ$QKsBY{L_&y$^pq(oq|`kQ|kwdX7v;WI)j&Yk!nW%(6m)b&le55Hk5F@;+ypfzH z&zOyh@eBqzk4Vy8;z$!5eg-f#X>m?}Nghjt$muZB5BcfjVPkSnO14VNiP=X6BBOns z)Z1&6FuJucy*K7b`vcE!BM2(4YoAtUx5`%y9ienQzZf37x?Ml>XCipPzqh@f*Q?2L z7_{tpA2{~m!**r;tc?%0MmBWdm(~ZiVK-O(GiI^y*+~vRB5sMNF6ETEq{oZyo1Sr(jmsJy`O zWn*r7UYiZ`g6#Zx*Y;Q?G9Kaxs$^`_R+}B$iZP?kLCXurZY87)m9O3N4-0(6-fhJ; zxNvCSDF>=g8f_>FI6Ziw@O|<_8zy#A5k2mX7@62f$C?L~>3|S|Gg9Dafa(x7O7Ml(8P3gT6?jtcZ z=i1u6zLRJF7QeObJ`64pJdEycv{^CgA0!#|DP2sYTnJLZpLBDE%!=kba}zVPRgV8K@!*O9`zQOGwD9jfQuJrV{*e@zZO`#S zVHgBKB{aAFFGkm+JclXM9;6q?cI}DKBm*VHgK0yfoy1l(TQiw)5^-)n{z`%MS^Ix7 z`?1Ps)!JH7Bp?@uMvr`JO;ykC^RT|3qve=AI>#HZ;$#wj*V7X|pN&_U74MaOX$T&V zlS9D?RlK8yJDD`}Li|(0<5~ElO<=_^Y04UyAbf%C(pac8e$alqKQs)@lJI*9yXJ2Y zsC!ybE#&;h#Wnr@-~3dt`*^uKJ%fK4-6&Vq+a7T|n5`pV$Ha$XPeL09k925?}yQ2z%1|Xt5#y^ZCOb?rhet|hy0E1B@Bk<~dFZ%tv{#w;_F@BQf zCwHeUZIo1rKwl}T(z=VSyJjJWpOZ^k-(Cbp)&DV{j&Yt&I35Xn2=sl>AlG`DNvGK4 zechWNG+)Zr))*GnfyauqByvB8yb?vBcR>0iG5-2W@m3c&lrgRA@$W?J$Jo0Bh@aLR{=Q=V^n#}g z48`j^Nzw8;H*qrpJ+h5Tq=7~@Th1Z+N?p^!k+sSK&J8m-r&Lhe$B#?a9p^*_C&znO z?SqG=>ZrTm1KV~W-4|E|?+(Spi%q`1D8squN2d##Q=jgvRf1^Y?RI(^=9;~<(QevJ zu0E|C9|-vpfE_fsKzVy*17Q>ce~5Vo~a<= zWy65c8PS9-SV2fiQ1Rf$hcgvaT22sY=$xOG7xsti(`LQX%N|?DZWlAEfA7a;Qye%Y zxNcZGayp#}%0{gMUULqOxsVS2tIPZ4_K_lqZZa58q|Dp`7C~hoxX-X{nu9v#7;NO9#KEBx>Ehx{l*37gJxDHxGmMGP56r(J`260mGJ6k zW)l&xv)=>FZW-X>V%O+8b$GwIFVE_sS}*JZ)nnxl&jrAcUdCiQ47`|q)40&`H0uz< zWp_@FLyvqY8?TET%o(U2T0112qPNpqe8Jn0g7|Mvqc17JN*)G(eFlYH(*-K$zHIxG zaT$^v^fDf#Nx!pNsl!*ZRg;IJkPhr}lw}P~{;Tgk+RSJRhm0{$6 zHx}+=n`emmEKI~UBL2arWII<+I|_zham|fb;3|gjZ|Qn8)SkIBiokdn6e-saTO>CV zH>l#h497%c%RXtq-!egVrD~H7SM-_#emuEG-l}>&)W4VlJEm@T_*Vae$BVl^6%G2# zV9g+2@|eaJbXd?VdB?e-vd;k^U>pX7@+3nEGJ9NuQbl!r#B|g7xgCU@dicO(J}04I zZ%h1{7VfmF=4Ktg!|4F!kwaLN{MrqkpxnGQ0LBFiR$}MY^wFf%=QF~zx(ItCm}&Z> zvM@V>rA2&~+5r<>1y9ae)ib3{(t6@`PsfDh;AY~$!lx@CvaR6+JDZ=m&o$lv_e+z_ zYdO2+qw>7Z%($elYex+K3>Iaq>fZJ=_^=WEg8@J4E)mbr{o@C)_#?S5C&fo@N^0|> zm{FnkiG6I5E;Mt?H$4jzFk=|zLHQqf*My^+oP8)<-gatgpNJM763o~Sb^zMWJGa(Q z9?W;1uC)|1YRs^fhONcy(zXPe+-T+V%zO9+H$3tC;Q6%du&a#t{rkUHAR;zBH?s2a z%ok65X#9EVF+hF9yMfxqo9_1#L)eHh;*U>cY{fDg$-3LgE9*Gz%W$2IFrmAyG&Ppg z;==6sgFgelJti&o2WO_-mMeyi7S{c$%x}I$uq+E&#eHv6p|fLOXl=Q8-q?3%ywP-D zd-_A2K#2Xn+IK^Ey3Fv-A@e^?L(ewUt6{+>Vcyp@o;2f^gJQkbzparBzkU>L6cN{V z*_v!F-&0S6W}yarboz6-laq1GF&aY~%G;1hQzj-$6eof&rtDcDhXF$VObO)Q%r!Ug zdd7WW`%L$FeM+_I*UKM&c@2qe6ibC?Z*oM>yyCo=y6feHAKCbmt55Xwy%drqiR9dI zbn%{iMBmVYB}Ei+I?wO4y<>j*&e9IzUU^^h+!nmltjb$^k3)RWCx`eAec(0gV@_5T z4MywLbi{`#>5U(0HL1Z~+iM34Vb#wD%S(P(Si7ul@&AeZ_N?WA!x!qOy8-XYQ#)(@ zF+N+%TO|{olaaA4-f*7GNdHtM2#}JJ{-Ue_ma*nNlPHy7Cb_()s9Ap}rjqlvSwoqp z$}!1FciVA=>x1@RaqExb1@t%nK~rh?cK)0;ytBW<1fedeFfygQ+)J*H0}hS6B>%TE z`fq(vRDv?M?0>!9nULF9GRcVcW*!j@7Q<(Rx>>e66+6w)1wIyu{`L7;?i__|?8?*I zJ9^gkSa8YnhOiRZzPD9m#N(pW^ErsMufTKuc-%>~*42{bb?FXZa2LDX>u)6x$}Hh2 z(5YP_Lqg9%N`=Nnp8?EqHQl^T40`0x?a7VjCuXI5_@yyb z57(_6xNdf)+QP9T{JCClDneEJLSW>ZydO2DH2or)J(ubB&CnY4f$tDcJet?PEM8fH z&;C|2F(BO%T+(?$0mDalPWbd1!_>Isy(Bj^u1~I0`rC26zaKj4cJ?_Y?@1qp{%Hv)shkxo9#dfa7E6EC0e~G9D){@V@+`=M9A@c-a+YDYi9g`fgeC zB?V#s)$Wd02v+%58GXwFCX5K1!YhWmo>qat-I%Piou!y;xS2p z8&S$L3B~p)gOUIQJj9kj>ED(;^Y$2ZuURURdRk_-x~1w)2ztG_JTqK46bAmzacvZ{ z$p}A|3WvbBV)7q|LU7(bCLG2HEI*CBGRP^ZS-VqyaM$qzcqhdFBz0drImGF1VFwsa zI>=aRiBATsS&ghJ2f48WjiWscvuaK4ewD~Ox)~b23w&YoVWaXK+j5oI4nXoOp|GTQ zGVno}M#UCIz9-0|M@arG>U8t&)2sy#em=98OK9!IYU9&?%V@dRYI_x_HmavcE^L3f z$__JpUrOZ#*w&ttN<(S=8C74r_F-MtY?n7?GYhB2^r#=SI<4L~(yPhk%1C}Y;4fvi9Bu_aU5Px@Z>5?jWf ziJ<@R?J~u;Gx^;ByTho|QpM^Iz~!0d^Mu1f<|kZ8I{{py*%EASZaA~}8#>kaRK`o> zQdsszmRGdM0ZC{zy~S6hg*fsx$W!Q*(ydD`(IxRcSa2NJj%{F!>wQ;LdxC^jD~X*p zF4R>kIcw`n@xDb)Qsyqy-Zos8{3N!khVEZW+qr&Q>FlH=jW)%=^5T$n=0E`C~s>VEW`fh0_hgi1up_c-E zGNj=E=gmW;E{iftaJ}f^F#)47y~;1e(XhxW;obM0j+tZv53TL-BgLabxi=D9>!m%y z)3Vqufk8iy(Z4m)ZfNusDO+5aiB6w^XDlL*4LL_&<;VY75W>6DBAF1U@eVSS$Qp2Xtq{t1q>aI|bi!*i3p45?|k$iNtPJ#(v1 z8%_Db8o<3j8XmIfXK3bV+q_uv1R2$>{P*^!`CrdHgq)zh(55`u^6RYMeO=RiyQ(;m zzCDJdH}x!E#xhT4ZzhR5E2=JbG6yavQddz}AD70^KmZy=jWeT#CrMbU+j2bYNo>)5 z=7&TI_Z@Y7PX;J!_4dDFZ@=f|n~xn^3!ls9q*on*Yr0rgeAQkfJqIWC zPJF5@z99J=^h(9I-ADD;!{j~F)qjuQpJoDwR#W`*JO84r<`tKX$)VQEwD7FG-`;kG zNV_SOFN^aQd~5r|7G6z9C zVjd#zlsdnLLjT0o&2^`fvzD$_Zv9Kod(wm1vj^R9S1$gxRT{U^9Al^MsOlA7f2 zzJogAlk1v$iI8y;vG6v|jh`Nf`GQ8Y1iEK>3&lelV2_bxlb~WENs`jS4;*CDAB;yD z2ez%YC9m`3^w|b+@R6HgEe?#iBPe(}m<+~Cdd-{WHwGKGeSK^EV?t9D$-@!6TEWhSzcvz4#B)ki247H9Q~(1maba z6QQE(q+KhMSt>D;le5&_ky@QX1#V04{SC~-kosq^m^I=%o8NIW3u<4CzlBGu`aw%a z=Wj8Gn06O_Cpdp8qtKk!YbiCIUFGc)9#rcl79;dDRBR_KzDiLJ?4OFeWK*E$uN4>( z?QJ~JmXGZ=lEnHlrn5xA`8OnXIljfVdp0qm!y+P5SkdRUbi-(;*Svz$;?t=Y*gKcw zLa!@#Ql0u|+A8zSB>ZUM)@P>I&49{!g@x_hQ%*uTWAO2CBSPJbWs(;&xq zm=QwXjIJ0kAfMvtW^jRrNx*6jK-BQ!m|(Nww->+rPvv?)NB6>h;X?3MB4)V~4Q=Wp zV4oQ;Itm!5t4_-=%pU4wR!@qrbEK|DlTIGq&FRg(g7`w&K4{NVEBZmR41eEcKz+Jn zcSVyDlEIbBCuTWuG2Y#S(6}8avWDRSMSx|w2O)KHu;)H)?@G=noNQQ^IC`drMj?bQ8D29xfuEa)H|pD0zCd*~k#s>~8Qxa#v{^ zYlGmc8G7n=WGQ5bmf7PWzFzw7%8y^*DW3lC=F(_#@hA*l%VB-W-DcL?#F4Evhw zX6n9lAeGn8FL2Y8P4g-7vD3!?^Lg`1nAe92XPAf!M0!q%={ABJ*30TLqSe;_Cze8n zMo}~WyQ%-^Q8~6J0-O&*+o}5jm|c-~oZaUIK=7G4DqK`uq2#r9p;8j;U^(!N zc<`hjt{2liA)z3u6I|ls$A~8!JrrCzN9bN;7Z$M6sF2aCEb!c)_i zw;56LEJl9{uGNmucv>1h+)Tew48WjsR!DYKbTrf9Hy2WuqwFfUSw-id8M_q|*#59( zarRUurb?0tQ0Ee7MoP|f-aV^6QTmiO z^B_kOA|DsLO>&@FRIf z$=xb0Y@)4S?=OXnNtxu8H?F1XJz&QCo`#j-wQ-{OP2s~)48*uQymV}Uz^sUld;8_q zLNa20slp@V^OFIVYit%?MAsjm6O;aBS~$#D+tm8w-jP2v3%KV!Z#N`e;;RMM9&zaV zP8ki%=B9lfozm%%tqnLru&@%;h>yagYV8OO@6#~HnOC*w7@JT(!#e77+(aB5jtArn z`mu7ylFNjN??QP3QY4vSZiE>y;6`4`{dmek8X2sgFs|%KAALEVNL46OVo?+jf+pcK zbxZj}OmWc(PQ6QceHhENIXrZQ71)u!S?kjM8LEmPp<TtGBhzR}YD$F=g2ysJ$zsC33eqct(|ZUki&wB` zF4Uv$Nk9FEZ27ln{S$Lh^kX-qmv^bzwId{)O<*t1XWGtt15c5ttrUNH3<9%Tc zVij!z4YKL@;f{F_dMY1?Vj2C00w)jKx*RAw%;2zf7hYq{er)MqOSQrs9wk&I#jDNVe700ghQ47P%=VGB}-vA;mrCZ_unJ>wBN!bIDb*YxA=@>7XuYkFAwR= zMK9178)meo*x`xqk4_Tw35xIZCS7XLq(!a^3F*6#=;qt##hb`ThQj*M5^0%Jw6#Vi};%i^z_s-5C$3}25r zQQuB2owhnZKi~T3Zfk2RC55CAd$sE098upCx%AA3OQ~ZZTS?cOJbh$+D^z@qTS6eG9D-#om z13Dz4{qO(u(S#GT?cZPe+#S`)=4dchYTR}!C9y}+h>NkZVu2_q4W+{wndfx21^ijk z3B!6lw>a~3zooD=>F?_m9`eXHZ{9S0IvN-p+;k8Sh++slt(6F6RKQ0b-=ZPMLS3r; zL)g@W-%cd+1-7xVgkYzU9-Xu|U@1K1CnNvHcWY~_?dj1&M~6IWQ;(gbQCGqT-`I7k zb{IwW73DAZu~!T`_9@@nTC`*S(2N%XzLXs+JT1YBUs zTf&DZE1Bns^{GYC(96(BG30?jC7)^GQ*}u|JR&doFqpWHA3qx9;_dd)FKOJK&upYa zvrKpkyeX>E2T2j9Q=F@pjfZ;i2n#3o-j?YVX)!xePPihr^gY_+QZ=YsPZ0AEySS!E z)m(65ag#?L%#`L3LQB=Zuky=THU(K2d*|}j%?2X&rnN2?O9qZz9xW;goM;_ys_pxU@PK0Dwe3x zX>mU{BMzUAmi}c{^`4bDy043qm7boSk+Hk44{gO8X3WBs5ntOg%dwW9`kk~h(B_U` z?iCgT^uLGOzYoPUFYrUnMl+7~_Q+r`T0#ur`+Q&2?}A~+mu=x;ZsNdq0qXH){GwK1 zsq2H3kX%n4CJDRV*TJoaVisVim>3w@+1W2pCa^WsX{Q29r9&BNBbU^3%CSj;98V9* zcCywy&>CfUQ&MDI;$AKG+G+%JoSw8jr6}hcgr5?7{5Rz<&|;tlLk>1hEZ+SU!yrO! zKU4CaRs?1|3rbSUDo85=n&pv;*}w#}%MG?1K2utT(L$}1vQkqQOQ*8M#KfXS&g5le zdh(Q#z_P*Mtcf!3%TW+JfGt4%zlnPA?ZTwzII(JiIX(x-=8P{dd!7C6H|dMo zau6UM$X9yD`c_{4moEZRZv+`b(xB_fioUvf`{p0k8(pzXA8_j0+S+&yodm*{3L1ao zcS=fwH|IOaNEC_HA~&9o_c!N#5o9I_ey~`^qH=J_n3x#*wuj4(05n``5&0Tb`_Gt6 zah9vCUVr}l!4E~wc!@QnCl{x12Y4KsEzh@Ib9|CM7jgG>;y~C*J#TXFv zCw|{^o>7=39%@glj{K{)HO^>NGv2-zu9X>07dCfSny}!BBImVQ?+m)gypmCixE(R> z3T|j<@VZ_PNwnEq``A#Rn%?+uxyZo4uznPhQjt!nLjB>F3(=5hAi0T&x9;}>z@seu zWc%^DC?zF@6%St_)~rg9kcRI&VcEWfhJ?#$`HJVN9G+ zuKogejdc)CYYAvi3y#jV2QFt(jEA7sLlVn;V0ZL>#AAB)?B?cXzH(A#+CJ{NnpqXw zz9A1k|IW@%*CH;x&GUDpbL?N0P_D%}clVi;xv#Nh@MW?2w#>2z=v{z3b z2poo)LS&@zyJ+$`-}7+JKn(&eDLaP1UXSQ3f2u~OC8oQmy1}hv<^14aRLoIjRaM!` zREWH?GGTa6;{8cNLV{Tp8S)xbpqo(UD=nmtlX;|kp>QVkuP1Sv%hv$51%bmVRu#KK z&hGH%RoYHdvGWc;MslSs`tppy3k;l}^7mJuBlw>dAVVcfXArDsj%*|%wgeb69OTpL zV>)E`jPHN_B2)Ad(n_)6KzTi3Wn)wQTiKu1KOn%Pz^Ry*1rBH3Bi;}&(9}digy7Y7 zQTo2LV0L~%gNHo6v}BW&crx2^hewAPR>Hhwn6^}01nE0_0!p;D% zyF{U!w{^OTl8R1AYOx3}GiVXw;n8AF7-*DaPPo0hYdafYVI@B3rLS{HLXFz|*g%O4 z@#_49%GJ5HV<^rS0Heoi$hp{=466!Ybi`z!*%8wIp&rZlo^|GIYmiN3UWZC`ZW5Wk zISSx(fnd$z>Otp^r3!;XZ@v``EP(RfL_r{nJCAd1K=_XntC7?UEQGc1f(uH>tlotc zF7g4i{Y$-w+Xd6Nm#jhN2kdjLY*y|RYI(2c>w^3H`x_G2!hH|Iz;gG=cc z-yvIM(!h@a1<{4;Exy#@#O8TcViq0*?&{c2mWD{eVP{j&#@BQ<E0r~3V!GuHBPPc{D!sewSU4h_?e88h{m6tQ$#jmXyVaj|lCJpJDueOZS zW3P+E*@FoHGfsR;&|)SEm8Jz~QVz8dd#y?*$ATIA2aX4(jLrIy0mI%#$3Pcf6X(x& zhkJXLS=k`yrVeNNJT~Z*)=K)4VRb1gDb?Ap+PN!h#WErxA<;E)eyP9|5D< zQJbzyjv?a~hmt)b>xEiB*mP@44BIlIo*5@F$4rd=H0cw&-OXuiq(;jkzz9M|M?bv` zfHoZv%T6|UB%r?eSzFzOKKXD$Sbfw~&d-zfHm#=PjRuF8q9R7>@NrC7;o)TxhoM2G zY5(KH-L#cU$f;tYC&f8sETfB1x*ogF)5DeP@scGExd*w{fDIom8CT%oR%aL)nqSR- zJs2U4v{+r0Jc7;_4U~K4krh`_Waksb+*61Pkp5I zea0t_F*zZ@Wq+y&V1~ve&>hZo{0ZC6X$*|8@L4v&r& z{1B^}Uz(VJzsc3wI51cW>M)X5P{6|o`kGH7?Zyz?k2?9uSo^1JK#(6mLZAqu(F01S zZ=biCq*{lEhN5F*4-XF?_0)>)=M_Bil1Iws`~<^kZ589-h{lkYe6DYG#Jnc(m&1u; zy7Wbe&a=4KSVW9Kv2_(FNsw%~ZjL?>Ww^MHFl6CQT!O+&5ML;!^2>igRfKvpR)C$j zRtuoJkgjzH0X_DlK3}0<|NShfQ6uT9c&VBHZVnYyuTD+=Rlh4`bWhW;b$8tQ`MY-{ zx>^s{VkGfNyN86#Ikz~OKlvP1NE3826DQYag!&JgaTqk$%W0yDJ<)+JH6-e--N74 zu3mLIi6qwd_yh!-*OqVQY%T$kVxXr7gj6T>r0Ikm%Z($b6|x0(#11%Uj`O9y35L>9 z_9e0Flhe?gRA7r(WsM%rSB2ox;9xkvqjWY3{?D>kIHWi_?gMoyF8_5R zzcx!EcLQ(wJiy6>vhTEXK3(7-W4BB?DVp zxUg^uf;o}rkBC=*+9>zvzSnv&%%^%@<~0X@>ToM?$C>WiWPTDx3LDq0i8Kna85tP> z42w-`p@MAELoK%2E6?Y1fdvUqXl(tfh~d`h&$`bkvRBUjZS|-7^8i$Au4Lu6$pa@S zW>v{O|EZ)2{AsybIX^o?M7hMA!(oFfblPvBy%d&m*+MlJqqymQ_5n3vfM1L7zoc6< zIzlQcDsuAATLTH{!mdB2t#n7WHI2=HDUh(fTRS*V8bRKeqcJ6{uy}+1T(TD|;n%W>|TC6re29N{NmA zxM*8PSIiWNpr;=Q%^H?jbd7%;>YcamZRY7$-K+GV!%{>NN@oZA5WHAeii?Uuhs|`0 z?Q;G7Que5o!%`nI;EZ;Pk*oVAvCKI2h_bj9?WB1?XervLelHLibYY3Z9s zii9^UyicaA*zKat{Dp2eu^q+Fj_{qmiU(PWd!n_qivW}*+0N_f8Wl{A)(feAyH5X1 zEFV7(TWgg@4op6wjcCL2YGhECUkKT(DY*J|wZ(6+__{d&$I=mLj?|;y*L-h7L1tgP zreIA0iMDg43oCe)&fUUw2&-Or#tWilbUkbJixk1cI`?deZ;+ zOvMl+0_%LAYs)TTgjY?=g)*!}`Ygw>U=jq05ZoY(ShgD*3qL+n6gaLfeZ{Bq7_!;i z7jaIFG)n55a{3mp5~bRfG4gZ>upOoA>qlhGDZ|4pj~V1G8?jtn^1cxW7EzZlB!+e@ zii0f_^CBn1QBOJR37K$)S*CL5o%)U{wC_ySgl6Jn>Eq&uKm(#?p&}HhE~&X*aAVjN zfF=Pz-(Ed2Swa?)4~7YyEKtw!@h2xF8Qf~TVh)N8g^pS8P}5nQny!n4Bx*VUzm z1W<6N9i99ckYhD|qYXX!uu5XW5$+K=YDKjPY9OnlIwp+Dr>NkpR0p_YN^OHcQu26f zK_E0BBU(qEgA12SePGiRjUnpqRf9(6_CWv>*!Ec_ZVU-`5lo&AyZ>A~i~ z-3=W;l3A?->s?zv;F!90LWUdumV4Q*VS3-sw5T%Dix1sYk8I>QK-S>&_XU?1X;qko z3MgS{0VmkRs4KVpKoI!Bv3%URiMO}+s%rL|0(Ub83-Xru809xIF@!wuwTY`()ll>;el|4f;bSj8RJ6FOydJ1d{=tF-2F=*1kz+-k(wUB4KW<1k+9XGbgG1r1WR1tfXhsO%+hziF;y%qM zBq*b_ayd9=G6UFrAm#wn0M0+9b!&xoGpXXT~czC{^ri!guEwyO}}+Nx;czDr;@+Fd@{Q+r--^ zv5l_gSo{=g&4ylN^=mEPNij%!aCY%OaYptW)|P`FQ-82NkPVf zqeX4t?eeB^KMYCe8JzI)?P2kSQPIlM$4cTvSELd8PSk+2xMq&MJY$r|ps2m1R|X78 z(seu9m_#UDO^6qGwK$JWeDSf>gK zpfUaNdwdQL9Mfu3c$m_puN{j%(Vf=|0?FXb`$yE5vnC=Jq<)Xnlt(AcRCmlyA$_VX$IquLo(uEKT+YwC~>dc)H)pG zXWO2b`y3)a0@=!0)< z|1;dm1Z5Q2_6vg@d{l|RxA~x~*s_7!C?IYV#j)D=KKn`L~-NMhxluzoJispI>`F|MXv?PzYB%>h$2AeS*c{S`Q*fpnzNhtX6(FP(N*~ zQ9l1f?=)awsm|7L&$J7xD1Z*D`#Ts}7bQYJVlF@k!$x+TlU0iGy{qb(t)KY$;rq6X@+y>0tUe4?l6p@5TN zNVo;(1h+9nY{J6oGp`q!s0s#z4}YEGY&8Lu=!28PrT-WAvQ0(BkSeKT^GC*(l-5JN z2Nbta2t}y6rTQyPa#2byrQU?b6sgQ`&odx6bRL&ZS!q0bT;Df2{ee&mTaTK^blEl= zxLlJr6aHJ>jqN7ld0Z8vHg=$d9(ymaQw4Nqr?m)PES=+YvJ+$NrL=xS-|jydL%lLJ3nR&h*-Sh2R5wgia>@ zCUuMV?(-pN@rTAUAcSwt9fIsYX7JX_CJcA2INy;9^t+1G*B}9HPI*X~N__XY98i(X za<44wdPCBXmVQ8q*jxazo#=7Ts>&ii#7FP*(m(ql?Dc$14wOm4{q40)y>j^vI>Z44 zIMMiI2D-=RQoM7blxgR0mx?hvch*T|X+$*cP9{l`K-KO4&;9<2?@_}gkbML@ z8D)AcvIBN$F_)HrKA0R=_))*d$+Fz@TUC^50@A-Stzi$%u&v5Vjtm>cHta)d6W~4O zIGy$DU&Ong0=a9VSu;}A4S?=rKnL+5f@KIJ1>Gqik$@g+fzx>zh}^%Sos(mR)#QQx zhpfDNeXq2rO-8eDATvQ-ImGiMwznL@1d?!%kGdvO=eW*kwO^l+o`{k!&5{I$!})RUN|d!TUW5je8Uy0B z(BAU<5xQ7i7js{07r>uA=g9vih#^NL1XwZ*ZXhzB_7$z&6=Fp6x%NL<`x)S`A5gsQ zY+hZDpzaQ{L1V^tLATH4I;e~T<*GuU82N==r5wEga!QVG=}no1N7egs{)5#aPc`ca zKdFCGy_b^-lP$iJF6JF{>};Jt4dqy-hwL{6OWsce%f)_`*8&Q3WznpTJSU7{K|Mg%i{m{hkS{UkSC}wDd)^lxgt#~$5 zbf6Y(H>`awu_mXcMT%~cD1_#@0jF;sa9bI-ixEJzUOGhLUh5%p;S=Z%lKxgYl6OQm z!RV0KV)O}FBCzaPC#y;mbTf<1N_=D4#8Ox;PObfzn*`6e>b+rpTwO+>@y}zV_V<_c z??{3FQLmlJWec8%gTJ$HlFUY*u>2R^{{9oA>IC&CHr+Th97l9K18e4N+jm@6u3#ps z(`Vm7{v9OdBXYe#g?#c8l+YP_hy3qQVyz1l4^Ed&dm8T*N*KaTgB@yQ(~BZ4-}hDm z3i>-bG%sa(6D9;xFU-6mTZuh4e$$s&TYhK2M1PNHpF!d_aLnm+m1wcG276ADm8O9W z*PYUkeu1@DSp^X&fU=|%TP%j`VB38}zDI7Y*2j4{nPw0Oy2lo9K2-Zd(2CHDNpYbL zGh)V~wjNG&+@VrygZB2ZTKoafbp61;!I#L&E1s|{sdh?gYu^f2RStNKhSVZ8C9Xpp z!`g`B=2~gdAyE9zEg$@*x3O3p42Q)!N(M&3E55tr>VU2=I-kO-*}$Ws2~mkaoGvE~ z%%!>ia7Nsj&K%`84tShnie@E9BLYJ!L2eM@)S-gY=`g>BWa1h?lh(%64O7J&g;3!_ zDq5g|0v;((=}V-SfEX%HkbCv5XqiuWnS!qHN{LE7!;kttzF#@Dr{cP#sdO`a#; zh(rh|Hz+GQP#(Utfnxt_>1}=kir+Du?QHm-8);tfcH@RIhvZtiUqNxR0A1VJ-*^X( zm8^N}l}Pw5zlt+k2;Kox?J4dt?DLld53bGIK!dA;VD}LwFYOD}&lCKtkUT!~s9^GV zi+CwxVv}wAN*4>q(F`x_U8y@1J$8ze0u0BWW;LW13FUDOW=>2;r3uw<%R#K*073hF z%VU$YmDV$Z1h@8^pi8Fuz~ApVr7v~%yNsM#OkiwL_j|RaYiLP@=*h#>SZToZ$cgqS zo8p3=6^d8uZWte=!x{LopC~CeeuL4E7ml-7G-Ws7Q*J0PS&+!oGP&yZ+tFmq0uUe( zt9}<-)(Br5uW;{_k+xt|E;nypG@YOibl%?Ha#ZSg|LM2#vQKsT?-*d!q1zVeSsAZs z|DuwjGYAYCz!)7hl2!ldNZOyX)4@UK43%rke2JR2K8pE#ILp3`?ILdW7dFMO-TrEN za6npUWOP~Htu|Jzv3fXGZzJ1A{~_=1rKR>C{Bp_{48?sz^~{K?i<%UDvSXlacLU@| zYXl_t>qeINZMHjwew=_q@@umheJR77o@LPKNW-Lrnpyu9>Mn5gsa(VP56R8R8pmQ) z&V~6x>!d4UwT!+JxaUi60?WI$;BaQyD2#U0+~P7qYo(VIQgadQdus$q1ncU1MxW8@ zVmPS^1(4aRo~;tso={UB@zP6^B8xc+;%$&#N{nGOURT|_%7>eC12h7f>zo|{Hd-Ul zCZvN~{CF}Is?g3U0?jEVYrGT^Uw;Fg?d}F`^tH4CWqpr~%nMWW{JT{dl35ZPe=QZh zHTAzS_r37OYv*p0HG)|pI{5DT*Z0nq)j&bnWWym)rZ!GWF}qz_5Wd;cC&9bBPk*t| z|84VQ(}1_}aS1}n$bOm&@;v!Y(&z?Tm<~kD(FKSPfXH_)fFa8UD2Alg%}(`c9P4j) z8*VY3|BR0bwno(Jf&z^C5b;0xK6^&tL()w)WF{NrlT3A9v!2xgAAzWHjTqFlC-0BR zcZpR<#0PFlR>mOI)$}@XmytuRd%TSROSs>^ChkQX^@T2~_Ec!2vFER%Culr4j_f$k zqSVVo1X|uqPY;b;o@-zlf1;M2ivfaz=-`q3aL{&Dn{W?RZFZ5DE*p8Tw6iV=Rr#A5 zkI#uBf<%Al?{19yj5Bz!F5%uy2SFFxK0)ogRm{L6g}t{C*VD3{9(1!*SOZ>0=026H zEMBbge*Xukp4L~+dTQ3Y_o!FylJ6Kc*}jtH0NdJgioJ>1q6k z3*h&Vj5O$2V%YA8DN*>GnSqTc5xC3+x?o%FL;GdywHdN%D#b0Wz6}p=i`z})z&7)#W%L)P=$2yJ$?dcHJiKhq)6il^CF4`j zV-$P0Me{~(mjOu5SVO&umoOlSPD(%VIRjgSPTjdJ#g=N@`i+9IiTOL_NkOm7%b7%+ z1g2F2P9jP~fJgB_2WA`fc`VxO=I9RwZ%-OCB%)mjAiD`GiT>UAF>ow&!6yJ2As{cH zoeg68p%TSxqPfQHIkzbRe0O**DgRalbqC@_)W}l)(@1V_R7~Wg=rJz4r!1rW_uYm* z6(Q-1Q<@_|{1x%<<XUoMjZw{V}>BH3d(}3i`WK!EG2m*qjQrI<)dbcTc!7UCD0wW6-Q^jO)H(4b#{mQGui=V|6+5nlrEuz9-jBthc*-5=gD&K zNDuf%*8y|}?FvF`O-f%)YniS9+(TV3F*$TxsavMuLE_p&e2@s7Pg#<&EO)ty5nsJx zZwm#IQdEKVC~BZG%jN=;x}eb0HmqMG_!8(uuaH%n7N6L+4a^}97GV1`rxt^@gqpT9 zbxmEqOPuaT+;3}McLJHO9z6|p4Ze*mr49yvbyO-qJNw4j74)gPHIIIasr(0qZg=&ytEaZCFv@;ZaWHcvS!`Xf zFCJI(>+zX_bAZ6rXoPC_iawVX&;5O}d*S{GK&AZhExqP{KmkK7mJurv2ZIPjH}AlL zUL|wK$Lagosmdtf&!9T@DAFCTMZl|xDc2*ZGQm@1H79|EN6)0yXOGa^xfxSa_Dfvt z(oN)&*6o4?<2ML~e>F{LZLBETt=?GTdFq$SRk)A?qD7A3cPq;@8>GbS7!>RjTyQ={ z4g688VkF4OOL)p>sjmKoskpEyfEQAGNqS$2NRtxMVNfwsh%7&;nqj{m)2d%zNCYm$ z>D|^p!e!DI(5fVc`vTI0Oc-RWR6KJG^0~i*z`qJO`}cnVUd@|)74E}c`6sNB&*8Hg z;nx!~^!Wqt`jDXEHF*wn@2-Hai??NrxXKsH=G_xY;~qC(68=*@$+k|FC;s%VQLY_d z`I0J=HdN_rw_*b|;NWOG*;C?YQ!pmu$;f^bYfsHD*!604Nz<)HU#CqjMp+jwx;TM1(dNNkR|4d g5u|@Vtu5S(f1(RY$iCP#1*3;3$f!z}!HxX?52@QCRR910 literal 15170 zcmZ8|by!s2_w|4vARS7B4Bg!yKxs+o?(URMC8VWGKtPaA>Fx&U?(Xh-58wB{-#o-K z_s*Pq?mm0(v-aBShJ2QjKz&2-1_FVgev%Ybgg{^n!LJh{EI1POHx3*8gK<)n5QdZt z6YYXGNcNH%P7nw(IDw1}14;Rb2M!`Qf07kLm_tT`A*ApMNvMH9-a|f#epGgwJ4|!+ z!kL`w85z7vSn@0U^^6{&N2ywQcGk@2)-b4Ux#34^KWD_> z7-N#h-mwR!_k;I6Kl*;a%}}IP-`_vAZ4Vk(4|XIk3y$!PxTAGdFu*|k&P6)lAdp`0 z%Ld-bfNO)f{hzmxWC-K}mQ@DQs-Ia?twLzECxdEJ&DB&TYc^RKpc1B{sF{Unxmgq!|GZ{sCFMdo;Q%G@rF%%SkQ*)c7{gQXXJh6eWdGPL*#?t8NYGvD&1&Ip4PO# z!h}w2J|zlbcqTS<+LSeqZBy#mFg(OrF-O%#P@vZ{o+T&Oj?h}8MT-NY~U&=9772SxNqD)O>7b#?H zZGB~fi}~o=Rq12_qJ`vwDJRw6u?(zX{rt%rD36DY&`$_3iz6T0*6t_`!h+>~yq znTT)Bt{(lhDSTPoQKy->m77yrFo9JvR;E7Q3_OyAv`kyr(h0xqo?g7Kvvf`2s+_h9 z^uUJvpsIB7eO$E=jBWg#F6UxpCD+uevWBek+L!G#Z7E^^*x%lCPd1{4SSP1)4~dLE zD}fPN@9VEu5PLL1w$gd1?bcidj;)Eu=DQEzqmJ(awcnP8PzJ!!!+Itvhl0;-_vo{e zSn#ADu#Rc%y_ipzEmfY>MzV>^*#8_=i;w8w4qiK&PwBD4LS6&Us<~ukQ{nmRM<$eR zDnI&)!aH=^Qq-F!J$&g>{oZa~$Gd{rKaf@& ziEMegIjKD55279$g4y&Ed@5It2}z|q3nY+m{&(I+W|KfE$c6LKC#2f)+JUloc;^jR zke`WzTjSbBaS~aDi=Mw0GOR$j00TSAN0gUzjY%P5Dj8De(T^$#$G2%`tLuUM^UE`I zPMVjpKFYpr9$hVs(MQdw;F}EOpFrSYOBU0P6o&9Zr&-vw1Jhe$SU=l5t|@y|q2ws! zrRI9?;|eY8>k_E-21gm)%(P^}v`?;585x`GbKhgPiW%rQelagTJ@_w=AB=xa zW~J9@V8FoScAsOjsZE~eT}cZ3Gi~(m2x7ISGdN6XW1>x-B{1>^_=?%h(8v8D)Do9t zl!2d?!&`IOm^$Foiz|z|w-vJsYnIo#dCb-2T*BXLCI%$7P+v>{i({53xpiqwS>@Df z^DKsle;7THjVny$XN6fDx4`xwtK8O=IcNhi`S!7;B7|DM#$^ zY%r~2NoxYpx#q%>s!npT8v7U516VH7r?}epDJ%51RU2UN;jmOu_HOcIPoF)R;SA$T ziJ_&6YDeMk!W4RVUM9>-Fl;c|(Vj+6ymREc^*;hx_h`18m5_~Ql`rt8o4GdVu60#r zTO&6hJnULwHNJ`$_}d~jsSoA#!A?XcFYyu1wO&0&tbJeoJ=OCMCb-~Fd~72wR#Rcg zzkF+Zx)D+*lO*GgP)Yf?9(yN`xj*|O8pHTXyvc=&MmQh{FL^%5zkXPpQp zWj*8BmR9?_ah%P#>wC@NyNm*t2I&Q^oW^~|%dcSFFcnkBKteB^QXV0GO$$2}?#0l? z_~bQ>yDQxuM+UX5Jz6d=cJ7f^X+RP58#t9zKAh|Q=U%u9T{@pn-~Ti{{@JLg~7*yJypIX?$Is@o1^TdSo_7QFHzgGXge=4CL^}SP6E~r z_JX8Jy*MH7FdnI!3b)U*M*8rBq2VAY?$h=D*>Um}X$#1vBoQskN_(S+mlkt_+)m5u zH6`x@I8sstY3Drj2Gcq=pQ8ewW?pK$=4Nn#lv)7f6rjhnx#CJ7Cm3tWiRac@)e6BpRp6xAI z8P2uhAE^A;|CzE|Gu&L<_h-0KL%XaV*2zhBxCCbP*=^}>_k2y39m9wb$sD7-G;U(L zGN=EGc|u84KQ+7}B&2TS?5wOqzU;DM(bZ<7)e2vUCS(BKRiO9*VKb1B*Q1{8dN1{mA-TF)F(B~=D zC(GeGGi*}Q3fD)g%Z$>k#-YO@5ll<>&WIYZC7DYtWyEAiN4%-@fTx+@7JVC*aPHl2 zut;__a-v{cC~my2OmVtlCtnm_z`Y-dwk5w|Cky?Bu&!D>q$`1VTj{bKi-dliMWXFH zZfN06f7~Qw^}K}^5ADK6Lf6^juhJ930XIO(U;t9a`8W2|zX_&C;lZ7c%`^OF_ag@$ z4zU^yoX{fi@R&y@661t2f z3_Sb!lMkOZ{@ii~GMIjNOm#)g*^5lKMLxM>RR=zyAClTc5|F>}khI z!-{cmcCj9I;|aa1IAvnd<0LyG|9H!e!G?Z&MCLr%S$_Db)LHkmo`D+|hyB zbIXodQ08f<`r9pnRF~Vzph!sd=iT|F6b&S!Q|E@ttew=4@4QTW?GuEwg8)TAJ53>EBjRO)YJRW z4A;2l&(qEC12M1)BXx;gxm!%|UpB>0j-+*zn@z~tz-8Jj;A9ofUzMoids{6hDw|bp zSTiokA>NidPB;3^0yM&FucAT?fD}Ky&u#S@wUehN0p&U5%4MCBe8r5xCkmkoSn8yS@$ysBSar-*r0&zGW& z(e~rYf}GW@9ZFNl(!>hoWj&qwk=sbQ$R8IpV^|bHl!1eM{u8mjNG}~2UqwdMMFIz< znv%}tFTsK|Go6fP=4@8e5$+E9&1Rv~HzB&~ksL(J3Qr=IMg-Ib@47weDYu=aC4Bw_ zcP`2wWn<756~#kwIx6@IOGXzu|Jd1|1`ZeKy=xY|c_LphTqeGr!+1ddDpL!+!u`FP zNh>AFTZ_g&P!Zp~w_NZ%r%@L+!axzwD^_9k6?Xy4=bQL~d%kCR^F`W2lp?%XdG-Y6 zD+|8R&%m$cuF}`msj z;0L2aDA3%sf=rt8XK1VVcTCi&J{$@4c{aYn#Z9vvetU;T#HLyQ-h63jb=oye>Yv6X zfVhvDXSlXa*kqkLYc8G}jL6@%#wYZT$Vbg>rEH`IZ2RcP661CT2!>?06E)}bwg`(* z-lzpdUK6oTQUmBg)VlVq<}`;$Hax^-qL~gS`pRBE+yI~6iV|&)!BBJIlfwd~tr~)! zyUoeh65eU9sp?9{>8-30fb16V!PZM>z7(2gB65N-YtHWYV#V!76nsXd?;A&0-01lY zNdyT`{9hw8!$wUD-WpF!TI{}mvL3@>%pv01V1h#FJ-JprQbv9!`;0SRrp+Dx@JZov z5_#$oZUJy<2f^?-^m8EU&@*S9B#E(vAdfLDPf+6&yO$QN;*O1toRgIoixBt988tS^ z(Nzk8(c(RP#zFq#NCuKB+!C?3ggiSf#JD{f6D_?#dhzl#bzH~?_Yc!8DR}4Gl+X7S z7lh5Nal#ZWybnQOqd^O#fbE6D6aHOuY%GCz`lI$dCo2ufPGdW>U&!*xN4YkqTfK)^ zF`$0oc?@Ks3q2nWKN%D?yo68Frf0$1g{T4xO-QR>!@P9bf*zvA;arLTgt)UD*&6uM z0yd@HKjbT@x_lERMDk(3iZ~0?_yVLME0%Fs^dP3LZ)uOq57XPpvJ$|RW*2oT=`!~u z5z@@`Y~VDy-43f1_O(>=okMmdEjh5j&sbU5g|9iy_#p9X;F@@rS%a6ov1ZQ27F6+apFuO1@1gC9t? z(Pw2dZx-Ze3XCX6Q#yMx0Ydm^l)}i^Ue7@EP$9YfL0G!mjRKw)mRx1^hBR2lPq!9} zdSpFgfC#T?zt|>jkrs(zAk;kir;9qTI4fl{+ezraL+*MDAsdVbCARG1K<9agyd}|J z{%;XgkLNd!S9Y2m7*wBP?OiX3BMZ1cDkztYGiC?K`1!A^yDo{ALG3G|I$Yc<*i^0L zv+xoq3CcZw&m;D1iB>R3ll3|BeCU7^359xpYRb>tzcs!Vl@QDjU}o;P+Ec;3q&*P= zOoG6c`Fe=$2(>3gD-U4C5Xm-f+>N25sBFNbei+zB@1~Mg9AeGeRWi%a0c+-1K`T=0 zL?IvJ`Sq2w<|{}tUN;fHd}nmP_gA1&>bdP^#$E(v8W<>_Z7QxR-c^7(vx|ITO`jBa z!-lztWzDwHgaQpUFt;CuaQzobM6$jkaswi8HUI<5A-yW*6J8SJ8|suZ%G<5ig9Yr9 ziGSmDR()^5etWy8@(F-;V_17h^V7rV8Vq4MPP20Y%-@J7F&^zNvmYj_yyaLvY9X@u zHuOG+(nZ(W#M-o529c{2EJdcj{r*4EYoFN@DbbLr2po!me7=dDoQcn~pU32rA{#-_(FN0?#&gMmWc zdO0N31kXq{s4nX#k8~Y%XckUiHl_az8~}6&EA}0%%>sQ^?G|lRe#GmETw3c_BPqT& zL0M^?b-yTLT76sAp3r;YjA3W|%ObBNuW$4UeE0WcF@(+Y<5cNr<}|o2F~exf5ufkz zQXwv6xP7g_@FA}@4`5NW`oEwJ`U09bPIIKGOGfs-`Y0l=B=41)THD>=0woD-zo;A$%b2#5p! z@Pwte-uwrESY2TdQaM~wJ&TVTyblQohTVK(LBy-j&o#2=8I5}arOF!-mHMlR;eH(! zM1X&WadxF|T;elRFvg6ha=CIilL!rn{C>T{aI}rQLaeC;WJLcuI#+TPU}d_I?Cu91 z^`Q{E_8f~!{IS2u534xGA@ZjGjTDpGmH*APl5Z1*u&wU><}`Uzpz}J2R^-3mM-O`V z?X9AakHLjbzfWub#p|ncHBd$gjoYMf4X|}~^T>L?4$VahmHj}~3Xg#%BXgDhc}_LA zX0J|!JA_Jf*2JAt*2;7^`gJg$Y!i}jm@(_jDOjsvk<8{=kRGc{{yN7=D3<=WhKV4%flr*9Iv!2lDlLQZ) zw;bqKFh7t!L~^F<&s3hLDHI_3)*ZKcl}uVzX%;gK(BXuZki#w{Z*6VCRDM&;kCUXp zO9j?GGRf+QKS`b;B_;E^y72JU)z#I|(6F(wF)|__s0e$<7#b0A$S3|II=b9py*oEI z7sk!jhHHMI&dIKRmnvU5lnJl~+pT2>(cKerudqQrACZ2d&dI3>U{*!kGrUqD4-E}% zetx{OEhzMGA`;e>rHK0ZatNDRZZL_J914YPf_;hJ z2Fk?c(E-`4)v-L1Po7)hB|k6}?JV7(tC@m60bPdq1qE}}Hd?l}C8E$wixdpun#4mh zjRb>w`BRS6p^H}4YoTRsV3NeG4;`4zX=|M6xjSRkjxV$^Z}F|SGqitTpymFkVPZl# zB{xTrw%rHhFCRp9D!@;@4>%$C_)=?L$-S@jGn_8b;dedytImKG0tX9HVMh#skxpiF zlGQlQ%eWYh^>s#$&cC(1j(;Ln&OVGvkmls(wts{C2r)R4LaZpjS`@GS#1VjDfY#MZ zVmsJoAu26B&>cZoSy{gsM|^C>_=_W&XLl2EQ@s;w}4PD3_L& zK7RZtl~WPBml+I5mBoonvVe`h&Kug|h2(XySM~=p>)BES==SGHxJP|7$ zsOBKiIkU5{<}~;t@&6IlNZ?GCxxYTH(JXFyV)G`#{AHoZl<2gMEZ|YN*x>4WIjM#h z_2C~(y+jgGTa;evFEFMd`o4q}Ke41sbDz~c~ z>%5Manldvp!Gy$0K{z_O(b&GUL*k%ad$lSH$sjd`<-}VRb{70Ad3)X9L@7fxKAJ5N zDB^e3i3`4cd$tKiC6B~oGuBA()Cv*BA|`&k7?*#(S&tHWx*_ji!Syx^%_>sezh3su zNJ`o{I*N=05i5$J)LBY;A9^D5$3Ns{8 z))f;IQ(-wt7a`uk%;kC$VSmhP&>j#>>N8`_HP0?g8!vTzIIs5kGb94sG;;K0hXq<_ z)?dwHxj&WLqlOw&rj@|2inZ%1p5d~_sk?i6mOYP~u4H^a<4|ISRNE~_+WrqCWL0q- z=P&9}%uIw~fF2lpJ=8$ba6~IUK0aB7ieE^rSatc!w2tvFENq%kLk*XOS_f<@nX>b{ z`uh5_Gsg>z-hjgFn(AU6kLQ*OU0rvtDtneJNj+;@z1LgR=jD~=DzdT>k8G?S7dw7_ ze*Q9wD_#M!m1bdGE)g#HC_#;l|2boUAXuI>I~hI%@J8Lv^u@ zMjAAVr@Fhv`Adn|O;L886kSR`$;qh}PF{I=n3&`YnTn|S%hWRE%UY;j9rTB>m`MJB zo>S2YkSe_i|-EO0CRbGfar^SvDR9JZ@HtAd;yTKya2_Ai(U#N_U%TU5JC&C$xwOTtOkogM+47`4z>NI#+@}Z@L%SVa~kIno*2379Fq#&L{tkxi!SZ2D3YYm ztTc^C!{xVJd38I7$oGN!r0W2*TeAPdx<4*IZt1zq!p)xHZbbS=eH2=+!>G7hbVu0n zE`+xHQ9k!Ey-~IB^5g`s({pfm_(=K_GFToTI#iX?ts*-X?xlAQ=M2yIX-{;xeX632 z`@5b}6j*wCdc~?%Bh2mw2CKCW>#ynEGHD753f_#ueMPe;vla9*1O1<(V#AK)DB_!& zS!*uObb)BexReyB*R)NcTCY?;f7btV{B0}=pa4<3$@4>Tr`R@A=iYg#oUzu{51u4+ zeZIIuC&7N51f=U?COz%Xr= zg=UW8?Crc&m1YzXFB?q>9GUU;(V`0zmI+?>_&B!rYa?u?9{;<@?bvyFbskfP?NfwH z)h4MT(8}cZcYG!h@b~fYF)W`U$tjz)MnEIN6Y%(}%|xo70p;YZDk&L$`c6(mgXsNw zDZr|&Y%n@#{ei575~0WGkM5Q*R#O$DPU`G9_x1UCWy|T96c+?S(EFkVz}U)H`cqL^ zSr`2kCX5H&Ja1e8cN?V;_~^=HukbTH-8rMt$q7b+`MXzd&w>P z&pnEmgv5LuI>yVc4cb4OK~g1*nUt}OBeWz|M*Jklpin-VWPZ5V9k)oSezw@A6K#4p z8_=c-G+KPqZ0hRnep^`DY{^03dGVx}&%wrqrJ?=s}-Z|zd zCdb7E*s^Q42IQ?HP%)h*kKJZMClIig7PM-FwDA)*(*N!EIJSL-f7$31u5iQ@-i2*$ z^u-pcl{TOC)0+(@ktGfUE;S1rQ#A&eXnbq?_wQes`}x+|nqi3On9`qC)5-`!ZaKPm z!}Zct*ua3<=5gaGT~~YF4z{0A>x_z!dBvI4S1=&9zM%_B%F7>j&Cw;8Fun^@%ny%< z*p$)_4hp(5XC47)c5Q8q-E7#{%xrsmTbd$9_rP;dHn2pmW$9_?GfX3r*@~nQ-5-F3 z)Mb@@&EYIEc7L*)$pK$wwp4Utu2E+IXcxTLlEdl6HV+REkB>>;z3UzlM5Ly5a&-Ky zXaNARmX?-=M)Hs;d$Nr7q*t=c6u(5PH5apb*@t}P_IN*;+IRQ+Xs3tyOR;yCmEz%e zm?R|Wv9aS}6=(}aeUDb<>J0C~{&Tg?W<1(m2 zBJgze5i3iR@SQA`G=pVsWcEDT64q??&xMLur!EDK-Uk$E*C|rP&MMiBk*oroLxJ@P zETkwjUdjZvHWnJ)s}OHW6Pf~WYO7iFtKQD{PFb{RdPsZp=OL5MOH$iIjn~yUHut$+ z+e*pLNB2hE6-DCN8ciNmY5acZ#I-hUx$+s2|H}|)Ttshcv1oC4w_Rf>513V2oDQMh zA)}^_-xWnAB=o#5Cjkb;3&ITRx?b|Mi&7K<{1U!ClIM}~&W0SWGJe5rwc_u%5TYOH zgJN$0c94{m+)*ItdT$w^&E$JAD#7ZAlx#2Ns5Ly1I&3OZ*xAoY(wD}J!La##ywmxN z^KPwGI%jOMP%R3N)o{mdhW=exBC{UJ+qdulE7U>}2>7%^0Jll;_xCuwJ8bgGfF*xt z=KFwu9FH?bQIu3vXd2w<(j+<+#(kY3*e{$6?&fj*H{w^=OSC%7y;h7X^W+}QEEidZ z9~CAFR0`MD*MnD%mQXqXq@A~G@!lFt{)gBb5T1&S0oT*T=p zaEB{b-hIUTpY7Y8v&6!@yuDoyW=5x`y4~@=(O?8%*x4ORkhpjD^dJzxB-=YA^9I@f9_-cyb1#Crj-D|;d@m2?1~n=o0u>#7WN`4|;d~@DH5KfNegZk#yn{0{GjHitZ|?7% zZZThOSK29WH7P-P!@ zO@~tA6aM##kIxz31%*WR1F|hSTjAv1(NX%(pZ@`*#cI@jQpHGy0g72fGRi|IqJSS3 zi(*7ya_<1aVl^OkwlvvtkZqQ!O!r4FjnAMR?pqtEPG3NKEVcaEK$FKQi_=~!3ySPG z5)x9S=}>4$hyo4H!aKq5Y=Da(0uq6jwmQEYw$rMA2;LV80k;wKnhU49i`|8a!ok78 z#>U2nhlik`*Cnc`*BjWvMXK~dIZ_HZ*C$RXP^=IzXQ>=1Il0KtQ1Ew4Y8^2L1}qz7 z+urUV4Jy@%b#=5e-;mbMLv18XEbv`VMLi=jD{tfFWi0%C$kYm$uef+}|8jw&K(%mF zsRaPq#6;W<7P@$;VpUIfcMA@J)dQq!+aIrc;dq$A-s(#&;O>unrp5o7>dy0_;*Uq0 zzy>gU_NX~WN_wP6tE)ZdkHA^IpS;x1Y?gW6O-lkBFW( z-SIzTLENxK!hqVz_AfpX8Op#`_165lY^OQUWw5PEogmExVC3915{{vy&3*>O>?^5= z@`O@xF(v1kFXo73uT#sjyyp#4vWzC8S%^UdzX>LcJU=wP28>6PEi?VUw`_-X;Xp!n zTQ`peC`5Zx<*zi^lUe-BCTU}T4Y*(C9kYmC_xN$0W0bJF#ix1TY@}HHRfGR38kMN0 zU(15)c&95pib{0BNr~8B_>U5p%Op^XImid9*K9|hb!I-6RZKzK2+r{+WC5}D3mY|8 z9C^GP8%V)Wxd1(2kLKg3*WBq6I_-uB#{+|NA|+TT0$7;0xje< zleoQJVk-qgGdi(1>gXS+ev*D0ay#_T(T;RUV%unm74(yZ00f67K@jN#$O8lQ*RR3w z|B{=&Al$grOgaFnt(HSPbr&#tPxiUXl(@~!bLt>TFZOF1@U3vXw_Ae}+5Bf~9>w#p z*$hZhCVL0KianeBXZAN4&;rY(&##LRR%C18d?yap=M(Sb8cqa%70L@5{7ptAn`Nwf z8$|aqt=FW*IqpC-#HZvvEi{=*fb85=tg5pNMOz09P2~5EYba*0{ZJlD-y2|!SHZ&t zKz)!lVY*Shm0`hXU5Q_NJ^;1V#;l8Lv33P*;LYX^fjok8J}F|>eu2}36GD&r-22ZR ztl96^@ODXj{lf?cBObW)*fa?WP{glce}wpFcdK7Z94-O&bgMdj*G1~gH)ojvy+uJd zqbbcMhyRB3T(yu%$CD2H=&9YDysgK=jpPX+bP~-c7mY()ME*c^M~rMF z@Y&bRY_lYG&Ux#rg}#3%L>KTkZK17(ZFlsMK=9Wtl2FE1s4F)R$b7I-NAy;dDDhs; zgH>LFdV=$U5t6*47tS?){Z`0ZV0|Fm6AJg;5connc=A@*d$h@9l#&snc}%0F}J zfYCd0;TF>8^7W6e5{$hCD#vxYa!Vj@AQUcHee(%%R?F4T_0w0-#9dVUZ&cu;a2R8F zeC`ft^@%UT%60sZp5kKPBBgWn);$vv96lC}qca4;glOB-PeIuK(+77I43F?A)=DGe*&r-O#Z8ecMh~~b-eIT8 z>-Kf9kwFnScUz`kIW^|xKR^=Nu)ViJKdJ^UX3sJwvwy~QQ1Vn6hQNAXvNJ!dwz^b4 zqdpJXD?*HSS#1kL_W9unJ_0s{nnm?H5fAE@g>eZFttPmWh{C<&$IBb?CianxVIcgP|mQ{%@4J!J8=vl8;?-ifw&n4jC{YjhKANHs6IzKe5se#BdGbIXxDJ? zSEL9(kb>c!&tIAG2j2pt_Q0{R9vW)7L0E|JzgqxRom_Pq57kZnQOZ2L@%}fqI|w z>9e}p7}DnumDon=y&{jNL8asovzm?n%^v-xal_Zk14?EbdM!%LzMRvnL_&B9@iqUO zoIdDA!dV4l+^&1}1wJ*b<}?^|)%S(p@nyh2W4d)cmeU7*oLb)<(RJ&i1q5s5L7_lz zV$EfoMCoPcmbMI`b`YZfFlr$bns(c7za2lx_{pOq|bg1&A* zh=NPW|8E{*{1s@U;^pIBNDdjD1G0ffR#(b;*TG|Jy1T}4s$59zPZ^xOu;&u~PEteY zXW_66X)j@>7>8ihsXMPJD-Si|3b1tvBi_v- z+^xXz(r4XD38t7*{M#9hj880x_C9uN*Ly8DjE&;8U7)RHFpm7scTwc%7GnJtrjNj?doGy$q%53&7(dp2UgTc<==n8-+}&VZUr8+%hoP@ zggF{0NA%epZRGP2eBOHg8p3&$eDLNDUS6S^%IlN@;H6l`7;%iU+avrdt*6wl1yXT{ z9EtPB-;L83w8cUR%s0GIoxVE%i3Ewo%(KT^C4{#ERWW|QRAtzLag6^IfO&} zLv+s`cGa#Yj{YE4@f{smeSL=c2H8HPdUM&;j;Hn2lD?_EUlW|B4k!!o=Jw5AntPoJ z*^UlT^yXcbvkwa<_S_v%Fw8W5(D6m&f*B^qj_aqX z`cWI*S@Sye_3Au$9s|tb8+SRAtB8L|?FY37%V7_^@?$tesR@qcD&Uc?lYyi&9rgEo ze{s=Umjm+lPbTC>0U!O?djm9!*5(-#RA_wZf0qXkSIMLu}~J?eEMD?7-U27-@Ik@1@@(XjqnfQyrJk`d;7sG5qq#WI9+| zCy$;to4yvv=Ota0OZVu95=@&JiLPPv@dm&ZJp^|@C5rE!|zwocb%Pt4Pq_$gT?kRb7So7dWu;5>x>V zqYo+#q*DjPpicOo=SnWTmZQW5efaJ9hQj9nPS2j*1nrlG9?5H%mlO9o{ros$cQF7B zvY0a?#<+%S1R+T!yjnqFTcy2B+Tn~8TF|{?M43b0LKyx8qVq?s5D=GA93z3ivb?CO zxB{{yd9$Zrp$Er)Dr?x+kj4p&y=3n%Fa1i>~wy81xdn8|rqPR_mD|Kb5FN<94RLr`~h`U1$^sQLqVf zzleo=i|y#0Sr?PH!*XNKPOm7IpZjDGL{I)OLH^W?Nd!$$F?7F9AHKpSVvssIK z(dqA`cS5XdZ2&$0JA^cX{V95Mtg6onXwNrC@NByIql%vY0oIHm`bdeZc4w8R4Btucd9Jjpf{k3=da#_LFB7%FB_b{ zCH2xftvoRXqVF_^Qd+S$)YS|f6?U5~*CJia!sz=EtjrX3QKB`kPN;?-A)!7rN9n=q zNAZX*{}}(hCB^uS$4mM;^|Y?2Dve{UYlHJ2A|#oI8Ts3G3-{9(gUH;(i4F}LO@qAp z=gxH?Q(PON5erQe7oaA}hk<;QxCVCuBvIf2GNs0DMzn?mXxuse1@MT(%4N%{J-Dws zbzHZzAT93Lya*fcY;v!W1$Q~#U;~vVza^?7V_TW{f3>7~W(BxPh9j$_em^Y*9~G}s z)80G)neAbIK=1~X_YNPF5i!6GXB7X+N=ZMZT}*5kXu0W{P1-TtJ?x*622hK|nTt(S zq_&qh^|u1}iuWEbLFeS!u(@|~cy~T{@r)(N#8u_F8Ax)eDplPCMKLcfdL=-eI@$!n zNAROkTp7la1iTh_w#5z)|HxRWTOu&S6#43sJm7PHCf&D| zD8A~bkpD)kZ~C_WYl&~u!tq!`mfdyYeHJ?a)-5dazNY>q=AOIUY|$5>3JCh;?^8H? z{*){a37{%6?DjBHt<0!UUATBEt!%18(W!9|7rma^2TV%8wu&aI2>V@@)AVgc5seZzATB}qGJ)`RFDzsDMn1lLi|9EN%h$J$ zD_*@XV^rgkO_M6KYljK)vFG#Z*2(_z@cpmxK2R`3ai>!RNLpAlQc+0}W71rsD+Sd{ zFD&HP(nSO9sRJeBg{pvvdhF1IwFMZZD%Yb}!xPLhF!Ssbvrxj5;`I3T%j~cm;6i|8 z3}<_4Kp{+JhX%6L`D5MOXb(VGQcIqGo{RlaoJWg(&&)&rj)W{5tk#F=}E!dh2Zg zB6y+!nk9OphtQgA0t;a|`NsnDe;+5+-#L5EPx#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0G-1|aoDCIA2*bxA})RCwC$ zop)dq)z-l8ow7ZxA$>z2y=^wxJ~Q|GV|;5!BiWK1jgF7Ph1$ zRi^w|c=}B7#X?;X0006Y=exJ#$$1!0&*+r~Rk6Mp0Kg#h*PuggJXcN&+m`HmbY$6s zl|o!F(4*hfkblLvMF)!m>pqrT`QU*nttLN{?l!2}oY+nnhEWNl^wSEwy&$mDUh&?cg$H1qHN-yLOQ7g^Mj&Wn77-e!LaT3R>$@+1VnvDiKJwx}FJ z>&xAWq`7xLq8RFv$OX^)41J0vYXU`u`r(ADQ(q4pH#dAb*DthX;|UM|V9%pN@77+RTmH9`1f4aNM#kpIX>>)>T9aA@1qk?T^C0IE1$A^Bvx* z*!ime%m0q}z`FcQGRWf@MRl%*(r^f^-+#DYnHrz={SeQA^E=M85kCMxH@B`3g1gG3 zS{^8e&{ClhJ^Rjo{6&6WcU^2degJ?_NeB+{Ejc@CWx*k|R1Q2oT4_|R?f$Jz`LP!9 zi|GGLR3`7r-Ou$4tt{n+3U{7c%jC5%@w1DiS5L(5lMoX2i{Z4;3esZy5JF3${!1fz z@1r9ojpww`{Mw`tU8*njF*kT-cD@=~BPPV0EwqAV`aEU%J2z&ve7-gLaR@DvQ$^Jrjgxd)2;0RZ{BLJpz%@>rASFZ64z+$Dqr0Hmfg976M@imH~kRCll* zM+P7O00f*Zv_SN^8E$vA}OivZ^v7(;9IC34ic004iXFNe^4apSp` z>C0Oys*qHfLO?u79KazoU;IUWS?Y(amHUN~%Q3E9*uOc1W=bb%c%C-DRa((vT}eiD z_Oo7tID{5SH`gu@z$>NKTR1ucA>5g}pMdyizZW=!7D<$>qW~9ND81T}(HVr0kumOm zba154b3Ba4UgVa=m@n<Zc`Z+j_9RmI|&;OP~=oBW#)}Uk;%~irDg*Zq3;_wf#i* z8tb08AzyP^GYH|ulB+bL<2(nnaPFB^*`Y4&YpOnH3(euTDsA2T#E*{`o^1IS2z6-> zo}A>*HKfgxVtwh%+Y275AH5m$>YVWROxj>;i?}Oyf2gE=Pxo$hZ((+=5cOYg_?Crv z?an>$(#1EGhRTh-zGL~ZcYl4jM@_43XOez>xMx**LQ>`Zdhgn?7OqJD{@0wnY|Or< z$a(Ik2OqI+UP_~C-rYqhm8pX~2Y%J%6AWOi+TBy$UvmGOVtt9C`r#*03vAJ7CuIJ- zS+Qfd&#<>c-f}dALR{#^uV+I687Z@FCGLH6XjOUw4(fW4?Rs6J0wexc;D zuh3^;yM9OWkH73cDoWbXTFoh&%YW}JIa7T8m4Fu)cAV2tTg}2w<_kWLYNes_MZXaXJI-c9 zBnYAPJy-o%c*@&m zT+QRpl9xSDrFD`;Z0z}cpxEE!zd#7(Jhur!;}JjrnYpZ&g?FL!N_^V)h1#O$y@xL8 z{1H1Y-?(rHpQu68&l#apT?k2-O9z&dno^Iv@T#xS=c_KC4fTp;FV=jX{Tl*kM8=8N zrjPb{enQarW@UY%@YK)Q+j6y!gGE7GdM8B6BAVDy;Sl~U3}C)b|H)72J3e3x0HC{D z*P5Kfgm`1G@A}H0zU2ssk#e5f>h$%$q?JSE75c;*_%K<0Z<{yX9Wc8yY*p<>DXZ?2Wy^cVSh@Y^-ljsgI>$fADixw=?aahw;UPs9pnXp0u(S9(erQPgz^klo|nq!M~$@0*o&t1ZF#v001E& zY|6uWTEDJae3Yyc3;tNa$rWioR2i#Z2^cd!Vg?RzVDYXoHAPCg)cdyfhmigB{wzGb zDt%owrJ)fWB=Uc^{iGpYu`M}#3IJdb!-jIQsuiCmf0bFSh;fTv9Q9vcp|>5m2t>=n zORvsalKge9<}m^wP~<;1{QW-ip3U;XtQCNv5hK9)pC*6xN5Nn1`0|=v_*Pl$($*2O z{JuQUGjTi4a1DVIYLv+@e>;d}Z=4>@t>Q>qehdz71JhH!8OA zFfv3Oyej5PIp4h=zqH`g)|?$#)j0y3Z*py#=)cs;t|^5=0C@1_TYD#naG~94UW;lW zgz&TEWv7eJV*pMG`Pbx-33cmt_oIV9XK&L|S_FWUlsxM-gv5x(AwU4`S3bzrJgl1? z50WE%pQ8}n_>0kqni)Dx)HL7K*&EX9<+qdu)!zPFI!eP#dc&&Ak-X#IXZ)yV3=~5J zOM<3^yfxG-u8GO6%lx@6LN*vSC;a^;dNd9ownA8QQpxWF_h1k+DIa_5fhzse z)sH(!f)jeIs2OnCoqJ$a`Z^LLL&QOAx_!+a#lXR9iwHYP!#4L`+gaABCZY7I%oS-r zK#2V-==EtGCULXK+af}Qkcz6hUUutQ@4;PMqiWp^76%RYdG-bGVeDBA)=)kO0jxU- z5poDEF@bMipYceO=O^@Ghe_F1^ylntgFRzIB!Nphe`Lk%0)&1&++yl_pb}i-R0ypZ zo!OOpU~T3`InV9TGuurU7bkyxruaf{_a5KIEVV_a#f-b=J?Ru$xISwW#Tb`%{>W4> zCkjuUEINY$Y#fGR9NG-dOmBSd)nUxlj`1DA65XKqg@nWa&))8>x6vGjuoc3qB8)1C|RcfksnvT z_R)|gS3aN^s=HfPmgw<XFijRvIgL7|(^2008b(+=CG7Bn>xvyVa(H zw>&fmZLqO|{b^yDz5)VF_P?p-_f@GRBzm~VSYKlhf)MI-y6o)i!otGX*jR%2`{i%n zzMYbi(p;r?cXuB?e7FO6Tb3g96tsIgWP)iw6h-wz1~6fg4%V_UrZN58+}!Qkw`XQ% zvR6ow95`?w`^>lBek*N$G{U2q#bT~gfu@H}G5yTZ85tsx2ynABY=Q7{>9zN7&RdhQo~>LhtwjjcIv2`3cnG0_f`Ww$7n-MJT)(iX zqM@{WjIT1P*q?gFfB-PE+uUN4H*k)9Hz<>c>US5HLfwuTQ>xM+l{* zrKP2%ktEqrQjPcL5JD71;W%E`XJHsdEEd1;!V9JaG|nEKfdI~rm}&Ys&VRJ!sskZB zc<>;PSAT1)(P;Yi?Hd#n)JO_^K7YlE6=%<$J#gRvkM~3r4MKSI=uxp)Jbd_Y$MBD= z;8+)Ei*nVEO+VABGA*nZwOW1e-o09S#wu21WaQ?}n`g|J;qUL?h$13{TwGj6j~<UAuObm6bWhTvjV|D&D#_ZQSLF?9Z)=oo`*8_FMi5OKRuh#fuV& zr0&AOK7IPknKQ@apg#3OK?w2r{N>A+*G)|z5UgFh)-k#$R?tFMQiek;Ut5R}av>!` zT#%wlwWM}3GBWD^yFehAHf>t7nm*G)oVj!78VrV-Ull?qCnu+(qQU_b(FzEI#eq1) zAF6X9gdL^f2F93Cm1RkS#l^*SRdCp_VW9a@X?A<8fPjGD;NV(YrePR%_wHTJ76Jf3 zh&YgdL}qn103cEt@kZd-DeWg(lAubZs&z%H)%NMr$80YNAru=MOVdxR`^9lQBO`mD=Xs=0sw%&$d`abo;Kge7)gvQ&=&ps&Z1CB``!IFS?euL%2e+yA0HoIU*E=I zYc!g=-F(g?{YRo*qj(rEQJD+?@DX~636VmbQ(;h=ms;I1JRXmyP$(MG&blOOUSZqV z%*@QXu>!7N_>aWMIunq%U6IKC9O@M(#D%}*|6yh?#kKan_d=oY%9Sfk+)$=or}W&p zb9J?_ZbnXu5FymvJ%&OQix467qVI4cG9Jo1W?q6mK0b9(oj!eFI@q zg?0b2R`ct;2qA<$-C~T4@s2Wy{n^c}ix?N@Y4fwH6=vv7hYlU;M%Qp0|Kf`;nq|FA zlQt3(66!|GX_{`|zCDNVABvr%5je!|Deto@r^f}n$jA7mx1lWsnzOQ=_7>l z^YfQ4U(R+Rny8Kt!ez^rX*8O;57RUq85zkT{73rAdyqeewXM?31O@fJPsr^;B(E<-fCJ!BcArKMfFb}c0( zrL3&1)_5;FqXNhA(9qDBn3%Y@xLT7CIfPBE<+`=DNwNKhjGtC_TNdll-@^23CSF-t zsn_e(YBfSgAQ13)Jef>Jl4QLPJET~7KT-(cCdIEu^8VPJd89u{hILMj0`aB0XY0FwDAtew%Ev>7o7JQsPA?Oth zVrohQ9A(~v10tjtBLaXu{I6cguFiSp^z)x3FXNW%wo!yx)j15Jy&7Jt?J4jO(QTVwA(PuYtRtSI1FI#-5`|- zUy=3$XB1l*A*+9HUY%B9P?8wAtKZMAHAa0K>a-z**7sORG1R_ChtHQ>;z+a_LIe>2p9e}=&(&+3n4@U zT2cKl%%y{^HQtG|5JLE6$_fl%^CM>2{L6xnfCS6$=j?5tBz?&RZmfWCv*K3`tqB$f z4)+<>lCSt@;py`wm#5yC%~3QTAwoz?>o#U@)6x3Hoj+>HdPE4V@3~5V^E0Zl-nl-L zqi7~V2;r*qbtEK*dBufFLR!)$K?q$**`EGegoH3#o%80^ciH8=9Jyw=GYFwlec7lB zuR#d^h}|!H+HkIobWuZVUcWM>imGUh#j4UBuf8 z9~y)rT(qzM)^>b(g+ck!rMJoq6@XL4rXWNJ6=(}j6`eOS##!OhTCoL#5b`kI-hNwx zMS)sM`^x2ssa5G5zo#RF5U$DiQHTrP2zsT}V_j_STl%c)E$=}w^xM~G9L@g&IQ>{Z zYj5`F2Y3$GIxerC{bJdA7$=7;U$ug~1HC1)oF;j>NT&^= zHf9UelxB-!$Hd^*C4{)`*N%k?`5mnxWjCZ06QUVmQ|*G^uH5~xXNF%byWxZu5o^fU zY*PHXGj|`bhZWBVAs50$LP$1b|KfygDOPWd6zYl-GB<6^{>46w0fZxcpF;@Utw>T+ zYR;Vj5W@YBj)-vKaPMdBs3fbpExT5B)ABQD9mFG5Q|cfytilGRMu&=`CXx%2o_V+qW#sKC#6tQ~wfkyW`XtT2W(ZSuh2bQOP=Tur) z&lx!5E=_#<`V4!lh3C8m>uBAnqH|8|kt(Vm=4lJc^cD747$2dxkI-99snabxK+(Pt zUMRi7$9Sm_nf3_@dUzTWeSa$&az0m{(if-Mp1}LtW0u7cP6bW zlS0NwX@p&Wzg=aW4UF-DD%}|oKGNhdi18EI_rZ*OoG&9?G?a$j_0)+F7HW$jguX%_ z2Pjp0NeF=vW}l2VQ-fcf=jNb?+a%Uif_ms(w@$ud) zzpFGO_qhws<&gpaVm$_o@O^H){}==iyZX!C-BBm;OKW?i zj!~+2uFoJb^2eU5oz520h<1Q;lvKv5~=#CgwXkt z%Y2;Q%e}kPAw&pul!oCDKT~|ser+Ly@J#V}08pIA0H>Qf10fvb@r<6)Unsp|$2~&E zc((YGfib+`Gt?R9&JY;USKd>E3->=hVyETVnzK_vhzEPdI>`*g+6WOsZv>8`5&dh< zZhN$XEJ7WlpAzzx^Q_p$Y8jIL?fS?_S&6>vT=7Nwss#w)+Kdetz+zmZ!(2Kz@8UE7 zzz3mI0Dvpg*V$1mAcXR@gyp*lPh}NsN5&=D$(f-h~$exH>)IQt34s&_DHD?Sy+kVGT7+^CCV776)o*&7^D7 z+wzXshw_f?d31SGUkg& zX-D9c@|68ZD-6n4ue`0MH1RQ?4`|n??Yy_rP^nNq?B>=5IK9B9o?c#ItUOV8>W#p0 zZRe#A=Ns%`M#Rd#HZIAQU1AtTI+Fd$8(7zu|r5 zJ-LINKywejyH<8{QPSsXN_}AVqM8*#g4=IzvGZVh=C9e^^6B4-LFg);sORSdPX1P+NHO9_wMdpV_c)z zW)J}c(BFFuCs(F)a*4noW?+mjoFAv7bv%qmBU;ZGg2jQc9{q=T4eBHf2i(b%wnkV> zDvhc?3jR7002ov JPDHLkV1h)u0aE|~ delta 7150 zcmVx{ zK!gyGWg-hE4}pwO78&8UQu_5@Lvw0S_JyN~4MG_Cj)(IG_y7c$p3>DNxzD!K?E(_RpK1tovA*bsX+J2^=MU=4GQv?(I|$($4R3F0+bSY4 zQE8s0pL&~hNs=-#j?jnz^|0>7K+9o&*U@9$CyVs?A`%fmIK*W`I`;4ae9#ecT2;ve zM!em$S&1vIO_()Fl{oO%+k)*!T*u~_XIqr!E80~V;Ry0O(NpoyUfUlpv<5SoGg z;@1({R#!c5qKwZK{ZdV+5I_(>IU*fppNC$n+pzlV8Vq7T$y^=|RsN+q11gq(Rh{xo z;gb#irdR9M0{{TJ(j{eym58E<5b6%~tg3vfeBXEX+V`^p8~IW6CqW49vmacr_m;|@ zvjZP5YhjG_uZwa1^6Ya>!PYmf{^NC)xdbGlQM<|pKgzh{tmk|O-6#M9Inx z`J7VxQf9hFfl)`hPYzJKkrBo~BvpH{`VaPy zDxan=db@dZkPr5IXUM62%t~lr@D=JPrPf#f-<3~3Rq*p`Ow(nAv6lcJ^z#9;!i2$* zGHJZXpX@o4qE1RtCCdn7EFu!O*|yjD8y?JA(YFLMI*Km)!eL2Jp#SMO6i3F55WmRG zvN_PwN!$AxY8m0M^n($9nC3uhkuDz$b9u=M>=i-0JH>KCn+s5!Wc^eWeoCvw4V=*blk*?d`qw` zUpGZY7-PxOW_mcUlXZ<;QX!qT&fj38JFZQbDI<*W;2J4CiIGi#){#ppq|+YnsWedf z6jgHH8%0Jqtie^`s(js)Hrh6#7jlMNd%O1En0UR0Q1`u2yk9JwSQwE&r}Q<9spo{ z{f0{S+256{QI9}&FI@#8B((xd|NiEexd(1K8s_~PdGb!${=kXHw%K;vlDsIAUrba& zIteRsO~Dob00uFM+_z}fzBSm^>+eDUm=y_XA_h}Jw7~3tvF}@7|5m0Z{h{3Za;XgL~dQ(8=u&A?3(`}feqyEzL~!8H`Tvmgiz!=91M$*5oxkDDhPN;J>VA;6^as{k6}W;G*RVq9+RD^8o;-dd~c{ zVPhNJerMWkpWF96U;Wa$(*IT9is4+Sn{_{T?tj|p&L{I9ooSkW#T(+Ai4cQWulQSp zL(z<|S!)cmym0>IhCtJbtb3NG+(O}$kN3ZPaov7@$CopVrEBwEOxL8AC(d79ao_Qt z%6@Ao(ZGWc9_v2&{P~wm3e)q&Pes`eCnpvC4JbeWfvYhVvwLP?U?sLWJGhh7_zT^LjXOT_to0hcX#f+-ZJ+W zd51=JLv@*1>PDgq^PUJh7&M&$Se%QKALuj5UdSZcP4j4>7ql~bhfPxo= zGoGrG-KT$;ak~;%-k*Ih0Du-~6|RC1lp0Ha?nu8)3PgJ?#g_gN;Db9l{?Qm{S(tRg z)y8S*>eK{K$pz z^5U|UxN>XTXB(UUrpA>kvwspzoD!*jLmtlS47uw3jn&?ZU5tYk7zm*XSDF>368+TK zrfYh|u0zX4CRl3!YVBX%Z}~fk6L7$~k`#}GRWUrQ1FTpF06~D|>!v)B|Fa~e1&4T} zzxny)I5OQdS7kLNuSo&p^HN?jS@zl0W;)9$l4#1A=-1O;Z`ktHF~E1-Z73P=zy z?+Xz?(+wq$PI)kr--H;M34@RknbYobIK%^daO2GNdP;Y>Bel3@+2MPCXo2Zf1bHg| zv6-gn9J*}JrxH`fk=lu(d=n4hMQ@$d{hk|_ZB-|$aOGgn5KqPHDSfP}2jfR3z^uLG z!wxzRQy9r3p)fulA5sJSmGck)CLF(sflwOOL(U2kBYDIZ#pe%Dn)w#W#OR&5=&kKw z>x?D6vbdr_Jy&JWgvvhtV;j@K|ZJ=6bX1d18iSSn#R;;Nj+;a!q~?$N4rl3`QQS} zoS#kkshk2%JcK@f&i|MCx7@79qBJ`h*F19>_}BdVw<-0ThzKPU+7xVYgq$}fUiU)v z%dU`XR>Hd!q>w0z!60Jj*t$X_h=>iz2xDnN0RT<@=GLI?YU8xctsj{v1BFol0Me3V ze&|uab1}bZlv4_1gs~JxV9t9lCQzo_xagu8> zN;0PyhV@;i5~X0+gq(&Pk3Skh=;i&D?lXCDIetEW7~q4YhLT>45kj@KwHr2Ua5|j? zK@4+wd)SZ42xBHkTt}6-vM@fcInWXi0$1sadogBN_O;huqiH%)?S>;-tu`ScK{li@ zlRftR7DYmaCVh{6KZPkK$7KTawHoGo@4eUE-5u%Yr08_ueqRj3q^;$`W>yqMlgYGl z<;wJb^mHJ1u*Fa=_-d@`i(bN zLl8t+Sy^#$u|lDUgxcBJxoOiTkH-Thri?o+l=R@?ocqlRlaqDYLtTrLzXL9{!$_a* z+_^KdTNgse^L$oT)=f9vR8mqxQB=RWh~xNwtXZ>ib90XzIkIKT7N5^2ML-ILqNb*1 z^XAPfR;-Yl5M#qnMB<`J<i53aA<006Mj9T>!a z(>1AItJ?JR`2GG!Eo52t_S0*b4<=wY@VN7Z zRi5hBU|T?-d691nv`p4!s&HlGu)18X$mm}bMZI1>b?VeGi6}3$c=6&;C=_XpYin!e zq#_2=5@;hKb(LYNoAWjXnseeNE581Jn1etd5NYlNLCDL?>q`zoNDxHiw#AY=3>#Ig zR-2rh9KIVMgbc$t91c008LlI|Dd~E%GT}PQ+&W)FW1#7#r0bMlf2Kev6pD0LQasYH zO)e`dqiH%ExGEHijEs!%rwAdvUfdsc6N4lc6PrPlKw9RrIDWsf*>2xVaZ3fPwzYaNPrIj z01_0YhjQ-Ajmz#8C92h5dp-t#13(cmTle-Mf1gS0RK3gF!|( zL@NpaXs0`CeD!4sGYO0c38AOYy|}OI5GVF-u&`JxN~JO!p+E>plH9au6VLM!k1IYE z7((dvdUx*J899}R5F$x(B9=+9aiW3%fHu0lA<%fOX(j;)MhtmauUTP#A~CWrgY4|= z$YLgjVO?EaYuB!oCj9yZ8Ce$%1OjWp(f0~(@2_~S77emU5>{-uw4WXG5myJQF zz^L0&ZxZthD0G6h)4&a#}gwv-_zw55M!Wm&?zydEXk@1VvZ?Ry( zg7Wh6UZzz>I5=^!-DlnBGBp`V%0vJ_mFGNx5mVx_2T%*i;+2<|FI~D+r_*s!$mc#etTbaY(2c+uf-^z`)5 zG|h3GMx)VaG-k6oH8nLmJ6ogCC=`lt1zZYm&2K%)YT9|$9pQ90xjV;YkUoVyvP^D=MDLsF*!y?7_Bq~T7Gf* z*MF&hEBsEaubvaRz8(0oj8lZCJyjtwl&#GQD~FA4R}-of)g+nC#6Xy(Nneyy{ypnr z>18Kv_i*06)4Z~b$8bi9DT?()`MNyv(iVL-ro+qmW*TRF^E}o!Pzxc1_5Q}Kw$FIP zg@L7is!~b~CErOdm%$8TB*)5L2}oF5v;Ljt&D-Z~u_(>G^0RM1x;DYIuH zhJIPLGo4kQ^K%l))c98mk-FLL_gX&q*tT7Nxco!xWr<}t6JEU#64;O1x9_zd=wLc& zfuRK&1K2#v?6(@;sOFmhGi!#5@_h)kW~7W$;6jL<)scj1Q0-zwONz3S<)M{K&PwIPP@XebrX;K&;$(5g#H^LFAG}c{=V`I~Q-#%>nI9-!=OY%*(TbGhJ1^U~owpVyr|~b#J1q*cS&<+k84-kkA%T6b<-@-<|Gg#HHdxJzM#XW6FEGz>hFlvO-|1o; zK%S;!ea3xmsr8oYE%Rbt`jS+6nRUtilkcr?o%mzz>$E^eztT*WD~wdw3s!q; z5|w5nWf+M9v~*5RY&tgRP&o?@CfymS@}{Qyi`#g%GyUw%=C2@=3?eNUJw;zeb5GpDuXZpwPeF zyyZl9rEE@P;0ym$`z8U2r74R?I>ZR)8+t1K*}^ATkzHT^7C`dc{ZI|5WWkyPt;f1g zq-m@-B+dh)(g`7{U9rA@@V}=#XlLx<84KB>4$&8GvuzIu^gZc!jOHLp6c;Bi1OPM# zTeY~xq%g|-25m?ognrI{+FfO&^b0I=!00w3N!RLTdtR%1b6wqE8G(^G4Gy8SdE;4c zbvx5hp11&m*qFo=IwCGboxHXEvkKPx##d)&95*F@UI)ga9}po#V8o)t z8?rT7(k&%Z9Gov?gixcu*+>~OG-+cJN=P0tJ1%SU>;b6LYka2}npS#d0_ktrltt%{0SQ4WSwH-zTYEZro%>V5He#S)xMN z;BWd(^~*N8W6Xb_q;?saG!Ai&kW&U0Ey7Nw>%Eo_T4~#VIQf-vgjPf%BZOpN(IQmg zN)4`Yvu<&Ggbx_R5dZ=bWMI*TGkLn)KV0=ffi4dlAK`~NtBqG>X)`x9y+7tZ(WEdfvo86M z%;f;cweGbe|A` z>;eamP`c|^PQJG=KL6A9o!dKhk3PkVzS?fK=NEYoV*nzV;^jyUygiPTldU@aD-Y{= zvf$Bq=Gmi`3jzST*`B6AOL2T5fyob07?LlPbS_P8U0twBi7THu_u}^U-O><`oGwcI z{@(U~$x~I&V*o2Pl*s&sjxcOar^e?$TlA!n(m!AQyO%D!3P)yTMR+IPEv;LBU-ReY zK&xy#V}K9;8+DQ?zdQWyb1!%w!up0D12-}pwurhQ!pi}gjT3m&toljW|%SR#xZ z>yK?a4>=D%oVzkvl{B1D4u99pdUkYv{$cAjN62-n_2&P`T&^SIWPVY~AR>kEfKoWQ zv-9)pxU73Ke~1t1F+btwx3L2EN&Aj}*uOaAuFBSCt(m&IEMW!#37OxBAPmd3rkQhp z6J|H~n?gcJO{hWC>z=}gNzZTFX4~;e`wko30T6m1d&R9OH){xW_yCzr> zA~3wdbs}Gv8%JobIPTJgc*KiHRN#t3jw27BdR$NGznk)J_h#O$Av9w?H~B3>7*?h; z?sJcwek#ZZ7be|sUEdD*TOK%t?i#3avZ7kH`-_$4sok0*+>~Rgc^ewE3yGTXlEQ0 zMsA(+&INDnA8Y?YVPvVX>}3EELbSkeh!aEs0fa#ehd7CmDqL<` k8pjCxf*jfbvJ6@NAID37cnRF~@&Et;07*qoM6N<$f(U%5wEzGB From c54a5ee1ed75d98c3c95dacfdf004ca040258b02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 22:36:32 -0400 Subject: [PATCH 0501/2835] update --- doc/logo.png | Bin 16760 -> 11980 bytes doc/logo_small.png | Bin 8163 -> 7828 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index 9667bbd8b034f9dc9f37b881379770281f1a7e44..ca9bbe4d3cbd1ba93937dc2103a853d2f3affcff 100644 GIT binary patch literal 11980 zcmX9^1ymIM*QP-P0ZHlZ?go*RZt0TlTyjZi;iVe{36U;omhMudW9eGDV}Wn{{(BB* zc7~l_-Foi5PmG3|0yZWoCISKiwvwW(7I3{rKtSZdKmxw8t$NY}7er4j1sR0uN%8~W z=9QbGfhPjOYy5wIhzQxaB)~&-FC|qu^rhEtkmz4oO`4V=AW$PH$-dX|T{_MKnd9#~ zc0=2bekYxDf>=_&Qh$&}ET@hRD-xF0h@_@fgVP|fV#b=`;UXr{u(B$@Uo>C*qQz|D zRVwD3(l}w~(fau0ekHtwWUtiVD;c~Wkm1;RKC-;GcHTiEr-^}(OF-^|hyX@LxWh*9 zK_hlSG@r7=a4zZk_*<5zds!f3h=hgjY= zdO!J<)h;oe;>q7jG=(F1s4S+zNDJh7O2LJk$6ga(_^sQD z%6GwF-5{jML_I0;ww@q$U=NV{V14C)w4MOw2C}i@EsoVA2OlxVEX(b26A)jJDp5c8#_* z?5H=PH4;PNc{vcxjUv|A% z_3|r*)&h^ALD9dkL+>x8ABD?>I}6vD+Qf$gx_et%eb|KVup1L;RYz;sl#CqydqxG`H|nDT~(fcXO0IVaWL^1VnnG^Dk1)I|Fyn*zowk=_y|syl-Q!gX35k5u*L}z>f_A3ts3+=$na!nGpPdiqLWUqBYy$QRT;OO3z%p?3XXk~8oT~VkDQ@4I!>OkQ zV@tE|4^GLHXlr<08mi+8pV4w-!_yS@$r<0QcAnZ_}k9@Tm8)YPYc|2gYr1Nywy=4h_WYf0&w@F9C&hRu4d>(!&M4k*N=yvfW2A`_Ta z>Ws`ya32d7g9en(1{iYz)Ub_uHySvhMefXauh2M8Kv(SxasRy&`OU98CW$``8q&~H z*)whK^l$obCH7BNKLEqC?Q0MTPh8T#`S;6$;SA1g!Cl}?8}L|OmAx)m^7uDVkufcI z1WiONKLd(f2sA{SW;Ys17K}j?M(aW50z7E=J;0IAnOmU>9Ifs5DE2JQVwRbIYw;a! z5bESMEEP4f*9Zk(+!*y#@==$;(2UmNpZ&KW9ySL&&-4ECye8|^A z|8r=<*eG)Qs{X(dWH{$cD8cX%o)-zwuNzum$2jzIF03oRxVPvc{tM+d_BflQK!K$o zVCnh~)K_xx>117%qMHdy?!N-wMth@A@xltKikzO#S+zDHX6QKnq)ZqNl_s;pnE1JJ znmMZ|OtM#Uw#OJH9&VM1Z%lFbB%M;K@*)m{9hQ>%^GRJloG+1yIVGCTYp2cpjn?sjsw(WY;EO?9!6M&i@LlfyJIjP06I-)` z4^)`ohxX0sj?DQ243Y!pDZGchh~C)n#!>6YuMa+|$@54;%Ho_pNnqu%jRPfTT`- z9#1uNan7#h3b1c{+?7?QIuNRy2xq(2eHmN+&(QPNwL<%LK;$D{}uK_xH~fz!A^#PuX|l3ii4#R3vGO-&0}~ zesT9KGs~e!c(G`#)EIbWuHZqI@oehVND=$AVQJ=0X z{){ZL$=?b*?rP`rVU4Qf5zSxfbCgGcPOfQQd-a?K`e}xma-H6@_sP=OD3;9>H?*hW zyQ%Z4q9uKd)aZ>f$I{}$3g1*;D}LCZ+N*#YPL2>)*{Mv@=Z;KAvkEbt148I_l`RA} z@rDW!EJjx5IQe3>>p|zocbZx-eLImk|LA|?5VU>Y*64aEm};k-C4t=WFd3gs#C!qS zgV1h#JladnQS6&yX!l8nR|^U71P8ArmGA1NKp+w6YL4$agFH^DP^%d_Gq3hjZv;MA zLo?B_EXFaHl<*(dyT8%I`#1fQ-*oi5YL2=Ktb^YvA~(#Wobm{4en|rN8DRyWKqv_D ztwqa)Ck9$Tq(UA{$<|8Ry^OjC6-+3JU(@9cw?Bu;B2EyBT!RJ*gV_=#D9`piY4G6@ zUFn^Nz&$wqNkA#dsi6SdkCnl^xYlox`fRzb5ZC*oyN$`Sjep4Bs8_F-oUYsPjpH9b z-d(0kFaQ>-UEnH2==4~#%PtfBExOYh{U+ljfmd90LdOxr3$4_?1xF7u&F* zuUAx@-S?o5PtyZ+!bSlWo*N0dk>bBjoGI<8u>rHY5RjYtq7t1N3A%}qQ|wJ=4D#*b zbPjsng%8q-GDC_xJe2&tM0o;uEl46DQdtH7A7B}%m})+Evh0_xdPQxx*+i@Fd?vvQ z9!DgzIwtsMY583E>A7robbkHGZEgghR%rBcl={Wb#vN>lO*YY~Y<@gPi-K&Y+Od4F z3!$_FR$Q5n;SRbNys%tbs>J+G3IE)R&Xx){_BS$*mbkN{HmXeqzo6H zf~*EgBzJNVY0RK#c(Q$D*@q-~nM;{{*_+cu&#%wFhKP^vtWXadWA6FLNZhct%!w## zVkIAvm`|#{`-&4mzUexV}EAc5R4_4o&2&BG&Si+PJ`XBTh)gt^Z`) z0h8X@AQ~%le3gd}LEuJ&VL%E`2c}#vGg&(=ou3D{W?y&$kmR0g5xFxBRzFq$ zVFR;1+?(&mC7^u|F>ShWMfBxiN;{iR<|~BUvgt`|(Vce8aO)T?$`ZE#&gO zW56u{7@Rqttha2dfl#oZ?Svq9os$nAs?H(-ET1b%o$^F%A(WSC_2u|mp^@_6!3WhZ%QtCM=IV9! zp0ar6iQqkf-Qcu_^woiZ%qOio&%}F4%Vi>CBT1V0%YcKmi7(5G+D?P$lIJoD^yLz& zya&E7V9Kh91#^h77zd->}be~on z!U}AvSm$-?K{8JfOQZsX^q86W5}gxSHX*C`&h%f&UN=9nzJzylB41n7fNK;~u^aUs zA$g-gj}myEvRsi;LXu}O=y3I{zPq6Bb>@rLX$DfQxX`3P;)hrrLe_ zFqQiXaT2=ZD$)2P5)TbdKY8H{rfcVW`(YuoTeC)?i`Hq1*%!O;sqA9-q#m;WZhi7^ zW0*zrAS%a4V+;?mNT`3x^X1X(&`~s2-`T{2-BCPbyI;g7fB#$l5m7k~-dB zT5TlY$G!~zLc5aO1o?r`XNO3u_c4SRwOqekk4Y7 zP+JGCTC3!=5#=nUc~F_Lert){P~lIjs)=;Bfhm+kje#!-><9=zCLV`|g&pT7~PgUQO%-yH^*WKyhkv@Mu z5&V3*B>0gdWvxW<-X-e%G3{*cG2mu^n*io<+VMg!cg*pUBSG_pM&<2=$K7_Q>666@ zs<|>G4J|E+HW0;hoRUw!vBk9DNq2H-P8+9>o~9;3opOJb>VvtUxi*B@*ubmqgjdE6 zaGN#q9d^6w2dWR(s8S~^j}>x1j@hojJVzoVZ0HMLpkg0=etXbGfKyKWL;~wYQslYV z`1C&40H$6CiWTnNlVMcB^gDb{a)~V*0tEn%>oBy}NnVNWgrzx&--UVMA}3HiuDJ~W zt!vR)h@GI2&i~glxf`MQ(zn&VYF83OAK7Qqf|Ku2#WqM}hK<&*P2o9nBaN>^pt~NG z4+JUC8pC)%cKU2W#HWCfr&A2h@62AENWjg;756c`OBI|6MGE0~ve|PtPOYsiW54G_ zf6+Lbx!Hs+@WeEC!=CQ$0W`nSgK)Edt5NVC?N4U;BW`0rzv6w{g1gXP#@>(h6e zmuF!MBHVkJwXoYU027=vZu8{;aqVcL^>@wJ*B?Xo5I+&+s4h`#uNEks-8v_7NxO7Y z!dx*mA*pcp#s{5T<*V@@_PY>*oBzC*rvto`N{bT})Rh@-(GgxhKIz{>fCxz!0c1k5 zihcQ5$ zX8JEDKDY?&pG@y^#zx<&Jp45dSJO@|>OeIW?tRa)cmDVoMwL~}N1Nal@p2}JGn11| zWMWjx{{vShBUN9GRV}ftFUjZ1G;HHLoC;Nna^-fc@?qse+EC_1Mqhy~4&i5Ar){Qr z{m-*|+yNb^`EF3efZG7=4Z%`JMlDiF8atJosHabjbWOYDk4^PMCSpM$%cf8kjD2Uk z0%L7klOOH*fv1RAxT9X$Gu`u z4|i)N$v_TU>OO^T2V{CwDtiIBBuJzMC~c4~X1IMv9P)+ekm@iImSs)Y<8gJXqT~*p z0+^0BYZ@qYnNqGAQE3;mEb4uT-Idod z2PEqLP5pc4p^X--ce}iwGgBleKLrncJQoT$Xuz;vI6frSJs|Wl7{>F8Dy|`duZ6AX zowctmXiSYht#e&j1!wDJOr~0y6roNSlp}QWz->TA0LnE|`PNZ7X*r7A`Pe6cO3waV zZS!^8MIw~9Yy(YOt;6w`Yd|D5Mi@17?AsXGs_&-$iH23bmhPhmEGZMw&zl9{M4qxL z23`S??>o!O+sd;Ev!lf3lyl_2k4#LYjqS|N&U$-$6Ucp~l9Ch^ZSyA2O-ib>=tC>f zvPa!eVVsG^qstGvF_5GR>zuq58{weQBF%?>+8^``;_zU7{LX74GP6rgebM}~o|DvG zTkAHJFTudTpsK2h79Q%C^$4qh*i7aw8dhGz=E=Uw$Mk5- z3IhIk)wqR%#;f>ny8!vug~3<)qY^fw4*#X_6~ii@gFhu7OWqFAT*AYn4b=0LF2B+6 z`(dMkM8~V2udGOH_(}Dv4CThKEnfYFg5uOOJ@6O%pB1gzbc5qQtxDgVskM5;<~BDi zf3y}pccCA3XHTy7`NUkeyNM-{_0{s(P8OGzirO`)CKr!<4%@!b`w2n4tCIA~Q*u=k za5ZY3b+{h{fP6o(;{jZysk^z^X=!Qc>HCL<&{3Kqp_EAPv|Z}Jg7_)s82Fyf zR0ZZF^2yuT0*>TtyXlJb_)Kb7S65gV7`Sl*99`Al?DqzlzCTfY*RC6AlVxgWs+02X z5Q)=uO{A=vXsKO-4q>($W$Qi-bC340N)Rm65@Oi!pw0P^jSx46(kx{`_P?N^rOO z^TNf!K{WVT{PAf#S3{PWKrM^cx;q4s$gki#bL?}II3*V^??pI{8pvu74HJ;Ad>-7?%}|zIQSHV^_hW%EKy(5=jc;A zTGoH`i=6?t)??`gA5Vzt|3LT{7_65X?MWiD@E5H}^mKKb0&ZPU;@S-xo=uaUCY0n@ zWtbZzVP##gg|Xyu^%v^q4K$g3d8?k4{dp&n++2x(Irwz19sdn%-u2$zbV6aecW%N8a(&ZYG?ex zkC>fVykaq(uhmD#B^g=%L^_s^>|B(VGVHp_K5yI-DIF-y+16 zv$to#aA1HS%RZ~L>ga=EO4RWZw)uIxqoX>yx>={7W|vhs9A5cu{ZG@y*+yThaf78y zo?1pu&g(G4kyG>YsxQ;Z2G>IWN;Ewe?bCYa%^yp6XY?r(g#cfrhS_ooPS(`aw6L() zRUv0o;$>rt%U{6pYJ(oN+8#ZS(Gcu-jGeZ}cOB}eFs1-+YV(3Xx6#oUIUV%W5%OP2 zttX-Hu=pvUXK0ZDW@RRh6S`Fk2xy3i;+Sa^?QQMOC zGO1%0{O$pu_9%waZ zIK5Jynr&&ROkpi@y}!Tz8lT}e=Gdt{Xammd)YKF?ga?c7Qu+0#Ue^qxRr~pz_JE@( zp8+xo47GGFGfGlY)|3(PlMofg%iG%msbI;dzShUvq4Dz=;e@dr54XRh=YSy6(+W6l zZ}e3naKHR;9)RR)ba`60mNCcnZLk|Q4v&X(5U%t|8^G=2+{d!GLHz6@nq5- zj10P`{u-(FZ=VbvBAKRnaFvYz*<6^#6RdgId=|8Sg&Tj|6*`0BB0td z*MUWuxOaT)JZVIiu!V{$NG2;?U?C@mibCmg&}>8wBjn3@O3LWw>skb3YASphduKF6 zmNN1=fuH;h69)(O;*a1cO1rdz0@qzLG7=;gT?oF3>-_}1vlYp>q48nMrj(;8TEW!q z?XA!d8fQjMQj$eV$Lp(ulG4&_k?@2fW>;5Nx4-`2sOa#_-?gGCrq!L!2zwgP_v3#Y z46a)_B#&3dy4_=BEp40?aQj<5_nfr+=pvq~UmFT+zpLJQC%{bnYR0Iu(6GeizKCEs%2+qCs2sl?Ec5%FNH=Y6I@Srbn?=v6)pTIGLdW@c0j0PV|CsnD;{ ztC-cx42-e(w`Aq+FqGPQT_-Vec3x!0I=Z>n2! zAfAB)2ZvFQ<1E5pFd!BWV3D83$L6LRm1tENw*skf6fGUygbEhQr*YQ^4bYtOXP90u zP)`c29LWx-5-ta?cZUimNZpJltd6*0VPN<&w)0C_!@nedEQ4z*PYiUr*6GmHHc)Bm2_}0ai7je0tTN^_ ztN%Q5bG-aj-YQ^^H%Fxu@V007f%}jSb>TQ-u9Fa@tZM@mqEu>4RxC#<1cWzQ|F{6s zv=vZFn*O1CZ!`uz>5?|3q+ZNXu_nwtH(`&~4j}THSY7RJ7!b%=+t}ob`_+H@_N}yZ z(3~3s1*uTOE+PAh$Dv7-iHS)-z#G~X)b4Za;N*1FP5RagtxpIVh(A}@87!J<*3cd> zKO?Z`8oB;6zAU`J$-Nt>CrT;b&J-#GU1ou6Z=WHJJ>3m)%G2KpR1;%kubmSyiP>_?OtiQmRQeIvjQ~i99CNJtnh!#%6 zz#zv?)VGN(c-fwz04mLe>P@DYyPE?DXAkyge*)-Gr!xh8&Qp$!i+hcYJux|X(#`F- z#Pk4>|Nh&KUr}8DdTY!|-pP+OW78dH}iA_M0TIl%= zNk{lI8~w(&r^~V~4ze%r^^~4Ij0pujc!f%vPm(L}eIR}l)wgx-D^6K3MO^z0r^jOY-evPnE{yilY8|+Any{of9hWYDL6=s={;2QYxxW8Yv{V7fz)UK8*TnDHx zbu&e$*`JP}o$o;Z`cJC<1=;JEk{?{Ow#9vo*j3XNZV$iA={T(tISnmhj*-0> zBy){_#|MaYL0wKwlOS;@N5s>nKr%?$CE$u|=#1MPW9(XN(#DtLg0M7q+09CMV@j`G zW{He5LktYRd&g%JElcZ3)N|m}z!H0Lb!A-t`4xp&=!y?%Feqh}bSr($p^fxIvQaO> zTu0Q$&Ze27KSdw%ol+FtpLyMda|=I2f#K&v0HUS_?RB+4YoIPB z&+Mz}88ttu*LH^IY0bKdJ7l=da9Ll~DOJVb#wH}Z!N%_Nzdi(Pde)&yKrCdRB@+<4 zcA47s^>xW|>-&?*s`-;kuBJcV{12~Qd#yd%*>6X#Rh)Xb6_(yj{doe~IDoPB^DOVJ zFE_hF%eBi$Nl7un_B(cb_w6EvwnjeJr>3SxQTk&T_;VP1YkICO&~&>wH;I?1iM;$^3E=9?y2j(n4@34F=jfF-g(<11q?{INfBaxhQj`@^ zm1#Qu4-7&Gqvh4fGDTANcV-nUpR}(w?aa=}$;rsjH!@PzO#N+FoF<2XhtjjcgVkr} zc)*1m)wki%`Zk6hFiIdJVqu9)F|xF2A0@=ajg5^Lkga=;(Eqx&{yO^N$+ur0!M66twko50 z#>L8u4Nfqv^r;lDPmvVTS!eMu;3q|ROh0kgdlwg&RUbM1%yk+N06zpNYX7(V>Pb>B z$RQ(8-KFmR zpuRqYqra}Mp~TwNb8O=eQ9MUg3yb6Zf-J;um5^=@o0(qbMh7q(^w$7Z0^Dp>8vA`F@@8hNN_z; zvpl8w4XAdBzOj}mPx2Wy;t`7^Pc5UFN8Ov}Nfqy#-uOiR2th;AR$jkywkcFf9CPAp zA+cQ_>w(SVO}A@z>_Rz=oqvsnM+$MWZd168ZP3|UU$3JKZtYcE8KB-y@k1CZ5xw>` z#PgHmZ;-F}`hLierEyW_gzLpz)29`51gCW&B~5g;MD84ul})ib))03ol~3L5rX_p_ z8K0}UlTpO!yVJo``ni^)fw@zrd1aju(cv}YqHvUn7@R zra7FctZ^7WxkN`8`&SzKTg<0@KqQe3*bEjC`r3;K&??5kC*uB4bNCfoxw?j1lhM~O zBn>WU>>YS;&9x8!u)PY1BnPz0W(PQ)cg{w}Z6aAc0e-_-BZA{yo}s6U-yCalTiQ!` zjctv3Ti|dXM^>i?Pyh!SD!e)Ws<>K7m^fJb#*l_XPs!GzBbZg><;trO%Icu@y2Eyd4Yz#@BLps1fhAa-CNJ@b-3SC zloE=oysfX^`}ycpg!t{*1sS{W{_Ife*UF`pt@qCrj6#4clBGG5IRP-AE-Gs(vrOOl zQo~Deo5W!iNUC@ZwjURYc@V!>y2WlGX;WSYMJ$S!D?XKMHm*5+B?Terr6}Tyik>xgp&!?l)dYo}EMkby z-GmxeyYRV*nPb(P*|0UrRa<#AYe8l2oj5)z+h!dk>DS|iQXD-3YR;9Qz4pmCzfK_H zljXaF85w)0(ALz2tg+AQ{`h+RBp%QgjW)i8#Nfk?!AT z?4fT4nS4-j_Bd4^Lh^WEu$88AfxUGDq~o*4K$E4>Tvv6L$%PH2_RAH?(_f!@3Hbs5 zk@Y?})*~hy-YN|ofU4Qo^ucbwo2(B&CWI4uvWmTHiNSI9YQJ-`)pYvy*}AbHFsxB7 z&*!15&^>|e<-Ky8vqLeV2fgGS4JB@IwQs1-@pdLz4Ur?Zy?uU92C(`iGAm@NjPq5_ zza}S}=?GQK-JMU80At`JDwqL&FP#LDFhlsfkFtbUkZU6#OW!sife$b4GtmNDX`*&G zA~`=$%y*O`;p}x0OBH0H957do8t`5B9_>x3MWND!2yUh@AuGHV>3!>bzznf?UxYB! zMuJ_qYIcqzGc2105bTnoF26z1;VvVJ5Q{hdJa4N3+M`i*g*3pEdJqLw#h*%&6nRFg z=SJdeLM;=Wjs2fq-{fMgzrR|RCIxfe2L+tF`fI0rH&ZV%TKOm1Hy(6K3%11U<_r)P z*-jw#U*G2l5jYumJb5y;6a#%0zCGXRG)rY2=|>^uLDT~jbIL9hn=gcBw@ z{YbOdHTkN?Y6!K2JIBH5i7W%!`deK6otvJp1=k?Evf*~mD8fdnIR*^2*pP2GTs>8C z3p9nTX3N^fk{7f^5N$65=Gr~AkmSWNGCs1cN%PwXumSW`z@K`%E1!9;5yQ||PxC5% z#R(UoZ#(crTwfoR&|QziN|R^-CRw6;3|4*rC~L^7gnQIjuq7UtedU~v?K^FP#p|tI zk&|vE1VEIN$wL4cP`JJRi@x(?dTyK5cCFhYR)I03vjQ!nSN$zt1^tN*+oTgC!7jalvh!Upo< zwcn`?yA2ly0qvWIO4ues64eU%+44K)8C~{{3(c(Ru*I!WPlMb$7aRkj7LoO4zHfqa z!M@%Hx_Z0ToP&Aq1iS;M@!bHTpl!Dzss860NgeOpPS5D6zSF^pa52=+&{Gq=%)`Ga+{9UVv;|bze zK;#g+CDWnsS-ZZbP0g+|y##mtK0s`dZ`Ou0@sve|z>}1;WVHgOPmlr>q8mUbm4S0-Z!|ztnQBlqwg-zD0ri zur7G=AMAjEjrb{AT-%xI*$g*XyQz1(?g4GrxmFx3w6#T)B7k_dyutE2T~LALL0g~}SNQJOSe*EQUpw;432^dO zKc=Kh1M5!ScG!hM)#Ot7!$x`*Wl1?%XEyQ|2w!k&Gq@;QQ%oSJO4jCt7q E0R(INVgLXD literal 16760 zcmX9`1zc3!(_VV%rMnT8mhM=(K~R+D1?dzJkZx&dX=#v@?hfhh2I&-#j_>&W_eWmt z-h0nEb7r1-<{3iORpqcT$T1)g2)2T}^lJzNffanlLJ`4{nN{oO;2VO|YdJWiY=m+T z{DSTv|K14#!6f+ii2zASBLxT1oE21L&=xRp5Ll=`bh%$bAhZw#X-O@&g`;#=PlCTs zed4b3IVy3-Em?7ik+f10WN#h5)6(M*5^3obilfM&XtnwiXux1(+4KYIFv&y|WDMJ^ z)Dh5?QHltGFatdwEu-m z1VW5>tuAVk_e&^Myp1&-lgsoh)t4{SO!X|Qb$`<;+;hJFSS^6j$60JhgnAz6>ojRI zV`w2Stm#Upl3Zp{)Ra(;L4;baVzAe99+(1y*hR%~roYamE*5qFNw+LP+9(nd;MDA% zVIZQ$QKsBY{L_&y$^pq(oq|`kQ|kwdX7v;WI)j&Yk!nW%(6m)b&le55Hk5F@;+ypfzH z&zOyh@eBqzk4Vy8;z$!5eg-f#X>m?}Nghjt$muZB5BcfjVPkSnO14VNiP=X6BBOns z)Z1&6FuJucy*K7b`vcE!BM2(4YoAtUx5`%y9ienQzZf37x?Ml>XCipPzqh@f*Q?2L z7_{tpA2{~m!**r;tc?%0MmBWdm(~ZiVK-O(GiI^y*+~vRB5sMNF6ETEq{oZyo1Sr(jmsJy`O zWn*r7UYiZ`g6#Zx*Y;Q?G9Kaxs$^`_R+}B$iZP?kLCXurZY87)m9O3N4-0(6-fhJ; zxNvCSDF>=g8f_>FI6Ziw@O|<_8zy#A5k2mX7@62f$C?L~>3|S|Gg9Dafa(x7O7Ml(8P3gT6?jtcZ z=i1u6zLRJF7QeObJ`64pJdEycv{^CgA0!#|DP2sYTnJLZpLBDE%!=kba}zVPRgV8K@!*O9`zQOGwD9jfQuJrV{*e@zZO`#S zVHgBKB{aAFFGkm+JclXM9;6q?cI}DKBm*VHgK0yfoy1l(TQiw)5^-)n{z`%MS^Ix7 z`?1Ps)!JH7Bp?@uMvr`JO;ykC^RT|3qve=AI>#HZ;$#wj*V7X|pN&_U74MaOX$T&V zlS9D?RlK8yJDD`}Li|(0<5~ElO<=_^Y04UyAbf%C(pac8e$alqKQs)@lJI*9yXJ2Y zsC!ybE#&;h#Wnr@-~3dt`*^uKJ%fK4-6&Vq+a7T|n5`pV$Ha$XPeL09k925?}yQ2z%1|Xt5#y^ZCOb?rhet|hy0E1B@Bk<~dFZ%tv{#w;_F@BQf zCwHeUZIo1rKwl}T(z=VSyJjJWpOZ^k-(Cbp)&DV{j&Yt&I35Xn2=sl>AlG`DNvGK4 zechWNG+)Zr))*GnfyauqByvB8yb?vBcR>0iG5-2W@m3c&lrgRA@$W?J$Jo0Bh@aLR{=Q=V^n#}g z48`j^Nzw8;H*qrpJ+h5Tq=7~@Th1Z+N?p^!k+sSK&J8m-r&Lhe$B#?a9p^*_C&znO z?SqG=>ZrTm1KV~W-4|E|?+(Spi%q`1D8squN2d##Q=jgvRf1^Y?RI(^=9;~<(QevJ zu0E|C9|-vpfE_fsKzVy*17Q>ce~5Vo~a<= zWy65c8PS9-SV2fiQ1Rf$hcgvaT22sY=$xOG7xsti(`LQX%N|?DZWlAEfA7a;Qye%Y zxNcZGayp#}%0{gMUULqOxsVS2tIPZ4_K_lqZZa58q|Dp`7C~hoxX-X{nu9v#7;NO9#KEBx>Ehx{l*37gJxDHxGmMGP56r(J`260mGJ6k zW)l&xv)=>FZW-X>V%O+8b$GwIFVE_sS}*JZ)nnxl&jrAcUdCiQ47`|q)40&`H0uz< zWp_@FLyvqY8?TET%o(U2T0112qPNpqe8Jn0g7|Mvqc17JN*)G(eFlYH(*-K$zHIxG zaT$^v^fDf#Nx!pNsl!*ZRg;IJkPhr}lw}P~{;Tgk+RSJRhm0{$6 zHx}+=n`emmEKI~UBL2arWII<+I|_zham|fb;3|gjZ|Qn8)SkIBiokdn6e-saTO>CV zH>l#h497%c%RXtq-!egVrD~H7SM-_#emuEG-l}>&)W4VlJEm@T_*Vae$BVl^6%G2# zV9g+2@|eaJbXd?VdB?e-vd;k^U>pX7@+3nEGJ9NuQbl!r#B|g7xgCU@dicO(J}04I zZ%h1{7VfmF=4Ktg!|4F!kwaLN{MrqkpxnGQ0LBFiR$}MY^wFf%=QF~zx(ItCm}&Z> zvM@V>rA2&~+5r<>1y9ae)ib3{(t6@`PsfDh;AY~$!lx@CvaR6+JDZ=m&o$lv_e+z_ zYdO2+qw>7Z%($elYex+K3>Iaq>fZJ=_^=WEg8@J4E)mbr{o@C)_#?S5C&fo@N^0|> zm{FnkiG6I5E;Mt?H$4jzFk=|zLHQqf*My^+oP8)<-gatgpNJM763o~Sb^zMWJGa(Q z9?W;1uC)|1YRs^fhONcy(zXPe+-T+V%zO9+H$3tC;Q6%du&a#t{rkUHAR;zBH?s2a z%ok65X#9EVF+hF9yMfxqo9_1#L)eHh;*U>cY{fDg$-3LgE9*Gz%W$2IFrmAyG&Ppg z;==6sgFgelJti&o2WO_-mMeyi7S{c$%x}I$uq+E&#eHv6p|fLOXl=Q8-q?3%ywP-D zd-_A2K#2Xn+IK^Ey3Fv-A@e^?L(ewUt6{+>Vcyp@o;2f^gJQkbzparBzkU>L6cN{V z*_v!F-&0S6W}yarboz6-laq1GF&aY~%G;1hQzj-$6eof&rtDcDhXF$VObO)Q%r!Ug zdd7WW`%L$FeM+_I*UKM&c@2qe6ibC?Z*oM>yyCo=y6feHAKCbmt55Xwy%drqiR9dI zbn%{iMBmVYB}Ei+I?wO4y<>j*&e9IzUU^^h+!nmltjb$^k3)RWCx`eAec(0gV@_5T z4MywLbi{`#>5U(0HL1Z~+iM34Vb#wD%S(P(Si7ul@&AeZ_N?WA!x!qOy8-XYQ#)(@ zF+N+%TO|{olaaA4-f*7GNdHtM2#}JJ{-Ue_ma*nNlPHy7Cb_()s9Ap}rjqlvSwoqp z$}!1FciVA=>x1@RaqExb1@t%nK~rh?cK)0;ytBW<1fedeFfygQ+)J*H0}hS6B>%TE z`fq(vRDv?M?0>!9nULF9GRcVcW*!j@7Q<(Rx>>e66+6w)1wIyu{`L7;?i__|?8?*I zJ9^gkSa8YnhOiRZzPD9m#N(pW^ErsMufTKuc-%>~*42{bb?FXZa2LDX>u)6x$}Hh2 z(5YP_Lqg9%N`=Nnp8?EqHQl^T40`0x?a7VjCuXI5_@yyb z57(_6xNdf)+QP9T{JCClDneEJLSW>ZydO2DH2or)J(ubB&CnY4f$tDcJet?PEM8fH z&;C|2F(BO%T+(?$0mDalPWbd1!_>Isy(Bj^u1~I0`rC26zaKj4cJ?_Y?@1qp{%Hv)shkxo9#dfa7E6EC0e~G9D){@V@+`=M9A@c-a+YDYi9g`fgeC zB?V#s)$Wd02v+%58GXwFCX5K1!YhWmo>qat-I%Piou!y;xS2p z8&S$L3B~p)gOUIQJj9kj>ED(;^Y$2ZuURURdRk_-x~1w)2ztG_JTqK46bAmzacvZ{ z$p}A|3WvbBV)7q|LU7(bCLG2HEI*CBGRP^ZS-VqyaM$qzcqhdFBz0drImGF1VFwsa zI>=aRiBATsS&ghJ2f48WjiWscvuaK4ewD~Ox)~b23w&YoVWaXK+j5oI4nXoOp|GTQ zGVno}M#UCIz9-0|M@arG>U8t&)2sy#em=98OK9!IYU9&?%V@dRYI_x_HmavcE^L3f z$__JpUrOZ#*w&ttN<(S=8C74r_F-MtY?n7?GYhB2^r#=SI<4L~(yPhk%1C}Y;4fvi9Bu_aU5Px@Z>5?jWf ziJ<@R?J~u;Gx^;ByTho|QpM^Iz~!0d^Mu1f<|kZ8I{{py*%EASZaA~}8#>kaRK`o> zQdsszmRGdM0ZC{zy~S6hg*fsx$W!Q*(ydD`(IxRcSa2NJj%{F!>wQ;LdxC^jD~X*p zF4R>kIcw`n@xDb)Qsyqy-Zos8{3N!khVEZW+qr&Q>FlH=jW)%=^5T$n=0E`C~s>VEW`fh0_hgi1up_c-E zGNj=E=gmW;E{iftaJ}f^F#)47y~;1e(XhxW;obM0j+tZv53TL-BgLabxi=D9>!m%y z)3Vqufk8iy(Z4m)ZfNusDO+5aiB6w^XDlL*4LL_&<;VY75W>6DBAF1U@eVSS$Qp2Xtq{t1q>aI|bi!*i3p45?|k$iNtPJ#(v1 z8%_Db8o<3j8XmIfXK3bV+q_uv1R2$>{P*^!`CrdHgq)zh(55`u^6RYMeO=RiyQ(;m zzCDJdH}x!E#xhT4ZzhR5E2=JbG6yavQddz}AD70^KmZy=jWeT#CrMbU+j2bYNo>)5 z=7&TI_Z@Y7PX;J!_4dDFZ@=f|n~xn^3!ls9q*on*Yr0rgeAQkfJqIWC zPJF5@z99J=^h(9I-ADD;!{j~F)qjuQpJoDwR#W`*JO84r<`tKX$)VQEwD7FG-`;kG zNV_SOFN^aQd~5r|7G6z9C zVjd#zlsdnLLjT0o&2^`fvzD$_Zv9Kod(wm1vj^R9S1$gxRT{U^9Al^MsOlA7f2 zzJogAlk1v$iI8y;vG6v|jh`Nf`GQ8Y1iEK>3&lelV2_bxlb~WENs`jS4;*CDAB;yD z2ez%YC9m`3^w|b+@R6HgEe?#iBPe(}m<+~Cdd-{WHwGKGeSK^EV?t9D$-@!6TEWhSzcvz4#B)ki247H9Q~(1maba z6QQE(q+KhMSt>D;le5&_ky@QX1#V04{SC~-kosq^m^I=%o8NIW3u<4CzlBGu`aw%a z=Wj8Gn06O_Cpdp8qtKk!YbiCIUFGc)9#rcl79;dDRBR_KzDiLJ?4OFeWK*E$uN4>( z?QJ~JmXGZ=lEnHlrn5xA`8OnXIljfVdp0qm!y+P5SkdRUbi-(;*Svz$;?t=Y*gKcw zLa!@#Ql0u|+A8zSB>ZUM)@P>I&49{!g@x_hQ%*uTWAO2CBSPJbWs(;&xq zm=QwXjIJ0kAfMvtW^jRrNx*6jK-BQ!m|(Nww->+rPvv?)NB6>h;X?3MB4)V~4Q=Wp zV4oQ;Itm!5t4_-=%pU4wR!@qrbEK|DlTIGq&FRg(g7`w&K4{NVEBZmR41eEcKz+Jn zcSVyDlEIbBCuTWuG2Y#S(6}8avWDRSMSx|w2O)KHu;)H)?@G=noNQQ^IC`drMj?bQ8D29xfuEa)H|pD0zCd*~k#s>~8Qxa#v{^ zYlGmc8G7n=WGQ5bmf7PWzFzw7%8y^*DW3lC=F(_#@hA*l%VB-W-DcL?#F4Evhw zX6n9lAeGn8FL2Y8P4g-7vD3!?^Lg`1nAe92XPAf!M0!q%={ABJ*30TLqSe;_Cze8n zMo}~WyQ%-^Q8~6J0-O&*+o}5jm|c-~oZaUIK=7G4DqK`uq2#r9p;8j;U^(!N zc<`hjt{2liA)z3u6I|ls$A~8!JrrCzN9bN;7Z$M6sF2aCEb!c)_i zw;56LEJl9{uGNmucv>1h+)Tew48WjsR!DYKbTrf9Hy2WuqwFfUSw-id8M_q|*#59( zarRUurb?0tQ0Ee7MoP|f-aV^6QTmiO z^B_kOA|DsLO>&@FRIf z$=xb0Y@)4S?=OXnNtxu8H?F1XJz&QCo`#j-wQ-{OP2s~)48*uQymV}Uz^sUld;8_q zLNa20slp@V^OFIVYit%?MAsjm6O;aBS~$#D+tm8w-jP2v3%KV!Z#N`e;;RMM9&zaV zP8ki%=B9lfozm%%tqnLru&@%;h>yagYV8OO@6#~HnOC*w7@JT(!#e77+(aB5jtArn z`mu7ylFNjN??QP3QY4vSZiE>y;6`4`{dmek8X2sgFs|%KAALEVNL46OVo?+jf+pcK zbxZj}OmWc(PQ6QceHhENIXrZQ71)u!S?kjM8LEmPp<TtGBhzR}YD$F=g2ysJ$zsC33eqct(|ZUki&wB` zF4Uv$Nk9FEZ27ln{S$Lh^kX-qmv^bzwId{)O<*t1XWGtt15c5ttrUNH3<9%Tc zVij!z4YKL@;f{F_dMY1?Vj2C00w)jKx*RAw%;2zf7hYq{er)MqOSQrs9wk&I#jDNVe700ghQ47P%=VGB}-vA;mrCZ_unJ>wBN!bIDb*YxA=@>7XuYkFAwR= zMK9178)meo*x`xqk4_Tw35xIZCS7XLq(!a^3F*6#=;qt##hb`ThQj*M5^0%Jw6#Vi};%i^z_s-5C$3}25r zQQuB2owhnZKi~T3Zfk2RC55CAd$sE098upCx%AA3OQ~ZZTS?cOJbh$+D^z@qTS6eG9D-#om z13Dz4{qO(u(S#GT?cZPe+#S`)=4dchYTR}!C9y}+h>NkZVu2_q4W+{wndfx21^ijk z3B!6lw>a~3zooD=>F?_m9`eXHZ{9S0IvN-p+;k8Sh++slt(6F6RKQ0b-=ZPMLS3r; zL)g@W-%cd+1-7xVgkYzU9-Xu|U@1K1CnNvHcWY~_?dj1&M~6IWQ;(gbQCGqT-`I7k zb{IwW73DAZu~!T`_9@@nTC`*S(2N%XzLXs+JT1YBUs zTf&DZE1Bns^{GYC(96(BG30?jC7)^GQ*}u|JR&doFqpWHA3qx9;_dd)FKOJK&upYa zvrKpkyeX>E2T2j9Q=F@pjfZ;i2n#3o-j?YVX)!xePPihr^gY_+QZ=YsPZ0AEySS!E z)m(65ag#?L%#`L3LQB=Zuky=THU(K2d*|}j%?2X&rnN2?O9qZz9xW;goM;_ys_pxU@PK0Dwe3x zX>mU{BMzUAmi}c{^`4bDy043qm7boSk+Hk44{gO8X3WBs5ntOg%dwW9`kk~h(B_U` z?iCgT^uLGOzYoPUFYrUnMl+7~_Q+r`T0#ur`+Q&2?}A~+mu=x;ZsNdq0qXH){GwK1 zsq2H3kX%n4CJDRV*TJoaVisVim>3w@+1W2pCa^WsX{Q29r9&BNBbU^3%CSj;98V9* zcCywy&>CfUQ&MDI;$AKG+G+%JoSw8jr6}hcgr5?7{5Rz<&|;tlLk>1hEZ+SU!yrO! zKU4CaRs?1|3rbSUDo85=n&pv;*}w#}%MG?1K2utT(L$}1vQkqQOQ*8M#KfXS&g5le zdh(Q#z_P*Mtcf!3%TW+JfGt4%zlnPA?ZTwzII(JiIX(x-=8P{dd!7C6H|dMo zau6UM$X9yD`c_{4moEZRZv+`b(xB_fioUvf`{p0k8(pzXA8_j0+S+&yodm*{3L1ao zcS=fwH|IOaNEC_HA~&9o_c!N#5o9I_ey~`^qH=J_n3x#*wuj4(05n``5&0Tb`_Gt6 zah9vCUVr}l!4E~wc!@QnCl{x12Y4KsEzh@Ib9|CM7jgG>;y~C*J#TXFv zCw|{^o>7=39%@glj{K{)HO^>NGv2-zu9X>07dCfSny}!BBImVQ?+m)gypmCixE(R> z3T|j<@VZ_PNwnEq``A#Rn%?+uxyZo4uznPhQjt!nLjB>F3(=5hAi0T&x9;}>z@seu zWc%^DC?zF@6%St_)~rg9kcRI&VcEWfhJ?#$`HJVN9G+ zuKogejdc)CYYAvi3y#jV2QFt(jEA7sLlVn;V0ZL>#AAB)?B?cXzH(A#+CJ{NnpqXw zz9A1k|IW@%*CH;x&GUDpbL?N0P_D%}clVi;xv#Nh@MW?2w#>2z=v{z3b z2poo)LS&@zyJ+$`-}7+JKn(&eDLaP1UXSQ3f2u~OC8oQmy1}hv<^14aRLoIjRaM!` zREWH?GGTa6;{8cNLV{Tp8S)xbpqo(UD=nmtlX;|kp>QVkuP1Sv%hv$51%bmVRu#KK z&hGH%RoYHdvGWc;MslSs`tppy3k;l}^7mJuBlw>dAVVcfXArDsj%*|%wgeb69OTpL zV>)E`jPHN_B2)Ad(n_)6KzTi3Wn)wQTiKu1KOn%Pz^Ry*1rBH3Bi;}&(9}digy7Y7 zQTo2LV0L~%gNHo6v}BW&crx2^hewAPR>Hhwn6^}01nE0_0!p;D% zyF{U!w{^OTl8R1AYOx3}GiVXw;n8AF7-*DaPPo0hYdafYVI@B3rLS{HLXFz|*g%O4 z@#_49%GJ5HV<^rS0Heoi$hp{=466!Ybi`z!*%8wIp&rZlo^|GIYmiN3UWZC`ZW5Wk zISSx(fnd$z>Otp^r3!;XZ@v``EP(RfL_r{nJCAd1K=_XntC7?UEQGc1f(uH>tlotc zF7g4i{Y$-w+Xd6Nm#jhN2kdjLY*y|RYI(2c>w^3H`x_G2!hH|Iz;gG=cc z-yvIM(!h@a1<{4;Exy#@#O8TcViq0*?&{c2mWD{eVP{j&#@BQ<E0r~3V!GuHBPPc{D!sewSU4h_?e88h{m6tQ$#jmXyVaj|lCJpJDueOZS zW3P+E*@FoHGfsR;&|)SEm8Jz~QVz8dd#y?*$ATIA2aX4(jLrIy0mI%#$3Pcf6X(x& zhkJXLS=k`yrVeNNJT~Z*)=K)4VRb1gDb?Ap+PN!h#WErxA<;E)eyP9|5D< zQJbzyjv?a~hmt)b>xEiB*mP@44BIlIo*5@F$4rd=H0cw&-OXuiq(;jkzz9M|M?bv` zfHoZv%T6|UB%r?eSzFzOKKXD$Sbfw~&d-zfHm#=PjRuF8q9R7>@NrC7;o)TxhoM2G zY5(KH-L#cU$f;tYC&f8sETfB1x*ogF)5DeP@scGExd*w{fDIom8CT%oR%aL)nqSR- zJs2U4v{+r0Jc7;_4U~K4krh`_Waksb+*61Pkp5I zea0t_F*zZ@Wq+y&V1~ve&>hZo{0ZC6X$*|8@L4v&r& z{1B^}Uz(VJzsc3wI51cW>M)X5P{6|o`kGH7?Zyz?k2?9uSo^1JK#(6mLZAqu(F01S zZ=biCq*{lEhN5F*4-XF?_0)>)=M_Bil1Iws`~<^kZ589-h{lkYe6DYG#Jnc(m&1u; zy7Wbe&a=4KSVW9Kv2_(FNsw%~ZjL?>Ww^MHFl6CQT!O+&5ML;!^2>igRfKvpR)C$j zRtuoJkgjzH0X_DlK3}0<|NShfQ6uT9c&VBHZVnYyuTD+=Rlh4`bWhW;b$8tQ`MY-{ zx>^s{VkGfNyN86#Ikz~OKlvP1NE3826DQYag!&JgaTqk$%W0yDJ<)+JH6-e--N74 zu3mLIi6qwd_yh!-*OqVQY%T$kVxXr7gj6T>r0Ikm%Z($b6|x0(#11%Uj`O9y35L>9 z_9e0Flhe?gRA7r(WsM%rSB2ox;9xkvqjWY3{?D>kIHWi_?gMoyF8_5R zzcx!EcLQ(wJiy6>vhTEXK3(7-W4BB?DVp zxUg^uf;o}rkBC=*+9>zvzSnv&%%^%@<~0X@>ToM?$C>WiWPTDx3LDq0i8Kna85tP> z42w-`p@MAELoK%2E6?Y1fdvUqXl(tfh~d`h&$`bkvRBUjZS|-7^8i$Au4Lu6$pa@S zW>v{O|EZ)2{AsybIX^o?M7hMA!(oFfblPvBy%d&m*+MlJqqymQ_5n3vfM1L7zoc6< zIzlQcDsuAATLTH{!mdB2t#n7WHI2=HDUh(fTRS*V8bRKeqcJ6{uy}+1T(TD|;n%W>|TC6re29N{NmA zxM*8PSIiWNpr;=Q%^H?jbd7%;>YcamZRY7$-K+GV!%{>NN@oZA5WHAeii?Uuhs|`0 z?Q;G7Que5o!%`nI;EZ;Pk*oVAvCKI2h_bj9?WB1?XervLelHLibYY3Z9s zii9^UyicaA*zKat{Dp2eu^q+Fj_{qmiU(PWd!n_qivW}*+0N_f8Wl{A)(feAyH5X1 zEFV7(TWgg@4op6wjcCL2YGhECUkKT(DY*J|wZ(6+__{d&$I=mLj?|;y*L-h7L1tgP zreIA0iMDg43oCe)&fUUw2&-Or#tWilbUkbJixk1cI`?deZ;+ zOvMl+0_%LAYs)TTgjY?=g)*!}`Ygw>U=jq05ZoY(ShgD*3qL+n6gaLfeZ{Bq7_!;i z7jaIFG)n55a{3mp5~bRfG4gZ>upOoA>qlhGDZ|4pj~V1G8?jtn^1cxW7EzZlB!+e@ zii0f_^CBn1QBOJR37K$)S*CL5o%)U{wC_ySgl6Jn>Eq&uKm(#?p&}HhE~&X*aAVjN zfF=Pz-(Ed2Swa?)4~7YyEKtw!@h2xF8Qf~TVh)N8g^pS8P}5nQny!n4Bx*VUzm z1W<6N9i99ckYhD|qYXX!uu5XW5$+K=YDKjPY9OnlIwp+Dr>NkpR0p_YN^OHcQu26f zK_E0BBU(qEgA12SePGiRjUnpqRf9(6_CWv>*!Ec_ZVU-`5lo&AyZ>A~i~ z-3=W;l3A?->s?zv;F!90LWUdumV4Q*VS3-sw5T%Dix1sYk8I>QK-S>&_XU?1X;qko z3MgS{0VmkRs4KVpKoI!Bv3%URiMO}+s%rL|0(Ub83-Xru809xIF@!wuwTY`()ll>;el|4f;bSj8RJ6FOydJ1d{=tF-2F=*1kz+-k(wUB4KW<1k+9XGbgG1r1WR1tfXhsO%+hziF;y%qM zBq*b_ayd9=G6UFrAm#wn0M0+9b!&xoGpXXT~czC{^ri!guEwyO}}+Nx;czDr;@+Fd@{Q+r--^ zv5l_gSo{=g&4ylN^=mEPNij%!aCY%OaYptW)|P`FQ-82NkPVf zqeX4t?eeB^KMYCe8JzI)?P2kSQPIlM$4cTvSELd8PSk+2xMq&MJY$r|ps2m1R|X78 z(seu9m_#UDO^6qGwK$JWeDSf>gK zpfUaNdwdQL9Mfu3c$m_puN{j%(Vf=|0?FXb`$yE5vnC=Jq<)Xnlt(AcRCmlyA$_VX$IquLo(uEKT+YwC~>dc)H)pG zXWO2b`y3)a0@=!0)< z|1;dm1Z5Q2_6vg@d{l|RxA~x~*s_7!C?IYV#j)D=KKn`L~-NMhxluzoJispI>`F|MXv?PzYB%>h$2AeS*c{S`Q*fpnzNhtX6(FP(N*~ zQ9l1f?=)awsm|7L&$J7xD1Z*D`#Ts}7bQYJVlF@k!$x+TlU0iGy{qb(t)KY$;rq6X@+y>0tUe4?l6p@5TN zNVo;(1h+9nY{J6oGp`q!s0s#z4}YEGY&8Lu=!28PrT-WAvQ0(BkSeKT^GC*(l-5JN z2Nbta2t}y6rTQyPa#2byrQU?b6sgQ`&odx6bRL&ZS!q0bT;Df2{ee&mTaTK^blEl= zxLlJr6aHJ>jqN7ld0Z8vHg=$d9(ymaQw4Nqr?m)PES=+YvJ+$NrL=xS-|jydL%lLJ3nR&h*-Sh2R5wgia>@ zCUuMV?(-pN@rTAUAcSwt9fIsYX7JX_CJcA2INy;9^t+1G*B}9HPI*X~N__XY98i(X za<44wdPCBXmVQ8q*jxazo#=7Ts>&ii#7FP*(m(ql?Dc$14wOm4{q40)y>j^vI>Z44 zIMMiI2D-=RQoM7blxgR0mx?hvch*T|X+$*cP9{l`K-KO4&;9<2?@_}gkbML@ z8D)AcvIBN$F_)HrKA0R=_))*d$+Fz@TUC^50@A-Stzi$%u&v5Vjtm>cHta)d6W~4O zIGy$DU&Ong0=a9VSu;}A4S?=rKnL+5f@KIJ1>Gqik$@g+fzx>zh}^%Sos(mR)#QQx zhpfDNeXq2rO-8eDATvQ-ImGiMwznL@1d?!%kGdvO=eW*kwO^l+o`{k!&5{I$!})RUN|d!TUW5je8Uy0B z(BAU<5xQ7i7js{07r>uA=g9vih#^NL1XwZ*ZXhzB_7$z&6=Fp6x%NL<`x)S`A5gsQ zY+hZDpzaQ{L1V^tLATH4I;e~T<*GuU82N==r5wEga!QVG=}no1N7egs{)5#aPc`ca zKdFCGy_b^-lP$iJF6JF{>};Jt4dqy-hwL{6OWsce%f)_`*8&Q3WznpTJSU7{K|Mg%i{m{hkS{UkSC}wDd)^lxgt#~$5 zbf6Y(H>`awu_mXcMT%~cD1_#@0jF;sa9bI-ixEJzUOGhLUh5%p;S=Z%lKxgYl6OQm z!RV0KV)O}FBCzaPC#y;mbTf<1N_=D4#8Ox;PObfzn*`6e>b+rpTwO+>@y}zV_V<_c z??{3FQLmlJWec8%gTJ$HlFUY*u>2R^{{9oA>IC&CHr+Th97l9K18e4N+jm@6u3#ps z(`Vm7{v9OdBXYe#g?#c8l+YP_hy3qQVyz1l4^Ed&dm8T*N*KaTgB@yQ(~BZ4-}hDm z3i>-bG%sa(6D9;xFU-6mTZuh4e$$s&TYhK2M1PNHpF!d_aLnm+m1wcG276ADm8O9W z*PYUkeu1@DSp^X&fU=|%TP%j`VB38}zDI7Y*2j4{nPw0Oy2lo9K2-Zd(2CHDNpYbL zGh)V~wjNG&+@VrygZB2ZTKoafbp61;!I#L&E1s|{sdh?gYu^f2RStNKhSVZ8C9Xpp z!`g`B=2~gdAyE9zEg$@*x3O3p42Q)!N(M&3E55tr>VU2=I-kO-*}$Ws2~mkaoGvE~ z%%!>ia7Nsj&K%`84tShnie@E9BLYJ!L2eM@)S-gY=`g>BWa1h?lh(%64O7J&g;3!_ zDq5g|0v;((=}V-SfEX%HkbCv5XqiuWnS!qHN{LE7!;kttzF#@Dr{cP#sdO`a#; zh(rh|Hz+GQP#(Utfnxt_>1}=kir+Du?QHm-8);tfcH@RIhvZtiUqNxR0A1VJ-*^X( zm8^N}l}Pw5zlt+k2;Kox?J4dt?DLld53bGIK!dA;VD}LwFYOD}&lCKtkUT!~s9^GV zi+CwxVv}wAN*4>q(F`x_U8y@1J$8ze0u0BWW;LW13FUDOW=>2;r3uw<%R#K*073hF z%VU$YmDV$Z1h@8^pi8Fuz~ApVr7v~%yNsM#OkiwL_j|RaYiLP@=*h#>SZToZ$cgqS zo8p3=6^d8uZWte=!x{LopC~CeeuL4E7ml-7G-Ws7Q*J0PS&+!oGP&yZ+tFmq0uUe( zt9}<-)(Br5uW;{_k+xt|E;nypG@YOibl%?Ha#ZSg|LM2#vQKsT?-*d!q1zVeSsAZs z|DuwjGYAYCz!)7hl2!ldNZOyX)4@UK43%rke2JR2K8pE#ILp3`?ILdW7dFMO-TrEN za6npUWOP~Htu|Jzv3fXGZzJ1A{~_=1rKR>C{Bp_{48?sz^~{K?i<%UDvSXlacLU@| zYXl_t>qeINZMHjwew=_q@@umheJR77o@LPKNW-Lrnpyu9>Mn5gsa(VP56R8R8pmQ) z&V~6x>!d4UwT!+JxaUi60?WI$;BaQyD2#U0+~P7qYo(VIQgadQdus$q1ncU1MxW8@ zVmPS^1(4aRo~;tso={UB@zP6^B8xc+;%$&#N{nGOURT|_%7>eC12h7f>zo|{Hd-Ul zCZvN~{CF}Is?g3U0?jEVYrGT^Uw;Fg?d}F`^tH4CWqpr~%nMWW{JT{dl35ZPe=QZh zHTAzS_r37OYv*p0HG)|pI{5DT*Z0nq)j&bnWWym)rZ!GWF}qz_5Wd;cC&9bBPk*t| z|84VQ(}1_}aS1}n$bOm&@;v!Y(&z?Tm<~kD(FKSPfXH_)fFa8UD2Alg%}(`c9P4j) z8*VY3|BR0bwno(Jf&z^C5b;0xK6^&tL()w)WF{NrlT3A9v!2xgAAzWHjTqFlC-0BR zcZpR<#0PFlR>mOI)$}@XmytuRd%TSROSs>^ChkQX^@T2~_Ec!2vFER%Culr4j_f$k zqSVVo1X|uqPY;b;o@-zlf1;M2ivfaz=-`q3aL{&Dn{W?RZFZ5DE*p8Tw6iV=Rr#A5 zkI#uBf<%Al?{19yj5Bz!F5%uy2SFFxK0)ogRm{L6g}t{C*VD3{9(1!*SOZ>0=026H zEMBbge*Xukp4L~+dTQ3Y_o!FylJ6Kc*}jtH0NdJgioJ>1q6k z3*h&Vj5O$2V%YA8DN*>GnSqTc5xC3+x?o%FL;GdywHdN%D#b0Wz6}p=i`z})z&7)#W%L)P=$2yJ$?dcHJiKhq)6il^CF4`j zV-$P0Me{~(mjOu5SVO&umoOlSPD(%VIRjgSPTjdJ#g=N@`i+9IiTOL_NkOm7%b7%+ z1g2F2P9jP~fJgB_2WA`fc`VxO=I9RwZ%-OCB%)mjAiD`GiT>UAF>ow&!6yJ2As{cH zoeg68p%TSxqPfQHIkzbRe0O**DgRalbqC@_)W}l)(@1V_R7~Wg=rJz4r!1rW_uYm* z6(Q-1Q<@_|{1x%<<XUoMjZw{V}>BH3d(}3i`WK!EG2m*qjQrI<)dbcTc!7UCD0wW6-Q^jO)H(4b#{mQGui=V|6+5nlrEuz9-jBthc*-5=gD&K zNDuf%*8y|}?FvF`O-f%)YniS9+(TV3F*$TxsavMuLE_p&e2@s7Pg#<&EO)ty5nsJx zZwm#IQdEKVC~BZG%jN=;x}eb0HmqMG_!8(uuaH%n7N6L+4a^}97GV1`rxt^@gqpT9 zbxmEqOPuaT+;3}McLJHO9z6|p4Ze*mr49yvbyO-qJNw4j74)gPHIIIasr(0qZg=&ytEaZCFv@;ZaWHcvS!`Xf zFCJI(>+zX_bAZ6rXoPC_iawVX&;5O}d*S{GK&AZhExqP{KmkK7mJurv2ZIPjH}AlL zUL|wK$Lagosmdtf&!9T@DAFCTMZl|xDc2*ZGQm@1H79|EN6)0yXOGa^xfxSa_Dfvt z(oN)&*6o4?<2ML~e>F{LZLBETt=?GTdFq$SRk)A?qD7A3cPq;@8>GbS7!>RjTyQ={ z4g688VkF4OOL)p>sjmKoskpEyfEQAGNqS$2NRtxMVNfwsh%7&;nqj{m)2d%zNCYm$ z>D|^p!e!DI(5fVc`vTI0Oc-RWR6KJG^0~i*z`qJO`}cnVUd@|)74E}c`6sNB&*8Hg z;nx!~^!Wqt`jDXEHF*wn@2-Hai??NrxXKsH=G_xY;~qC(68=*@$+k|FC;s%VQLY_d z`I0J=HdN_rw_*b|;NWOG*;C?YQ!pmu$;f^bYfsHD*!604Nz<)HU#CqjMp+jwx;TM1(dNNkR|4d g5u|@Vtu5S(f1(RY$iCP#1*3;3$f!z}!HxX?52@QCRR910 diff --git a/doc/logo_small.png b/doc/logo_small.png index baff7eeeca3ec7247eecc1bee4ccc54923a87938..0c79cf46ca845eda07ebc4c4e9f3bbb0ff4b3216 100644 GIT binary patch literal 7828 zcmX9@1yEG)*ItlV5OtA|mIditLX>9dmPSfiy1QfP5|D1B8|m(j1!?K-rKRD&zwf&< z?|tW<8*}IM^PI3R3X-p|Nw5I`z$v5gMAhx@amIHqRW-s+ zc-u+aeu}iBl++uR+AkU)P#%5v->RwrU`0SUx)TBJPgZQI3Sc9!teJ{lSBwFh{?7-H z$6KkBpxP8U?`=d#;U*}KUD`Bm^mH84$9QLqY4>@&X*Ycl1V;OCX51Br&97@FjczRi z1OvhMSthgRBo+OgvlGec%GkasS<#cTyvoq|;ND^FQcZ6U`Tq4kzoZFv&*jkZy8%Ej znlKOr6(-@t)7HfcW1n;PJC<_+ja1Q1VU1W$vXpv} zsYkUo9HeYeI?BEMPAid z*h%Aw{k|Le($<~%eEt6=vUrXB&T0-bLH|msG4i(_t`K5V>USar-+zFuFerc8 z^9L3(#nZqt8FPm9xyy%(hTa0%kYCw9%nAjVOm9hXD=Q}8f_ngK;eu)T}i&;CxmX^YP1)BuEvt+ye6n2qxqGb+OW%E)k z2a`TFlL4nl_{%dm|I*V*^Z-yy*q6DL2J8gF`~G`+mDCUSpNSKKmj0a2<#D>~;(}f* zh%vBZuR~pg;mAS^SUz&Y-$rZs6^_n=m*Ejdy{vTlD5){`!$Wnki_T`mFA5R-9nb1z zB{QG0M&*~R97yXUC(fHLMpA z*xG8L{ChKv3`X4&VbkA^xgO=VZ;iGKmt2i@8QECnZ`t9hlwd2VVQT`;xbNR*w%C)2 z-+xAIzn(fWkgcd*>#KP#sZwL4@B4Lp@~p9&be!l9GW1*EJ$1A*ZyWPrr2eku?mU*+ z?JXl?|E)#oZFjJ-#JM&x_t&sy#6@!(aW3#J3w&7nyKb^?#X)XL0}d9+jYnZpAc4P2 zOYI^eOZ|u>{9n9RYWvxJ?xR}0g;2k%_{K&>^Z46t-f81!I8rI8k%j!NwTXR<_&r^;V zukwAkHoB7dC{79jb;l1X+lhr!<5DO-#=fjW&3Y5 zc+8%59ND%H-%5yx{SU<^+LM`FPaWjqOpxpE|?hF*_$F$W`c@K zBsXAb*wJjH%cAAn&QtzXfAMQo!@SN0{{Yuxo#RZ>foPGDlTnYvnZg@ks9(%ait<{u zujr(bpD-XN*XgT^Y_WHo>x*$U-nu*QHn5Veb>@Y#6*;Fq+4oud1Is$~{uCZD-F+^m zY8q6I#0?hmVgSR$in?14o1Si5di>EVylKUs?wo8;k#Gm6iSMUkKU(W|@o<;tsP@ z7wV|{b_1E{gDyV$qiDMwBK=VzEXB+>W~7`zRz;BC<1sHz`SSGmP6RG47+GL$)3H7~ zd8SB{|Ap6kmjyQQG^%~!;+@i#A70+?hXurzY06$a@_F*kwhk4B(W~X@7LF)Cdsp3r zrtmaq1bhEzq?aJy_8MIX-ynnmbMUqc=kja^*^N1p*U{QTYW4-JqN-I1M(tFjkyO{!*05rD_2X7C+C8@#QrG}4GcEo0nUAk0IS7R_c@pwBXn9rU z@)$AyYxb@Q4TKg70K$BaZnmgou>zAPbEtP*J*Obxi_q3V6JXzfURt~T_b!TRBXxzL z73pMd%Bn1*7%fuVYO~zj1eJJGJJ{i7+Koh#M(#{3g|SzK7`>olah+Ayw#jZOt8!>A zuD9>a9x-`V$7l3frlMeO)Y{YnAb2BVu=*i`wFZIJ)*}BotiLUK%qm4p{6Xhoje86l3b5fpR`dJ72`Fpr&OlK{nT!^iU@emrv5F~AQ(`6R))hVOtSvS-tVlAj|E0U z>Rw{`0W&P3*`mcQcf0BBdVTA13%1V>gZ_|NujCsW%q&ewI79a)(6{2eyL)7W(MpXg zmq_k@$m{<0+K1!7{_j2&3+kolnZJCK_8}jo3_&%AeQ{93r)LS+V4&I2ncBvVpkK!l zoJ!N;)8@K?C)ohi{4I^z2e!tC_DpU7Ktf7kXxHUdqr-1SUdpuY3Mc(`QyPRvd2I=O z@@S!M<#QXX?C~k8fhp;;YB^gkvzc@ZRWC8l^O_m4Vy?&2O zk_yv}p^#--4xdaA$z{%o0pQ*?=K5z2RzC4%UC&)5lL7YRwv-`hXSs)HqgB7lj(tYY)JRP~m6LcdlDFm!M3_a@o;XzT42f^DH84hqa* z7V>UCcnd9x`L;oh@<>Y1N(lq{abIC;7?}~!yakAb{+6VQV`?P>1IncC(ExWhG_QfJ z{_#Sal?7_YY6eW6ZHu+@zwFk@937NP1)7Cg1?G@+$wrI-@rb~1Bdp#YS6@tcrz8+?-zph2H%;5bxO&kG`QgP zaAMvzDAmm|6&y(Spp8|zkh7C=9`bzNSw-#3Q-y)0r!9G7^3T-&)Iv*O3r-9M<~*Bv zmlvqwcQ;srPNabiyC*r2Wj$ir0f$M0L~SM?W?mrD`~u}pn#H1VAK#6Gph%cd*69Ev zCFGw+uqPFi8MBKT-T9wFl(A~9CG&tLv&Jv{)T)*jZ5n|ygDJ|F=70%g%OpiXxnChq zEe3d3T1g43tyRqC7v^eoGAi&dtv*d`z|*PGpf#}WwE&l^g24fR6#*Ao@hk}-F}_!b zfb|WnD<&X(uD4DlxUo0lP6(1h>io@8;4@fw|DhExCv-Pl@D<=~ZN@XR2{1&~H{R&$ zx98rS-2g+fJo3-Y&hJWx8YCsg(YDi|d9L+WIQPb$@wxr zA_@bw>8;8gm#Q)b8?;guRgfB#@nVDJfLJQ^ctQ;9@2G@Hw2r6N+;rxMFrfgue@6>b zg^IhoyN6dE;ki5G8HXn)Y^dX7W4euYIWrd48~~`4nc3c-d|C6zMoCbQ z{Kn~V%OQBGE&vsD<-E%s^~eB(-8&+F^W9(t_(Akq-Rx~_D)#nBToRNGa#rki;7y=i z=7(>YCCDT%9Lf7vp5;9_udK|fKazN1Vd0;?WWe-g=w(a5`$C#ABF9U}eTeA_)Q`($ zCG3Lbo4ov3+1DC5T0C*3w+TB`XnvN#I5heAK-&$u4LY)QfbhzVnv#uq01T8i_V3@K z?T9glrz5?zBd!GzHwvnsm;)uN9-7~B6*m^tY^Fqwo12?ujUNk4HV_tNV`s@MtYN27H)4N1OW2a7FWoY)Q?svUH>GS9h;?t~E$1rKN)IzVq=4)0 zPqC5|!~!0*@$YD0W_5qpdqQG}AciL^ZHx8R@k7Qu%*$aQfU2sh*WG?0(rd1x^U&uc z5rNZ+6hco<6%5~SNaC5;1Rw7sJh88v?6^BOHa0NiL}=q#v}*U-?-6*VoKYM$(7|Mg zk)GRnP)SKi+Ap2(+_?Doh=_=`&s6AsZQ}P4EHs59@X)Y-a2iRDsD%?9k2$pm5cKx; z*7=Ti@=4G0xNc%%qF~0bH}L-IkjHUHt_AMIFC?zSq+WLF08=`U0t~X7M!?DPqqnfw zdgPkwmel4HWMxaIM6m;YBqtN$;Zb8e%GY>Kx?k>34Zz`*3Y`fF2{be`dOuXBBfjVi zE0Ku=f)297Owb8mB$$sntBDun<>j@%pjAkpAI<*eM1j*2Ny0sB^>GeLllhT_W$a~Y zj|Xml$uGcKoS@nR=VYa4IxX!N#Hw!6T8U*+V&d#=Q%6UKLZRc#zt_}~(PcFuDY^8M z@t)RVB<0aae#v&W2E%?VQ{L5!7St6K4`fMf*08|F#&#$;EA49>7xX^!D=I3|B4m5| zNJlr2wg|OP@FjsrC+r@@Sntj~(25;Zm$J24-h}x3`#ZbLV4$JV<2&Pgd5wcpHWja- zq2XCWP_GjFmJ+AHmrcmg^;G|?x`S=$*>Q-C>Fv9gUc$qg8ZDI~nHbt_JDHS;qvx_DJw zuk>NCAJKyLj51)*xo6*3C32&l6SXhc{>WS#O$C#>z?fPgGnrTGaKA9!<)bG_tn7HY zGgaL%w;jK0!_Fz{vQr}pI?W95_I00I0|QP?#!UfcJFmAO5Qy_fU82qRz7N%{t*t{3 zM?%c>^z;l2O-;RwdJW$9|MU=2D3nlwXaG6e^YYVB)tU%eXhH)y<858aTy#`ad5xMV zkO4oOMmF&i>#JqWcQQa+G=rfynt(6Y7I1!eNJxn1XDDm}S*10*!G5K|PLtNJfk!d8 zi7Myiw=KtK&?y`me*RX$$8dwK^v{kJ-#T8uCO;cb%cca5B(Zfg^&ivQ+uN_$1_i!2 z(MKl@DpKv?p0rjD^`LcmOLv-7s$$hUn@&R8!~RP$re3Su*(UHbBLyk>KAIexnW?ub1%QTXl}Cq$o}$90A8s#5-7*qt&xMD;c;D3xo4jqSky_?^ z+1`Lv4|S~G3^tNCtKfA-c&&B%^A?99+ucINEJ5c3RTC4M9`>iFr>lHCAUJ-=30YID zks0G}tu)!D>UFR$(4OpiBZ3VB6VubtxgXIkf%axOz2y?2<#?)jR>Y$6*OD${et5umss{{g?d3m);XC%bL^dlkt z$V%Yu?k+Aau3oA}R%Gx#QfoB|9RtG!v9!P!EVS&gWSaW)w??{Btfaf6d6V-#q^bWQ zBu1#XKy_-PfFJitDxQr^&-;99$f2oKFp9?o4WDaXRvcKZRgP4foSfWc^z=$`PD2X! z0yvzKAm2fCBJJHyR-;cUbUG(zm7Arhim5p}TmR|dHi|@d(imxQqCR1OyZ$*($1^v=}x5|U;$tuAu;59>gg?81vXBe$s|NXpUnS!u+^$~ z!B>13p{C)(BCzubzbyWvpfZ6!?o;s=($%;nuV6ngDctn`+C{tT_ zq8?mlUFGFdA@>{IR2Q#)wI|ZCQVN%R?FlYpORKyy=n0N`FSH^UGoi*v2=BS@GB)07 zb-(CMAzc%nDyWKQoZt||&m|k-`VS`&gC??|H>K&eY#N1JQ(^%9t-)Bw-SK>S84zq| zPU^UW01eE_YJ*s6`nRyM@?oC%F}#TqwYFhTn1y`o=%%&NTAWk9Ujc%AX5qJqQGXS%iV{A<#T!t1mPD4 zGKEtN3B!JV03o4{wKe^dJzu+*P!y@0CVFIZIdGn4&no+P{YD&9zV)`S#pQN_#8q>P zWjTAZv3T)&j>PwPY}my3gMeaY>OZ|bMMEVQixthRieduYcu!W_=sdhp+Wt+i-VK@cUhbKxeE_Id&s=@3 z-amtH5nUi#P~o`xFm_>tqMxRlrFW$e`gC}2-#Dg7Tx~5ifySOGmgfj{sr4QA{-Mg% z+y@`)mj*%QZ&8$$hqWo1SPvt{niij(eOQ^Q;=&g}-BSeL0hGcnht)|jb-sWX6{fG? zS?&a_CMWiA33`tC)CN*&vcj8V!J-L+>v~dT3UBc_A30)q+m|RRqeG-oV z*0Ux^(b=;M7Ql;W{>Vum4TVH)mdm`opE0a$enx4&{4c+c^qgn4ddka}OT&=lC z&TqNDWQ~0@HH%~^#laS4r`h7mJ5v0q8&bH?G{=T-6$;k3mUvLT@K2Hcwa|) zW%GwKvv2|%_&uVhb_Q^|cyQ}4 z8r~uyqF`R98~f-w=w2hwn<}9@1HWW|(_7>EtAx}cT!IpY^OD%B@v_LDQ-?b*%}vF( z7#|TyAc!wW_*V)V3_!Viu(dSgo&;3UxDIH)Bb1@Inzh(O8*i63D1C{7idGhI)yG*` zNK%38r<5bQ`=M0Fj7y0fL zvJp$zcX@qxFd2$z9Wd6L_8-M|;~q&c%Q2uVyZ^Ky8Y3wPx#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0G-1|aoDCIA2*bxA})RCwC$ zop)dq)z-l8ow7ZxA$>z2y=^wxJ~Q|GV|;5!BiWK1jgF7Ph1$ zRi^w|c=}B7#X?;X0006Y=exJ#$$1!0&*+r~Rk6Mp0Kg#h*PuggJXcN&+m`HmbY$6s zl|o!F(4*hfkblLvMF)!m>pqrT`QU*nttLN{?l!2}oY+nnhEWNl^wSEwy&$mDUh&?cg$H1qHN-yLOQ7g^Mj&Wn77-e!LaT3R>$@+1VnvDiKJwx}FJ z>&xAWq`7xLq8RFv$OX^)41J0vYXU`u`r(ADQ(q4pH#dAb*DthX;|UM|V9%pN@77+RTmH9`1f4aNM#kpIX>>)>T9aA@1qk?T^C0IE1$A^Bvx* z*!ime%m0q}z`FcQGRWf@MRl%*(r^f^-+#DYnHrz={SeQA^E=M85kCMxH@B`3g1gG3 zS{^8e&{ClhJ^Rjo{6&6WcU^2degJ?_NeB+{Ejc@CWx*k|R1Q2oT4_|R?f$Jz`LP!9 zi|GGLR3`7r-Ou$4tt{n+3U{7c%jC5%@w1DiS5L(5lMoX2i{Z4;3esZy5JF3${!1fz z@1r9ojpww`{Mw`tU8*njF*kT-cD@=~BPPV0EwqAV`aEU%J2z&ve7-gLaR@DvQ$^Jrjgxd)2;0RZ{BLJpz%@>rASFZ64z+$Dqr0Hmfg976M@imH~kRCll* zM+P7O00f*Zv_SN^8E$vA}OivZ^v7(;9IC34ic004iXFNe^4apSp` z>C0Oys*qHfLO?u79KazoU;IUWS?Y(amHUN~%Q3E9*uOc1W=bb%c%C-DRa((vT}eiD z_Oo7tID{5SH`gu@z$>NKTR1ucA>5g}pMdyizZW=!7D<$>qW~9ND81T}(HVr0kumOm zba154b3Ba4UgVa=m@n<Zc`Z+j_9RmI|&;OP~=oBW#)}Uk;%~irDg*Zq3;_wf#i* z8tb08AzyP^GYH|ulB+bL<2(nnaPFB^*`Y4&YpOnH3(euTDsA2T#E*{`o^1IS2z6-> zo}A>*HKfgxVtwh%+Y275AH5m$>YVWROxj>;i?}Oyf2gE=Pxo$hZ((+=5cOYg_?Crv z?an>$(#1EGhRTh-zGL~ZcYl4jM@_43XOez>xMx**LQ>`Zdhgn?7OqJD{@0wnY|Or< z$a(Ik2OqI+UP_~C-rYqhm8pX~2Y%J%6AWOi+TBy$UvmGOVtt9C`r#*03vAJ7CuIJ- zS+Qfd&#<>c-f}dALR{#^uV+I687Z@FCGLH6XjOUw4(fW4?Rs6J0wexc;D zuh3^;yM9OWkH73cDoWbXTFoh&%YW}JIa7T8m4Fu)cAV2tTg}2w<_kWLYNes_MZXaXJI-c9 zBnYAPJy-o%c*@&m zT+QRpl9xSDrFD`;Z0z}cpxEE!zd#7(Jhur!;}JjrnYpZ&g?FL!N_^V)h1#O$y@xL8 z{1H1Y-?(rHpQu68&l#apT?k2-O9z&dno^Iv@T#xS=c_KC4fTp;FV=jX{Tl*kM8=8N zrjPb{enQarW@UY%@YK)Q+j6y!gGE7GdM8B6BAVDy;Sl~U3}C)b|H)72J3e3x0HC{D z*P5Kfgm`1G@A}H0zU2ssk#e5f>h$%$q?JSE75c;*_%K<0Z<{yX9Wc8yY*p<>DXZ?2Wy^cVSh@Y^-ljsgI>$fADixw=?aahw;UPs9pnXp0u(S9(erQPgz^klo|nq!M~$@0*o&t1ZF#v001E& zY|6uWTEDJae3Yyc3;tNa$rWioR2i#Z2^cd!Vg?RzVDYXoHAPCg)cdyfhmigB{wzGb zDt%owrJ)fWB=Uc^{iGpYu`M}#3IJdb!-jIQsuiCmf0bFSh;fTv9Q9vcp|>5m2t>=n zORvsalKge9<}m^wP~<;1{QW-ip3U;XtQCNv5hK9)pC*6xN5Nn1`0|=v_*Pl$($*2O z{JuQUGjTi4a1DVIYLv+@e>;d}Z=4>@t>Q>qehdz71JhH!8OA zFfv3Oyej5PIp4h=zqH`g)|?$#)j0y3Z*py#=)cs;t|^5=0C@1_TYD#naG~94UW;lW zgz&TEWv7eJV*pMG`Pbx-33cmt_oIV9XK&L|S_FWUlsxM-gv5x(AwU4`S3bzrJgl1? z50WE%pQ8}n_>0kqni)Dx)HL7K*&EX9<+qdu)!zPFI!eP#dc&&Ak-X#IXZ)yV3=~5J zOM<3^yfxG-u8GO6%lx@6LN*vSC;a^;dNd9ownA8QQpxWF_h1k+DIa_5fhzse z)sH(!f)jeIs2OnCoqJ$a`Z^LLL&QOAx_!+a#lXR9iwHYP!#4L`+gaABCZY7I%oS-r zK#2V-==EtGCULXK+af}Qkcz6hUUutQ@4;PMqiWp^76%RYdG-bGVeDBA)=)kO0jxU- z5poDEF@bMipYceO=O^@Ghe_F1^ylntgFRzIB!Nphe`Lk%0)&1&++yl_pb}i-R0ypZ zo!OOpU~T3`InV9TGuurU7bkyxruaf{_a5KIEVV_a#f-b=J?Ru$xISwW#Tb`%{>W4> zCkjuUEINY$Y#fGR9NG-dOmBSd)nUxlj`1DA65XKqg@nWa&))8>x6vGjuoc3qB8)1C|RcfksnvT z_R)|gS3aN^s=HfPmgw<XFijRvIgL7|(^2008b(+=CG7Bn>xvyVa(H zw>&fmZLqO|{b^yDz5)VF_P?p-_f@GRBzm~VSYKlhf)MI-y6o)i!otGX*jR%2`{i%n zzMYbi(p;r?cXuB?e7FO6Tb3g96tsIgWP)iw6h-wz1~6fg4%V_UrZN58+}!Qkw`XQ% zvR6ow95`?w`^>lBek*N$G{U2q#bT~gfu@H}G5yTZ85tsx2ynABY=Q7{>9zN7&RdhQo~>LhtwjjcIv2`3cnG0_f`Ww$7n-MJT)(iX zqM@{WjIT1P*q?gFfB-PE+uUN4H*k)9Hz<>c>US5HLfwuTQ>xM+l{* zrKP2%ktEqrQjPcL5JD71;W%E`XJHsdEEd1;!V9JaG|nEKfdI~rm}&Ys&VRJ!sskZB zc<>;PSAT1)(P;Yi?Hd#n)JO_^K7YlE6=%<$J#gRvkM~3r4MKSI=uxp)Jbd_Y$MBD= z;8+)Ei*nVEO+VABGA*nZwOW1e-o09S#wu21WaQ?}n`g|J;qUL?h$13{TwGj6j~<UAuObm6bWhTvjV|D&D#_ZQSLF?9Z)=oo`*8_FMi5OKRuh#fuV& zr0&AOK7IPknKQ@apg#3OK?w2r{N>A+*G)|z5UgFh)-k#$R?tFMQiek;Ut5R}av>!` zT#%wlwWM}3GBWD^yFehAHf>t7nm*G)oVj!78VrV-Ull?qCnu+(qQU_b(FzEI#eq1) zAF6X9gdL^f2F93Cm1RkS#l^*SRdCp_VW9a@X?A<8fPjGD;NV(YrePR%_wHTJ76Jf3 zh&YgdL}qn103cEt@kZd-DeWg(lAubZs&z%H)%NMr$80YNAru=MOVdxR`^9lQBO`mD=Xs=0sw%&$d`abo;Kge7)gvQ&=&ps&Z1CB``!IFS?euL%2e+yA0HoIU*E=I zYc!g=-F(g?{YRo*qj(rEQJD+?@DX~636VmbQ(;h=ms;I1JRXmyP$(MG&blOOUSZqV z%*@QXu>!7N_>aWMIunq%U6IKC9O@M(#D%}*|6yh?#kKan_d=oY%9Sfk+)$=or}W&p zb9J?_ZbnXu5FymvJ%&OQix467qVI4cG9Jo1W?q6mK0b9(oj!eFI@q zg?0b2R`ct;2qA<$-C~T4@s2Wy{n^c}ix?N@Y4fwH6=vv7hYlU;M%Qp0|Kf`;nq|FA zlQt3(66!|GX_{`|zCDNVABvr%5je!|Deto@r^f}n$jA7mx1lWsnzOQ=_7>l z^YfQ4U(R+Rny8Kt!ez^rX*8O;57RUq85zkT{73rAdyqeewXM?31O@fJPsr^;B(E<-fCJ!BcArKMfFb}c0( zrL3&1)_5;FqXNhA(9qDBn3%Y@xLT7CIfPBE<+`=DNwNKhjGtC_TNdll-@^23CSF-t zsn_e(YBfSgAQ13)Jef>Jl4QLPJET~7KT-(cCdIEu^8VPJd89u{hILMj0`aB0XY0FwDAtew%Ev>7o7JQsPA?Oth zVrohQ9A(~v10tjtBLaXu{I6cguFiSp^z)x3FXNW%wo!yx)j15Jy&7Jt?J4jO(QTVwA(PuYtRtSI1FI#-5`|- zUy=3$XB1l*A*+9HUY%B9P?8wAtKZMAHAa0K>a-z**7sORG1R_ChtHQ>;z+a_LIe>2p9e}=&(&+3n4@U zT2cKl%%y{^HQtG|5JLE6$_fl%^CM>2{L6xnfCS6$=j?5tBz?&RZmfWCv*K3`tqB$f z4)+<>lCSt@;py`wm#5yC%~3QTAwoz?>o#U@)6x3Hoj+>HdPE4V@3~5V^E0Zl-nl-L zqi7~V2;r*qbtEK*dBufFLR!)$K?q$**`EGegoH3#o%80^ciH8=9Jyw=GYFwlec7lB zuR#d^h}|!H+HkIobWuZVUcWM>imGUh#j4UBuf8 z9~y)rT(qzM)^>b(g+ck!rMJoq6@XL4rXWNJ6=(}j6`eOS##!OhTCoL#5b`kI-hNwx zMS)sM`^x2ssa5G5zo#RF5U$DiQHTrP2zsT}V_j_STl%c)E$=}w^xM~G9L@g&IQ>{Z zYj5`F2Y3$GIxerC{bJdA7$=7;U$ug~1HC1)oF;j>NT&^= zHf9UelxB-!$Hd^*C4{)`*N%k?`5mnxWjCZ06QUVmQ|*G^uH5~xXNF%byWxZu5o^fU zY*PHXGj|`bhZWBVAs50$LP$1b|KfygDOPWd6zYl-GB<6^{>46w0fZxcpF;@Utw>T+ zYR;Vj5W@YBj)-vKaPMdBs3fbpExT5B)ABQD9mFG5Q|cfytilGRMu&=`CXx%2o_V+qW#sKC#6tQ~wfkyW`XtT2W(ZSuh2bQOP=Tur) z&lx!5E=_#<`V4!lh3C8m>uBAnqH|8|kt(Vm=4lJc^cD747$2dxkI-99snabxK+(Pt zUMRi7$9Sm_nf3_@dUzTWeSa$&az0m{(if-Mp1}LtW0u7cP6bW zlS0NwX@p&Wzg=aW4UF-DD%}|oKGNhdi18EI_rZ*OoG&9?G?a$j_0)+F7HW$jguX%_ z2Pjp0NeF=vW}l2VQ-fcf=jNb?+a%Uif_ms(w@$ud) zzpFGO_qhws<&gpaVm$_o@O^H){}==iyZX!C-BBm;OKW?i zj!~+2uFoJb^2eU5oz520h<1Q;lvKv5~=#CgwXkt z%Y2;Q%e}kPAw&pul!oCDKT~|ser+Ly@J#V}08pIA0H>Qf10fvb@r<6)Unsp|$2~&E zc((YGfib+`Gt?R9&JY;USKd>E3->=hVyETVnzK_vhzEPdI>`*g+6WOsZv>8`5&dh< zZhN$XEJ7WlpAzzx^Q_p$Y8jIL?fS?_S&6>vT=7Nwss#w)+Kdetz+zmZ!(2Kz@8UE7 zzz3mI0Dvpg*V$1mAcXR@gyp*lPh}NsN5&=D$(f-h~$exH>)IQt34s&_DHD?Sy+kVGT7+^CCV776)o*&7^D7 z+wzXshw_f?d31SGUkg& zX-D9c@|68ZD-6n4ue`0MH1RQ?4`|n??Yy_rP^nNq?B>=5IK9B9o?c#ItUOV8>W#p0 zZRe#A=Ns%`M#Rd#HZIAQU1AtTI+Fd$8(7zu|r5 zJ-LINKywejyH<8{QPSsXN_}AVqM8*#g4=IzvGZVh=C9e^^6B4-LFg);sORSdPX1P+NHO9_wMdpV_c)z zW)J}c(BFFuCs(F)a*4noW?+mjoFAv7bv%qmBU;ZGg2jQc9{q=T4eBHf2i(b%wnkV> zDvhc?3jR7002ov JPDHLkV1h)u0aE|~ From 9a5356664ec232492c88ac0de0ee3ac5dc9675f7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 22:38:05 -0400 Subject: [PATCH 0502/2835] update --- doc/logo_small.png | Bin 7828 -> 7621 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo_small.png b/doc/logo_small.png index 0c79cf46ca845eda07ebc4c4e9f3bbb0ff4b3216..4ed97ac00dedf917859dd9082dff26a2209345bb 100644 GIT binary patch literal 7621 zcmV;$9XjHPP)Px#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0J%0be7u7XSbqS4l)cRCwC$ zU3p*=RsR3JnaQ1|O`6`NcR5-p2o$*wQM}l75qCiraak3w#pMtXp#maOL=*+l<5dZ%gVnKb9jyx$)Kv9?WVo1|^h`Tmt;CU0imXFlKSecyWs zV+_1>y42hB3V{#C7$Jl){=V!#!Q#N-{>ccT_BMrh2s=|a%r@TmbWh0-(<7$-E&c`P z+obqz?M(2_R%d>AV@t8IR6nKESOWqhOk8xq+1mC}IYD^j-rlsW+J4$cnCc;7Q9$<1!MeZ@Pj;ro+^j58352%5#RY9J`bU5paF`0JcOPqT9Xa{#reeY5PGWU89f3}36(sA zo+>tm1_TsA@(_BeNGS=%U}8)>gq|uYp%MU>7|VDFJypUbAsFC>{w5Ehr^*1|L>9A| zHCa4_9;--2N?(CbiLunmT6qXPRfhQwMSuznMSPCbQ{};+@c@urm&4~sJyI~n$*TLp zq#@zbu+G0aU;gBJI6Ik$JPTa|+pnD96{#nJXJ*VxF0U*w6n4oy@TI`c>dbt7!7CRQ zHqfSiKD}e)Q9Oj6gC77wXj9(SZ%g-?8S|earp$_(-X(Y9G44!T+yo350qUGPDK(a@ z%wBi7Hd`t5O-WolT0Ok+ISgh9Aozm(rgGa{Rb{PqbW}hHV`PzTMUVrJ!OTyZL#wja z$w+BGMeomse%SbE18w{ye_Mf}=(C|8xOG~gM=H`ol^L7!wmITotUPAt{hzrTzj~YD zwTp`j425Dsw5tEoaRH<3hY&(L3%~w%@wa+fZ)53$+Kfp-6L?$r8zB)XB!!OpSAg96 zEC`_=%m2GR=VKbvgZu`(mAE)W63po^gwVyBtPQzaZs_ynlWW;Z|<)-wO%n^)?tk%%0s9JF!p zS|K6y6)1L>e77WP#pWUF-KP?0mj3IxxjcleG7k^{Y{}p8ec68+=!X0Khc52>8b>~k zTbuH??khW>r}dH2UT-EW9O&1d>)%)7p1)9&RnSo6K9z8=B4bnDKWnyX?A_}!cj^7S{LiJCqka7=rt$(>wiEdILqyF=Ox z18q=Hiut|gObMUN9pxeXwVPOAuh-?})aO3c>k01nMBl!4gt&8rq;P4!`3iy0n1GQ@ z3Fla>jJ2?#WcihMqGgdQ5|@NaLnWl-p6LjCtnzK?zBSi2$SA1~C4VB~vDbRP*!U>L zgU7*;U6->qXJd)6w1F{>2^bM14zzdExvET!x$^g6k8-~}9{O;*n1m2ok+`I=q4ryTBK@nzRpthAI-t>0V5-&5!^3flF)vNK67H99jqKE zr({j8-Zj~o)|HPV`L1;T$NB$o)L@)EW_$9-?Y$EVV?A~I?C#R<&sAR_5E`Z$Iwxk< z0KY`;Xqi#7wcwuzD-O#j+0Wy@cmBc8RAuff|L=vG%R0+#3>affARcgQA>h95Ec&|4Sl;$_3LA=Z4f&2P;X}#C z299J{rj4x;h{({MsHf{mM3RW~75J8;D9FmeD%dLlLim~=J*N#C_);Z4!JH{B(m~Ka|2PgoH0Y4u4@RYF0y&Fwx*+P|*0L{pO2EQODGN z>$lqYtB?=?2s+#v1AG&pD++{K z4A)r9vKWU-LO84ZRneX*YfXqaXkPDE#s`jq-&Vx*EC>r5iWg(LxG_XAwoz9 z=EOWZE#k@MA)vj3D$Tb$VMM!?d%lxxAp{!+tw~nh*R7O9`(8(vX#DR=_piIYQA~<4 z;NJ4MZOI=wI@8A5a_VoqoxQHiq!E!K=iRv`K2+Oj2pDXvZAbFvzKVE{PB6KEg$pk& zJ6nB0Op0x+ZOwp`#We}znetoLyoB#|o`)2sI|{lfOKppyWS|-9sWm zLW-McPBUZv{8rkITWNAqMgR(t1iupZ+}MDT7-M@t2E23$gc##$Yt78lbA(91U>3C$ zT@*GHy?lP5kuk|hnT55ymAH6P&;zd8=#3X=Aw&pm$lc5UV`j|@`n=kBxS2y}SMfKW z<$pmS!pN8=28~GCu&FHR{^l}IHdgs;WF)VP+CL+Gum?;h43ClMhe1aI_T z_DIM?Z!$UFqe$n)zcf2)hQCPFL@{2d$vRY#AtFVAqJS+!HmIAOz)M#`h%q+M4Oi=O zhN=cpi1HVynsX~b(RW6S(y9s;s)J&qPj}6Z@CO#*e`) z#w>|Q5g|-XTItq+fx(Q0u{cP!U|UzL`QkAU{!sQ)W=$4FkkQR%vCcUCd{INm>i)|f z3Lfun%lBm1?;Z_)sFl#3hp=T8+Q1k$-uP5V2!h0c;{!%{W0`S>x8W7am%674J*+*(V%Ef%mM6S!FU~;-oxXiGPhS9l#;QjQ zRt{{}3|oW{1~6SbPuJCTS!xDj+3=A_BK3Y2et2UGLTHqFc#>aYV>=fn3H22!5*2;d zB&BrB_t1WjYPy)&HAKLGF&ls2xUg4*mqVCVwA;qo5JJ-;p0uk_4#g;0uOCN$<&6JA zNF;<@-Zg;-#w=!m@CGNAXq|!?mB&d$3JKwG|DpDBE}i+d)?5i4Ez0Jl^B}CV>GKRX zX-uz5Txth3UtjRp@jpF$bZV8Qng_4DL-@t5G>oxAph#8?wilaxe(OsaB^8lEu2_hd zZVlnS@&gn>g-b$&gn%0hEHh~iS02R}&+GlN@b0`wUb>?@gE1~LmN1yHVfw+K33i3K zF6Sd5A*2ZE;ot}P6lk}G5FvESPy_&0*7``u101M&+K_L!i2*11CAK$~=cNmRipzD` z2*6*c@)h{rsT8fxGtdoY#*)%+k!u`#i-k3ry6n+03*KEjlT)9I0Y<8a*~5mfiuWQw z{S>{qQC#P>sCv%d=$fjogAhoizjs2&!IM*;3xE<7eYhWOtnFmg88IOmq8#LWEXesm z@=;^E_Tjz586#RWgeJyppc@cE5z=sO@b5xR7Kun2(?gVloOeXqKIx?KwYbkNbc2ys z)FHO4NQ;dn+}Mqjlyb@IoccTpk!HqnpEAj{iBD&mEjsW=z6z!_D?HED14<}BXIC=7< zPNy@Q%_K>tq@<|TY7XOqf`YYc*U~iIVE{13etv%I)~yo?g`SQg<%kC-A2`tX*)(P? z03_XA3c?-av**yypFh8A*Djq-Cz0G)Mv!3`nx=o%^ClBP5YF^I#@J%9Fbw0JEWGlVNppEt)4YUCOqTB$X z-T0N3md=|uud=d|AP5hdWZO+W(mf?VF1)n-vEzTT|7d0`7_d9`I_03!YPHLkFSlB) z1i|mS;538)P-HBYk`e-t@h%a$C0)d%*yR+&7@JI{HEY(0#bVAJ@~F4~LLW-bg@d-c zi}Bz)rV+V=h3nU^He@-W5+l)wAgIpHA34$XpFJdY840sj&AacFTO}kO_fL_&}fNDFI1!m z&EOk)l#lQ|HW? z6B!w4u~=FJ9%KCW+izPe7LQojspBlXTkOjcK-ddJS{w@7lGCqNwK2rPu4XY}qn@{(SF%(4hyvDtp~i zC!f99PzV?>#=qP?`}FDOzbe`5l4s@`(P%V|OduE9r>3U19ls_xO`JG!!GZGOG~{Y^mGW}G2JiK)|v~|m$)eVRMlCnx$?c7k6h@;l$4aT7)4-=8HQQ3 zXpyh4Z^v>m+{uZFiA$F*Wm&e#e=)`+Nq+d@hu$1i*b^}Y4TW4t5iN^kFe4&Gm1do* zuAQ5k+akAEmQ7Ah4i66}h>oxP#YLP+Nl7Y|s>xfzHCR_yr_pGk=o zP1Eegj-}ZJn-VLaWtkxm?}~e(l;JVC>kjBuO^^3qcU2rKMh= zNFCPBL`r)BU<+%lvDPVtz5wvq&=0vK4qbI^lY`nPe}Dhhak3BJH$JDuG5tH!Kcb?z z(4~nnE-o(iW?L962?78CqR!V`v_G~TKj%#d#@J{yItuSv|6@oTUg)#bYEwK*HziXX!;_Y6o{R0EC< zc?a9t4Iuzb3>wQ|rrKIdV+H`kq&QHdrVw>Pcgn`voWFK+{l-a6O-(Dhx|PIOlS5ow z0RY}%^Ev|Iq~He_%-*(CF)YpX#k5{iEUcBrbg{A2d2{Fp!itItmxOB^mRGM{b)+z5 zGMP6+h%t_lMad{Bi}9(db6j^k95PW(Nm?=ReNV|>$fG>-7!`)3bJ1j9VF)f-dk|clr`DbUi zX#Znzaj{mbZDF4ygfPZoVPRgbNCBXiGzKX#J8P-48yElySBx=jQyOLm>BP8;TZJ@8*3vG zab2IssZE5?OL5Q9EPbv1x`{D4lf$Ga>WL?wXwlc;_*YX?vwZpTBS((dY_`^i@0+|# z8jWV|+_}ZY#jW6{P$;~Eh2+YWD_eP;MWiUb@~DAs5RpP|6H@?)l}CSH_LG&h7B!S` zrm@53Dn33wJw3feE~2|~>C&a+$B%2ZT7f{IR;ydAQsmCm>-A^OoJmVd`|i8%0Km~e z2SUhZv(2ABpUdQVZFa`4z8(x5pH{T{LQR(acM%~R;G1x&>fABiFFMO@wa~uLGd3kwdWu?_>Wf(>v z5ZFhO9kDPdC@3Z-#+%EZ8b9rXz%jdu_XLR4jk|ST@0SjrI3}Z{|J59t74@_;qk7Y) zPd{|%Q0wK}9FSb@)MPT{<>j@kQb@A(N;vL5tyb&InKLPh@{XF-HkIC%nKUY)pO_Fg zuJ3#6>bplPPXa*3gnbMTW@s^Ay}53Fk!-s88f_Qxra7&eW*BCLQ3p0 z9QR>X799flx-}2{U7Mrt2}rlO|1i;e{8RnKIaxNHI7#I3*>8Q`%lr`QKIr z_-f{&mD#Cfrr!*lt{=Dx01UL@ll<*Yc0EF9(3VcaKp1gg!@gcRyf3$iy0mPn?AFtG2bCMzeFlyAORjXEs#bW#9O()51|B#T7 zuxQaDFPPKY5eT_w?(GwY0R2aEut^^+=<}+cHrTMu$$|_!iV+bJsi~w6_)jPlrlh1SSg@d#fktl*D(o`Mf5^$(XZ!`KCjG7!`5zd&r_@-U=-bzMbH#m- zNFSfTCEf) zTj>R^Vjo2$TVwK|=yy1LqCv(?npa5Dv!N~J_1 z2?z*KDwQggDj*;LdXlsQLKfJ3eL=W1OiH$}kimP`KmJK`&`28(jNPjg`nh%jTjP+l zjr8p~3pKXY2SEsZQ}X?s^Yhl`e9&UoJQ4P&g|SFT$vZjkySDG_Hte3xNMq!B&T0$o za3gh=Dh#-=q2*GdK0fiYqyJ1}`burK8(0My#*9VxNrV`dc_ z%RGI9YHx;6B~oI*j&@B#2q8f43|Q87Cy1BMR-__A2mm&G?+VgtOU?T0o3d)I^7wTx z2&*jB2#}PN-1E#Gg*$&N|7q!!Rb~g2<)v#NJX@U!0NmzvvKN3I~=!O^0zXsm2 z_qb<6$YHRzbRU6;SXuNvPY#oWObMUNV5X>{^vj}e_%)rn7(&h^*^s;0SD>)5^w@xr zZE@@8^?tdJA|3&JeB&Q^`kTB_>@o;B_PlfL{k^68FyMHfmy)fYa$mBypkc@Sd22%)sX-G?eN z0OS6?{l2_^n}86sD^%wId!yg{IC&HT^m#$r*{V$5D7su)UbzP$bfzkEQC1296ebP* z^8T&R(exv?Q_!?ivyHTwBFL|XZx0j)@cr&y2mzqhS~un7Oct{wB7YdY%SRx0@-jK~ zxwFr{Dk4PwBGtE}cJhfgw=2@UW&2Ew$%bto4_@aZkUKw5v!9~RyxyxsV{x(QC0GuAd;r`y=w_mZbw6LML#8_5O>qn@Qc`mwMkv5(H z*81AI9pfSZ%sTT@v9XlF?DB+#kA(1>Ai5etD{FoK`leWU%%39ufZVfLIgRNlCuZtt zL%mIZaNM_Qkw1@tD{bN23yTj_r0=-3v)-zA$Hg$l6rw&E^1g+!V!$=lS{?xB<<}wK z(^cnl>+>1Rj8xz6D^$3J@(2Mygf#qz(O=ft^zlA1J*`y9y)4{Q@;wV|usCRC;u3ee zi-F>Rc%K*mFfyhyRhjm!6nJsU7M7VbpWjORYxJMPB%z%~i>FS`(wMdJ^4J~2Ho11~ z_8tUbhvzN0L^kDZ{i=8miOA9F;cJst@OX4Yks<_XMZ2@ySvlB>cQZGB&cwj+1Q16m zPiEC#;qmAQ3wIZP|Iv+4F~G@*2f74B7TE0t|7T*%bK_qm5x?hF`#sXn@^?@KHAFeE zYu+G)2u!>fJKG-nYfRb@Ne~ZAI}|AcEQ6U>;-2p^J6}(2hcel|j&23@d9Rvdg z3|7`U_rj9w`s)Z_LeQ8M2}?*s@=?D-2)p4s&S~|R&o8)EpNkOcujsdF$a}8Zw2gdU3&xrj*y1$NZu?aMLb|F*url5W+6l%VnyQO34vo+%$6F!tZc20|U(!yGejOkik4v(M5u&@aob!Oc`ZN?vaO%Wi9N73_A$$WtC zk&3jrQpFfon6#hdZ!iDtDz`ifJxSJF+ioBQ z0!2@DQ$4;u zg^w{jzHSpCwDsmc=U!OMV5Vm}HQrEqyS-p%R_)bv?XjLoM{j^|sOmlh=-9dmPSfiy1QfP5|D1B8|m(j1!?K-rKRD&zwf&< z?|tW<8*}IM^PI3R3X-p|Nw5I`z$v5gMAhx@amIHqRW-s+ zc-u+aeu}iBl++uR+AkU)P#%5v->RwrU`0SUx)TBJPgZQI3Sc9!teJ{lSBwFh{?7-H z$6KkBpxP8U?`=d#;U*}KUD`Bm^mH84$9QLqY4>@&X*Ycl1V;OCX51Br&97@FjczRi z1OvhMSthgRBo+OgvlGec%GkasS<#cTyvoq|;ND^FQcZ6U`Tq4kzoZFv&*jkZy8%Ej znlKOr6(-@t)7HfcW1n;PJC<_+ja1Q1VU1W$vXpv} zsYkUo9HeYeI?BEMPAid z*h%Aw{k|Le($<~%eEt6=vUrXB&T0-bLH|msG4i(_t`K5V>USar-+zFuFerc8 z^9L3(#nZqt8FPm9xyy%(hTa0%kYCw9%nAjVOm9hXD=Q}8f_ngK;eu)T}i&;CxmX^YP1)BuEvt+ye6n2qxqGb+OW%E)k z2a`TFlL4nl_{%dm|I*V*^Z-yy*q6DL2J8gF`~G`+mDCUSpNSKKmj0a2<#D>~;(}f* zh%vBZuR~pg;mAS^SUz&Y-$rZs6^_n=m*Ejdy{vTlD5){`!$Wnki_T`mFA5R-9nb1z zB{QG0M&*~R97yXUC(fHLMpA z*xG8L{ChKv3`X4&VbkA^xgO=VZ;iGKmt2i@8QECnZ`t9hlwd2VVQT`;xbNR*w%C)2 z-+xAIzn(fWkgcd*>#KP#sZwL4@B4Lp@~p9&be!l9GW1*EJ$1A*ZyWPrr2eku?mU*+ z?JXl?|E)#oZFjJ-#JM&x_t&sy#6@!(aW3#J3w&7nyKb^?#X)XL0}d9+jYnZpAc4P2 zOYI^eOZ|u>{9n9RYWvxJ?xR}0g;2k%_{K&>^Z46t-f81!I8rI8k%j!NwTXR<_&r^;V zukwAkHoB7dC{79jb;l1X+lhr!<5DO-#=fjW&3Y5 zc+8%59ND%H-%5yx{SU<^+LM`FPaWjqOpxpE|?hF*_$F$W`c@K zBsXAb*wJjH%cAAn&QtzXfAMQo!@SN0{{Yuxo#RZ>foPGDlTnYvnZg@ks9(%ait<{u zujr(bpD-XN*XgT^Y_WHo>x*$U-nu*QHn5Veb>@Y#6*;Fq+4oud1Is$~{uCZD-F+^m zY8q6I#0?hmVgSR$in?14o1Si5di>EVylKUs?wo8;k#Gm6iSMUkKU(W|@o<;tsP@ z7wV|{b_1E{gDyV$qiDMwBK=VzEXB+>W~7`zRz;BC<1sHz`SSGmP6RG47+GL$)3H7~ zd8SB{|Ap6kmjyQQG^%~!;+@i#A70+?hXurzY06$a@_F*kwhk4B(W~X@7LF)Cdsp3r zrtmaq1bhEzq?aJy_8MIX-ynnmbMUqc=kja^*^N1p*U{QTYW4-JqN-I1M(tFjkyO{!*05rD_2X7C+C8@#QrG}4GcEo0nUAk0IS7R_c@pwBXn9rU z@)$AyYxb@Q4TKg70K$BaZnmgou>zAPbEtP*J*Obxi_q3V6JXzfURt~T_b!TRBXxzL z73pMd%Bn1*7%fuVYO~zj1eJJGJJ{i7+Koh#M(#{3g|SzK7`>olah+Ayw#jZOt8!>A zuD9>a9x-`V$7l3frlMeO)Y{YnAb2BVu=*i`wFZIJ)*}BotiLUK%qm4p{6Xhoje86l3b5fpR`dJ72`Fpr&OlK{nT!^iU@emrv5F~AQ(`6R))hVOtSvS-tVlAj|E0U z>Rw{`0W&P3*`mcQcf0BBdVTA13%1V>gZ_|NujCsW%q&ewI79a)(6{2eyL)7W(MpXg zmq_k@$m{<0+K1!7{_j2&3+kolnZJCK_8}jo3_&%AeQ{93r)LS+V4&I2ncBvVpkK!l zoJ!N;)8@K?C)ohi{4I^z2e!tC_DpU7Ktf7kXxHUdqr-1SUdpuY3Mc(`QyPRvd2I=O z@@S!M<#QXX?C~k8fhp;;YB^gkvzc@ZRWC8l^O_m4Vy?&2O zk_yv}p^#--4xdaA$z{%o0pQ*?=K5z2RzC4%UC&)5lL7YRwv-`hXSs)HqgB7lj(tYY)JRP~m6LcdlDFm!M3_a@o;XzT42f^DH84hqa* z7V>UCcnd9x`L;oh@<>Y1N(lq{abIC;7?}~!yakAb{+6VQV`?P>1IncC(ExWhG_QfJ z{_#Sal?7_YY6eW6ZHu+@zwFk@937NP1)7Cg1?G@+$wrI-@rb~1Bdp#YS6@tcrz8+?-zph2H%;5bxO&kG`QgP zaAMvzDAmm|6&y(Spp8|zkh7C=9`bzNSw-#3Q-y)0r!9G7^3T-&)Iv*O3r-9M<~*Bv zmlvqwcQ;srPNabiyC*r2Wj$ir0f$M0L~SM?W?mrD`~u}pn#H1VAK#6Gph%cd*69Ev zCFGw+uqPFi8MBKT-T9wFl(A~9CG&tLv&Jv{)T)*jZ5n|ygDJ|F=70%g%OpiXxnChq zEe3d3T1g43tyRqC7v^eoGAi&dtv*d`z|*PGpf#}WwE&l^g24fR6#*Ao@hk}-F}_!b zfb|WnD<&X(uD4DlxUo0lP6(1h>io@8;4@fw|DhExCv-Pl@D<=~ZN@XR2{1&~H{R&$ zx98rS-2g+fJo3-Y&hJWx8YCsg(YDi|d9L+WIQPb$@wxr zA_@bw>8;8gm#Q)b8?;guRgfB#@nVDJfLJQ^ctQ;9@2G@Hw2r6N+;rxMFrfgue@6>b zg^IhoyN6dE;ki5G8HXn)Y^dX7W4euYIWrd48~~`4nc3c-d|C6zMoCbQ z{Kn~V%OQBGE&vsD<-E%s^~eB(-8&+F^W9(t_(Akq-Rx~_D)#nBToRNGa#rki;7y=i z=7(>YCCDT%9Lf7vp5;9_udK|fKazN1Vd0;?WWe-g=w(a5`$C#ABF9U}eTeA_)Q`($ zCG3Lbo4ov3+1DC5T0C*3w+TB`XnvN#I5heAK-&$u4LY)QfbhzVnv#uq01T8i_V3@K z?T9glrz5?zBd!GzHwvnsm;)uN9-7~B6*m^tY^Fqwo12?ujUNk4HV_tNV`s@MtYN27H)4N1OW2a7FWoY)Q?svUH>GS9h;?t~E$1rKN)IzVq=4)0 zPqC5|!~!0*@$YD0W_5qpdqQG}AciL^ZHx8R@k7Qu%*$aQfU2sh*WG?0(rd1x^U&uc z5rNZ+6hco<6%5~SNaC5;1Rw7sJh88v?6^BOHa0NiL}=q#v}*U-?-6*VoKYM$(7|Mg zk)GRnP)SKi+Ap2(+_?Doh=_=`&s6AsZQ}P4EHs59@X)Y-a2iRDsD%?9k2$pm5cKx; z*7=Ti@=4G0xNc%%qF~0bH}L-IkjHUHt_AMIFC?zSq+WLF08=`U0t~X7M!?DPqqnfw zdgPkwmel4HWMxaIM6m;YBqtN$;Zb8e%GY>Kx?k>34Zz`*3Y`fF2{be`dOuXBBfjVi zE0Ku=f)297Owb8mB$$sntBDun<>j@%pjAkpAI<*eM1j*2Ny0sB^>GeLllhT_W$a~Y zj|Xml$uGcKoS@nR=VYa4IxX!N#Hw!6T8U*+V&d#=Q%6UKLZRc#zt_}~(PcFuDY^8M z@t)RVB<0aae#v&W2E%?VQ{L5!7St6K4`fMf*08|F#&#$;EA49>7xX^!D=I3|B4m5| zNJlr2wg|OP@FjsrC+r@@Sntj~(25;Zm$J24-h}x3`#ZbLV4$JV<2&Pgd5wcpHWja- zq2XCWP_GjFmJ+AHmrcmg^;G|?x`S=$*>Q-C>Fv9gUc$qg8ZDI~nHbt_JDHS;qvx_DJw zuk>NCAJKyLj51)*xo6*3C32&l6SXhc{>WS#O$C#>z?fPgGnrTGaKA9!<)bG_tn7HY zGgaL%w;jK0!_Fz{vQr}pI?W95_I00I0|QP?#!UfcJFmAO5Qy_fU82qRz7N%{t*t{3 zM?%c>^z;l2O-;RwdJW$9|MU=2D3nlwXaG6e^YYVB)tU%eXhH)y<858aTy#`ad5xMV zkO4oOMmF&i>#JqWcQQa+G=rfynt(6Y7I1!eNJxn1XDDm}S*10*!G5K|PLtNJfk!d8 zi7Myiw=KtK&?y`me*RX$$8dwK^v{kJ-#T8uCO;cb%cca5B(Zfg^&ivQ+uN_$1_i!2 z(MKl@DpKv?p0rjD^`LcmOLv-7s$$hUn@&R8!~RP$re3Su*(UHbBLyk>KAIexnW?ub1%QTXl}Cq$o}$90A8s#5-7*qt&xMD;c;D3xo4jqSky_?^ z+1`Lv4|S~G3^tNCtKfA-c&&B%^A?99+ucINEJ5c3RTC4M9`>iFr>lHCAUJ-=30YID zks0G}tu)!D>UFR$(4OpiBZ3VB6VubtxgXIkf%axOz2y?2<#?)jR>Y$6*OD${et5umss{{g?d3m);XC%bL^dlkt z$V%Yu?k+Aau3oA}R%Gx#QfoB|9RtG!v9!P!EVS&gWSaW)w??{Btfaf6d6V-#q^bWQ zBu1#XKy_-PfFJitDxQr^&-;99$f2oKFp9?o4WDaXRvcKZRgP4foSfWc^z=$`PD2X! z0yvzKAm2fCBJJHyR-;cUbUG(zm7Arhim5p}TmR|dHi|@d(imxQqCR1OyZ$*($1^v=}x5|U;$tuAu;59>gg?81vXBe$s|NXpUnS!u+^$~ z!B>13p{C)(BCzubzbyWvpfZ6!?o;s=($%;nuV6ngDctn`+C{tT_ zq8?mlUFGFdA@>{IR2Q#)wI|ZCQVN%R?FlYpORKyy=n0N`FSH^UGoi*v2=BS@GB)07 zb-(CMAzc%nDyWKQoZt||&m|k-`VS`&gC??|H>K&eY#N1JQ(^%9t-)Bw-SK>S84zq| zPU^UW01eE_YJ*s6`nRyM@?oC%F}#TqwYFhTn1y`o=%%&NTAWk9Ujc%AX5qJqQGXS%iV{A<#T!t1mPD4 zGKEtN3B!JV03o4{wKe^dJzu+*P!y@0CVFIZIdGn4&no+P{YD&9zV)`S#pQN_#8q>P zWjTAZv3T)&j>PwPY}my3gMeaY>OZ|bMMEVQixthRieduYcu!W_=sdhp+Wt+i-VK@cUhbKxeE_Id&s=@3 z-amtH5nUi#P~o`xFm_>tqMxRlrFW$e`gC}2-#Dg7Tx~5ifySOGmgfj{sr4QA{-Mg% z+y@`)mj*%QZ&8$$hqWo1SPvt{niij(eOQ^Q;=&g}-BSeL0hGcnht)|jb-sWX6{fG? zS?&a_CMWiA33`tC)CN*&vcj8V!J-L+>v~dT3UBc_A30)q+m|RRqeG-oV z*0Ux^(b=;M7Ql;W{>Vum4TVH)mdm`opE0a$enx4&{4c+c^qgn4ddka}OT&=lC z&TqNDWQ~0@HH%~^#laS4r`h7mJ5v0q8&bH?G{=T-6$;k3mUvLT@K2Hcwa|) zW%GwKvv2|%_&uVhb_Q^|cyQ}4 z8r~uyqF`R98~f-w=w2hwn<}9@1HWW|(_7>EtAx}cT!IpY^OD%B@v_LDQ-?b*%}vF( z7#|TyAc!wW_*V)V3_!Viu(dSgo&;3UxDIH)Bb1@Inzh(O8*i63D1C{7idGhI)yG*` zNK%38r<5bQ`=M0Fj7y0fL zvJp$zcX@qxFd2$z9Wd6L_8-M|;~q&c%Q2uVyZ^Ky8Y3w Date: Sat, 13 Nov 2010 22:41:11 -0400 Subject: [PATCH 0503/2835] update --- doc/logo.png | Bin 11980 -> 13171 bytes doc/logo_small.png | Bin 7621 -> 7321 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index ca9bbe4d3cbd1ba93937dc2103a853d2f3affcff..1f51d1890c406ada11721f17f29f8c8061df527a 100644 GIT binary patch literal 13171 zcmX9_1ymIM+g(x`1nDltrMnw}rMnyHF6od|O1it3?nb(iZjhAjZusW?e{;^lVU~&C zGf&=o=ckf_6gmnK3J3&3he%7PfIu)9z=sP77T81nc*F+$f^kui5(8C^lNDZCohX_F+3J^p>RLx`gB-6s@!`#B)+1PDd z?LkKvhnSdD3{_5mB$kAz@&{$5a5|F*`tuxEFgQG%4(Z1SV!ZG`q-iv1WSZ|Wl7~eO zag4+EjpG8$p=jc@$m!Kfw!G{3KBr>}NiFPmO?Ru#DqC_oJaDu4ww zhM@lfRlCee%11S76xmFsa9HOTEtPzH|JgjEnP)KSWViPUF>GO(GU~ zh^3mdCh5_^aK;QX38Sv)h&>{PhDEXFLv1u5Hq_+g{R7FuB_gOpy)j8o*=fcZ z@dl6W?ePj=uRp$V>fP5pS|-r>_;jUUi7H{fFuS!r+V%7MdaNma5g6#^rNHg-YbkBl zD1s6kam#dVt-Sgh8df{`?@$IS2~QN<#c6(XFee-Ui%h7@sdPXsSFDz_gD1^AZ!i1I z8au-Xls~8sY5Z$XyYW9#T{zcnuk=GMpNA6+`N?s@@Z>c95R#yPB_=t&NbkoLt9 ztEjYNQ`+ucRY5Hci^kH$L}$(PLU9%!Sd>S6f@W;`BGo~_zaF%`rrT8CcD%>f&tGzR z{3EsALj){xCKB@xbK0;m`Irrh`!R;VX;#)Z>+C3Y@#wCxWq(C*$NRk)PpJR=ThMPZ zWD49IcO@dVvMOJX+zU3hq287W384$=eerK>53(_RSA?>tX_eH$dHwIx-`N=%GZ2)4 z8b4~PpRb1CTV(>q>?C|%imbM}t&~i3M`_5Pkb1|*c<`;9zvt(u^ z8D+5P;#hSUH|8UU8i9$!-NE2hD(>rk)RgSQL@UO(nE6 z`s`m>rY&}+rsly4I5ifKeCH$0|Jzb`nXsF(cz0a$Tf)gg%6IZYyxd+VKa&fOt~Di_4#F0vE-o=CN5#; zXBCg0zZknnoDA+|I*KSbfK!>6bF`{_K1ByhtUItv9pd2VRJpD3NsmAV7l={7Pt(eo z7qzBTLa?^#?#tqn+gw&jTKQu0SW08^$LKYZ1JyuBjw(rh<+n8Iek~J$(#p;G3v_Y> z*bGPoBlK0}cC1#wWpEeut{K$!9thT~SMybFMks*Qg0Wst5MDn^ zYz|RSg(4{vob*0h+{DWDCq`Go`{Q=0{PXJXym^Z{Cp_#=wR9+lhF>?^lA?3CxD+&<`!J)B$Sb{Jojt+hdLNDjHGJ{)Q;ecF)M9IPU zubZbyD_E(TV)A$wj-?BJ% zuR7VbcT<_Vp`BXOPa8B9n6WFvUW8JXF>goitpR=x;iR{;Z(h5jsSa%TTK^q2&TXy~ z(ss_#eO0#;v^R;JJQYK8p60?mw zf%p5YPpd{ChbuE@2KXVnjE+8Ryug-z_2Xt9adck;AE7kDe&p7?p(_5d3YOUEF>&;y z+5&J6z5+>IS=?t2Ato(fC!0vQC{o{ z-pp>MRVL;*LedB<^ohiwVKKC>-G@!}uPR7OVu1As>o>axw1LwViq8F31#)brXzRbq zExwZ|UK32+d0?YPJlpMgO4+j+Gdar%Q*A$J&4EhK1*DoX2DN zmfMD;mI4V@3b!!a^#sB^)KZy7y<}3YMd$AXr$GUV`vV#x6x2jHz=$GFAXcKLZ%-3d zsd`~;XO~rqMcFs7*xL~XEO~|ED{>$_3=$=^6@z8#DIcz|lIIUF+NBI`&3}*O_WDM; z**J#dYqNABdhkV6nt#cqFUnPrRwg(mYtqm;^=5Dd7Wp(Feb(B{##wkoK1>Jbf#2x2JpOHX-5EG6|v6u^6gG`<>SX2@Z!6?&^G>+Ccl0P?~`)3 zL_Q}az;n71>SN4x#sxT`c&NZt(5m8*>&>ij4298`tuo|OCsKXGJSJ3}&}tT`D9pxU zpEKYnEHRm&QU!WAs1{5djZJ+6Nrvg>jpQ_q(Z$~-`;?$(0vq|(s)(jH4W}3(_Ro&b zTZ<23NnOV`GC_ywTW7E3uTOyAV==G4=IUeP)qQ@+)V!ZTGi}Ff=Pmh3 z;wkYN_s2EC`PZ^lTeYh8*4sFN2tJZsK}+@Nx7Jv8!h$WZr5vMf__yP-$DmWWP!Dt; z#@8=(y1Y7a%YtUYZ!|Aa%Ow9RAc~iP7KWP3D zHq>7)VxfbwXOaB+QZo^CM?Ujv(qd3k=rOpNmS69=`{%}s$?G;a!0E(4^bD0CeL?Ns zrvH2FZ}+8K1%=^pTe|yj6uR$NC^(_*>CYNKGMczTT z+j2XjhuCv(v-3T(P?)31-fDB>_<&N?TuNYm*!4W3=4ww;s9Vn33=;^)Y+S!T8DSnk zVUr95A&rVppz1?4%GvPo=Y0VeF8NvYKA%i9p_WiSCC~k)5#j5KQK5u>PVU3u zpD^3rmf{@J{oh$*i&SgbZ6p6qtyM%zC(=wrr)ZZ|q?7T{YLM6^rX#wE)#A1{%Q*5ZPXTI`1KSHj(2bl=FuhNOw^G|Bk1ZZu0RE?>UiMV}Lz zSDYkT5RK39A>+Xd5%Ia1Ypd{y*QrBvlLubRbq?3nibayUKM zTO<`h?fp2AC-EL=@Uk7o?prOrm0dn_2DH#jw9-s$Z5(SkjVYadkIg$La8d*2B5L+w z5rVP?&cef5|Dy@o<~s;##^eB|iTQr-O)DaERzk$nVPpH%+%Hy;@agm3J>3e-#~3_U9tg(z_=xHtlZpM3x6SuN==9aoFff< zmUg%+1X7F+BVU*5!X|)Fhq46Hq(}XU^5X4i7hlr(BY6|Q$)2)Ken$4xAAo81_ACk} zHEB`BM~^mU7#k6Fl#VN(o`2SNc@~*7_Cg?{?6bHaLs3Y_eMQRTNiR~%pSRpKIM$<@~k|=u*j=PP>c$ z9HsRy3bCP}ZJ|%j^Z#-nJoD+_%ZY9-(LBv7OLHXjHHwnB4tGf3RySS+){Tsnnps?P zXxy47uwrDmfp+~}8Af(O=4MTEgvsugrdI6!1Py(_Zc=+U{SE$Ue2fhV;mDQG1uJjS6DpH$nB z7G+)9w7Dq&6LHq6v)c`Q#pi9UG3 zLxCOa6p2i#6%{YPZO?Mk@)3Kw1gcmYc`l58a-DvHk9;xgQK?%1(p7qRV^+yivJegf zl8?}1YGq|T2!lCMr{cJ6`V1Je@}nGxU?PN>AFVWFWGIAlUW)m|I@^wm$w&PQNpTSv zUbi=Y2@vtkAx5B4;PQ)AY92E@U5Lz4dWA7W@eiKF#(a(sI_tbTA;9>~U3GH)qIEL- zr+fUY+L?Tg23wHZO@Es^h;&x{bP1P=gH@%)U^pI92%qyLM}U%u6z5je|+{; zJ5c(ovIQR)cAOb^q$d5r8-mkymyN?~)Ba{T7-h2>H1DP1(imAyvC*HEw^a=O>BnQ{{K^Hqa zzH-kfJL#MUDN2rgSc&(?l`b5#Vzp{YK7K6R5=pP&&+gC0znKjZjX$3JzLL})bAfW@ z_+z09sO(}P-IG50vS7`Gg|$(A!#R~+0p!lKq1BH=SfZ0#i=N_?QAuTxei4Gkq~k}( zBfZm6^!GO3&<-qi?!~F3lJfF+-Rz$+v>xEg3#)F-R*!c_;VLa_dhNT*gS3aWT)lD0 zkL{Cl|AJ_!>Tr2!51lzpRZH2sdt3O!e9>{bE9nx~!5Dtp^@%Ol(Ui8%=LPtA*#{~l zL=DPUnty86>ECipqIqWi5rqC-&1#}L;GFz93i)C)?HALu>0z@djN<=${pdor^+TIT zQQ5j~O?C@88s)6tm-XDujSRJk-=cwC(#kybY722PLBB6<%GDRrBT_$oLX6ZU;Wzy1 zbl$)gb^TP^F?guZXxKlvWz8@_FA+zBYH{II*5Y2rdJE*yzuVl{Y0y!K znazblk%ENnrmQdCs!ffmN2=tb-EdZXtv|9$G~C^NR6Da#Isb}vZnj*(!BRv7BPN;@ zMP8L|&(+j~)~F#uFAIM5ilaU>!>+lDqg6;jUq0+%Kch4$jkM7FwNh}+fQG8^d}8sFlRp(o#zT>lP#oBS+etr`+9M{Ne^H&a=A($2&3S5BKYdJ^@08HQo$Q+?ZY5&= zy_O26+WT?<5C?B;o#Z20SchUjy!4k!O#5Hk znIqbFX^=(iNNqVTJ~1)Qm2Y;q2g$c7*zu2v z04nr3lOotJ=`pae`ec8JOE8~xV`Ac|crE4`i5G#$0Z)a&} zNswrf1mvX+9%GrN$2KIUgPi=^atQBS(Z6M%*}vR*Xiu%K?<^p1C+-?9WSe(L$YS|N z=K^?>;^jUAmsqHPi{_Hz7`y57sR3RSL52~GCgHpy8UZ(}(W)IYKO_IBfFAbK^K1-< z2W^1wbozHv$p?($AXtCa4e?3jjei{|KZk$>^-_H5+OB&Xm}0?59)THCU`yDZtEMm1 zXOq*Q!Uq%ZBD3?Wlp3g#Qb@QQ`oFNH1U@_m?D!Kip`qi4E}zU;u{Jm|8r*SUe-zA> z`Vp*wpu#E+z#wILcGt%7xzKp}?6&(ny($6egOFL&jxIb>-1W}G z22$962lei5IUMlV!{YgiZFtlrTgkWLYAy`jpaf6Sdd`1dN`;W>SxNQspk;sXVCHz= zh0^<|r{2BJIe5^^8>`5|{hr;={wIJfj;im>SonCZo$tfo$Bi_`>0cij*Yh}g4SlUk z_SJ3u)bUEW^(jfyi8xna8}mkZhfG#!wVJ*{A}`O=?*Q** zJ&~9`ix9&jWI-`$GfSY|4=qLm;`$ZJ4@p)PeDZV_Sk^mtKS9=!~@4fryq9a#c=!`A+Hgd|6gD8py zf&w3SOAUV*sRFHEdO->i!w7>j#eEka9z&Zm8mOC(4S))K`lO}=I0ss$W(_*}N=g)o zGDnApM8w39xC|~k9a~%5Csy%q-)@r_b^kfHpf6Vre@$aC?lJu(IIt7Qd8DKQxs{H@ zlfvUn_HbMUknGXtlG?&>gHa@G_~kc|ka-9MGW^^3X=HTNjW5%ns(5hH|M_At5CL_$ zX{xI8b=ipJhXj|*oAbeR;Z(kql!Sx`yBiJE(Q&-!J9%>H=*xp28HxL@lo`=yeGHt? zMgL56L~%E_n=C=^%d0D15{&3=B+7P98f-`$ylBdcc!$BS6)tBLDRf{`a7J$| z;mS_(`8Ee9O0v%D#=1o>dOL;&XdoBg2qmOcC=7c>0Yy`vxvNzMlrwmTiZL&A3(ZZm?4!q|@9p06hCBUR84gWB4j?k~2bqDh7~9T{v_&5dic0CV|1 zUK6sK1gn8xHp9qKP*FVvvS|HB_pTJP1PF%C#>^)7Jng6Ebj!wZ) z5si^xHpfkpDdfu>?}_L)xp;oKimkw7qC2wb3V;bhM6W&fN|n=4R)(>gy(qw@K`I!< zWmM-j1oxyU>!gU%*~!`2RqUt$i3%ABsd~pK;E;ac7x{#_?$^7<`uYzaK8O&z2|$BU zf*fQ*rphRSP5PD5pb6LWT3>RNv=*;9v&PJ;^NNH!-0b*3n1|f2QMsWM{yA3 z#R#q=Z`ZFYOc`W~+UHjX<0rcmTC6JKo&qrow)n>g_Yler1O==e7-a1N4 zN4~O&vNybIWzi;~Lh50S43946plkTZn8~F+hG$21Z50h>)*}Ge?n6enu{chl(zo1# z>4@z6coPfg`(r$_V#w;-cV6lw*rHZiLH02_O1-@5Ij~k9OZ0`go4{71v(fG zPEKuST2ZXr+}yva`33q1o^sBh{xffPDAIt;cKK0=+u zCu=J!3{)k$AQ1M$e_DX2#$7B(oF3C%BR?W6EG$DTN~kg3A$c`BwE3@|!0-81Dl1{v z;he+Jn}%;E%jf5gKP9&2V6cDEu&_7+g@OUE%;mFAvkT#%6I2c5&usSq7%7)Wl43qk zSLzmt5<3=1wS0{HD-+aC2G=usdot3%AQjQCQ)3bmg0E{s0pE5(ssaXpc%D#IZ>mvF zl?LTMV-0Z?LrXgC@hY+J7Pg1a3BRpfdspJOp7di$3=E^Qz(kNUp%qHqj$N-Jmq#2 zS2peGS1kq&NlC=RLRx9-^{>jxu_3+`&E-6$YH_Q8OGb~&u&2ec-vl2U>dps`?jRx} z>P%NJ+I|DJDtPofduBnwzhcSeaWuZZzPYJ<1!_IPl-Eg5c<~ygEoRC0T5rm(mz#R( zqUnm>jURcKuzxZ#+=&MDcC zmp&?jPg7x1c9ucL1iO2C7RES05FZ>IFfuYWayziGv3+Gr-I>=^Qo`a;4NE|2kj zDv-QU{GH?#bWO?0h+hpl+P4*d6@Fn85n1P|v7l``^5{0&#>x~nDx;n}W+x|GJQzY~ zu+wQd5X+epd76Zt=Vx!6JMyB=!hbNfuUOFedQ=}){{d6uMRas@FxuV`gr>?Bp%Zh_ zCCU)<%$pKuXlOLL>?w?zCdASZl8~UK4v=VpNis6W2rBjBEE-xizULsjKAbJ%$OMg; z-w+Di_DZFSSBXp~)_*m0(& zrZz5|?k=Qi`J4za&|IeOXvV66sdk&2h|3PXiv6yv&!#d3m`rDx1Wx)al=;0=30??v8UXH4u+0kjw|n zf7X6Ae)_i!=+QLA4N~NaVBioPTlBOlnsBtrG%5}a^2d~Srl-2Z;6)*k=*&ozOyhR5 zqX+UPZhYEArl6U8Qq+k4IGI9DPEP&h4%w4N8VK8`PkXaa)&Bneo0}W2j;z7$iw7TB z9bUI673kQ+grcINu;WJ8T8DRfdU{(dZ)CXF^W`YB3l9lP*KzP^*xl*b(wW@L1930z zms{#w%!U>tYsY0kxAA#}K~vpP-7~7?+mZ_@YFKZ}-I_PhhP1su3Vw76>W+#3Z|mK@ zpi7o?aB%SVf2*!$vDOIZ{hpqjJhdtzljqM2}W4&j!C#Z4rl z*7~vO2)@m$htVX)psDr<83am>PnQ?E{CLFuZ$)CxjM}v{LPBe=L0{{ZfR4#R>dTk! z3K<$;pbd(iKD$eJDW|?pa%BLJn#FQcQKU_t*Tb@^kF>N)6%cVdwiH&=0id4}{WQ*7 zUhKhXvjgysgqP)j+AG?``{}aXin}U2sZ`Ve>gQvajtulS2?+_neU+AiQGGjYt=yej zR*SR!y!w8;&%v=cRY!^gSW3x6fV8*eD&;@aN*R5~#$P&8TN1TX$A|UQp96C?7x8eAlkjjmki#@JiDz+%p$5r$W z{*CPA?*Um&*{Y-Gi*d)LZZuYSR1`WUW~cMc2oTMQGKGu_bf51Y7Z(>>T=pKHo>Y%M z<=}*Mq1f*@cWiC4E2?}BH|d|!MReHzw}tJt1cX-*!uManLcyjT| z*@&V9lZoQknt(beK?RYOZ9A%Li%}O3fQzQY4W0y)FhJop**@mpG9+4%MLunv@_{Cm zPFb?C|F^j~I|Bi-WZT&ANC@62oU*(=KbKZePyiY?d3gjw&*UT&G3lZ*3sYOtTd2aIulC(#PA0x29x6s4sJxb5}i`rb_S2${` zF^zxs=k3shvTxJFN_s8Mg*iDn)z#Ixx!-9ZLERz^e!($QmIdTQp~eXyI{g|g2Cj5Q z?4NP*@%edqz)z^-9R+602x-m!w2fxs0uRq%M$Z zTYp!4#Xq50MYfIXD44jq-h^mB3cD3=_<{iR-<)Tj>vK4tiOeJ%@~Q8BFCgmbeF9|| zf`1E;x9SxqYG3=CE&>`aOGQ&w;w>#WCJJ1KN__n!?FDqpJoqL#n9tyIEmjOsc{nJ~8 z&3A#PN85Pfgf{kpp8WpO6g0-Kdg_{RZ?QyQVIm#o?C2upKKx^PYUCFaK^!R z@H{`&6owRL%$UZ_%)^ht#yv4#Sg-bb0rq!&5!^=xx-M4(*fDqK`KhpcU#W9l)fKm2 z^*P7@_F9DLANT6lxGB&M?gG>i`4xqK0o@}{D8Ryc?lmL3)u_z^B&NCf1{`<`U%j}_%nOb3J%*avIKXeHqTbg z;e<;3d*?}9kcQ%67jSamOb`xG1qg&ybZ9b~EW-Q5gOKqVwZ;@|SB zV)$I)b?VZ4_D}ERYrFLo&=uMM<>BLb0>jJTrQzld%r|hNexY#D!kZL%^c{I`M(&}N z!B5=}jL|umo#17^+zae>;jYtyxWeYv%vcg0NfN-+tf98}=iw7u9V>T?MejySGcXI% zWIRFwd?984a3NtR&fo1s#KcbvFMtpS?ecIv!aZaE zyrOJ(uvCIfZP^+u{*FT5U^B4^S&faYQF!7w8!v!VCt_MeAOOsz~G-n7M9N8|Xd zb_7fe3Lh>1mDhYJoO+n1o__q2)!N1ymCT0=^B(%^;FS3Se(iVZpJ7&H3YD4P=-yns zuB44sV|~|N09sVm#KAlZ2|uKV7{Lqa%6nIMd|i6*W@{s3Q%L-z5f<=?WJf_J4ujL9 zSfneX0Qo{^nQx&j-*-}nJ_RCtztww(goSY(4f8FOH8J7e`!~ydaSY9}yxnPQf-N{f z#>D%1IUSOWm(JkA`k=P_Y9asaRn&;Adk3-t7&xzakT=~uj8KGOB{by&3zG!aKdqbp zvB_^@NU72@$qP-+ju;H`u!210*#s=(qDRywr~n8ISoFV94^W%LISNYkN+a$g!;MXO z(r@DKR_@I$2z_nMhv_1Pd{Nx*WPUSqUz*T~AD=1%?yi&Sf|GAB;ycUoUMO7GMS&RA z(ZO8a)E4LOf!h2%Y&uiz%_@)g4xBixqT&Xgs9Zs@Wve@xz z3h)Vg#B6Z_TVY<9f!ye;P_?wEL&h)M;(su2h&DLfWN+jYZsbDU(X^z%0IfaK9S+ok zpYek@G&NO+iY*4Ly!)^Ad$@3VcuDp|3Jwt&03VgrE>*CySM$F zr?cS34V*r~{LJPKux@toVqPb8mxLBI(Mrwsj0baQD~W;dBm#7@Fobo&*)bYNGqDBh z^8$U6o*&#StgANoh4w}AacyT+J(G;j%vzerw}hj_p#AyrX5JAGA~8A&r3_t6Y7PjMwou7hA>M=nk#JJ-r7l%dQgN3B9+ zEwiDz5^?O@>qi%pM!G3#vNQyv0Wzn){Wy)U{yVvfsNAgiWD{uK9)8y|jcAvc0cs>c z!1+3ppI9?RFezQD-UO>YG2}K~z=5Q95COWds=2kOzo2jeEvBIT@14KBEvo2aClqgK(#2WZ@#Z80+Cje+QjSzQLI;L}g z6a+T`|DR0CYyU&5TwxZlo&dsWf}=ZtXjPG?{olWaA}MRtN>TNqsrInLKta3OL`{$~Mk_#6fdPR4-82Y#O{EjBF#3ZM5qQIc zJtMAAQ9k=d=@G}Z+7Wo)BV{vvE0ghK#e9J;_0tJg^gUp-W=Km>-W=LYR;4g1=6|=g zcX2MYe`y6MIRRmGmsV%2>x0QQEx~$_8tAI;NB%z62A`O}au)LcucKsvqSdGI6;=QX zSR|P_+aH%G3S2-c3=_&Qh$&}ET@hRD-xF0h@_@fgVP|fV#b=`;UXr{u(B$@Uo>C*qQz|D zRVwD3(l}w~(fau0ekHtwWUtiVD;c~Wkm1;RKC-;GcHTiEr-^}(OF-^|hyX@LxWh*9 zK_hlSG@r7=a4zZk_*<5zds!f3h=hgjY= zdO!J<)h;oe;>q7jG=(F1s4S+zNDJh7O2LJk$6ga(_^sQD z%6GwF-5{jML_I0;ww@q$U=NV{V14C)w4MOw2C}i@EsoVA2OlxVEX(b26A)jJDp5c8#_* z?5H=PH4;PNc{vcxjUv|A% z_3|r*)&h^ALD9dkL+>x8ABD?>I}6vD+Qf$gx_et%eb|KVup1L;RYz;sl#CqydqxG`H|nDT~(fcXO0IVaWL^1VnnG^Dk1)I|Fyn*zowk=_y|syl-Q!gX35k5u*L}z>f_A3ts3+=$na!nGpPdiqLWUqBYy$QRT;OO3z%p?3XXk~8oT~VkDQ@4I!>OkQ zV@tE|4^GLHXlr<08mi+8pV4w-!_yS@$r<0QcAnZ_}k9@Tm8)YPYc|2gYr1Nywy=4h_WYf0&w@F9C&hRu4d>(!&M4k*N=yvfW2A`_Ta z>Ws`ya32d7g9en(1{iYz)Ub_uHySvhMefXauh2M8Kv(SxasRy&`OU98CW$``8q&~H z*)whK^l$obCH7BNKLEqC?Q0MTPh8T#`S;6$;SA1g!Cl}?8}L|OmAx)m^7uDVkufcI z1WiONKLd(f2sA{SW;Ys17K}j?M(aW50z7E=J;0IAnOmU>9Ifs5DE2JQVwRbIYw;a! z5bESMEEP4f*9Zk(+!*y#@==$;(2UmNpZ&KW9ySL&&-4ECye8|^A z|8r=<*eG)Qs{X(dWH{$cD8cX%o)-zwuNzum$2jzIF03oRxVPvc{tM+d_BflQK!K$o zVCnh~)K_xx>117%qMHdy?!N-wMth@A@xltKikzO#S+zDHX6QKnq)ZqNl_s;pnE1JJ znmMZ|OtM#Uw#OJH9&VM1Z%lFbB%M;K@*)m{9hQ>%^GRJloG+1yIVGCTYp2cpjn?sjsw(WY;EO?9!6M&i@LlfyJIjP06I-)` z4^)`ohxX0sj?DQ243Y!pDZGchh~C)n#!>6YuMa+|$@54;%Ho_pNnqu%jRPfTT`- z9#1uNan7#h3b1c{+?7?QIuNRy2xq(2eHmN+&(QPNwL<%LK;$D{}uK_xH~fz!A^#PuX|l3ii4#R3vGO-&0}~ zesT9KGs~e!c(G`#)EIbWuHZqI@oehVND=$AVQJ=0X z{){ZL$=?b*?rP`rVU4Qf5zSxfbCgGcPOfQQd-a?K`e}xma-H6@_sP=OD3;9>H?*hW zyQ%Z4q9uKd)aZ>f$I{}$3g1*;D}LCZ+N*#YPL2>)*{Mv@=Z;KAvkEbt148I_l`RA} z@rDW!EJjx5IQe3>>p|zocbZx-eLImk|LA|?5VU>Y*64aEm};k-C4t=WFd3gs#C!qS zgV1h#JladnQS6&yX!l8nR|^U71P8ArmGA1NKp+w6YL4$agFH^DP^%d_Gq3hjZv;MA zLo?B_EXFaHl<*(dyT8%I`#1fQ-*oi5YL2=Ktb^YvA~(#Wobm{4en|rN8DRyWKqv_D ztwqa)Ck9$Tq(UA{$<|8Ry^OjC6-+3JU(@9cw?Bu;B2EyBT!RJ*gV_=#D9`piY4G6@ zUFn^Nz&$wqNkA#dsi6SdkCnl^xYlox`fRzb5ZC*oyN$`Sjep4Bs8_F-oUYsPjpH9b z-d(0kFaQ>-UEnH2==4~#%PtfBExOYh{U+ljfmd90LdOxr3$4_?1xF7u&F* zuUAx@-S?o5PtyZ+!bSlWo*N0dk>bBjoGI<8u>rHY5RjYtq7t1N3A%}qQ|wJ=4D#*b zbPjsng%8q-GDC_xJe2&tM0o;uEl46DQdtH7A7B}%m})+Evh0_xdPQxx*+i@Fd?vvQ z9!DgzIwtsMY583E>A7robbkHGZEgghR%rBcl={Wb#vN>lO*YY~Y<@gPi-K&Y+Od4F z3!$_FR$Q5n;SRbNys%tbs>J+G3IE)R&Xx){_BS$*mbkN{HmXeqzo6H zf~*EgBzJNVY0RK#c(Q$D*@q-~nM;{{*_+cu&#%wFhKP^vtWXadWA6FLNZhct%!w## zVkIAvm`|#{`-&4mzUexV}EAc5R4_4o&2&BG&Si+PJ`XBTh)gt^Z`) z0h8X@AQ~%le3gd}LEuJ&VL%E`2c}#vGg&(=ou3D{W?y&$kmR0g5xFxBRzFq$ zVFR;1+?(&mC7^u|F>ShWMfBxiN;{iR<|~BUvgt`|(Vce8aO)T?$`ZE#&gO zW56u{7@Rqttha2dfl#oZ?Svq9os$nAs?H(-ET1b%o$^F%A(WSC_2u|mp^@_6!3WhZ%QtCM=IV9! zp0ar6iQqkf-Qcu_^woiZ%qOio&%}F4%Vi>CBT1V0%YcKmi7(5G+D?P$lIJoD^yLz& zya&E7V9Kh91#^h77zd->}be~on z!U}AvSm$-?K{8JfOQZsX^q86W5}gxSHX*C`&h%f&UN=9nzJzylB41n7fNK;~u^aUs zA$g-gj}myEvRsi;LXu}O=y3I{zPq6Bb>@rLX$DfQxX`3P;)hrrLe_ zFqQiXaT2=ZD$)2P5)TbdKY8H{rfcVW`(YuoTeC)?i`Hq1*%!O;sqA9-q#m;WZhi7^ zW0*zrAS%a4V+;?mNT`3x^X1X(&`~s2-`T{2-BCPbyI;g7fB#$l5m7k~-dB zT5TlY$G!~zLc5aO1o?r`XNO3u_c4SRwOqekk4Y7 zP+JGCTC3!=5#=nUc~F_Lert){P~lIjs)=;Bfhm+kje#!-><9=zCLV`|g&pT7~PgUQO%-yH^*WKyhkv@Mu z5&V3*B>0gdWvxW<-X-e%G3{*cG2mu^n*io<+VMg!cg*pUBSG_pM&<2=$K7_Q>666@ zs<|>G4J|E+HW0;hoRUw!vBk9DNq2H-P8+9>o~9;3opOJb>VvtUxi*B@*ubmqgjdE6 zaGN#q9d^6w2dWR(s8S~^j}>x1j@hojJVzoVZ0HMLpkg0=etXbGfKyKWL;~wYQslYV z`1C&40H$6CiWTnNlVMcB^gDb{a)~V*0tEn%>oBy}NnVNWgrzx&--UVMA}3HiuDJ~W zt!vR)h@GI2&i~glxf`MQ(zn&VYF83OAK7Qqf|Ku2#WqM}hK<&*P2o9nBaN>^pt~NG z4+JUC8pC)%cKU2W#HWCfr&A2h@62AENWjg;756c`OBI|6MGE0~ve|PtPOYsiW54G_ zf6+Lbx!Hs+@WeEC!=CQ$0W`nSgK)Edt5NVC?N4U;BW`0rzv6w{g1gXP#@>(h6e zmuF!MBHVkJwXoYU027=vZu8{;aqVcL^>@wJ*B?Xo5I+&+s4h`#uNEks-8v_7NxO7Y z!dx*mA*pcp#s{5T<*V@@_PY>*oBzC*rvto`N{bT})Rh@-(GgxhKIz{>fCxz!0c1k5 zihcQ5$ zX8JEDKDY?&pG@y^#zx<&Jp45dSJO@|>OeIW?tRa)cmDVoMwL~}N1Nal@p2}JGn11| zWMWjx{{vShBUN9GRV}ftFUjZ1G;HHLoC;Nna^-fc@?qse+EC_1Mqhy~4&i5Ar){Qr z{m-*|+yNb^`EF3efZG7=4Z%`JMlDiF8atJosHabjbWOYDk4^PMCSpM$%cf8kjD2Uk z0%L7klOOH*fv1RAxT9X$Gu`u z4|i)N$v_TU>OO^T2V{CwDtiIBBuJzMC~c4~X1IMv9P)+ekm@iImSs)Y<8gJXqT~*p z0+^0BYZ@qYnNqGAQE3;mEb4uT-Idod z2PEqLP5pc4p^X--ce}iwGgBleKLrncJQoT$Xuz;vI6frSJs|Wl7{>F8Dy|`duZ6AX zowctmXiSYht#e&j1!wDJOr~0y6roNSlp}QWz->TA0LnE|`PNZ7X*r7A`Pe6cO3waV zZS!^8MIw~9Yy(YOt;6w`Yd|D5Mi@17?AsXGs_&-$iH23bmhPhmEGZMw&zl9{M4qxL z23`S??>o!O+sd;Ev!lf3lyl_2k4#LYjqS|N&U$-$6Ucp~l9Ch^ZSyA2O-ib>=tC>f zvPa!eVVsG^qstGvF_5GR>zuq58{weQBF%?>+8^``;_zU7{LX74GP6rgebM}~o|DvG zTkAHJFTudTpsK2h79Q%C^$4qh*i7aw8dhGz=E=Uw$Mk5- z3IhIk)wqR%#;f>ny8!vug~3<)qY^fw4*#X_6~ii@gFhu7OWqFAT*AYn4b=0LF2B+6 z`(dMkM8~V2udGOH_(}Dv4CThKEnfYFg5uOOJ@6O%pB1gzbc5qQtxDgVskM5;<~BDi zf3y}pccCA3XHTy7`NUkeyNM-{_0{s(P8OGzirO`)CKr!<4%@!b`w2n4tCIA~Q*u=k za5ZY3b+{h{fP6o(;{jZysk^z^X=!Qc>HCL<&{3Kqp_EAPv|Z}Jg7_)s82Fyf zR0ZZF^2yuT0*>TtyXlJb_)Kb7S65gV7`Sl*99`Al?DqzlzCTfY*RC6AlVxgWs+02X z5Q)=uO{A=vXsKO-4q>($W$Qi-bC340N)Rm65@Oi!pw0P^jSx46(kx{`_P?N^rOO z^TNf!K{WVT{PAf#S3{PWKrM^cx;q4s$gki#bL?}II3*V^??pI{8pvu74HJ;Ad>-7?%}|zIQSHV^_hW%EKy(5=jc;A zTGoH`i=6?t)??`gA5Vzt|3LT{7_65X?MWiD@E5H}^mKKb0&ZPU;@S-xo=uaUCY0n@ zWtbZzVP##gg|Xyu^%v^q4K$g3d8?k4{dp&n++2x(Irwz19sdn%-u2$zbV6aecW%N8a(&ZYG?ex zkC>fVykaq(uhmD#B^g=%L^_s^>|B(VGVHp_K5yI-DIF-y+16 zv$to#aA1HS%RZ~L>ga=EO4RWZw)uIxqoX>yx>={7W|vhs9A5cu{ZG@y*+yThaf78y zo?1pu&g(G4kyG>YsxQ;Z2G>IWN;Ewe?bCYa%^yp6XY?r(g#cfrhS_ooPS(`aw6L() zRUv0o;$>rt%U{6pYJ(oN+8#ZS(Gcu-jGeZ}cOB}eFs1-+YV(3Xx6#oUIUV%W5%OP2 zttX-Hu=pvUXK0ZDW@RRh6S`Fk2xy3i;+Sa^?QQMOC zGO1%0{O$pu_9%waZ zIK5Jynr&&ROkpi@y}!Tz8lT}e=Gdt{Xammd)YKF?ga?c7Qu+0#Ue^qxRr~pz_JE@( zp8+xo47GGFGfGlY)|3(PlMofg%iG%msbI;dzShUvq4Dz=;e@dr54XRh=YSy6(+W6l zZ}e3naKHR;9)RR)ba`60mNCcnZLk|Q4v&X(5U%t|8^G=2+{d!GLHz6@nq5- zj10P`{u-(FZ=VbvBAKRnaFvYz*<6^#6RdgId=|8Sg&Tj|6*`0BB0td z*MUWuxOaT)JZVIiu!V{$NG2;?U?C@mibCmg&}>8wBjn3@O3LWw>skb3YASphduKF6 zmNN1=fuH;h69)(O;*a1cO1rdz0@qzLG7=;gT?oF3>-_}1vlYp>q48nMrj(;8TEW!q z?XA!d8fQjMQj$eV$Lp(ulG4&_k?@2fW>;5Nx4-`2sOa#_-?gGCrq!L!2zwgP_v3#Y z46a)_B#&3dy4_=BEp40?aQj<5_nfr+=pvq~UmFT+zpLJQC%{bnYR0Iu(6GeizKCEs%2+qCs2sl?Ec5%FNH=Y6I@Srbn?=v6)pTIGLdW@c0j0PV|CsnD;{ ztC-cx42-e(w`Aq+FqGPQT_-Vec3x!0I=Z>n2! zAfAB)2ZvFQ<1E5pFd!BWV3D83$L6LRm1tENw*skf6fGUygbEhQr*YQ^4bYtOXP90u zP)`c29LWx-5-ta?cZUimNZpJltd6*0VPN<&w)0C_!@nedEQ4z*PYiUr*6GmHHc)Bm2_}0ai7je0tTN^_ ztN%Q5bG-aj-YQ^^H%Fxu@V007f%}jSb>TQ-u9Fa@tZM@mqEu>4RxC#<1cWzQ|F{6s zv=vZFn*O1CZ!`uz>5?|3q+ZNXu_nwtH(`&~4j}THSY7RJ7!b%=+t}ob`_+H@_N}yZ z(3~3s1*uTOE+PAh$Dv7-iHS)-z#G~X)b4Za;N*1FP5RagtxpIVh(A}@87!J<*3cd> zKO?Z`8oB;6zAU`J$-Nt>CrT;b&J-#GU1ou6Z=WHJJ>3m)%G2KpR1;%kubmSyiP>_?OtiQmRQeIvjQ~i99CNJtnh!#%6 zz#zv?)VGN(c-fwz04mLe>P@DYyPE?DXAkyge*)-Gr!xh8&Qp$!i+hcYJux|X(#`F- z#Pk4>|Nh&KUr}8DdTY!|-pP+OW78dH}iA_M0TIl%= zNk{lI8~w(&r^~V~4ze%r^^~4Ij0pujc!f%vPm(L}eIR}l)wgx-D^6K3MO^z0r^jOY-evPnE{yilY8|+Any{of9hWYDL6=s={;2QYxxW8Yv{V7fz)UK8*TnDHx zbu&e$*`JP}o$o;Z`cJC<1=;JEk{?{Ow#9vo*j3XNZV$iA={T(tISnmhj*-0> zBy){_#|MaYL0wKwlOS;@N5s>nKr%?$CE$u|=#1MPW9(XN(#DtLg0M7q+09CMV@j`G zW{He5LktYRd&g%JElcZ3)N|m}z!H0Lb!A-t`4xp&=!y?%Feqh}bSr($p^fxIvQaO> zTu0Q$&Ze27KSdw%ol+FtpLyMda|=I2f#K&v0HUS_?RB+4YoIPB z&+Mz}88ttu*LH^IY0bKdJ7l=da9Ll~DOJVb#wH}Z!N%_Nzdi(Pde)&yKrCdRB@+<4 zcA47s^>xW|>-&?*s`-;kuBJcV{12~Qd#yd%*>6X#Rh)Xb6_(yj{doe~IDoPB^DOVJ zFE_hF%eBi$Nl7un_B(cb_w6EvwnjeJr>3SxQTk&T_;VP1YkICO&~&>wH;I?1iM;$^3E=9?y2j(n4@34F=jfF-g(<11q?{INfBaxhQj`@^ zm1#Qu4-7&Gqvh4fGDTANcV-nUpR}(w?aa=}$;rsjH!@PzO#N+FoF<2XhtjjcgVkr} zc)*1m)wki%`Zk6hFiIdJVqu9)F|xF2A0@=ajg5^Lkga=;(Eqx&{yO^N$+ur0!M66twko50 z#>L8u4Nfqv^r;lDPmvVTS!eMu;3q|ROh0kgdlwg&RUbM1%yk+N06zpNYX7(V>Pb>B z$RQ(8-KFmR zpuRqYqra}Mp~TwNb8O=eQ9MUg3yb6Zf-J;um5^=@o0(qbMh7q(^w$7Z0^Dp>8vA`F@@8hNN_z; zvpl8w4XAdBzOj}mPx2Wy;t`7^Pc5UFN8Ov}Nfqy#-uOiR2th;AR$jkywkcFf9CPAp zA+cQ_>w(SVO}A@z>_Rz=oqvsnM+$MWZd168ZP3|UU$3JKZtYcE8KB-y@k1CZ5xw>` z#PgHmZ;-F}`hLierEyW_gzLpz)29`51gCW&B~5g;MD84ul})ib))03ol~3L5rX_p_ z8K0}UlTpO!yVJo``ni^)fw@zrd1aju(cv}YqHvUn7@R zra7FctZ^7WxkN`8`&SzKTg<0@KqQe3*bEjC`r3;K&??5kC*uB4bNCfoxw?j1lhM~O zBn>WU>>YS;&9x8!u)PY1BnPz0W(PQ)cg{w}Z6aAc0e-_-BZA{yo}s6U-yCalTiQ!` zjctv3Ti|dXM^>i?Pyh!SD!e)Ws<>K7m^fJb#*l_XPs!GzBbZg><;trO%Icu@y2Eyd4Yz#@BLps1fhAa-CNJ@b-3SC zloE=oysfX^`}ycpg!t{*1sS{W{_Ife*UF`pt@qCrj6#4clBGG5IRP-AE-Gs(vrOOl zQo~Deo5W!iNUC@ZwjURYc@V!>y2WlGX;WSYMJ$S!D?XKMHm*5+B?Terr6}Tyik>xgp&!?l)dYo}EMkby z-GmxeyYRV*nPb(P*|0UrRa<#AYe8l2oj5)z+h!dk>DS|iQXD-3YR;9Qz4pmCzfK_H zljXaF85w)0(ALz2tg+AQ{`h+RBp%QgjW)i8#Nfk?!AT z?4fT4nS4-j_Bd4^Lh^WEu$88AfxUGDq~o*4K$E4>Tvv6L$%PH2_RAH?(_f!@3Hbs5 zk@Y?})*~hy-YN|ofU4Qo^ucbwo2(B&CWI4uvWmTHiNSI9YQJ-`)pYvy*}AbHFsxB7 z&*!15&^>|e<-Ky8vqLeV2fgGS4JB@IwQs1-@pdLz4Ur?Zy?uU92C(`iGAm@NjPq5_ zza}S}=?GQK-JMU80At`JDwqL&FP#LDFhlsfkFtbUkZU6#OW!sife$b4GtmNDX`*&G zA~`=$%y*O`;p}x0OBH0H957do8t`5B9_>x3MWND!2yUh@AuGHV>3!>bzznf?UxYB! zMuJ_qYIcqzGc2105bTnoF26z1;VvVJ5Q{hdJa4N3+M`i*g*3pEdJqLw#h*%&6nRFg z=SJdeLM;=Wjs2fq-{fMgzrR|RCIxfe2L+tF`fI0rH&ZV%TKOm1Hy(6K3%11U<_r)P z*-jw#U*G2l5jYumJb5y;6a#%0zCGXRG)rY2=|>^uLDT~jbIL9hn=gcBw@ z{YbOdHTkN?Y6!K2JIBH5i7W%!`deK6otvJp1=k?Evf*~mD8fdnIR*^2*pP2GTs>8C z3p9nTX3N^fk{7f^5N$65=Gr~AkmSWNGCs1cN%PwXumSW`z@K`%E1!9;5yQ||PxC5% z#R(UoZ#(crTwfoR&|QziN|R^-CRw6;3|4*rC~L^7gnQIjuq7UtedU~v?K^FP#p|tI zk&|vE1VEIN$wL4cP`JJRi@x(?dTyK5cCFhYR)I03vjQ!nSN$zt1^tN*+oTgC!7jalvh!Upo< zwcn`?yA2ly0qvWIO4ues64eU%+44K)8C~{{3(c(Ru*I!WPlMb$7aRkj7LoO4zHfqa z!M@%Hx_Z0ToP&Aq1iS;M@!bHTpl!Dzss860NgeOpPS5D6zSF^pa52=+&{Gq=%)`Ga+{9UVv;|bze zK;#g+CDWnsS-ZZbP0g+|y##mtK0s`dZ`Ou0@sve|z>}1;WVHgOPmlr>q8mUbm4S0-Z!|ztnQBlqwg-zD0ri zur7G=AMAjEjrb{AT-%xI*$g*XyQz1(?g4GrxmFx3w6#T)B7k_dyutE2T~LALL0g~}SNQJOSe*EQUpw;432^dO zKc=Kh1M5!ScG!hM)#Ot7!$x`*Wl1?%XEyQ|2w!k&Gq@;QQ%oSJO4jCt7q E0R(INVgLXD diff --git a/doc/logo_small.png b/doc/logo_small.png index 4ed97ac00dedf917859dd9082dff26a2209345bb..bb9c2e2fb2f4981045b3816cdecfc545f78ec7f5 100644 GIT binary patch delta 7319 zcmV;I9BAXkJDE9<7YeKh0ssI27J|Zlks&XCDF#O0J$nEE94kpgK~#9!?Ok_#6xH^> zciPsJO|mHvNT*0k2tkS=3QzP0`hxrrl^+6vB8Y-Kfd}&Bfno�fV44F(}BRiXw_s z1Ja}fB#@8>sk`ZAcV=hK?~lQKYqAO3Qg_ZD``Mk{ow;+q_j~R+=bpJ20|1hwEej!k zsi67AwX~rmg*Fdn7-KVd7~^7tKCSXr30qoWs`RG40_57Bp55i7T>RGjdkMnUf*{7Y zlB?R1x&2rDnM$szhOgxWPCx)4AcQc&7-Iqxa#HR=ss7RB)rYhLBo?+J`KjRe7nxhL zs&fSp5Q2fe{bMw}LscPOw1`_6$e!HFKU z+L^O0Pg}DyXV1KDv&M8vYy?Hgkh&5)jPcg2uQ%S_#)IH5(~j&g+?f;3>H6A#iK3L1 zH<27qeUkp)L~m6TxVBEZu?bAxUvx^ zo(@%Y+A(l*{U1mybPB(DmlvHVPU-K{w~6o*iRjB{`up@PG!&P!<@G5iLFl+l zBtz;NNvpWEE`3v|s`IC@%bOZMk?~4W2?)N={|VKb=#U_EgcGwby@?Q7+jm(r;3tZV zWFGX1t>$ZXzRiQ5*l(6azvC~{E>BxiXegF~3TL&+3apHj zH5&!ZQerH9;?y&}y?TCs)^CNxLZ?P187ULrH1nl>WSYbdBhKj0Ni1}bd;$`(s&gaM zp)DJX>Wq3n#@9SmC9%*B@@>vco_uyj%eDa_2Yrc&)6U$zE!&fh7M(0I>K^frH%~ZAW1+3$#dr>X@QpJF#+L0#n=_Nul=2^4 z|2yX;L1+g9l46&mdd~tjX9M=#J$S3~_H!ZsYk6EUq;@c%&YR|!uXA@RDf!H(mn~~8 zq{0hZ<@1cKfM9CGq!w&LEpGr2#BfkDq;`k^!h$PrA1^)?qv_qUEm4qiE9OCBVIyM? z#`t>qjj0!ZW(&Y4XdimN_agD2#jQT)%Kr<*IMR#`g?+e1zq2CPLXk7`dnT4FdB^(_|W4dmm5h5`8 zQ^FpSAe)GMg@)qEXJ4%_RdiBz+!XhzoRpbY{W+9>|MQBQpYk9ad3aw_+kl@6j=p{U zBPFT$E`A4vDXU$UYW1d!AP6gNGe!hz0t$+Xd-wXsKNcLRRe=1whNtD=<4Z>*Wq?X{-EZ+*t* z@$@f+#|Qa6_*25}so@g=;N$3}BqCO)uXp)>c$`s{m3Zp8v`YWxyX zP=PSUspWsnxG)DH)WfU$mi|fRuTsnZn122r1SY&0uXV92LRFms-5U|TX|Ll$54zTJ z8hPWgv^C%7A7%(cZpqrt3T&h*d_&wSIVrdCwC)I@<lvpZVftwIPD8FiD+PVcMf^KP%Vkex|X z6Ywi;tUaMiDPc?HgnU)>2Yr2d+Z8f@xg&%a;~hDB&X!zQ+H0YLRA7v|ctna$iKsP< z@#>fl8G;E`c4!bh3ki^GLsgv*7aToalrpB%lTQRa9_bNYr@L6~MgZ`)V~N$Knh<5D zH@m;_M2E+a{b_WnO_K>>`)J1r+-d!}cdvh3WvcR%`Er6u@n>L}R(W&f&2@8sy1p9a z5rGg2|MLknTeJxP>ta`&EIz#_dbV^`TGEBmONR@N z9?Uxo2m}C(5W)x}Okje>bVvOK5kidd^P$f;YgC#cJ9D(~KkB_iMJYMKG~(nK zBhT6hABo9^x`KcN+j;^}1kDS)=u9qr;ElfpJWOIzq~?mWPY>iB;(;H3=o|M=uZ0Z3 zxMY+SKrn&HB=Gr$f+9mPk9Z}i3Y7c%Fx~_v#rjWIMI%ki!p*mn_vigkY}6wJ+p@ly z7C9LJ2u#|Z7;9)?z!;N=w9$}az!>8-F(3BydVs)$*lWMb|M9c*%_VF}CuQ(QeU^l( zIy;|5FgKOJ1cModKoRPHun2Y7eQ&1~L1=E`*EzekW$mcoDhW)OfEyE>I5BLT1Nj!` zZx|2=NI(dMtHZ>FcyCswFHc*WT7KP&_FCL?{$qi|?wO*dpLJ-8ygcRdCjO?s@aEMe ztiU0J6r^H!;Ly34+24*PAuO6uIU%bsReqMSX;^O3M_t%?Fv&|-y z3&~e()V+LewvN?fgc*Vv9ys)$-DV++6?0q0aYY~o;QR8vUw&h)jAS10?7FG{r{>|X z&BsWtBFD#RYxm#%K~&kp+K1liF_$Lj`Z{8H?#WZnvcNXyO$ve_zF`P@L%&s_s*vWK zux$e@EMZHhUw9*buQnec6etf^75!dEWe4=vh}f*Ji)EKvBo_k!v%9|XWQYGD|9m>B zec+}?Vu=#-LF zn84&G^9@&pHWcmEAI8p?{%##_fLKkmZB8V3Gv5DtU{1Tu7divs!$+UsLFnPx?X!L> zEQc2Hz1~>zuhgYi%dU%nc`=@|yS!pfEZH`O@$tt-SPi!cL>@`lYdfjh&B*5hLu#pu zU4y>Vv-&Q7>c~14Ph9c%Y{^9cATg<7ypv)-4w468jO*-@(zuwQu>_h4@f0F$Z*zM^ zk+K)RTE6zRi}M~1h<`rh89$kipUkI@rv7th|Ave$wY)(9AwaHO)@NzBI>a0oV%LpE zwx7y8|KRQ{1OOA5rktCZZz#yPo&9`B-TZ8|srt=-tM6PWy+mP3LCWWKn>8#Tp~d6E zb}x27h%sK7mXvQO;DP_5-|9Nw{$6_J_21vb7y*JI+ClI1TG*snB}p7Kq$*0u5cKjs zA9nSOvSjDB>~GekZ(;~WK`OuKw>r`zydAp_ZQGE(67k}+h{^R+Q{Uw5mXWe>Rp_Sv zYumSf``0ER6!BPgW6hZ0#4vSd5|Q<7jqp-J^~X7+QF2SJOIXsz#t)M8-!WaIp<0)B7_n< zjhJ{ZV6#S&bG|)X zc$CD%sNg5;QVjqAYqW9J7hLokgfU_mcZ>@G;8NMu+?u--L4DL`iRD*kbmu8dKI}jE zrSJ*%am!@7nb04MhUU9P04gvj>d*;yWFf}*i`&~sOa{pV1LWGj1c)*36&i{<}KoeB3vV*uGduzW%0MxFZ zWNr+??K!)Yq{3h3r(raJ008dP&*;wy0FMSfa?e^~l$5^F zlk!kf3L7Wr=qO(;YQM>M$|?mh#x;BmCvee!-o4Ph z>7=h2g6`)NiR$7-UTV_*vv zblX|9*d7dNYWW`sp-#$R^Zi$G)kc9G7BHlFh*F<49LG7_11O4ey9%~Qb(n!S#A^D8 z)G4ej;((J8j8^8?lx_z;a^%Q=wQJY%^71MwDn^YOHGKGR5ze%|A7z+Rs^{gKk?OIFf3WJM5oh#kt8YNCxv=0 zaA!<{>7yH;hb4PT28s=##Hgq!e}8{J zKfmtXySrUb@nZOdYvq4D7C2P=_;SRg)wkBiYWlcD5de^$o}QAD!Z=sYVs?lKf~emx z0N{Ck;>3w{mg{mCL)t+RxMR@fKHfdVkHZ2I_C2({i$|nO@WU8?@7uSpK7N*aeH)EN z4-b#~vU&RS>GARLEX$ffvivqPGjr+ErRKbhI~HEldHT?WIjYN-FV|P7cz%PyFlWx3 zjT<+P9)174tNs1`Crp^IVZ(;ry?a|FSyooo-o1Oi zE`6g*I&yMyvP>p_v*JSuLQqi9rcIl=bm>CVwC%Q(qNo`&W<37*<0ezxauAXv4;(mP zG#cA+xY8gCGpn;t7N4#*)nJTkOf}cbZ+w=q*(FvqBO{}}BMJaYrE?`Q0Y~-0fCZK%1OC@jF4qjXFKVO0zgSgiB;Dl zF)`6ek7LG+8Fe5OyTaYOciqmAB7_vAoWfK_Ri?Q1j+#=Dn8-I2Ie9&k$;5G7eP}6) z8ai~S!}y7R^z`(MjEt=RZB0#0Wo4xsLMS6Kd9yNI)Xs)}t5!t6H_$iEc?hkhW5UA1 zToTLo?%lh8{)ou)e0qAidqIdXj`5BbkRWcpjWO2B{RaBQId6_Qj^lYNiy(M=dpB}& zj4{V?!e52wc`>tU`Ln2~D5J4%RR=*3w{G2Xuj~v$Xoz;;?z{W5YjVvUyo4*eQIR&- zZ-DbjuZGsAZ0KKu1`X2d^>r|&C@LtZZY>?dF!j5C0>-!!3`}@Wsgo=#CoWRip zjWJ$y^}P$Fzt8SEV{Gs!CkL%Y{MCkzKl$X7HX7>sClUQpw;Kx)LY)*HDNOxTcI^#4l3}aQBdcD4((>8uN%WnbT`t|EpgHCt!ND)Gzst^iO-{l<=t>6Cy z{f!lWSe;Slj7BJmvI^nt+qYd3bf%`JQj`_bqS0vF4nhFv;n`I{LZP8ZjIR3o#(L5o z2ElkJ|A@mxBuSEC7^`I3zkk2OJwwE=q@*MxBg5+GYc<$t=MZ9y2WtlydA6LZG}CT$ z@ZSkUth$-x%q6WlpCm~hJa~|0SqD%w|7HDu`t|kmK4Ly0CML%1AVdg7sKaC=V-$>O zmA6Di4D*lY1kNBBiwwmMyE9QyQGy`UflB1c>#x5qMs@ZAi7`HX`gD4Fy45*khGEod zwL3zH5Q_I3NFX9P`&&^>ksc9#GGBzymds>_E#A0s#fWU}_^(cBM0?LdUk$>P%j+)Jsv(`z9B?7sV51y|wlRpa8~ ztVVBQR4{-3{FN(Lnv*#7gAL2cBbH?sE?oH8XP-3`kcf4pZn~Hy0Ce$)2$i%($buqoV>ie?Ck9P{QO2r7cs^Pg<{mGQOKRWP$tK)s002TlLKZGuh}yLf_hpQnl#BRT_T>caV1&`GJNq1Cp-83=KKMY) zhc(G|xBwXAxpU`=zOMU1C~BO46HL2u_gQjf;RA0l1fw^We4eq@0R@S$`T6;M^2sMI zXQ7B0&-2TcEfa^^_FW`yQzh;br=D4HGwE=_FO~-rJB}bQ@$H>`j?qX|t4gI>ym+ze zccP-GRjXEcdV0D;g}+S@Mtk+7F_oe}Yk6>H)XO9$4ZLCb4F?y%5cwB>6cn^_<;uv& z$eNlO=kKSswzg-_o=Hhb?KqZq1j4bMo)VDooBn*A<8M7M*OT^$(e!pWQzO30WU{$) z=gyrwSEW*kCEgD720;+ya`}P<3uexo*}exq`#e%JSH}E0afo)%%q}lCT~ovhUAuPe z+i$pZ2+ zvTS8#mvzo_ywuj4$gd z3y{BDD<|ZoT=~khBp39ldfS3f6s1zBJUl#neSLj>eLX!r)oQh6xwo4?SZXVT=SnV? zv1Lb#Pu5d8!U&HK9V>uvF#j-;q}j#RO%5W|;Q|O{to7N6$zkK~)Z|%i{v%1NgHS8; z1q4RH*w6sBsK-2iNt>|Pq#dELejir)iTOEWs)psx=l$5IR zpY6SSFuf{s;@N2?fp@F;x(z^hsNiRcAOjj*{RseOc9|-GP-UuKbt_3)fut2d2mrt8 zej^a^fM>VH;{h+)>)$>8DS+^w-2FfQSwk#IOF}3b!?CA-CzY~gRebgI$jSGeA|4iy z(8H@cLoiFPFBd?N%xOzP2mq|Wj{f!eB7=?tZqWntRFuk=^7wVJANw#G02p`X#rFN2 zd#i>}%v!y4epU%vY7~tB=<;e}#}T$?w83-}Pvo6lhuFeI3 zEf206)9LSjwx=R7#*ymqw|dOwK{!%);&{=ol55kf5Q>cX@Q+U}mR{z8U({n>*Lr(q z+ae=^h7a|RM+nWk^6!kQOi9oiLN?{x zYrGUGHm4!on|F{0{vp2suS8C9M(-phcMsY|6EpySOgj6DWKNqxtT-CtnDd>GA1 z-I+W?J^if^0o+8ajIS3(vaD+Onn`f7nQs2mb zpy6^#kzZSocxoI6oCJy85OQEz6Y~*jxw7^y9+4kMFBOnb#+Hvb{&bD0Mnc6t2!{tg zqNY_X*#wOGX`(;xx7sMM9GD(IF{a8?C0SHE^jBgm^`^aALzX17I{UfPFUm<7jp;4@ zlR78@rQD58EYvXGtwd8{s*qiSwi1|s;DDQOcG{(~tCE{)6NIg*HNnabdj}_bQXVG3 zH1qd=o-95sxvBRzq^(Use?AvJq1dPg1TROt@P8psOS8mn2|}~Xy7H$ zacd$iAg1?crEIsksTfb1Qx&4Y=Z5u*K+V)r|%Ax$9`yNhMb8CYHUR#AQxAv}@ zQvFzP1WB`rZ4|;uq2mm^p`5EIW6PyMu}wlq6LdeH7>w}ex4)F|Ym*QngdPhVh7tZD x|FC&8FbTr8BmmIIyC*Af0)kSuOoH(L0TAE;0GL*7(*OVf07*qoM6N;tV1k0<8@&Jk literal 7621 zcmV;$9XjHPP)Px#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0J%0be7u7XSbqS4l)cRCwC$ zU3p*=RsR3JnaQ1|O`6`NcR5-p2o$*wQM}l75qCiraak3w#pMtXp#maOL=*+l<5dZ%gVnKb9jyx$)Kv9?WVo1|^h`Tmt;CU0imXFlKSecyWs zV+_1>y42hB3V{#C7$Jl){=V!#!Q#N-{>ccT_BMrh2s=|a%r@TmbWh0-(<7$-E&c`P z+obqz?M(2_R%d>AV@t8IR6nKESOWqhOk8xq+1mC}IYD^j-rlsW+J4$cnCc;7Q9$<1!MeZ@Pj;ro+^j58352%5#RY9J`bU5paF`0JcOPqT9Xa{#reeY5PGWU89f3}36(sA zo+>tm1_TsA@(_BeNGS=%U}8)>gq|uYp%MU>7|VDFJypUbAsFC>{w5Ehr^*1|L>9A| zHCa4_9;--2N?(CbiLunmT6qXPRfhQwMSuznMSPCbQ{};+@c@urm&4~sJyI~n$*TLp zq#@zbu+G0aU;gBJI6Ik$JPTa|+pnD96{#nJXJ*VxF0U*w6n4oy@TI`c>dbt7!7CRQ zHqfSiKD}e)Q9Oj6gC77wXj9(SZ%g-?8S|earp$_(-X(Y9G44!T+yo350qUGPDK(a@ z%wBi7Hd`t5O-WolT0Ok+ISgh9Aozm(rgGa{Rb{PqbW}hHV`PzTMUVrJ!OTyZL#wja z$w+BGMeomse%SbE18w{ye_Mf}=(C|8xOG~gM=H`ol^L7!wmITotUPAt{hzrTzj~YD zwTp`j425Dsw5tEoaRH<3hY&(L3%~w%@wa+fZ)53$+Kfp-6L?$r8zB)XB!!OpSAg96 zEC`_=%m2GR=VKbvgZu`(mAE)W63po^gwVyBtPQzaZs_ynlWW;Z|<)-wO%n^)?tk%%0s9JF!p zS|K6y6)1L>e77WP#pWUF-KP?0mj3IxxjcleG7k^{Y{}p8ec68+=!X0Khc52>8b>~k zTbuH??khW>r}dH2UT-EW9O&1d>)%)7p1)9&RnSo6K9z8=B4bnDKWnyX?A_}!cj^7S{LiJCqka7=rt$(>wiEdILqyF=Ox z18q=Hiut|gObMUN9pxeXwVPOAuh-?})aO3c>k01nMBl!4gt&8rq;P4!`3iy0n1GQ@ z3Fla>jJ2?#WcihMqGgdQ5|@NaLnWl-p6LjCtnzK?zBSi2$SA1~C4VB~vDbRP*!U>L zgU7*;U6->qXJd)6w1F{>2^bM14zzdExvET!x$^g6k8-~}9{O;*n1m2ok+`I=q4ryTBK@nzRpthAI-t>0V5-&5!^3flF)vNK67H99jqKE zr({j8-Zj~o)|HPV`L1;T$NB$o)L@)EW_$9-?Y$EVV?A~I?C#R<&sAR_5E`Z$Iwxk< z0KY`;Xqi#7wcwuzD-O#j+0Wy@cmBc8RAuff|L=vG%R0+#3>affARcgQA>h95Ec&|4Sl;$_3LA=Z4f&2P;X}#C z299J{rj4x;h{({MsHf{mM3RW~75J8;D9FmeD%dLlLim~=J*N#C_);Z4!JH{B(m~Ka|2PgoH0Y4u4@RYF0y&Fwx*+P|*0L{pO2EQODGN z>$lqYtB?=?2s+#v1AG&pD++{K z4A)r9vKWU-LO84ZRneX*YfXqaXkPDE#s`jq-&Vx*EC>r5iWg(LxG_XAwoz9 z=EOWZE#k@MA)vj3D$Tb$VMM!?d%lxxAp{!+tw~nh*R7O9`(8(vX#DR=_piIYQA~<4 z;NJ4MZOI=wI@8A5a_VoqoxQHiq!E!K=iRv`K2+Oj2pDXvZAbFvzKVE{PB6KEg$pk& zJ6nB0Op0x+ZOwp`#We}znetoLyoB#|o`)2sI|{lfOKppyWS|-9sWm zLW-McPBUZv{8rkITWNAqMgR(t1iupZ+}MDT7-M@t2E23$gc##$Yt78lbA(91U>3C$ zT@*GHy?lP5kuk|hnT55ymAH6P&;zd8=#3X=Aw&pm$lc5UV`j|@`n=kBxS2y}SMfKW z<$pmS!pN8=28~GCu&FHR{^l}IHdgs;WF)VP+CL+Gum?;h43ClMhe1aI_T z_DIM?Z!$UFqe$n)zcf2)hQCPFL@{2d$vRY#AtFVAqJS+!HmIAOz)M#`h%q+M4Oi=O zhN=cpi1HVynsX~b(RW6S(y9s;s)J&qPj}6Z@CO#*e`) z#w>|Q5g|-XTItq+fx(Q0u{cP!U|UzL`QkAU{!sQ)W=$4FkkQR%vCcUCd{INm>i)|f z3Lfun%lBm1?;Z_)sFl#3hp=T8+Q1k$-uP5V2!h0c;{!%{W0`S>x8W7am%674J*+*(V%Ef%mM6S!FU~;-oxXiGPhS9l#;QjQ zRt{{}3|oW{1~6SbPuJCTS!xDj+3=A_BK3Y2et2UGLTHqFc#>aYV>=fn3H22!5*2;d zB&BrB_t1WjYPy)&HAKLGF&ls2xUg4*mqVCVwA;qo5JJ-;p0uk_4#g;0uOCN$<&6JA zNF;<@-Zg;-#w=!m@CGNAXq|!?mB&d$3JKwG|DpDBE}i+d)?5i4Ez0Jl^B}CV>GKRX zX-uz5Txth3UtjRp@jpF$bZV8Qng_4DL-@t5G>oxAph#8?wilaxe(OsaB^8lEu2_hd zZVlnS@&gn>g-b$&gn%0hEHh~iS02R}&+GlN@b0`wUb>?@gE1~LmN1yHVfw+K33i3K zF6Sd5A*2ZE;ot}P6lk}G5FvESPy_&0*7``u101M&+K_L!i2*11CAK$~=cNmRipzD` z2*6*c@)h{rsT8fxGtdoY#*)%+k!u`#i-k3ry6n+03*KEjlT)9I0Y<8a*~5mfiuWQw z{S>{qQC#P>sCv%d=$fjogAhoizjs2&!IM*;3xE<7eYhWOtnFmg88IOmq8#LWEXesm z@=;^E_Tjz586#RWgeJyppc@cE5z=sO@b5xR7Kun2(?gVloOeXqKIx?KwYbkNbc2ys z)FHO4NQ;dn+}Mqjlyb@IoccTpk!HqnpEAj{iBD&mEjsW=z6z!_D?HED14<}BXIC=7< zPNy@Q%_K>tq@<|TY7XOqf`YYc*U~iIVE{13etv%I)~yo?g`SQg<%kC-A2`tX*)(P? z03_XA3c?-av**yypFh8A*Djq-Cz0G)Mv!3`nx=o%^ClBP5YF^I#@J%9Fbw0JEWGlVNppEt)4YUCOqTB$X z-T0N3md=|uud=d|AP5hdWZO+W(mf?VF1)n-vEzTT|7d0`7_d9`I_03!YPHLkFSlB) z1i|mS;538)P-HBYk`e-t@h%a$C0)d%*yR+&7@JI{HEY(0#bVAJ@~F4~LLW-bg@d-c zi}Bz)rV+V=h3nU^He@-W5+l)wAgIpHA34$XpFJdY840sj&AacFTO}kO_fL_&}fNDFI1!m z&EOk)l#lQ|HW? z6B!w4u~=FJ9%KCW+izPe7LQojspBlXTkOjcK-ddJS{w@7lGCqNwK2rPu4XY}qn@{(SF%(4hyvDtp~i zC!f99PzV?>#=qP?`}FDOzbe`5l4s@`(P%V|OduE9r>3U19ls_xO`JG!!GZGOG~{Y^mGW}G2JiK)|v~|m$)eVRMlCnx$?c7k6h@;l$4aT7)4-=8HQQ3 zXpyh4Z^v>m+{uZFiA$F*Wm&e#e=)`+Nq+d@hu$1i*b^}Y4TW4t5iN^kFe4&Gm1do* zuAQ5k+akAEmQ7Ah4i66}h>oxP#YLP+Nl7Y|s>xfzHCR_yr_pGk=o zP1Eegj-}ZJn-VLaWtkxm?}~e(l;JVC>kjBuO^^3qcU2rKMh= zNFCPBL`r)BU<+%lvDPVtz5wvq&=0vK4qbI^lY`nPe}Dhhak3BJH$JDuG5tH!Kcb?z z(4~nnE-o(iW?L962?78CqR!V`v_G~TKj%#d#@J{yItuSv|6@oTUg)#bYEwK*HziXX!;_Y6o{R0EC< zc?a9t4Iuzb3>wQ|rrKIdV+H`kq&QHdrVw>Pcgn`voWFK+{l-a6O-(Dhx|PIOlS5ow z0RY}%^Ev|Iq~He_%-*(CF)YpX#k5{iEUcBrbg{A2d2{Fp!itItmxOB^mRGM{b)+z5 zGMP6+h%t_lMad{Bi}9(db6j^k95PW(Nm?=ReNV|>$fG>-7!`)3bJ1j9VF)f-dk|clr`DbUi zX#Znzaj{mbZDF4ygfPZoVPRgbNCBXiGzKX#J8P-48yElySBx=jQyOLm>BP8;TZJ@8*3vG zab2IssZE5?OL5Q9EPbv1x`{D4lf$Ga>WL?wXwlc;_*YX?vwZpTBS((dY_`^i@0+|# z8jWV|+_}ZY#jW6{P$;~Eh2+YWD_eP;MWiUb@~DAs5RpP|6H@?)l}CSH_LG&h7B!S` zrm@53Dn33wJw3feE~2|~>C&a+$B%2ZT7f{IR;ydAQsmCm>-A^OoJmVd`|i8%0Km~e z2SUhZv(2ABpUdQVZFa`4z8(x5pH{T{LQR(acM%~R;G1x&>fABiFFMO@wa~uLGd3kwdWu?_>Wf(>v z5ZFhO9kDPdC@3Z-#+%EZ8b9rXz%jdu_XLR4jk|ST@0SjrI3}Z{|J59t74@_;qk7Y) zPd{|%Q0wK}9FSb@)MPT{<>j@kQb@A(N;vL5tyb&InKLPh@{XF-HkIC%nKUY)pO_Fg zuJ3#6>bplPPXa*3gnbMTW@s^Ay}53Fk!-s88f_Qxra7&eW*BCLQ3p0 z9QR>X799flx-}2{U7Mrt2}rlO|1i;e{8RnKIaxNHI7#I3*>8Q`%lr`QKIr z_-f{&mD#Cfrr!*lt{=Dx01UL@ll<*Yc0EF9(3VcaKp1gg!@gcRyf3$iy0mPn?AFtG2bCMzeFlyAORjXEs#bW#9O()51|B#T7 zuxQaDFPPKY5eT_w?(GwY0R2aEut^^+=<}+cHrTMu$$|_!iV+bJsi~w6_)jPlrlh1SSg@d#fktl*D(o`Mf5^$(XZ!`KCjG7!`5zd&r_@-U=-bzMbH#m- zNFSfTCEf) zTj>R^Vjo2$TVwK|=yy1LqCv(?npa5Dv!N~J_1 z2?z*KDwQggDj*;LdXlsQLKfJ3eL=W1OiH$}kimP`KmJK`&`28(jNPjg`nh%jTjP+l zjr8p~3pKXY2SEsZQ}X?s^Yhl`e9&UoJQ4P&g|SFT$vZjkySDG_Hte3xNMq!B&T0$o za3gh=Dh#-=q2*GdK0fiYqyJ1}`burK8(0My#*9VxNrV`dc_ z%RGI9YHx;6B~oI*j&@B#2q8f43|Q87Cy1BMR-__A2mm&G?+VgtOU?T0o3d)I^7wTx z2&*jB2#}PN-1E#Gg*$&N|7q!!Rb~g2<)v#NJX@U!0NmzvvKN3I~=!O^0zXsm2 z_qb<6$YHRzbRU6;SXuNvPY#oWObMUNV5X>{^vj}e_%)rn7(&h^*^s;0SD>)5^w@xr zZE@@8^?tdJA|3&JeB&Q^`kTB_>@o;B_PlfL{k^68FyMHfmy)fYa$mBypkc@Sd22%)sX-G?eN z0OS6?{l2_^n}86sD^%wId!yg{IC&HT^m#$r*{V$5D7su)UbzP$bfzkEQC1296ebP* z^8T&R(exv?Q_!?ivyHTwBFL|XZx0j)@cr&y2mzqhS~un7Oct{wB7YdY%SRx0@-jK~ zxwFr{Dk4PwBGtE}cJhfgw=2@UW&2Ew$%bto4_@aZkUKw5v!9~RyxyxsV{x(QC0GuAd;r`y=w_mZbw6LML#8_5O>qn@Qc`mwMkv5(H z*81AI9pfSZ%sTT@v9XlF?DB+#kA(1>Ai5etD{FoK`leWU%%39ufZVfLIgRNlCuZtt zL%mIZaNM_Qkw1@tD{bN23yTj_r0=-3v)-zA$Hg$l6rw&E^1g+!V!$=lS{?xB<<}wK z(^cnl>+>1Rj8xz6D^$3J@(2Mygf#qz(O=ft^zlA1J*`y9y)4{Q@;wV|usCRC;u3ee zi-F>Rc%K*mFfyhyRhjm!6nJsU7M7VbpWjORYxJMPB%z%~i>FS`(wMdJ^4J~2Ho11~ z_8tUbhvzN0L^kDZ{i=8miOA9F;cJst@OX4Yks<_XMZ2@ySvlB>cQZGB&cwj+1Q16m zPiEC#;qmAQ3wIZP|Iv+4F~G@*2f74B7TE0t|7T*%bK_qm5x?hF`#sXn@^?@KHAFeE zYu+G)2u!>fJKG-nYfRb@Ne~ZAI}|AcEQ6U>;-2p^J6}(2hcel|j&23@d9Rvdg z3|7`U_rj9w`s)Z_LeQ8M2}?*s@=?D-2)p4s&S~|R&o8)EpNkOcujsdF$a}8Zw2gdU3&xrj*y1$NZu?aMLb|F*url5W+6l%VnyQO34vo+%$6F!tZc20|U(!yGejOkik4v(M5u&@aob!Oc`ZN?vaO%Wi9N73_A$$WtC zk&3jrQpFfon6#hdZ!iDtDz`ifJxSJF+ioBQ z0!2@DQ$4;u zg^w{jzHSpCwDsmc=U!OMV5Vm}HQrEqyS-p%R_)bv?XjLoM{j^|sOmlh=- Date: Sun, 14 Nov 2010 03:26:31 +0000 Subject: [PATCH 0504/2835] update --- doc/logo.png | Bin 13171 -> 8699 bytes doc/logo_small.png | Bin 7321 -> 4506 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index 1f51d1890c406ada11721f17f29f8c8061df527a..f601656c61a4d215fcd2c05e939766b5b50d9afe 100644 GIT binary patch literal 8699 zcmZ8nby!qiw55?|q-$ttq`PwfsgVZhPU%J@M>+%)RE83eEcYp@gTZo}25)uy4(-Rpf^9wofC8n2(x+3Nr4n8UcL%u)BKEM(i-)5F=~B`o|~YwW?DXl*%X~&S5I+X zJ7YdNf~he$-c_F6loqE&_20pJq2Ez?2fEGJ3X^4jHt)IXSt6OP!B7-5Ts-x|aZI}0 z_i8E3lZJb54%f_A=!TP4PnWw~NSdE3co`|rmzbwIFS^c@oWo}NE;;l0I{z4? zGUS>%?Be06M6S^<7TMLmnQ`<(e+g-N6KSSCJxMv1a!gk8mh&knS(vMolK_z?fz5_QE@j$A;|3!MbsJd_0vb|x(t-?ggX74 z#=P&3j!-0?S&{LLONj{~FA*`nIv56>FBhApk(DHV@kz2(4(TNK)okw^@p$NZIrBz2 z^JYd8UlATT(MmvES5%{fDOWD?_pb5x5)(hcb=`N*f$6vcs`mt3{$f$^RZxy2t(Kz~ zkWP{x$rp+u6}3p32b>%F*}R+Cn7Uw>d_nLnbZ7iEPmjp&JQ6w>t6zhYZ2Sjutnj%p z;ir8l>KTYRa(aKFl%$x*D>S#%!LQ9t*N?GBN};!M>KXypwVQyf4KEvA^d_`p?Z1L@ z=}xghAfx@VetwrHUbeLJZ?wXGD zd+hJyG2Bfff~I05D%Xx|dt?(N1a+3E5;IX(qaM-T%sAB3IFxk9ggAa*OY0r(7$lUcps8ov7a4qA_u!ntzu zg@uRP0~c`n$ATaCA|XE(LpffTA0wRj^I?$j+n^t$vDQ7xY1w%lj+?VA4E2-27fO8A zL7sP*&EDH@CcP$C?Gd~S-|CX|Lt(dVzv_ExX&yzcg$9fX9))Sp1t0u3|0R>Bw3o=_ zYvJkfKU8emnWXs9d)?PE=u|{aE`1L+@ITHjT=?!K#$qc51_u+Up(M*mnq9I>N#6=u zUhsa(79*%xdpEI#S`)~X&t%vsX#5DXI?(053fzfnyBIjfSD57Qn=3hvF01$Q*LXS1 zLo8fqwsn@W)4E-k;0c^x@UBEXqNVvAbA}BI6rP5u|Mis4-G9$w!MKvxcfXZ3FzXWC z2S|Ol)e$8Sk@;1#bwOrbhCw!r_VIJfD{Pfj)<6V!#W`YG1AN znic91!P2yrp0vv@7?m4fFtqSCAS}nS!=l3`>=So94%+!wRS<%SB*W-@p@7gVFT>cu=7Yw#lh&6A1JkeEHr%mCsFg)Qms6EpN^6)CN zq}AM;kz!Lxue|M#Ht@E`qpn`7MJPBj9M45;z$W^tbwD@c#v4bLQ!9dBAKw9qwjdeA zi@~;!AJjITkN*?wL7|0Y{!`F$WkfogPcIx6lwmhF%?w8}w^}4!sJ$0j7bfM+|H9Yx zhgW*|{uLGwtYEH?Oqv~q5UoY+@IqBnk%OOh;ls^ z4(p3uarCHzvY6wFFgfvYNDuG$hko|DUb*+Xzf=7$pTGcQPcV8u0dxL(}@ zDHvpM!lp)7wSq@$P%!svUh_w1K@h}c`V4k!RmmBi1zt<>)ECA1ob@`B0img2^-O^6lU=>b>*Q~HxdMcOqk z)(RQ@uCmZM2t=ywnD~zHsQ| zZcyoW75vwFZs3m!>JqSA1Ir_*c6NT@P@GeOh#=_U*zO4wKf%PSce$pqDhX0A0CRgqs2b6$uP0aEA0 zawHWtO<@?r#`-(y#OEfkaL9h_D^DHrV4viskBxhj4nbCk?}l`Ql5uL^)LgXEzj%Iz zzG{WnT2(4Sciu-624?lMdG~mZsNX0MtV+0)q+0*@>GWeo|LA=HT+(}zK$&<+h7Ou3 zd=p;+EjO1YL5~!R#i~DOCDT3Vlf*A7tftjlHy?)MvLW#p?zx*iKmoi4OWLEQ$KT*E z)PXoRmV`L0Df+~J1h?vPp^kkp0$f4&>`s21sA=fNuXfvwyU+v1JRur~w(l}(@!LC#X9nr_WUmCj{r+=kGhuly{F*QaygqT_GyDEC@5Hh52wxEz ziDr?e?8~(lM%uOE()3VjCzuL_C#*h>AHHCsv-g5vhb@x z&R%&MT&wC5_ftFnf|Hz?{eD89zZj0O(fD&&dV=DiJu(4>D<^`5}HY#7QQv= z$G41%jWpTeLI0B01FO38PnFlvKP@+x?GO~E6Vrw3@$C*+}RW7QklC3H|R&mJ5M0-oto~ZZ6*sWu?i?)+{_GS+r za`k!`!=&v%uL4Dtx#+Faruy~Sv2pkdco;W;C9gr!;;t0suYRHAlKCccrjR^UH+w%9 zJ73UUWc^5R^39sJ9T8`SASM_iEwi zp3AVG#iR6b@WrSUjr6PtXr{n460~B7daK;s6xcFf8`u&=lJHaCe+KtC8lYmRh4bIg zOm>M)c9~ji@w)VpPWXJW`4aJ{{y-~bDj!nQX9BCo zZ;6$ZdiB?S62CW?@#xroST#x~^b?4m#EP+RIqkk;P5k9iI#Rpsgpjk4i@cs6hAX6n z(vFhl7({idqT)%hBy|o3^bo@9&flHCO61Ho!l@Ip`v&`Vb7^Pa{%`!2nwg>vf;ZdD z3Ibtp9c-ikG@+em;;>}mKHt__Ud(9aZ0&I28e<*EAjKQUw)+*0)|R>?ARKzqBE;M!Al1OMjMyl)eWRUt4*Hlqp1!!nrb#rs*n^&6`PN^AZZrV| zO~dGg`<@Pe6uBQ8XPL#PXIu~O{>|_DS{mPLaJ~#iXY8!u5%R!wg)~tbX%7;C;ee9( z>pT}F(501eCNQkh_;)Qy%<~v+;bRCFK*j!H6wt%?c z51=D+2CeFa_vLC@HfrZpGY2c%>ZsoheWKOc>Y~Llf`t)|`Cjgp;Q_(|eQ!_5Dnpdk z=XXNv^O3$cU2c1qpPm2*pw>517jKo;4TdzcqW59icPqU4K_%7FVw)kd++Z5)+!!rVVs ztZ@8V>dwodUE?AqZH$G^cO|4^;bA?!$ z45Q4Y>!cSqk(#irp3EyMM*D2x0NJ-qeua$kRNcmV8=Fjjc@wDv@CLW)HJ45pb_Vl( z`TP|_luq__&L-J^S7R77X!ks_JHG>g{u=kU#5h<`+omXjC~t$H7TlTqQ}ts#xc7a! z$9fiyxE)o4=)KsJy%~aL`t)^G+yr{RcdNhq8&3~6)8Vq&Gz9;Zi%X=8zNpKpoHu#a zKn_6TR|3WIJe5!0)Y7ic`mw3-nL!~X^xC{8V_`~Samr-vyo&sJ%C?93 zCV^lZPd+*HJdgOv1GAmkIY&7kKJ({eWa6xj%0nS2s##ZS1_<~5wnj7B z{#c32!pSjfVfs3Pt-h~&4Een)fG~QJm}J5>5IpA$=8e_rd=n&@td@iB#iD6>k-q`Y zqK!F{Tp=4Or|+96i<&4)WnGrxYEi`ZMnz)8d5lVyUfqyyao?gQ9US2(XiQPvRtA1A zXjLpApo2qs^-4qbMZL^rI~%;hC*@0sqwRkN_IZs^5J{4A3C; zRC4HlR$4+pAl`f!3}|*IF);jJlV0(^CQyQFOD}UiTd(p_7O&0iEwnmFLDl<>9m?@K@?5~i*3{=;F_CdHnu>;DBee$%_wHmN}Y%s7e1292y#b0-DyS0j(vbI#IG6(YI znb&hPWct}PxC&(Tw>d<+%|k@1%~p`I?HVntU)9|X;vfJqxFu!?=}WSNXg}7^)EU8Y z=m`}@w(kEy%c9_$H_tNpMqbPOcgVm3|7-qJ^Z%Lu)C8^-^jBPmx`(FP0N06^?|wWo z!x;SJ$f5bLfJ)f0=SZ{=+vhVCZ_$_R+kDX{5@>_8G$ZaB9mazne{IY2+zqd=!QuBf zLqP2QcT)B;R(+yhJ6-F$TBHI$w_!?E4Z4|YUGP(|GzCUo%>1)rSwueO9J77((^rhN#q`CDa$>7)U~kPW)d>(@kj7VG@D)!VUK0bciHZx>tWyR~Z zBtb11KrHTisNn9-iv(Eje{V!jPjArV!^Nc1x$w)=Fc4Vpxs!Gn-SS>V+ohV>$6?zp zxNJC&(=_L+uVN$tElY*RZxsum^vulYT~T;lwGI=2-=Rik2NZO*Bp2N>LlX6}YoawaKAiO0vs zcifIjusry(tvkak1WP5QIlxSo(7b;B1enh1e0v-QgPC_cW7to{Mv;EFwo3dqn977@ z_r)Cp9JpU)N>DU`9(Q){;pyqV7B4JEh*p94geEKF?JX#U*stO405D;9>pK`}ePiQh zfvINJS>$Ypzv}IZTtsxV@8`LK3a1Dp9`{uRPQz+6y^^Ut__$-;^+tx-+vRrTOLg9S z2%EZO&1_tHI%&nbkJX%AD{tw5L%<>B+^&Bswzx$p6(mB+Wwd_rRa%x14lS;<9i}A3 z2VzfAQ4yV!G*+`ODl&3qGs{g+LF6m70FZ%q*C(Gu+;cViW~QfetE*EY!|4@Owcflr z@Px!u3FDufoW$(4xlhAvhcnUZ<`9A62L}hNTwFFQ0A#&OwV|-BnJr8jw-@*%>iGgq zR?GS^o^!B&Xh^})@yARoI}mrPtE(evZa{F~cdxu{cNRhrBPAy%Lo_w@_kBGvwj#+) zx4qT12_wLQ;UDM~LoDm(wav_qCfr3uMZdl;u%4ZreTIW`cIyci_rb-F4v&lgW?E!A z^;yoV!Po>8d4ukeOX}!s1Q#2d1-%7el<-w%R#uCYMhX)X3k!maHPBd9HD=}>n#-Eo z*qHg>c$E~U+WL9~F((j8FOjVTC|ex$eplm`>FrHDyScvhtB_YEM9W6`@-ce!5WV4> zz|7tFWZ>ON-^uk_VqRU{)WIIm`>RGYvs(I7{N8QHzpUYb<-!1LBPEO?#nl<5Dh4jbHJuojna(+-Lt-Ew!TFDW;lpt^GZ! z)D*dBAu%xivtEg!oLtz9v{Ic>JbXPeB$_;7p%Y7E! zvHUu|!?oUOFr6!=ToIW?FZ{&j1)53FL#3yX=D#xk3sG3MnoDRUi-1`E7?nS!1LyV?AFC+I{o? z?s=zUHuXcpwBUPn_485SUr8}Bv4HajqcTzl2M21Tfg}dzBpZrc*3|{i(a2%jvm0*# zO1vRk{-vNZ_2$z7jyK#2ardnZm&QBjsdpLjW`W#y7YIfYp# zr>0g~5r=yxo!BGdR*rR)WN*Ac-!-%G!#m!pPW+yK4{uvCOSb_cw9A@rrL2CbGedVClsKil(Te%8u~0_k1oOv-dsc34B|}3B0ZR2m1Y86EFPT{oC?_X}lUz3JOa;Bm zEb|eUMdb1B4B&vx3|;7#3Wu6mEghYR++2ojuf`EW!FVM`Aghy7QvUc0AI#L>si~^M zryQTi{pU|19UYx^Aki(Wytsu6CU)-h=Btd#`iF-V?d@5ev;nTn1u-g_Z;$1$vay{E zQokCZ{dLynB18pPZDeF*&iA>}?O*GNL$HmZRKd}}4aJwb{8Z`GHq>s>P6=V*3}g~( zKu9U5OCcFlnj?FOeVy?P3#2m5WtA&{%I1yB0FRO>xxxwWn?%nn)q>rN4|`~`?fbJe zIWv0{TEOy-shWUZiH^29S?#GbDnqsX@qEj%4+sTwSpd@L&~%lQFt}UJ{~&{)BjU-i zq+Gk>2D||E(_*Bt#sK^1=}%@16!2juL{SA81n`EnohXpdWjA?Kj#YjqmT}~~xn~+i zmD(j!4VQCH$=|+>0Q{*9pZXq3lpWE&1bDb7UnP|sunT*=63~M< zWZd0Abl>xZsA8#0#4Cm0oNcXr@f@*`!He)10&R8tJ10Uz284&xOnDDWc2L+D;MO~f z!Rm!smBUgRJ~%m)*xI5lfGc|^r@a%;kl>qH)EIijj#7Xq z_omCb%HK5ITElHLfgH0Jpd9)lO60jY^j@#TMus<^b)d^q4YS$AOJ#G> zZ~)u^C{AWnUTr=bk+-#dAtxs%HUqLg_&^R_ulCu{(b5Y2^9TAdEv>_v&szhwg-f~I z9n;T3#w<%H*40b#7C=}aDFK-Ta1X#!%050mTTH>Y%zzgYQ3qGv!pq9=1O)}{U0e>R z2h4@Cnj}K<#_f6hj?Ez{XyF~90CXuRDA4iooqlZkfXf05tBBNl*NXq7&~It77a5=G hfhR>C_U94xVt+}*bDZCohX_F+3J^p>RLx`gB-6s@!`#B)+1PDd z?LkKvhnSdD3{_5mB$kAz@&{$5a5|F*`tuxEFgQG%4(Z1SV!ZG`q-iv1WSZ|Wl7~eO zag4+EjpG8$p=jc@$m!Kfw!G{3KBr>}NiFPmO?Ru#DqC_oJaDu4ww zhM@lfRlCee%11S76xmFsa9HOTEtPzH|JgjEnP)KSWViPUF>GO(GU~ zh^3mdCh5_^aK;QX38Sv)h&>{PhDEXFLv1u5Hq_+g{R7FuB_gOpy)j8o*=fcZ z@dl6W?ePj=uRp$V>fP5pS|-r>_;jUUi7H{fFuS!r+V%7MdaNma5g6#^rNHg-YbkBl zD1s6kam#dVt-Sgh8df{`?@$IS2~QN<#c6(XFee-Ui%h7@sdPXsSFDz_gD1^AZ!i1I z8au-Xls~8sY5Z$XyYW9#T{zcnuk=GMpNA6+`N?s@@Z>c95R#yPB_=t&NbkoLt9 ztEjYNQ`+ucRY5Hci^kH$L}$(PLU9%!Sd>S6f@W;`BGo~_zaF%`rrT8CcD%>f&tGzR z{3EsALj){xCKB@xbK0;m`Irrh`!R;VX;#)Z>+C3Y@#wCxWq(C*$NRk)PpJR=ThMPZ zWD49IcO@dVvMOJX+zU3hq287W384$=eerK>53(_RSA?>tX_eH$dHwIx-`N=%GZ2)4 z8b4~PpRb1CTV(>q>?C|%imbM}t&~i3M`_5Pkb1|*c<`;9zvt(u^ z8D+5P;#hSUH|8UU8i9$!-NE2hD(>rk)RgSQL@UO(nE6 z`s`m>rY&}+rsly4I5ifKeCH$0|Jzb`nXsF(cz0a$Tf)gg%6IZYyxd+VKa&fOt~Di_4#F0vE-o=CN5#; zXBCg0zZknnoDA+|I*KSbfK!>6bF`{_K1ByhtUItv9pd2VRJpD3NsmAV7l={7Pt(eo z7qzBTLa?^#?#tqn+gw&jTKQu0SW08^$LKYZ1JyuBjw(rh<+n8Iek~J$(#p;G3v_Y> z*bGPoBlK0}cC1#wWpEeut{K$!9thT~SMybFMks*Qg0Wst5MDn^ zYz|RSg(4{vob*0h+{DWDCq`Go`{Q=0{PXJXym^Z{Cp_#=wR9+lhF>?^lA?3CxD+&<`!J)B$Sb{Jojt+hdLNDjHGJ{)Q;ecF)M9IPU zubZbyD_E(TV)A$wj-?BJ% zuR7VbcT<_Vp`BXOPa8B9n6WFvUW8JXF>goitpR=x;iR{;Z(h5jsSa%TTK^q2&TXy~ z(ss_#eO0#;v^R;JJQYK8p60?mw zf%p5YPpd{ChbuE@2KXVnjE+8Ryug-z_2Xt9adck;AE7kDe&p7?p(_5d3YOUEF>&;y z+5&J6z5+>IS=?t2Ato(fC!0vQC{o{ z-pp>MRVL;*LedB<^ohiwVKKC>-G@!}uPR7OVu1As>o>axw1LwViq8F31#)brXzRbq zExwZ|UK32+d0?YPJlpMgO4+j+Gdar%Q*A$J&4EhK1*DoX2DN zmfMD;mI4V@3b!!a^#sB^)KZy7y<}3YMd$AXr$GUV`vV#x6x2jHz=$GFAXcKLZ%-3d zsd`~;XO~rqMcFs7*xL~XEO~|ED{>$_3=$=^6@z8#DIcz|lIIUF+NBI`&3}*O_WDM; z**J#dYqNABdhkV6nt#cqFUnPrRwg(mYtqm;^=5Dd7Wp(Feb(B{##wkoK1>Jbf#2x2JpOHX-5EG6|v6u^6gG`<>SX2@Z!6?&^G>+Ccl0P?~`)3 zL_Q}az;n71>SN4x#sxT`c&NZt(5m8*>&>ij4298`tuo|OCsKXGJSJ3}&}tT`D9pxU zpEKYnEHRm&QU!WAs1{5djZJ+6Nrvg>jpQ_q(Z$~-`;?$(0vq|(s)(jH4W}3(_Ro&b zTZ<23NnOV`GC_ywTW7E3uTOyAV==G4=IUeP)qQ@+)V!ZTGi}Ff=Pmh3 z;wkYN_s2EC`PZ^lTeYh8*4sFN2tJZsK}+@Nx7Jv8!h$WZr5vMf__yP-$DmWWP!Dt; z#@8=(y1Y7a%YtUYZ!|Aa%Ow9RAc~iP7KWP3D zHq>7)VxfbwXOaB+QZo^CM?Ujv(qd3k=rOpNmS69=`{%}s$?G;a!0E(4^bD0CeL?Ns zrvH2FZ}+8K1%=^pTe|yj6uR$NC^(_*>CYNKGMczTT z+j2XjhuCv(v-3T(P?)31-fDB>_<&N?TuNYm*!4W3=4ww;s9Vn33=;^)Y+S!T8DSnk zVUr95A&rVppz1?4%GvPo=Y0VeF8NvYKA%i9p_WiSCC~k)5#j5KQK5u>PVU3u zpD^3rmf{@J{oh$*i&SgbZ6p6qtyM%zC(=wrr)ZZ|q?7T{YLM6^rX#wE)#A1{%Q*5ZPXTI`1KSHj(2bl=FuhNOw^G|Bk1ZZu0RE?>UiMV}Lz zSDYkT5RK39A>+Xd5%Ia1Ypd{y*QrBvlLubRbq?3nibayUKM zTO<`h?fp2AC-EL=@Uk7o?prOrm0dn_2DH#jw9-s$Z5(SkjVYadkIg$La8d*2B5L+w z5rVP?&cef5|Dy@o<~s;##^eB|iTQr-O)DaERzk$nVPpH%+%Hy;@agm3J>3e-#~3_U9tg(z_=xHtlZpM3x6SuN==9aoFff< zmUg%+1X7F+BVU*5!X|)Fhq46Hq(}XU^5X4i7hlr(BY6|Q$)2)Ken$4xAAo81_ACk} zHEB`BM~^mU7#k6Fl#VN(o`2SNc@~*7_Cg?{?6bHaLs3Y_eMQRTNiR~%pSRpKIM$<@~k|=u*j=PP>c$ z9HsRy3bCP}ZJ|%j^Z#-nJoD+_%ZY9-(LBv7OLHXjHHwnB4tGf3RySS+){Tsnnps?P zXxy47uwrDmfp+~}8Af(O=4MTEgvsugrdI6!1Py(_Zc=+U{SE$Ue2fhV;mDQG1uJjS6DpH$nB z7G+)9w7Dq&6LHq6v)c`Q#pi9UG3 zLxCOa6p2i#6%{YPZO?Mk@)3Kw1gcmYc`l58a-DvHk9;xgQK?%1(p7qRV^+yivJegf zl8?}1YGq|T2!lCMr{cJ6`V1Je@}nGxU?PN>AFVWFWGIAlUW)m|I@^wm$w&PQNpTSv zUbi=Y2@vtkAx5B4;PQ)AY92E@U5Lz4dWA7W@eiKF#(a(sI_tbTA;9>~U3GH)qIEL- zr+fUY+L?Tg23wHZO@Es^h;&x{bP1P=gH@%)U^pI92%qyLM}U%u6z5je|+{; zJ5c(ovIQR)cAOb^q$d5r8-mkymyN?~)Ba{T7-h2>H1DP1(imAyvC*HEw^a=O>BnQ{{K^Hqa zzH-kfJL#MUDN2rgSc&(?l`b5#Vzp{YK7K6R5=pP&&+gC0znKjZjX$3JzLL})bAfW@ z_+z09sO(}P-IG50vS7`Gg|$(A!#R~+0p!lKq1BH=SfZ0#i=N_?QAuTxei4Gkq~k}( zBfZm6^!GO3&<-qi?!~F3lJfF+-Rz$+v>xEg3#)F-R*!c_;VLa_dhNT*gS3aWT)lD0 zkL{Cl|AJ_!>Tr2!51lzpRZH2sdt3O!e9>{bE9nx~!5Dtp^@%Ol(Ui8%=LPtA*#{~l zL=DPUnty86>ECipqIqWi5rqC-&1#}L;GFz93i)C)?HALu>0z@djN<=${pdor^+TIT zQQ5j~O?C@88s)6tm-XDujSRJk-=cwC(#kybY722PLBB6<%GDRrBT_$oLX6ZU;Wzy1 zbl$)gb^TP^F?guZXxKlvWz8@_FA+zBYH{II*5Y2rdJE*yzuVl{Y0y!K znazblk%ENnrmQdCs!ffmN2=tb-EdZXtv|9$G~C^NR6Da#Isb}vZnj*(!BRv7BPN;@ zMP8L|&(+j~)~F#uFAIM5ilaU>!>+lDqg6;jUq0+%Kch4$jkM7FwNh}+fQG8^d}8sFlRp(o#zT>lP#oBS+etr`+9M{Ne^H&a=A($2&3S5BKYdJ^@08HQo$Q+?ZY5&= zy_O26+WT?<5C?B;o#Z20SchUjy!4k!O#5Hk znIqbFX^=(iNNqVTJ~1)Qm2Y;q2g$c7*zu2v z04nr3lOotJ=`pae`ec8JOE8~xV`Ac|crE4`i5G#$0Z)a&} zNswrf1mvX+9%GrN$2KIUgPi=^atQBS(Z6M%*}vR*Xiu%K?<^p1C+-?9WSe(L$YS|N z=K^?>;^jUAmsqHPi{_Hz7`y57sR3RSL52~GCgHpy8UZ(}(W)IYKO_IBfFAbK^K1-< z2W^1wbozHv$p?($AXtCa4e?3jjei{|KZk$>^-_H5+OB&Xm}0?59)THCU`yDZtEMm1 zXOq*Q!Uq%ZBD3?Wlp3g#Qb@QQ`oFNH1U@_m?D!Kip`qi4E}zU;u{Jm|8r*SUe-zA> z`Vp*wpu#E+z#wILcGt%7xzKp}?6&(ny($6egOFL&jxIb>-1W}G z22$962lei5IUMlV!{YgiZFtlrTgkWLYAy`jpaf6Sdd`1dN`;W>SxNQspk;sXVCHz= zh0^<|r{2BJIe5^^8>`5|{hr;={wIJfj;im>SonCZo$tfo$Bi_`>0cij*Yh}g4SlUk z_SJ3u)bUEW^(jfyi8xna8}mkZhfG#!wVJ*{A}`O=?*Q** zJ&~9`ix9&jWI-`$GfSY|4=qLm;`$ZJ4@p)PeDZV_Sk^mtKS9=!~@4fryq9a#c=!`A+Hgd|6gD8py zf&w3SOAUV*sRFHEdO->i!w7>j#eEka9z&Zm8mOC(4S))K`lO}=I0ss$W(_*}N=g)o zGDnApM8w39xC|~k9a~%5Csy%q-)@r_b^kfHpf6Vre@$aC?lJu(IIt7Qd8DKQxs{H@ zlfvUn_HbMUknGXtlG?&>gHa@G_~kc|ka-9MGW^^3X=HTNjW5%ns(5hH|M_At5CL_$ zX{xI8b=ipJhXj|*oAbeR;Z(kql!Sx`yBiJE(Q&-!J9%>H=*xp28HxL@lo`=yeGHt? zMgL56L~%E_n=C=^%d0D15{&3=B+7P98f-`$ylBdcc!$BS6)tBLDRf{`a7J$| z;mS_(`8Ee9O0v%D#=1o>dOL;&XdoBg2qmOcC=7c>0Yy`vxvNzMlrwmTiZL&A3(ZZm?4!q|@9p06hCBUR84gWB4j?k~2bqDh7~9T{v_&5dic0CV|1 zUK6sK1gn8xHp9qKP*FVvvS|HB_pTJP1PF%C#>^)7Jng6Ebj!wZ) z5si^xHpfkpDdfu>?}_L)xp;oKimkw7qC2wb3V;bhM6W&fN|n=4R)(>gy(qw@K`I!< zWmM-j1oxyU>!gU%*~!`2RqUt$i3%ABsd~pK;E;ac7x{#_?$^7<`uYzaK8O&z2|$BU zf*fQ*rphRSP5PD5pb6LWT3>RNv=*;9v&PJ;^NNH!-0b*3n1|f2QMsWM{yA3 z#R#q=Z`ZFYOc`W~+UHjX<0rcmTC6JKo&qrow)n>g_Yler1O==e7-a1N4 zN4~O&vNybIWzi;~Lh50S43946plkTZn8~F+hG$21Z50h>)*}Ge?n6enu{chl(zo1# z>4@z6coPfg`(r$_V#w;-cV6lw*rHZiLH02_O1-@5Ij~k9OZ0`go4{71v(fG zPEKuST2ZXr+}yva`33q1o^sBh{xffPDAIt;cKK0=+u zCu=J!3{)k$AQ1M$e_DX2#$7B(oF3C%BR?W6EG$DTN~kg3A$c`BwE3@|!0-81Dl1{v z;he+Jn}%;E%jf5gKP9&2V6cDEu&_7+g@OUE%;mFAvkT#%6I2c5&usSq7%7)Wl43qk zSLzmt5<3=1wS0{HD-+aC2G=usdot3%AQjQCQ)3bmg0E{s0pE5(ssaXpc%D#IZ>mvF zl?LTMV-0Z?LrXgC@hY+J7Pg1a3BRpfdspJOp7di$3=E^Qz(kNUp%qHqj$N-Jmq#2 zS2peGS1kq&NlC=RLRx9-^{>jxu_3+`&E-6$YH_Q8OGb~&u&2ec-vl2U>dps`?jRx} z>P%NJ+I|DJDtPofduBnwzhcSeaWuZZzPYJ<1!_IPl-Eg5c<~ygEoRC0T5rm(mz#R( zqUnm>jURcKuzxZ#+=&MDcC zmp&?jPg7x1c9ucL1iO2C7RES05FZ>IFfuYWayziGv3+Gr-I>=^Qo`a;4NE|2kj zDv-QU{GH?#bWO?0h+hpl+P4*d6@Fn85n1P|v7l``^5{0&#>x~nDx;n}W+x|GJQzY~ zu+wQd5X+epd76Zt=Vx!6JMyB=!hbNfuUOFedQ=}){{d6uMRas@FxuV`gr>?Bp%Zh_ zCCU)<%$pKuXlOLL>?w?zCdASZl8~UK4v=VpNis6W2rBjBEE-xizULsjKAbJ%$OMg; z-w+Di_DZFSSBXp~)_*m0(& zrZz5|?k=Qi`J4za&|IeOXvV66sdk&2h|3PXiv6yv&!#d3m`rDx1Wx)al=;0=30??v8UXH4u+0kjw|n zf7X6Ae)_i!=+QLA4N~NaVBioPTlBOlnsBtrG%5}a^2d~Srl-2Z;6)*k=*&ozOyhR5 zqX+UPZhYEArl6U8Qq+k4IGI9DPEP&h4%w4N8VK8`PkXaa)&Bneo0}W2j;z7$iw7TB z9bUI673kQ+grcINu;WJ8T8DRfdU{(dZ)CXF^W`YB3l9lP*KzP^*xl*b(wW@L1930z zms{#w%!U>tYsY0kxAA#}K~vpP-7~7?+mZ_@YFKZ}-I_PhhP1su3Vw76>W+#3Z|mK@ zpi7o?aB%SVf2*!$vDOIZ{hpqjJhdtzljqM2}W4&j!C#Z4rl z*7~vO2)@m$htVX)psDr<83am>PnQ?E{CLFuZ$)CxjM}v{LPBe=L0{{ZfR4#R>dTk! z3K<$;pbd(iKD$eJDW|?pa%BLJn#FQcQKU_t*Tb@^kF>N)6%cVdwiH&=0id4}{WQ*7 zUhKhXvjgysgqP)j+AG?``{}aXin}U2sZ`Ve>gQvajtulS2?+_neU+AiQGGjYt=yej zR*SR!y!w8;&%v=cRY!^gSW3x6fV8*eD&;@aN*R5~#$P&8TN1TX$A|UQp96C?7x8eAlkjjmki#@JiDz+%p$5r$W z{*CPA?*Um&*{Y-Gi*d)LZZuYSR1`WUW~cMc2oTMQGKGu_bf51Y7Z(>>T=pKHo>Y%M z<=}*Mq1f*@cWiC4E2?}BH|d|!MReHzw}tJt1cX-*!uManLcyjT| z*@&V9lZoQknt(beK?RYOZ9A%Li%}O3fQzQY4W0y)FhJop**@mpG9+4%MLunv@_{Cm zPFb?C|F^j~I|Bi-WZT&ANC@62oU*(=KbKZePyiY?d3gjw&*UT&G3lZ*3sYOtTd2aIulC(#PA0x29x6s4sJxb5}i`rb_S2${` zF^zxs=k3shvTxJFN_s8Mg*iDn)z#Ixx!-9ZLERz^e!($QmIdTQp~eXyI{g|g2Cj5Q z?4NP*@%edqz)z^-9R+602x-m!w2fxs0uRq%M$Z zTYp!4#Xq50MYfIXD44jq-h^mB3cD3=_<{iR-<)Tj>vK4tiOeJ%@~Q8BFCgmbeF9|| zf`1E;x9SxqYG3=CE&>`aOGQ&w;w>#WCJJ1KN__n!?FDqpJoqL#n9tyIEmjOsc{nJ~8 z&3A#PN85Pfgf{kpp8WpO6g0-Kdg_{RZ?QyQVIm#o?C2upKKx^PYUCFaK^!R z@H{`&6owRL%$UZ_%)^ht#yv4#Sg-bb0rq!&5!^=xx-M4(*fDqK`KhpcU#W9l)fKm2 z^*P7@_F9DLANT6lxGB&M?gG>i`4xqK0o@}{D8Ryc?lmL3)u_z^B&NCf1{`<`U%j}_%nOb3J%*avIKXeHqTbg z;e<;3d*?}9kcQ%67jSamOb`xG1qg&ybZ9b~EW-Q5gOKqVwZ;@|SB zV)$I)b?VZ4_D}ERYrFLo&=uMM<>BLb0>jJTrQzld%r|hNexY#D!kZL%^c{I`M(&}N z!B5=}jL|umo#17^+zae>;jYtyxWeYv%vcg0NfN-+tf98}=iw7u9V>T?MejySGcXI% zWIRFwd?984a3NtR&fo1s#KcbvFMtpS?ecIv!aZaE zyrOJ(uvCIfZP^+u{*FT5U^B4^S&faYQF!7w8!v!VCt_MeAOOsz~G-n7M9N8|Xd zb_7fe3Lh>1mDhYJoO+n1o__q2)!N1ymCT0=^B(%^;FS3Se(iVZpJ7&H3YD4P=-yns zuB44sV|~|N09sVm#KAlZ2|uKV7{Lqa%6nIMd|i6*W@{s3Q%L-z5f<=?WJf_J4ujL9 zSfneX0Qo{^nQx&j-*-}nJ_RCtztww(goSY(4f8FOH8J7e`!~ydaSY9}yxnPQf-N{f z#>D%1IUSOWm(JkA`k=P_Y9asaRn&;Adk3-t7&xzakT=~uj8KGOB{by&3zG!aKdqbp zvB_^@NU72@$qP-+ju;H`u!210*#s=(qDRywr~n8ISoFV94^W%LISNYkN+a$g!;MXO z(r@DKR_@I$2z_nMhv_1Pd{Nx*WPUSqUz*T~AD=1%?yi&Sf|GAB;ycUoUMO7GMS&RA z(ZO8a)E4LOf!h2%Y&uiz%_@)g4xBixqT&Xgs9Zs@Wve@xz z3h)Vg#B6Z_TVY<9f!ye;P_?wEL&h)M;(su2h&DLfWN+jYZsbDU(X^z%0IfaK9S+ok zpYek@G&NO+iY*4Ly!)^Ad$@3VcuDp|3Jwt&03VgrE>*CySM$F zr?cS34V*r~{LJPKux@toVqPb8mxLBI(Mrwsj0baQD~W;dBm#7@Fobo&*)bYNGqDBh z^8$U6o*&#StgANoh4w}AacyT+J(G;j%vzerw}hj_p#AyrX5JAGA~8A&r3_t6Y7PjMwou7hA>M=nk#JJ-r7l%dQgN3B9+ zEwiDz5^?O@>qi%pM!G3#vNQyv0Wzn){Wy)U{yVvfsNAgiWD{uK9)8y|jcAvc0cs>c z!1+3ppI9?RFezQD-UO>YG2}K~z=5Q95COWds=2kOzo2jeEvBIT@14KBEvo2aClqgK(#2WZ@#Z80+Cje+QjSzQLI;L}g z6a+T`|DR0CYyU&5TwxZlo&dsWf}=ZtXjPG?{olWaA}MRtN>TNqsrInLKta3OL`{$~Mk_#6fdPR4-82Y#O{EjBF#3ZM5qQIc zJtMAAQ9k=d=@G}Z+7Wo)BV{vvE0ghK#e9J;_0tJg^gUp-W=Km>-W=LYR;4g1=6|=g zcX2MYe`y6MIRRmGmsV%2>x0QQEx~$_8tAI;NB%z62A`O}au)LcucKsvqSdGI6;=QX zSR|P_+aH%G3S2-c3_CX>@2HM@dakSAh-}000pONklSD%0FTRv=#wWdFvAmkK0q886;Ol#vc!OY zfv_7e8nSQQq?2@2SJ!=ikS@EDbkf;7UH5xVPP)6QYq|GRzpZZFCX`Z&M!l2&0k&TG z2LOOliaQf-$A64i3_=K)m^}KjMGPt?jVPtasmTRTRCF-Hn$m24J#{G|1j!)-;ZkIEVX3p6wIq{9gb)-rl&A(r zp$dkF4TG2C(}D#6!$Jo`QY0_1%2-P>HB-4*slIk8G6q@_n|U@3SOO($ph;jyI zM&IBcSi{1Gzzs!xUoF{SlkBWjnE(Jq4W*j+YIxXC|G)x(TVno2eQn?bd{+E9i?f(u zlWefLZ0dVARWKxMu>WA)8aGLQ6K5T+%w};Gi?8r$(LW)OUjp^<{l@uRzeTXpBSvZB ztb)3qSj)2b3g1>9)5KR(<1^a_){xLa;JhOL2DcN(t8+oY+@YRTSbs$mUtJeIvVC9y zz`cogfl>-tkr8c-!qMs+5HNSBOE(CLoa*n<;CB834&gBst~L2_7CqdsB~CnXzm)@me5v;d89HMFP?cB zPPYg`2;$ALm^*M5158JMWrnr+AL6&kvk`5K^4n{1ZEg5+T>)H*3j)W3Cmgn3*}m_BH>k9qqud1y*v~w@R1@Y!MuP<|0l)HpM6-3tRN5@Ynb(Vt=f*O|% zzd1M;m+K1J-N#&ecf98Mmmw=MD5Z!s_ruPK|I3Cg2a&aZ`u+ER&*X+X&qFStho4T=O{NK>u9N#lwv2sIjyF&o@uI78} zDg6e^Mm^Ut>qk53$2^d2`uYEG;c7mXk6y|~aItcO|5;&cbk8~b5GuuL0Ki%MIgAP) z$t+r5fd>?SFP>h7(Gh8wmo!@gQ)l|HQx?lj-I%oRE*OXaLI^0O;0T9rrtDzuvX5A# zvsce!;fX&q2I{f+f<0O~@)^XNbXvB%YQm;%H7%{%f49CDoos>!hvaYw@K*VgHRyuQ;h zmbYYoMaI;udo`o&DW!-qMq>Mfk61t-08l+}Yyqw`nY1|C7=>-OY`pfkUl;XHmx2Xc zd-VGDtXa+cv{eYhj)@;LxOx$)+21~hYNZIcVyO%?2!v>isy*gaV%?Z)1vlw$sO3sF$_ zv*t>1Q#X<<1F`Am_X4&LJfE3vb4M zj)@<`z!|#j7`W~{K|%=rHf}BMO`M_r;stPRHs)ey3GO^F2iaA}1JZhxIxA2iRr&n0 zxvmE6ML`I`gU1%&wF_@GgK9`+&`jnA08T~3vnQ9~ z-@bb?ARS^~a)h1>^H|k!+<9OQt~3;XYZ54hDwvgc7dDJr+at*n*ArI{0NfaT13sIu z8Ai^iu}%Uf;8NX>$lN~@ubh9AMcbW-3m?l_glA8_0B7R^#@?CwYhzx*Q>jn%*zs^Z zdGh^?(bzp{8y*?*fW~?e5D-G}Mafr~mNgr1{qSzP18F_$)$?!R*8O+m2gi?p8W%?? zg^4p^`}j>57oE|g$JC=>d92q%gMW)pZr*@MLxd*w;s^&9bz$3;oygoj6MsJaGDNq? z$nlY}zWu{{n3gpgpO@?gA=-Mmq)3=Q_@(}pqnvJk5wD(KgZ&kUpil)I;WPodLKQ?Cqwv-ZuOKNTQFB2D zt;K4|di^?fJPu@jiG?YT!s!;(_={4CrD@M^zTZu1 z;}Sx!rSLPHw4KJXy#K_MeKYWR$sQHZmO7y`(6trY@f_dnFdeEqil=jz;zUiZ=JP*} zcuI2tif$24q&$KHnO|T^?8IKPQUT!#Tatk@U~bYZ%uSkwb5}3I*re8SqUIzBArOcF z(Jdl1Y?wMvDj@{#{O~@1YFu{Q7<~iMBSs<7l7Ne~`FJk(PtCT~W}?wXTyoNO8gssV zs9oEXs0;ry?p=f%!kcxx$GsCmu%q}(ym8?j$kYvi@UZY^jBT9phf<13apO^&=RgQ= z!S7QZ!|Z`GyS*;er@``8T6n7F{qEAesFmvwB3K%&9FO7FVVZS+c9!gdQxUPdbT8`M z_1HIMCuGHqN~x;dGv#n?7;~A*sB+b`4=lF?iOP+?Qd)}B_Om!qa}rU;NGupOU$vB} zVZ$IPE|~ahw_CESPiR62BQav*xc|m=5orPI#se0trkdXWHY8dSu&?|Ob{6l#jWOC^ zow7-G)JQgj2q6%EI050riU>nEuArn-F<>{Ob+Acx%s4a;p?nBouKv@I88;EXiX7d4rxK4o1(vrJX2u z378T$u}>WD(9pr?XNti{Ybr*D56AV9>CMUt-R_ndSiKwssqauRwE<(2s^a}*ZiW_xuUArycgm0#|TFXig zAB7d^%i0(J5FmuN1YFIDS*(tnAjc!Z*#1DiEo;OR9^nRS`|YPr9)sh+#Nt2(783)D ziGjt$#K2-=Sz%vRgWK8cex$b~#l;$_wIqeIX<%A^)EWD^YeiBYvhX2D4oL)?a8-w3 zu?a|+E9{=Dh7d?4#i-u=cMu@}PF0o>`@|w}A)ySczAkrFC2BTp0B17y`mrdIh?x10 z>jEs6751oaX5|JZCI%K01B;1)#l*z2!d4?@swkj}()&k36$Vz&oM1vC`ak~{5T5US zu-uA&E=mmmyQR}hmMi%02nei`wVmue35oanF1?y{aGObBP!TjVG$1=W8@ai;a5|k> zvSf+6oKlJf3l`w&)vKL`DK#|}FTL~gD zAu=+u)B7zhE`}(IFquq2-^PMop<2IwJq{m#J`A4cRj8O4SRICnQi`QZm!i758hm4> z$e?0ibr>iQK+in$4C?Fadpyq3Tk@@o%DX3C+Prx)>gwux)Jl0n?RL9$1DS4IVSau- z_U_#ap@#?j@jQ=3ixwd@H5DAk>BjZy=3e3Y_3K-Osv}m!lR+x2+%8fS1z&&twI-f_ z(}Tqddax*^C@3gEadC00Kv7CzwOX-i)hfTY-LGFitXsDZFTC&qBuQ!!P)aFIojQdr zTehf{idn26DJ|^Tv&U!c$kWMhzWF9zfBkhtM@ReJ@g9f<4<3vS8#cgfZaxDaAp{)9 zVaJXgC@n2zlY#_G&<+5{k0195hN38cShsE+LPJA4>=HfSS+iyhEEY?P>x6`a;Egxl zV3UFbi^Zy{s)8trK9}Ed#~o^YtD|np6E{&x@!WIIX}0IRuk!M8*lacyV+Di7vfJ%G zVIhRzzWeS|%Mo2s(nv~5g2iHKwmm`!1VO;1OP5%T6%2AX9BP{1R936C%lAirDMfmE zdW)GN&+|Bc{ycMq!48(uX!Of1xm+&Qy*2%7T0S6zAR{BgX9=RHs7M!J>3%#`TW(1Z z1gu`Yxc6p z6HY}@7+AsN4sW{YrdFS_;W!T2+1XgUcri|$I^~!3@?N%e>sCDZ`WY@$vB(G-wb2s6%iH3JOqB zQGueOB8Z}>j<@m!VM9ZI16HnFiKL_?zcNk+Rv)oik|bgN{Q0fk#__I8d78x&hJ2DK1^QgcU1Rz-Tn~sKv0`?bx_+BMb%u8#WV+8DdIG z3Lbv=VThvGV};84`g*Khy;@gL=?W|lNST?Lc;JBt(9qD(BTy}WU%ng(2?@Hp#iP5~ zW;}7!nKNhb>Z`B9Y&Li4Glrh#bh%u3<&{?u7Z(RTO4rjEd19ZOoE(@;rcT2}DTTw~ zz|5I5F>l_yE*U_`!0Llm%cIMeFJsrPT{v>&2n+@T42Jgow64qLf-K9JFku4jx#u4A z@84hdzlx!2i`BGk^WAPY^78VKlaqtIygby`*Q-TO?{b6ZzsY36m@#9Jk&%J1W5;S{ zs?odHjDBsvX1Ckna=B1nUk~8gFdRV;V6|FR$h=3AGN>3>e#6yO>oTyIm^!Kd2dB8X U#Xwo9hyVZp07*qoM6N<$f{9Ole*gdg delta 7320 zcmV;J9B1R2Bbhmn7YeKh0ssI27J|Zlks&UB0x1SY-#vQ(030hxL_t(|+U;F;d=%C8 zzjxZ!lufcJ5J;y;O9(-VAqr3Q2l|5i5S1SSf+C25Jb?%D{{4b=xiovE@gpagGO9AWc}6vES0xBt4_-M}vHQ+}F~Nx* zwAz`oEl*psGiT4dZnMU8N^As0$&k7dJdE+ytgkoT-o}IAFVl|fFx;6F&guGp+KHl+ zl{b?JOpFeGQet6~ONp`cg>y4X*)lbyUKhJOMAg~(a|$4gO_@|^EZ+Cf*BaW(DiI|J zT_Xqc4lhqzV-$?9M@=8w`6(BYJG&-#Y|5l4kBH;}pIbGjQfTjjAU!F4fiVtMb)FFVTyD*O-K^?diG@uv{=Iee-J?Y(Bh+DAA6#cwNN)R8O1QES zC!P*fb=om-bNwGkEOZLLd6yTRC{F3`)3=H66N%`{X!`r~Ei@FDv*qa{LC^vVhTbi-%u!xg)Woi?45h=9_;Pae90Knmb zUnE278cD0TwJv>AsH*d)vCEqpKaufDQV9sY&;JS4o9K`rbc7SLFTIHnTHAM7GvFtR zjAS14iLK^qcI51-b43Y4M}!r)B|YbBWxmaWpV)7fMZe=O(=Jb2Q)nobf(mD~$qKBD zlr=3XwU}y@Ahbtz=I-^8Xc1Rglu3gmwr+FeZU( zC5^dUmWmLPlQIc^LOVo7DJx7>t)wx(>V6{;p`?^ZDh0M(G>muAUuSt*DniYy&SnUD zXh4E^L}H<>60GP@*a{%T7+)y8Okrw9CD>4)BdL2*#G}Nt-j1)s*rd zUH?1hB|&Hh1CnBwqk7K*HfIC&-931#^7eBf|7&?%GNg7epw64-m#=eoD=GQRsFy8k zEu_K=TjleNt$<)^#H1E%LoIIr5X5j$GNg8h0K$SRZyzr{6{G3hvMo`Nax3ORVqqg= z561X<`HiW67iJ5x}w9c|bELAw_p~_U{YA)S=%D+-O}1t(k?Kz$F|VNn^ThqY)x7 z`BTCkk|3Lie1(SM$!A}!FjaI?cH9*AshpIVSN%DEl>hUJo1gL^9C>(OQ`>-_3XZ;g z{Uar*_%412g(<6DmTL8;j35XrZZk#%Y61$1ihK9^$3GSvspSoed(0o%VK@MY$8T1q zFHZd+yXHe)s$-cfG=o*Zs^CYKd69a>n;ePYZW)f zrc642nEzu>ukHuqcZ&F7jLW$4MXB$PKQlGE=FY^h=PnLE9;J@hoA>?Y($uCfV}Q`V zuD-i1Ye#2g@XkS7ylJmSek{d>_rR;=4J*<}Q^`FNaBm6dqvx#PvB`up_#YQP3HscQTZ zQc!^~#;N6h%(yTIA=JaG`_uc?P*x8hu`1h}}c5cbq&I)X#DttrSDmf{)@wDy;q2pIj6&ZDt&Q9;E>GN){w~(Dl zR1@$kZmd0_ODSPX<%E1y^ap)?dfOF$GPxs!7~>r|d(M_zSlVl$f>dCPyLd#3PKl^B zjPdH24;g|9R(5C*JPQesYeQ9?4;LIgUX(JX)00mGJs#;1UZ=ZQ?M49bw_}ObrkW6C zr#HL5@kEEmko{?Ns!fv#Vf$#u3EXM@xp%LBTxF{AllgLjN%3c3nO1po<;`_}bGp77 zcb8oH9OStjMqmu5#XgufqJ%dRa(-8(o2U6 zjvmZA3Olo)~LrV89rYh_um=V!#;VH8CId^m>56gxG7p%m4AS^vxw~Nhf9SM}3xr zsyaKLMKCv&zyyOChCmU2>aYlP*nMxO6+vij;@3I5wq@<8;3^4Bn1CA-oH#LToCEn5 z=WiGg2uMH(g{#BFg?MjPrY}!hn_7O|i}qUFbN*w2!|s`)rk{0aio87K@+SVKzwqYO zC9J?9gcPJ=c;L{v-CjekUfJJ{CLt`EP&py1Fjan*v1wQCJ|(GtOzb#ve)rd#PP5G> zlncpMY}CDcZnloqV}u!k86G(FpWS95ixqQQ#&Ja;2H^YhzF&T0t&C(I@a(#&|EK2R zu+7Ivt|G_BX>0f2{XtaO!`g@5>M@rl==wThdG5(m&$7Ta=S>QNAiiM;dPBcep{kJP zoUm;JEG%J5r(bw~Bd<0eArvSNSQY(VM`Z`}*NE7xuZv}uTO=0)0JFQk@??kqA^&_j zseRz*6ojH`cHcR$?8X`y$(Vqj6*cYI&ZGZy4gR8_+}gX_vUVIU_=Oc%nxJie%B%6z zhK(noG2K}m{A7p63DiztbD1Ii;`W!DGLvac1t|hQiGDwSK;d8iH@7M?Hr(ENtmu@I zRG7fzC-V(gg*FuJ)gQ*rm;P=YZ-7`$v~5l#cr)JrdSFhw%@;ZY;loFt;6do&+3mA_ zD=dc=@x9(y@~_mTSIe%8fO#>Vv%9=vPAu6rhVk*oMpzBE2}B-A*lRnf+Re!40z+!4 zi(P}h)U*13F6ziS7EfI9_-x5V03b1`VZ4)KKMs-yV2tbRlG3=Cps@s+3GoynZEtgX zMUk=>zgoWbwTtr}4v2p~0q zMz){IJpbVCECc`(n5LYYnQth_xSjocNZtHwwW<1l&8zQRD7{2sNB0 z{%hNRxBJ&7Ar$dgc4N($;KVR>XA+V1ZR3)j3o5wEVF3v)ANCsZS)>kP2ZR{oIhWo# zqd%8#C|D8wt|hK3(vqURdkqbU7wZOGPCfz?3EIJwl{^5(h`=BrX&Z!D)j8)%E+T{y zJC3l#b#dwkzZ9J~Qh01=KzviUEdW3flvopg=48O`?s4IQD@zbU5$f;+?O?M;l5@U2 zTzHhk#HipW>{1N?0Bf{y))!p#8-y`p7E)!dr96hVE|XNl!kXLRQ&Og`*C z_@(d(_HoN(x|z@)jE3gBMF1)=DC*D&cVr>P_>0@yNK6LF0|VsRzXXUe?-d$~Ix2&I z?A8kamNU()Ni1ubJqeme8k_Vfs4e`h_1x`F!c@4& zf6G`mXC^C1xv$KpgCa;up|$q05CD!9pQH#X(xMVMMR(Q&rir1?wK4oKM(w({HORtB zu8I@5{(nAkb4GW*f~%C1GRqu|V+v)0Lx648#pozsEo#5XcgiXSF~&7~4JUAa(cZn# zz3HT{8G`QT6XTqF4+Kqzef6J~h%CU3;zEpZ1y^YlSQGGpihHZ$pE#iQa5sX&m*OYDtK62!L$hB+N^78U3Dk?^e8Z~_Qa1qY5w6yo$d(T`~ZdMNt@Ip6B!O z^32fc^?HRu;qX<;%F2)%RmH&Yd3RVQU7C-UTU@$CMvP7qU(~%@8;wOcA zE^uc|g6X3hpNM;_A%ya33oya~a(~yFO@vTIMaAsdvr9`$Ns>fTt(yx&`vq6uK9F~4 zd-g8zV+~)65UR5nt&_j-MhP8Q1N2;glpx0JQg@q{P=Rjq}8|9$7=exL=gaxo}QkPlEOGw&ti6n2!g2J zFaY3re&WQ5b(ZUL7em@X5x8T}=04s%#E-)Q681f`y^BYrOYp;g81LJ+uRea3dwm;? zMh_2<`?7iZ^y%^O@hr=lL9+ZdGc$AP(xv9Sj5`)y)Oq^Qg*mFrmoL{>sCa&Z!7yje zoQ)edjvjsgy{rBG{U=PAuwlc7-o1NUC0SNh*518)+i`XcH8R;W`RuE&|GvO{^Dm#D zy)J#DOFD9LaeX%+REQx2O;G7ow*i5YRLV(zxr~ryRcAZtivmDNNr_e0 zBQY`2NsnX3j2U$x6uZLRyLa8rkRpT>q@2Q3MpdS`_KuoTk(kIg6ghc4lgY$!TzzOM ziW)j}sKfY)fAsY9jEsz||7}f8O=V@J8$u`}F?q8xUDVEoeydhQzc za*Q#@al&7P=Xo)+YWcIMs3@bcZdC_C5VvmKa-BXorYI^XsBSGC!!Y%Ky8_0z9kZ}O2>biSB7};J#hk#= z1dTCXboIRprN7VaI%90`C?^N4M*P)=jz9V2lQtUa`X>?nQnwom5kj359VtxxRCw%O z94+&c5wg8^4m#sBG7Mu?n|i&zq0=^gIm>SW;QICJR)bD=^hgmxp{fuHQ{Uws60P6= z1pSSF6f+W4!bi^QBi^*)PYLm%ImMcE=G0s0*Ns`efo5Idb-s)WQJkX zYPCB;h!BeR8%Q7`Is02tO_3fEellN#(3Z?(hb`W?apTO>Lv_3oK@eumnpNMmsgHMm z9TR%-;>AsyHd!470YIbC$Yiqi>e1W}LhV3=(8=P{0^CcfywhtT0PMc|T?JR+@KxjD z;;cq*VpK4H{`{3ISDKSJ^@9z|$s?9!7cN}**=L_M6p)B@q;9&HB>;5sh~$7{FjLM| zXc%t*==3}&W;PuQIK)%uJr4nQcLBs@GkHa7Okl`B>mCVDqwu&C4N1VIpgvuk2z zjvxrJ;*=ywj4@5q4OvJ4;5csl`0?#~*8V=z*#KbR4d+WQi5At|mxmu8U2CeH8ZpU% zEr-lCFH@&ZCCMh-X#fC1LP8cUT!`AW5cg$_oRo|BS@z`w?O=q_t~>i2W1&c<4?g%n z%!f6}cDMi-L8PD^}mMs&9+xA@~Zc`=h6Q`b8aWm;~!7r8v6FZI|F!Al3eU8ycRI5s*TD*9% z>vy7}s8y?0d3t)fLxsOh5Jr3Tq%oDEKWlk#X4K0hCJnq{`3(mbz!3R=7Zen5!4;?Fja4`QclBC(i)=dr~)ZqdMWvuntiOFH(@6_a3ZvG=ltAkK0 z^92M(!Pw9Mwy4K{JV~3a+K^HNg%IMw)W`sa5V~4+{l#;y>5Zk5W?LPEIn{SCB9xS> z@t^IzdoaB!bK==)CV_XW___^1c&OlKiXa0TUHu6FW_Fn>fKX+sUUe%;T7je$K?nf9 z>V6{-@qlNy#^V7m+Uwsv{waX)pWOXF|5-yUNlQW~8pE-Frze%NWmSCj^vKEgogy9< zkkG@cJ3}x_uP+xskj!aILI?n?z>fa)`67dk18&g+^Hh||mh$*@u^;;|8UPq~=Ee5? zoO`Q=P|RArbbeL|TWS=H|LF2+V#g7-XXBPdzs(A4UTwkK*FKa`Y#9hK#(fWlds7+Go0aLqkB@%#%w(6dB(dX&NulE~!dtVpS8-L6-d!kvnQWSW z?Ep z6Lj0y!Lkh*(xCxEdU^HucaM2;Qr;ArHJ=k>{I}zy9zQXr$f%RT3>PR?5QOGmG&k|o zxN@$7!sK`HJ6yHoxBIdXW1Lx?y)Jzdpk^X}1ORM|TMGb2f!&d_N5a5a78V(Fe?R#w zLomq$J`YxQXtoRMjQSlpyQf7?Au%Dr;1mnjXKZE&MnHn)w&%?telp(~Q7;h)BZPj) z{~5LG1)7=`!XFEM0R*4-`-CRw7BB)BuSolJaq9ceo}MDjNjV51fpCO6tea<-mQvq; z$e`hJN|9e%ka%hw2b=_n-4JqMS`+gTYPquZE*_B|M=uqSP{x*zIR12vsYXJ@J_v^g zKBA^oE!hN&`e~v+@3-10upF2kKQX4tR3%wdJM>p#EcK?nT0@p3vpV~^(=WJ z{gXN<0;SxIO)S(f-mOGaVXBZ_gSHZXnBah$aCX|Iva6DtY7>O5sx`sN4tobDdr}@I z!8G&tf1WHpExD=pH>9miL4Q6MKB3sC2LvxiyzqY^PfN4JZ3#lN%)0o`OJ%7Dp$Lz# zO>t`_6x$etB4FR->|S$g1Bpo`sdz2wrI8&T0|XdJK(-MUiYWf3KfmD0+dS}pCNS-J z=qrDjR>G`RyEA|3To@yS`uoNn9I`Xcr*D`l#D~^MfVCwS)}doVA0=r66Frve(PUI* zO8B)s2+i{Ni7^v?o0g(GE5(Uz6~ZWwa0*k~vcHjN*j6E2+Vfw4;Eeu!rBsP(AygY` zs~7D#L^~+2w!lNGL~UChX&vu>6uJG1U2PjeN!s>UD9WMypZgw8SaWNG1YTQ(Ft_%u zno|8(a0E%SiER|ZNulEmyrG<{C}Yc|L9tCjNE38FpBRkr=eNI<@N1J0B7`0b9EK78 yA^)&>GcXClwj==1$GayhZ~}r-woHQX{{ax-0RWg*ZPNe%002ovPDHK)LSTX`@*8{r From 54d0f73e67135ec675953d608d0780dfbb130f5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 23:32:56 -0400 Subject: [PATCH 0505/2835] update --- doc/logo.png | Bin 8699 -> 9092 bytes doc/logo_small.png | Bin 4506 -> 4713 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index f601656c61a4d215fcd2c05e939766b5b50d9afe..38d335a4514173cc21b1049c3de4b75055a502bf 100644 GIT binary patch literal 9092 zcmZX41yqz@(EfsSDBZO*64D^uDF`bFD4o(Eury1vbcb{b2$IrW(k|U05=wX1_wqac z|2f||-?G5o_r3SdotbBznR(BKYp5&YVo_j$KpKPgv17X-oQWu{`RI#=mY(9^!W6X+g6+iT)}iw(sKcUunC`jkwEF0WWYsqS7kMM^abqa z&*+%=8?)SjOXRNdx~{U0_VyMIt{~aZ7ACG1=5+4XuAk@>mDMzK0-lqCKy)BwIcY7A zh5cnu53T7niKELEJ>Ee^MzFVx%=?+(V6`}T{i>4a8^U`DqG}TTDmZbGjloV}-}jdh zs`k?QPO`Sdk#!|*7OEJk12V`+GD>vjU(9Q)>b+sGF=b3}twqAfn-#?A8QT^YpUdGF z_g}9sS6oo^QYbo4u;p(mF|etj%$Tse||Z0@++N=JR+ zZJbra8XyhZlK#>dn`~5ik;+MGfdSh&E0-+%hid4h%SzmkBgc=HJo$QK%>zj5y9Y1= z4gcIOI=K}t%c*ErQQH7st&6*9+;It89cUq@z*Lb*z~? zuB{ylh7M;+d9a*^x3e9bH5*eIx6NDLXvE`q?yCWcS-)^UCjDjp-L*9#(I&l%Eg^hi zn9Z3F-k2PEb)?>>Sk?Jw<{nz6+~zlReVff3`bH{=xkgdr_2-Z{xKGbQkK^xGibA=g zOIs5ck)ItJXdxo9&xRIXdX;nY(yeU$;**`Iw!T3diERF{xJt|FlPN9FkFp^|VtQIy zNhb^Sq%9VCd+C)brK7YzvDrocSeXvB-_yQY`@`8tf5V$0x;pETTjnVw;A%R%s+NHH z(YAe^obklDlG@-f<-YafvF0H-c~uu4O`OsEkXEkNUHRwlGtfjJ#O~lq&Rcu7<1zEv zyum{8L+mjQ!FA(|OS%V^bih<6WASC;#IT^1%(qfJw@@z-pcFxeY8&vFESBTR)wU&6)9V`i!!vB5d2YHr= zxOHa&nWI|KNp8G(9RCw9M9icGjYoj|$GLSsc$C>F6E^Z?KhZ(%y%O5VdB_0P z;BP>k^SFo(5}{tB#kLT)sv+oih|t4O$0O#y|_=7mH}04v6v*!r`jj7O)~+#A4le;p6z6D91*M>lWJSG4M5 zO)`f2aVeM25BnG#T zIl7;c%`llE!<%W>na{!|`QX(lb|+iTq0+dB89Dl!O!RO>@?ur0Rrn{RFfEgu59YpD zY@jr%bPJbj2hpGk!uUgxY0c>LUkhDj%1E6*s@Mcm|Nc7}v)6+qt1G0kjy``~ z$Q<&_}sXID>OoA4&4 zLXffL75}33LSyCIAS3+v%j(%c2$ss5&v}znPDe4PwWC-ac$Ku<5+hX#d0ju&5_#Bt zM81Zpg@6LAqCa5(vsGTuV1JL46?#n#8BbR#L}f0R(TF8qo8*3)7;Guc`OWhyJd}={ z7bU4w~ZY}p;H+UoC#Wbstl z2zZH*!XR8c`g+bVCn)Q@34Vs%-^^x^92VZkXM6>AxoIL;NXjrfw_F z&F!6uln9a0I4UQ&hVT5M;23J6<~6@!=T9T#A?%3yVs$xhsB_lzc4z2clVECw1s`+= zdJ4>qgXzE)&!Z~pUfj6f(snvRU-c-mNQC?v`~4a$;C96rd>b;a|971*=NOob5oy<@ z$nxO4V!^UdR&_Af;&DD8g1}_c#M__w8j+!@IpY;&l{$dFH?<$CCwN&JB9IG41r~`LZ^7s`* z3UdUXAX$9x*I3fGX&~AB)rGzU&6&DuEIW2tR)+_|BfG7PrSIIx`0R)fFXLZW3pJP( z3ephUC;v65z#|%nY3u%8xjkcbDIs#q#ubycdA0$Q+YmP&7?|Gg>QBzOOv5?Lhm-3H z0BHi3jh^CzO2&)z^&!Nu%e}f%O>k;$wlolmfaAqf-TJNXbQ`F#2_2kn+sPqv8{=p` zB_u)Sy{l#IQU#ibg12%|HuRkk^MR|IvrCWLEA*26@}9w%ca1Fe*od$H6 z=*4q@vma7p0tpTck4SrO`j0s9JWCV`XdX-NF=u-XF*~Hw@xK|#?iinf%Ox)yVuzF7 zlPZpMudhFLCP;1M8FG0)Gs?WficiPjP}UFj98Z0I6+M4X0!iRln{9zbzN4NNob`75 zlXGj~flj}Z0gSIk4@6tUi&G1!!$lKFTf{7y*>AqD>l4W>B{U8kcvVvCA*|=a#l~*T zD80p$^jF>opkwQ-t=xqNi~xtH*y7xyL@wzY{U|7I_cD9X!>)5>*gqsmnz#EC51kTmFM-qsle+uN zpUobav_p>}$X`Wk?R#mZNieMZFoX!-v+~de_S05zW-WERqtkv+=*2%udByEYE4?EbbptvJ0L! zc0|CJWD}Umxjd1EFPFm*>JGklshSzZSH^dGB^ub8Q9MSsO1 z|B6aFI+{zyIHh!OUyfy$>}N^NsV&5Bl1g9_(>rOEP0Zy`{t!IXX&#z1?c~P#KH3{k zW&9x*$C1hxTNKfRr6hOpUGySGb~S4-Pp`3>b_;R36Nl`7++|ixfBrmd#<%d3+ib0z zocSuS@wyrM2e64>&oLnJnM6#JPB0P3yx(x`2&d#Nkz{1|*B}mGIp|(FqK{H>wWcK- zqGdYWh|Z%=y`y<(boS$!*GB-a3_j`0=OUb09#u}n0Q2c#g?q&>R*}XMeTRNpCN6b1 zZbes*_~Q&_!lI|4^p;dWEm|;WSYC>eBoYhojYij{2>+!CxEv6;JJt;FiWGE@nCAMm z+*_Y_epr&_>M#_ZhjHkvJzvAxTE1kSlu<84`OK(qwCG77AQw8Dg)f2Iz;wT0CK>s! z)RCk66&2#G2Lbq-amy|R2U__63`5kdMNhu(+htx(%@_76Ra&7Gh6qw(%|<%b2$M|-9nN7Ng7F^#KW+1U8LD@8N=89w+C_T@js z1~G$6N*z_-(~9Y(t+5-S*gIkpmgRq%shZ(8i|We`VvsAAuo+xU)b%=L1j8@MpX-vT zvpBT+3qE9*#A2a>=s`Bns}K*@rTb;eQt*kkQT9Sa>d=bQKhGAEZ*=8@K3Hr4%7(}D ztu|4of4&mj>Zh5PU#-gY%`6Kuw~L@yk(h5W?)^Jy+Pw?8`b2lkP+yWhCkNs#j)m zZpnEQKWH2Y1A#B{qn#Dr$h6J8l;-s8wse4Hiqf@F^if*NJrgRNn?3BY#Naq{HW-Ht z%Bn_sL0taX#%Jre=clpT4K05Db4xb&dOh&4@Uq_eNs$e?e@=-4{3TgnCv8OgT;;3I z#6DR4>!qRjK<%ERyI_{KZ{dFlf!w&C@?6aDjQ@gvOY|WCZ0z=vmWN>vIR(m+L9*VF z9b>%7omww6OfnLl7EW6GUKY&iA!UH&MNXT2MdX;=ad_nK3KTkGLMFrn$O<0&e=lwZ zrAHIUZHFHsUX^t~ZCZ@v3<6swQs|Y)2PO`UmDFw>xB|7gLX*t9MvkmKUao7hie3Gb zep@6YxE3p4mgV(cfRmdX@y3J7WM|Fre!i{!2dg}jpX3eKQnFXeyN@#xc*N(4S(ut+^)WEp%>Q=$9xGRw(fBIgr_`Hh&C4-=USdq zL-GZ@t!{s5whZN%_hnhByh5=Y9C1X)ZKkztSWjtf$;4s}$?Y^KZCJl7I6V2{f6@f7 z36A`o>6Q)0n?|I3LY;V=1boV&)t*1CqVp()((jh_g0%Y(CevMifr{<5_Q9-q#m{qw zisL4;{Ifp7?|4MI>35WGo{#qOF};$uvUBoSOr)L$^muJ2asCQOx*M+FqXvjA2cy)I zbJp%7Vol<5dfmV<+V(8+*#2wU{2a6qdHEz6+Fd(RFv1W%onMdTh?=DFH zMXRaX=ael5=`YjhlL-<(l0H6OdEhpUBKhL4NIW1tZ{#*MOY;(@q42Jhb$QQQ*xI`X zCr`smh?$LT4R_3H{eTQ=arK-l8kYT_f#smqib)!*w#6X0kuZ_M+v;*ByRmc8_~YI} z)1?YvOQ2~9?L9uZ-defdvmfan4qKJd?q^HA)@KbjtbN^S*B0*ZkOyZ+)pBg_}?*V`0Hi)v(Fik{w~x?~q{| zqst(Mgj5lTdFdjCKJJ0*=qOAEiF$ zLtZ<~eTt$KOkb7R(KhUnXoXxfNX*V1lGoWOor&`ldf|Vii9uIbjlOoC*M-?JKt+Z5 zPJn9Bp6iwE>xh$jEWmbd183A4?jMR~HvO?QuGY7DLmDLxY;u};62a5xr2&H;4&8*T zb|UbGVaZNE-+)o$3__mCtp~xJ8`Ab~4e-FqY_jC*%R5Rw{A;|yv2n_UY;xkQx2QM=^902UtyR|ewWX)E+ z@Wj2X+B(`LBTNd#1aCJAY%`b03%qDdwc)8VPUU)`ybv$gZ=Fu-Ue^P5>pB{26u-7C zL}NY$@fGf601i&A$^O~3B|GQ(hjuP+IDm2Zh;X`*`9dXQ@xXd1mc(V(Gc~zBX-(+%q(}Zii-!Zcc_Sw)gL7DGJ2= zNHM#s;=wrGZ5wio=e(-lQX~>e7CFz%XJ^z4?2wr(_7_?3qOyJ7v2u8s8XgGu)J$!* z;BYK3?*%-qnfpq{T2(K0b8!c{w(>*N`@0^yJX1w<3l_k3t7dvYH=n||0>|_@ivVvv zuYTas&GyQW4e?U+jTv>b3{|Uby8GLlb+uUS4$36|>@K4u0zpoU11jHOe|)8JW*Me$ zbb~e&Yz%&x5jZ%c&JExJzv*#FlSG9xMgjeHvV1AwH?|(K7AeC zw7N7QV>HvrNtv!Ck2U)+m&C`Sp&`ThP24MeuQo`9^9&}L3DJ-tlG8Ph9JN|-4Migo zN~IUz#;k68lfWj`3G}yse`YO5=ie)#mW8f;OEZ!JvpN|pz4G?uG*!ZdAKJp|n|y*{ z9Qpk5g0pNkT-8lt(v*m2`x=DWp)$fQ4R_=9>%!xPApr8J>^T(!+DmMI1OP3-(|H%m zuNSaNsn1_?t0-g$jJ8Z3{Khm<@(lG>;Or?Jdt+C&81CbXKT#SEH8hp1wnsSt4}c>by>2^mw7IfH7Qur1H>}375!y(_F*K>RLL_(s))p; zfa~W{Nu)_UC#%CtX%*vythk!{vTU->ui3Tv0`VFcSi0uJo~c3<=fd-v(ZZCz@`*Bh z-oZ%_4SIG)>RJX%?p!j+%3CliRr8eHtHL?jO)(^7f;vp$g%1w3XtwNy%wd*mImHjS z-yxA~d3tHY64aADU+LpseBmn$O=r)4C7TJudEP+PRT?uHhiEc998DB^{mcSyXfXT-^j&92^w>7kaQ9Fwl5r0N3AITktv}rTuh0p zO6OUfPSEtry4LLku>4;v4K!n)Bi*OfLF2P=B5!fE{QY$wKSg1wY%;Lnu;q6pm#aNJ zx>B%Nj|V^&N4w;!0!!v}tVB_Nh4Jv%J4Xo7pMUw!T2!16yT<`M2t(539QMCcC-b&(dF$>Tf-Euz-%<-rMi* z3C?SvNy6+}KVD4wcynJ(BaMgmmo^9_57MHCh3O^ccio0uAExp+fI=U*RV~xJJST&%MSUXwxOXpANqLy7)7B`}i1VWX!9 zhCGFE`2QQ4D&hY(1k20uV+dm*zb@~YK&mVGZ2hs;JV28d`S*a-22~7;{PTEoOrxBK z=PyZRRT3=Zu0j~t8M9tI5CH1?px*-f0UK<*gl2m4=p~Hl>4Vk>U;5vR8K6k}!xWgc zc$~$^aM&ocNU$jc+hV`M{LrM$iyGMrwO;m?dp+;!FjwL!{B*RqYa`GLf??42pFS9s zzL(KB@wj?J{ufO5bmk{R|6dh)f#*H{ONEX1KAm&JgEc6+j7~n&kN9%L)XJRtJ{*!|B^6ccmK z>f&jmmcVhwNr3yX-rLL>4NPcf#o+uC2K!xn|^ktI-XBrtZF zR{LMF)-ub}P-vW=pU+j9JR`#Di^E_mf?$ToAW2F}Vg&kcY?w1LGScz#CWHin<+J7R zP?Qpa-F9cBiTc3!8XvxsV3Tl2>FSc1nwjC_;W@4RW){30klona%&V#SJa8skS67!; zUXD{*T55pg2IY%I0$ljQW``UH&2F`Qd0+PHnc`{;YsAMUZn(SOTM4G#fDy(Z)r(p1Cf#lETKf6rkpBxC_d-YIu6^BykFjs3EGiu{mP0Y=d+Y5 z7;hw^sRtGUS{hH4Xr{e(VSdTZUTrQCe=ZofMZ6m;}j<14%iNvC4QTQ&UsMr(2`TfvIE={sA>{(7@oJbCa?q?-!>oR8r&l z8Vd|8tR2B0!&0*tNXA1+Y`RUKnW7X5CZ!ABmmQxXB-j6q7wlyKP%Gv?Kl2j*@bP2T z17xTMC$cXvI2eV)pxJe-d)w#G_UzxdZo5y@q|O6?ESbE{uWA{(!hr5qScQ$h%pa3&Ilg1OPc~nTTDE#`Bo`+|A zTbYk|d3HpoH!fe=v|2ZJcGm#QKQ0cBk%h&3&#D&fZ?cZzwyT;ZUa*8fP&ioL z!Bj8wg8*qA^ZwkU7xMaRxiqzM$Zu0h&r z)g){Q>wo-63JC_w2Q4qzT3b7Ni7F@H(b68)=k7UTVq&Uas)pyo)6*$P2ttW)nUxd9 z#>dwnRiF4N8c^Qo>ZZLLeW{m{_r5HvSynX;#Lh(h z{W9%OMYFjj23ibq$C5thpOgT;o!=p*zs4C>UZ$rjC@Y6A_WuBw92b zv;LW@HZy!TSyoYDIT;O>Us=$v)#Xr!1rTGQUK}j%rZ5}9W{E8HevTI?8hqpV{Duu! z&s+B)IxUT?xFbrXhyWE8l}3Cs@tkjeuDYklB;G<#K~4_kKLeg`gwy}{@Lg2^t#&?Y zX-Pk}LLSM=GRWD25d|j$ z&JtHu1Nh(2Yn#ax3cc7qpSBT5NJ#K_#wr^j7QitK_&HVoGv*uS#Nk`bQmwJwbC2Ec z4M#V-mF|>Y4U)FnGj!JnEr&;YwFuzI?TA|(3=AnRFA?mpSUmHUlwHD|-QD9c*%#-Y zrl`BgJVlW3vWjM`TFcSE_htLzuAeUy#$0AoEy$^8`9^-SRa2E~}%!a@`a3kx89`5pv`h#DJtZW5!Qpi~)OVZ^X(Vt4RAwq^Z3 z8Gfaxr}yQfrE>Mbq&aV1V^?9fkllwy&cu7>Doe**(c10g}mEe#;TED5}>4MMXsp-2H5&h*`GKdaRP2~#7$;p{C3lya^%x9I!(~v5H04YfN`+Fc#D;FQjhl#M{ zJ6`CsU?6XeW}rta^!Ps!s^GP`o`Qx3-urT0q=Io_^U#;ozo25?7j`5lK$yyE6crZ! zUa+mCCIDNs5j6Q+pGYjTD*z$hnM)v#KR-WDCJMF7ImkK*^`8K$j>GobIlW&%$z%Hf z7^Gac8rke;)OK^imTi&>lSBKpB|v;nRn&%k9oz2_?Yi5;4%+qp*vnGOxO1*sRBRIC zovBhBRzR|gE@q&=#c+xV!08T6upz>94|itA=CBti$~kVIr(jEZ3x6Q?uIwIe^nLtk_$o{ z9uZ+G#Zx$t3NV++?#%bO4%nmr(x-V+l1NlQoXgfo*WL9gK#b%XKvlE3K;-nba_AF( zepMA7t$n22Gk`!S`uRyjL`IG;EMOHu<_`rp zb~#1>h<9~$4GauOSz0pJ)YJ$oyyht^VF6%OUoQxh8YVzF0URCHfF7}ouRCL$z=)a8 zYPvU9-MCjX9G9Ms1yraNo+x_W0Ljjm8|V>}ko-AX>tSVQ4^l7QrNBc_@OOS8k4JE+ z1k$;NL<{^T!a@Zc_TvL5kbmMbGoyf4h4c-X3XsPn&{O(MrMqjrqHbVMurdL#?1{aosG{QH;%IaT$<={ne%Rl;mR>xy7iiQu%MAa&ZugmZM7M!D WD9US+Ndn)80V&I?%T>r22mTMqt$z~$ literal 8699 zcmZ8nby!qiw55?|q-$ttq`PwfsgVZhPU%J@M>+%)RE83eEcYp@gTZo}25)uy4(-Rpf^9wofC8n2(x+3Nr4n8UcL%u)BKEM(i-)5F=~B`o|~YwW?DXl*%X~&S5I+X zJ7YdNf~he$-c_F6loqE&_20pJq2Ez?2fEGJ3X^4jHt)IXSt6OP!B7-5Ts-x|aZI}0 z_i8E3lZJb54%f_A=!TP4PnWw~NSdE3co`|rmzbwIFS^c@oWo}NE;;l0I{z4? zGUS>%?Be06M6S^<7TMLmnQ`<(e+g-N6KSSCJxMv1a!gk8mh&knS(vMolK_z?fz5_QE@j$A;|3!MbsJd_0vb|x(t-?ggX74 z#=P&3j!-0?S&{LLONj{~FA*`nIv56>FBhApk(DHV@kz2(4(TNK)okw^@p$NZIrBz2 z^JYd8UlATT(MmvES5%{fDOWD?_pb5x5)(hcb=`N*f$6vcs`mt3{$f$^RZxy2t(Kz~ zkWP{x$rp+u6}3p32b>%F*}R+Cn7Uw>d_nLnbZ7iEPmjp&JQ6w>t6zhYZ2Sjutnj%p z;ir8l>KTYRa(aKFl%$x*D>S#%!LQ9t*N?GBN};!M>KXypwVQyf4KEvA^d_`p?Z1L@ z=}xghAfx@VetwrHUbeLJZ?wXGD zd+hJyG2Bfff~I05D%Xx|dt?(N1a+3E5;IX(qaM-T%sAB3IFxk9ggAa*OY0r(7$lUcps8ov7a4qA_u!ntzu zg@uRP0~c`n$ATaCA|XE(LpffTA0wRj^I?$j+n^t$vDQ7xY1w%lj+?VA4E2-27fO8A zL7sP*&EDH@CcP$C?Gd~S-|CX|Lt(dVzv_ExX&yzcg$9fX9))Sp1t0u3|0R>Bw3o=_ zYvJkfKU8emnWXs9d)?PE=u|{aE`1L+@ITHjT=?!K#$qc51_u+Up(M*mnq9I>N#6=u zUhsa(79*%xdpEI#S`)~X&t%vsX#5DXI?(053fzfnyBIjfSD57Qn=3hvF01$Q*LXS1 zLo8fqwsn@W)4E-k;0c^x@UBEXqNVvAbA}BI6rP5u|Mis4-G9$w!MKvxcfXZ3FzXWC z2S|Ol)e$8Sk@;1#bwOrbhCw!r_VIJfD{Pfj)<6V!#W`YG1AN znic91!P2yrp0vv@7?m4fFtqSCAS}nS!=l3`>=So94%+!wRS<%SB*W-@p@7gVFT>cu=7Yw#lh&6A1JkeEHr%mCsFg)Qms6EpN^6)CN zq}AM;kz!Lxue|M#Ht@E`qpn`7MJPBj9M45;z$W^tbwD@c#v4bLQ!9dBAKw9qwjdeA zi@~;!AJjITkN*?wL7|0Y{!`F$WkfogPcIx6lwmhF%?w8}w^}4!sJ$0j7bfM+|H9Yx zhgW*|{uLGwtYEH?Oqv~q5UoY+@IqBnk%OOh;ls^ z4(p3uarCHzvY6wFFgfvYNDuG$hko|DUb*+Xzf=7$pTGcQPcV8u0dxL(}@ zDHvpM!lp)7wSq@$P%!svUh_w1K@h}c`V4k!RmmBi1zt<>)ECA1ob@`B0img2^-O^6lU=>b>*Q~HxdMcOqk z)(RQ@uCmZM2t=ywnD~zHsQ| zZcyoW75vwFZs3m!>JqSA1Ir_*c6NT@P@GeOh#=_U*zO4wKf%PSce$pqDhX0A0CRgqs2b6$uP0aEA0 zawHWtO<@?r#`-(y#OEfkaL9h_D^DHrV4viskBxhj4nbCk?}l`Ql5uL^)LgXEzj%Iz zzG{WnT2(4Sciu-624?lMdG~mZsNX0MtV+0)q+0*@>GWeo|LA=HT+(}zK$&<+h7Ou3 zd=p;+EjO1YL5~!R#i~DOCDT3Vlf*A7tftjlHy?)MvLW#p?zx*iKmoi4OWLEQ$KT*E z)PXoRmV`L0Df+~J1h?vPp^kkp0$f4&>`s21sA=fNuXfvwyU+v1JRur~w(l}(@!LC#X9nr_WUmCj{r+=kGhuly{F*QaygqT_GyDEC@5Hh52wxEz ziDr?e?8~(lM%uOE()3VjCzuL_C#*h>AHHCsv-g5vhb@x z&R%&MT&wC5_ftFnf|Hz?{eD89zZj0O(fD&&dV=DiJu(4>D<^`5}HY#7QQv= z$G41%jWpTeLI0B01FO38PnFlvKP@+x?GO~E6Vrw3@$C*+}RW7QklC3H|R&mJ5M0-oto~ZZ6*sWu?i?)+{_GS+r za`k!`!=&v%uL4Dtx#+Faruy~Sv2pkdco;W;C9gr!;;t0suYRHAlKCccrjR^UH+w%9 zJ73UUWc^5R^39sJ9T8`SASM_iEwi zp3AVG#iR6b@WrSUjr6PtXr{n460~B7daK;s6xcFf8`u&=lJHaCe+KtC8lYmRh4bIg zOm>M)c9~ji@w)VpPWXJW`4aJ{{y-~bDj!nQX9BCo zZ;6$ZdiB?S62CW?@#xroST#x~^b?4m#EP+RIqkk;P5k9iI#Rpsgpjk4i@cs6hAX6n z(vFhl7({idqT)%hBy|o3^bo@9&flHCO61Ho!l@Ip`v&`Vb7^Pa{%`!2nwg>vf;ZdD z3Ibtp9c-ikG@+em;;>}mKHt__Ud(9aZ0&I28e<*EAjKQUw)+*0)|R>?ARKzqBE;M!Al1OMjMyl)eWRUt4*Hlqp1!!nrb#rs*n^&6`PN^AZZrV| zO~dGg`<@Pe6uBQ8XPL#PXIu~O{>|_DS{mPLaJ~#iXY8!u5%R!wg)~tbX%7;C;ee9( z>pT}F(501eCNQkh_;)Qy%<~v+;bRCFK*j!H6wt%?c z51=D+2CeFa_vLC@HfrZpGY2c%>ZsoheWKOc>Y~Llf`t)|`Cjgp;Q_(|eQ!_5Dnpdk z=XXNv^O3$cU2c1qpPm2*pw>517jKo;4TdzcqW59icPqU4K_%7FVw)kd++Z5)+!!rVVs ztZ@8V>dwodUE?AqZH$G^cO|4^;bA?!$ z45Q4Y>!cSqk(#irp3EyMM*D2x0NJ-qeua$kRNcmV8=Fjjc@wDv@CLW)HJ45pb_Vl( z`TP|_luq__&L-J^S7R77X!ks_JHG>g{u=kU#5h<`+omXjC~t$H7TlTqQ}ts#xc7a! z$9fiyxE)o4=)KsJy%~aL`t)^G+yr{RcdNhq8&3~6)8Vq&Gz9;Zi%X=8zNpKpoHu#a zKn_6TR|3WIJe5!0)Y7ic`mw3-nL!~X^xC{8V_`~Samr-vyo&sJ%C?93 zCV^lZPd+*HJdgOv1GAmkIY&7kKJ({eWa6xj%0nS2s##ZS1_<~5wnj7B z{#c32!pSjfVfs3Pt-h~&4Een)fG~QJm}J5>5IpA$=8e_rd=n&@td@iB#iD6>k-q`Y zqK!F{Tp=4Or|+96i<&4)WnGrxYEi`ZMnz)8d5lVyUfqyyao?gQ9US2(XiQPvRtA1A zXjLpApo2qs^-4qbMZL^rI~%;hC*@0sqwRkN_IZs^5J{4A3C; zRC4HlR$4+pAl`f!3}|*IF);jJlV0(^CQyQFOD}UiTd(p_7O&0iEwnmFLDl<>9m?@K@?5~i*3{=;F_CdHnu>;DBee$%_wHmN}Y%s7e1292y#b0-DyS0j(vbI#IG6(YI znb&hPWct}PxC&(Tw>d<+%|k@1%~p`I?HVntU)9|X;vfJqxFu!?=}WSNXg}7^)EU8Y z=m`}@w(kEy%c9_$H_tNpMqbPOcgVm3|7-qJ^Z%Lu)C8^-^jBPmx`(FP0N06^?|wWo z!x;SJ$f5bLfJ)f0=SZ{=+vhVCZ_$_R+kDX{5@>_8G$ZaB9mazne{IY2+zqd=!QuBf zLqP2QcT)B;R(+yhJ6-F$TBHI$w_!?E4Z4|YUGP(|GzCUo%>1)rSwueO9J77((^rhN#q`CDa$>7)U~kPW)d>(@kj7VG@D)!VUK0bciHZx>tWyR~Z zBtb11KrHTisNn9-iv(Eje{V!jPjArV!^Nc1x$w)=Fc4Vpxs!Gn-SS>V+ohV>$6?zp zxNJC&(=_L+uVN$tElY*RZxsum^vulYT~T;lwGI=2-=Rik2NZO*Bp2N>LlX6}YoawaKAiO0vs zcifIjusry(tvkak1WP5QIlxSo(7b;B1enh1e0v-QgPC_cW7to{Mv;EFwo3dqn977@ z_r)Cp9JpU)N>DU`9(Q){;pyqV7B4JEh*p94geEKF?JX#U*stO405D;9>pK`}ePiQh zfvINJS>$Ypzv}IZTtsxV@8`LK3a1Dp9`{uRPQz+6y^^Ut__$-;^+tx-+vRrTOLg9S z2%EZO&1_tHI%&nbkJX%AD{tw5L%<>B+^&Bswzx$p6(mB+Wwd_rRa%x14lS;<9i}A3 z2VzfAQ4yV!G*+`ODl&3qGs{g+LF6m70FZ%q*C(Gu+;cViW~QfetE*EY!|4@Owcflr z@Px!u3FDufoW$(4xlhAvhcnUZ<`9A62L}hNTwFFQ0A#&OwV|-BnJr8jw-@*%>iGgq zR?GS^o^!B&Xh^})@yARoI}mrPtE(evZa{F~cdxu{cNRhrBPAy%Lo_w@_kBGvwj#+) zx4qT12_wLQ;UDM~LoDm(wav_qCfr3uMZdl;u%4ZreTIW`cIyci_rb-F4v&lgW?E!A z^;yoV!Po>8d4ukeOX}!s1Q#2d1-%7el<-w%R#uCYMhX)X3k!maHPBd9HD=}>n#-Eo z*qHg>c$E~U+WL9~F((j8FOjVTC|ex$eplm`>FrHDyScvhtB_YEM9W6`@-ce!5WV4> zz|7tFWZ>ON-^uk_VqRU{)WIIm`>RGYvs(I7{N8QHzpUYb<-!1LBPEO?#nl<5Dh4jbHJuojna(+-Lt-Ew!TFDW;lpt^GZ! z)D*dBAu%xivtEg!oLtz9v{Ic>JbXPeB$_;7p%Y7E! zvHUu|!?oUOFr6!=ToIW?FZ{&j1)53FL#3yX=D#xk3sG3MnoDRUi-1`E7?nS!1LyV?AFC+I{o? z?s=zUHuXcpwBUPn_485SUr8}Bv4HajqcTzl2M21Tfg}dzBpZrc*3|{i(a2%jvm0*# zO1vRk{-vNZ_2$z7jyK#2ardnZm&QBjsdpLjW`W#y7YIfYp# zr>0g~5r=yxo!BGdR*rR)WN*Ac-!-%G!#m!pPW+yK4{uvCOSb_cw9A@rrL2CbGedVClsKil(Te%8u~0_k1oOv-dsc34B|}3B0ZR2m1Y86EFPT{oC?_X}lUz3JOa;Bm zEb|eUMdb1B4B&vx3|;7#3Wu6mEghYR++2ojuf`EW!FVM`Aghy7QvUc0AI#L>si~^M zryQTi{pU|19UYx^Aki(Wytsu6CU)-h=Btd#`iF-V?d@5ev;nTn1u-g_Z;$1$vay{E zQokCZ{dLynB18pPZDeF*&iA>}?O*GNL$HmZRKd}}4aJwb{8Z`GHq>s>P6=V*3}g~( zKu9U5OCcFlnj?FOeVy?P3#2m5WtA&{%I1yB0FRO>xxxwWn?%nn)q>rN4|`~`?fbJe zIWv0{TEOy-shWUZiH^29S?#GbDnqsX@qEj%4+sTwSpd@L&~%lQFt}UJ{~&{)BjU-i zq+Gk>2D||E(_*Bt#sK^1=}%@16!2juL{SA81n`EnohXpdWjA?Kj#YjqmT}~~xn~+i zmD(j!4VQCH$=|+>0Q{*9pZXq3lpWE&1bDb7UnP|sunT*=63~M< zWZd0Abl>xZsA8#0#4Cm0oNcXr@f@*`!He)10&R8tJ10Uz284&xOnDDWc2L+D;MO~f z!Rm!smBUgRJ~%m)*xI5lfGc|^r@a%;kl>qH)EIijj#7Xq z_omCb%HK5ITElHLfgH0Jpd9)lO60jY^j@#TMus<^b)d^q4YS$AOJ#G> zZ~)u^C{AWnUTr=bk+-#dAtxs%HUqLg_&^R_ulCu{(b5Y2^9TAdEv>_v&szhwg-f~I z9n;T3#w<%H*40b#7C=}aDFK-Ta1X#!%050mTTH>Y%zzgYQ3qGv!pq9=1O)}{U0e>R z2h4@Cnj}K<#_f6hj?Ez{XyF~90CXuRDA4iooqlZkfXf05tBBNl*NXq7&~It77a5=G hfhR>C_U94xVt+}*bU-_mr`egiOE+RFa8^EpIQl?@Bo0{L=VE!8y0HT z;k*!?bDl?q+o3KXgkW32PAogQg0&>mRZ(F1FYl^Q0l?vkoI0yw*OUpXO}^r4)paIy-;MEYHS*V@p`8>RboP ze!zF7nROTm6+)0vvR`AAH~`pqVGAgw5ET(d&ZsW*P!E2eG#OSQ4Br;-#pa9KSnKLk zOHVAve;ru>6K||{`J&^mLsHyV!BVJ#FABeehbq`KbPaCnKJ;pw1p?6{V*Sw7a49aV zJo^!AVV!Dw!IyvF2nP#q)>tO5%)A-f3wB-w>ul9|oT)m8IoHiVkC1o_in>AFkRu#U zRQ`f6Arvq7n}fuV1lGbj)f2rR#((twJ?^xP2E|pd1j0k16mIH9a@aLjjS{Ve-x=pQ zJeD~PCf;<}fAS&NcH8=^Nr9g@tfeY0E{4r!V?EbJ^h|&DESz$lRatm7Z2>;YU5DeB zPGa_T)A3-!IBoFmRo}2a@IX;NmrHioC2bj^EoHOW>NTxQfvLQjLiPWWA|cLf!Sl$$8rtnQ>i-R#fY0)`w`=FOHLOhSBfDgW-EH?t?iBP1>y1!DDE`oQ zD$G~BoL_&iZnurp#98}GvsmZrT$ECLnzs!C5!CO7bsyq){d&Pli|7wYk<`z*Rp(jD z>Rf~nd{>$Yj&LeyV`9^r2UefZUf_JHt2G`M4wfI*WCKk4(}n7bn)vGa@P5q$3jiKW zxDS+4$cl_;V-&tG&jtZ=hq`owpvW%&5j7syRd9a@k6xjPezy_)`O|yj$DmTK!t~^) z5M%1bT363*5CnV64j{@Hi3-UMp7406=lF)Vp$(*a6qH#hCXI?*C3xrbdw4AAw@9%j z`%UT^1k0Z_V;Q2}T_YMbH(X;7c~v5o;hj_OF{pG{W~l2L8<)^nig}cQM`eZ;r$5BQNfQxmjB2vi;u_oW*{WQ)6*mNq2TwR`ytsuyr6X&# z{?|{j{=%o2@ZBT0D`O(E$__TE{~L9MV`9@GDx&&1zor01t`gSjI?u%^ZWlQ8xkrV$Cu5>*-F4=`j zj}zUE-I~<*EdmRm__aD85rP%r1}lRsh=cc*_N~y|dPb#OiN~{^g#?#p|!4n>Y5ahWEaL#!iy+f0j)e53a@agPX zC~}n`+8Bj3x4er`K15yCRB_EY=yld)#F}kB8*`X>crJq~h^*B)=Xn&mia`iLh1-sM z_fE#ys@!JxG1uG!FTe3s$chX~DPn)j-LZAV|FSXbL1eA|_~VP3fd{c>8~TRzY5u-i z61*=u{yOf7zYBvRZ)moOWD0yu-2WKL#R_<+2OEZcggA3-$AHDk4Z-aW0pN#*b)qgNX3-Ii5i$%(}%6HSRU%Z$Zzk1fe0XkfKm#MaQJ5Q7UnLyh($Vn z=>(=9dhtr29)tBmSGP&uwb~UHx{5G)|5OMZub!-lH(}?EdgGF$e@_XXPV3cjR=A@l+6{Z|`QWJ|Y1y0sCZtoO`S4IVUHLu@n#F}kw^8P5- z7UWY(@z?%y)uF52?~7c;cqH=~bseU_aDP({JPD^;-D4=1DzxoSfiFjGhQJAJdQ7gZFT%(f@#VC ztL9GeP!AFeNHtWftCy zEh9dGfitw*IdJWHfrJozG-L%HOc)2n$2IbVN4~27_w1R3tg?dvX}ttAcWwt%xQT2JyrT&u`n%u28i*Sl)}7mGTGt_rOJ($=m?IrHFXp$b9_g z4>JPNA$BE4=)He2Gs_O*o;{Oru_j-WKq*wggoOLBddP|nNuIclxO)KL=IER7+3
4H5;y^8s?H&O_jtT^;w=_!w<0clI(sHwIPx-FS6(po&D38X_zIp&dA7sOhwI3T z?{197&XJq&WS>Vh){B6E5Q5JOzQkP_6Y=(`cbgqZ>sfzGPrQxWci)dw&T|?UM=6Dg zGhy?vwHOjTxI@pWL&5S|ug7~ojPEc$T?;el89D5a_M`Yyq z$ynbx^&alZn27BKJ3)xHUM?vTruKdWJ4SBol!^0%&T60pOFT9Y{}_8q_T#0Tg%F5A z1jyf|)n_&3g1Gmq*0X6U(u?WFHi&fFqnHKv$@OXk!%KzUejfmqa?Ft+>u= zAn(QA7BdWc(!ap;mlpd%Pv<9Vhs^{7)ycv71 zTz~D1&#kzzaKKC0Jp5zywDjoIbLRq~0Xf29a^imk>`C8_>B&#Qt+?Toofy{r7PWB+ zA=q%?GaRuW!~C4TVf44-u)ScH3TSH9(i|u%hZ+DpXm41)s56{D)io=@k|8(7R znj28`hVZIbSa>rAU77KRQi_qbVW`Y;B80c# zkI7GCV$bpIUYF|9VEHOFJVo<)XW`eVl&gObB3Q0iIUd(q`)bzNTCf8yMa0g+uTkZx z#Q|pr>d4R zC9E$b#jXA}U{?7dO$cEmMywg~4_qIS8nA9WV8N=bv2k}G!4i*ei}zt`{tn#S?IwTC za=YX}g=9yF5CVY{5KgR!FofeG3R)EdwnJJ6yX3&QeNzz1hk)mJSU9r^76C!O@Dy-9 z$FFo(;;W(z+!;Gc{VtH9>QZ09!HLA#y;MN<6w0SiV-Ib{S%D)Qwn@ zwgA6Wp2qRYlbTKnL?u|_P~a-MoCpvehwV8S6?1iX_u{la!*b;syJF@mAV?Hp2*;?H z;TUZj(Iw9Jn$X_pZt8}9))e#$zZN$}rqwAcw7XkoV0Cg_)DBvdHos0r7)*Z(DzN-T z%QE$VSLEh33=tZfDPN(A=5bV0BvTLS8x;|ZPZ8R6pkE+JRy?SZtD0^3hJQ1*!CF>Y zcz-NPo8P?nhX5hGCE#jK%wo0V0y!QL#^wj|jaeg}@CY|pn{PjL@){fmCKd-Wu$UNF zObje0CI%K0%L==)YCNtw&m(_*B`I#!NDU<^luZLuuYPj;e7Q*KM;1N=Ng)Yf6RzqI zEH(iNbA|15*AN1!ARp!H{s|%kz@^GEVxFA|E+mwJ)z#%GD@DcH)!;z~aS=QBhF=j^otXUVLYMk!2Yi z$00H@ven1U&(DV_iZFkfOhMnqf?lCowQ3di@81ue=T)ef7+5Waic*Ss^X8$vyd3JE={^p z;yFE7te^*rQi|N%T;%8HHwYA^6jrMhOO`BY^1i!w?~au#SK{TDUxp+}^#V#M#nGcj zv0=jo)lxBw6(psFUAuPqtsQwg`K`C!!W(bAf#~SyCU?9SqTaoGWA*CQFq`Ynz()uH z$8p%QWeW-m3)z39Ai)x}1Hi$92mOMfC<<1tT#3-o&=$Ky?>5VqFNeiqsdt@_kPy82 z=9_F%kYKS`Wo2a$MbYo_yYIeRt#7r|O?l%cN-5^dnWNdC@3D%Di($9hS&S777R%vq z_=SZKf`=Y@NG(USMM)zuF%cGvrOy5cArJ%sXU?2qF;;&t$mw*dX?|^4t=2X_9;Fm% zX=(LliagKb#EBEk6$U$4Mx(JwZprO-tM0A#U+wY%Aq0a55B6Jv$ji&q1z5TtkJXr4 z5(ELumMyCnz=(*5dLIonR4*$kDq>&-yBf1rUT7jBA{ty?TNf@|(EWq$pm&9#p`m`= zlGEvIl{J5>?-#r%rTG5)@BO-8R>ukwb?ep*vMkrz!J;VQ!i5WM>Q)IMC@CrN3uSzK zJOe8jTwP>jWW6zu7lIWlREQ&AKKRxr84TW-0f!MAKUjzd;f7G}+wg`-E0HpzPVF59?qBWBE)p$_Ep7hfeM zCAG;*8C}9#=;ejgwQJWRBO{~HOiz*|goTBnM~@z`*=)Gxnrjdn8;f4OdI5ks1SdB) z7bSlsCCJOmgD8sXcq@Mp*3{Hs@#4iuOiXN2#>v3yB34V1But$;wZX?YzI7>Yvv>n> zZ^?q=IQ6eNfTX6T;<@LZgI=T?i$y8LnKNgwXwf1Vjm8eO7!HR6Yu2oR!C+v+W`Z$8 zOioV56Hhz=Q4~9@P+47Fjb+Q0=?W@cf#rV%DLp+Mk3RY+YHDga1geD#7a~4BUU!dp zbT`|KH;(%0r=PHN=~9@@<~DuD(A%7Dw;Qj$_8M$98}uk$Ph;ebeX_H&VKSLo4Hu;p zPNx&&$B)O9DO1{H03idb3tBC&&YnGs9Xod5z<~oW7z{8Nn)lPXZnqn#57pMDQ#n83Is@?e>j|Vw9Imph=Movx+s;jHjqNi`U!TYbtWWvCK12K5;U31B;2NmHK~0 W`VnC%q(6TE0000(k&zUCSxG_&30nvR$)LoC3Jeex&=C|7 z9y7=7;8S@XMsSSD%0FTRv=#wWdFvAmkK0q886;Ol#vc!Oaup2NMvTxm_lXO*A z*L{DGF1wO+(%Cy*_j^uGy1S}tx%X4Qt!~{Wlv0XDy_5d|wqE%M0Dw}8I}>ile~efR zLI{|Dm^}KjMGPt?jVPta zsmTRTRCF-Hn$m1NbtxeP$sq&bQe<^usk5B5B$Gyj5EM6*s0K%&3WkRbgO}sef&~D> zLI*=qBrmYaSW7ZBQ@L2FzIG`x23iuEc{UAySOO($ph;jyWA)8aGK3XC1E0W^opaukdNnKOvA`0`>9z#`#>oMX=H%Mrq=#g1Vnr%d+?i-&P*e z#8*?}GusE&kkCQkydwVww-d*!b3ws>+@YRTSbs$mUtJeIvVC9yz`cogfl>-tkr8c- z!qMs+5HNSBOE(CLoa*n<;CB834&gB_3zMD(me{wtQ3<*WzGt`o&O#lO#WA-T2p)`b*+Mb<;$9} z3{mf{5p9|qt}Tc>st~L2_7CqdsB~CnXzm)@me5v;d89HMFP?cBPPYg`2;$ALm^*M5 z158I{hPC-0;^_V9JXHB!JyKSwff}ePw`3NrwaTW9*I?q@uEav7?m8eA7X68|;!HUF#~?ZB`FR&vmvoU#+OZU_2*8DsqF`woEx zQ2bn9j0nMsaD$aW7R14ONBdT2Z#|<{uEj&yPrxSGkzkI;$G3dQdLYPPs+DWukn1tP z65rwN_ze~z1Q%-a@mS6x@Pr2;1Vzpg{OI@zgF};<)e53a@Ufi5D05aI+8Bk6H@%Bc zK15ySuekQ?bc8hp@#Z*xuP<|0l)HpM6-3tRN5@Ynb(Vt=f*O|%zd1M;m+K1J-N#&e zcf98Mmmw=MD5Z!s_ruPK|I3Cg2a&b<{r7*+3_OT8$6;95koIq@Bfqk53 z$2^d2`uYEG;c7mXk6y|~aItcO|5;&cbk8~b5GuuL0Ki%MIgAP)$t+r5fd>>Xo?eB~ z5ows0G+P5xXZo;H7RycDn6&RM7>EEu2q>lC2#0T`>|pM)k65I$SI=YNi9a+3>aqCb zmJMCfcb#^HrOq;c%so6`o!yO63KMU_?n&EN&-DQ}ckBMUp-@FN3^H{iEAyY4`58JQ zu2TmGivH$!EKPeFo+rZyArRdnW*nTydN06B3)2tWr$Pk)3KVP`|MyNq)oHK@NaI|k zQ!zPig65zdavkO!c|;S7F$GT6H{;+uh>fFyJz6^Q8N{1^bXvB%YQm;%H7%{%f49CDoos>!hvaYw@K*VgHRyuQ;hmbYX@#?-8PHKXk* zrHC>{V*7-TSU?{DP(5&L0j@Nev^d%rg>AQNy!NkRS3h5 zi61k#dJ(FB+21~hYN=@EKwI`9{L@27Viq^px)o$ zwJ$`CwXy@#56nhwlMC8S73`U`9Y)UBJy|F1#@?lWlw$sO3sF$_v*t>1Q#X<<1F`Am z_X4&LJfE3vb4bi66tj8M^Hlxb8eb zLJ0mgZY}OjoT2{W1#oRP=3-|F?mRFD*;U5_(t4FTD^MX-`TVoFt_JKyK?uQv#}?qV z3vV@lgK9`+&`jnA08T~3vnQ9~-@bb?ARS^~a)h1> z^H|k!+<9OQt~3;D5-5c#n3Z@JHjG=_Bgqrj6ITxa+!%cWKAW%^M$V|QP68+3Qr(Zp z+&>eqoPU!=+ntCDAIn*UXHUKWXX69L-kJJ;Yhzx*Q>jn%*zs^ZdGh^?(bzp{8y*?* zfW~?e5D-G}Mafr~mNgr1{qSzP18F_$)$?!R*8O+m2gi>Z7e^_Di8EpQ_)QoWozbJm z)T3Z|tk*+>e~V9U-hfC$geLal2nQE+VcV6R$lN~@e?I*(M7PMu@sY8<{lj~hmNgrH zpO@?gA=-Mmq)3=Q_@(}pqnvILuby9n{S}9xPz4;}Gy%Fo6+|1O@YW5lASonKb3q5K#cIiV{W^9$4rG3b zg(;81=@!-ai&BcEY0qk4`mXu}7JvVLM;xd;(ky5VO6%0ZxmkEK4mO^D(~QrhxUhWm z^VmM&BMh_*=+$HA0-*sp!eMUGEF8%E91Bw(g-db4Av-X>-%V=c5<;-0@H3pWoyM}f z|HPDiGw^xI9u?4*I-xVrwH4d(9N+CQ9jZKvr*oI$L`|;d^FNMwN^=2EMqR~cNa?*AhbH073UE7qX3;#3jU4$FL zn{~X$y%R#Pqxeg_ap4`v)D3}u@UZY^jBT9phf<13apO^&=RgQ=!S7QZ!|Z`GyS*;e zr@``8T6n7F{qEAesFmvwB3K%&9FO7FVVZSzmh6I45wW{;FY4U&*f(V-WW|k2sjA&G z<#25nbD7Gha@DjCEVl%S%8kELT8h*5vp7+65>duTEEqOlwUnu0!yqYtE|~ahw_CES zPiR62BQav*xc|m=5orPI#se0trkdXWHY8dSu&?|Ob{6l#jWOC^ow7-G)JQgj2q6$S z0pY}o2tzooprlhVU^k?7ut|2zI5ZETds z)z1PMsy+o4aP9uw=!j8&cyjpTaC%K<4*(}@CpDGkroR*$NQyK@z$#c(c&elt*j-l< zVF*{(85f;_(^t>ouKv@I88;EXiX7di5|2IwmbVpYT87Aox*n_2U&O`Qe4MSlpy{MQ zRHCInN*aH2nWD(9pr? zXNti{Ybr*D56AV9>CMUt-R_ndSiKwPAbWB7*TQLbnd|3k1oE8+CGByDi^@Z>G0e%SsO)g%#<`+86&2AcVIB zT+NADtd5)@$0Ndj*#1DiEo;OR9^nRS`|YPr9)sh+#Nt2(783)DiGjt$#K2-=Sz%vR zgWK8cex$b~#l;$_wIqeIX<%B^8T+|wMN%KK@F7SJNd%j4Rfk})2}qbL?4GNJ5J)A( zsNVc{5Fr3gRhAL^#3FDZp$x3PE_YQWYBp^EXEOKtu_%&%h?x10>jEs6751oaX5|JZ zCI%K01B;1)#l*z2!d4?@swkj}()&k36$Vz&oM1vC`ak~{5T5USu-u9+N(}(JrPE86 zEBNmS2&|K}o$NgciTC?1y_$7!n@M0$5i~S3AUiu7xw*M;I-OXuWQn?*Qi=r&7U1gD ztDS}^H8mA~FTL~gDAu=+u)B7zhE`}(I zFquq2-^PMop<2IwJq{l}44&szsF)a79fpchils}JqPn^od}F4_pkiQk7$^@w&ph)C z>g(%!JkHTu@~w-?yC+`Sym>R~>gsybN_j)=cDr?d1DS4IVSau-_U_#ap@#?j@jQ=3 zixwd@H5DAk>BjZy=3e3Y_3K-Osv}m!lR+x2+%8fS1z&&twI-g^gT)GZuqdS{C@4U2 zadE3aQA%O8TCr-?D!;eguU|i`Tel7`yzl}fNoo;LN-0jAI)yD;wy2hhS*#!_E$rE| z$7k(-$kWMhzWF9zfBkhtM@ReJ@g9f<4<3vS8#cgfZaxDaAp{)9VaJXgC@n2zlY#_G z&<+5{k0195hN37~w{9ImLqj|45WLa*pgGEt9VPRpHx>Z65Dk>^`LYa_|z`zOyR~H!>*<#G&fne?0 zwOzI^N-4H%*#fuQ?b9-)rKK^jf*~+}4+{%ZtLiN+udJ-Z2OoUUDOctBjwp)QvuBUb z5vj5)BRM&lffX#8GiQ$P2H-dxJ9Z53zyE%x_900Uo__jia2)3oPDN1|Si$5DZ@THG zR-dxrI1bs_*;u@IF;1O2<(KvHUbc1XRy_ISlj=Y|U-4CPa&nhEl+h==g`WY@$vB(G-wb2s6%iH3JOqBQGueOB8Z}>j<@m! zVM9X$R<2x$q@*OjGEN3oAF*1JBw_yi`K{i@@vcjGn#B`{drB4@$Eknq0VFLg4NpDw z6!aq9SS(5@E?v5W6)RT2Xf*bJsKv0`?bx_+BMb%u8#WV+8DdIG3Lbv=VThvGV};84 z`g*Khy;@gL=?W|lNST?Lc;JBt(9qD(BTy}0z8nb&3A($*qr2H=JaN>SGiUJXtFOXr zHh1YWhMwkhxm%%$YNPF>l_yE*U_` z!0Llm%cIMeFJsrPT{v>&2n+@T42Jgow64qLf-K9JFku4jx#u4A@84hdzlx!2i`BIA z-EKGX^74?AlY_jxJk;0Mt3^-ma)al;$z;NqF=LRCk%6&e$7*J((Yx4;er>>Jx7*=z zxlmtU58&D`96=CZwOUmv$h=3AGN>3>e#6yO>oTyIm^!Kd2dB8X#Xwo9hyVZp07*qo IM6N<$f-2Bwvj6}9 From a5c4dd974396d1dc9c3e55381215a904fa997cd2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 12:35:05 -0400 Subject: [PATCH 0506/2835] find: New subcommand. --- CmdLine.hs | 3 +++ Command.hs | 7 +++++++ Command/Find.hs | 24 ++++++++++++++++++++++++ debian/changelog | 6 ++++++ doc/git-annex.mdwn | 7 +++++++ 5 files changed, 47 insertions(+) create mode 100644 Command/Find.hs diff --git a/CmdLine.hs b/CmdLine.hs index a683be5c53..caf7279463 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -28,6 +28,7 @@ import qualified Command.Fsck import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit +import qualified Command.Find subCmds :: [SubCommand] subCmds = @@ -61,6 +62,8 @@ subCmds = "fix up symlinks to point to annexed content" , SubCommand "fsck" maybepath Command.Fsck.seek "check for problems" + , SubCommand "find" maybepath Command.Find.seek + "lists available files" ] where path = "PATH ..." diff --git a/Command.hs b/Command.hs index 40a21cacc8..60d82d09fb 100644 --- a/Command.hs +++ b/Command.hs @@ -158,6 +158,13 @@ withAll w a params = do w a [Git.workTree g] else w a params +{- Provides a default parameter to a with search. -} +withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings +withDefault d w a params = do + if null params + then w a [d] + else w a params + {- filter out files from the state directory -} notState :: FilePath -> Bool notState f = stateLoc /= take (length stateLoc) f diff --git a/Command/Find.hs b/Command/Find.hs new file mode 100644 index 0000000000..db0589feae --- /dev/null +++ b/Command/Find.hs @@ -0,0 +1,24 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Find where + +import Control.Monad (when) +import Control.Monad.State (liftIO) + +import Command +import Core + +seek :: [SubCmdSeek] +seek = [withDefault "." withFilesInGit start] + +{- Output a list of files. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, _) -> do + exists <- inAnnex key + when (exists) $ liftIO $ putStrLn file + return Nothing diff --git a/debian/changelog b/debian/changelog index 64973557ff..07eceae6cd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.07) UNRELEASED; urgency=low + + * find: New subcommand. + + -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 + git-annex (0.06) unstable; urgency=low * fsck: Check if annex.numcopies is satisfied. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 61a5962f1f..c496ca0d8c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -166,6 +166,13 @@ Many git-annex subcommands will stage changes for later `git commit` by you. With parameters, only the specified files are checked. +* find [path ...] + + Outputs a list of annexed files whose content is currently present. + + With no parameters, defaults to finding all files in the current directory + and its subdirectories. + # OPTIONS * --force From 1af400b98be0033d3c31f68b909368ba9b1ac469 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:07:58 -0400 Subject: [PATCH 0507/2835] add --- doc/forum.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/forum.mdwn diff --git a/doc/forum.mdwn b/doc/forum.mdwn new file mode 100644 index 0000000000..ce5787adcb --- /dev/null +++ b/doc/forum.mdwn @@ -0,0 +1,3 @@ +This is a place to discuss using git-annex. If you need help, advice, or anything, post about it here. + +[[!inline pages="forum/* and !*/Discussion" archive=yes rootpage=forum postformtext="Add a new thread titled:"]] From f078cea3f62f0424db6d6252373a16c8e48b7179 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:08:20 -0400 Subject: [PATCH 0508/2835] link forum --- doc/contact.mdwn | 2 +- doc/index.mdwn | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/contact.mdwn b/doc/contact.mdwn index 1238ca0403..1f122fdad9 100644 --- a/doc/contact.mdwn +++ b/doc/contact.mdwn @@ -1,4 +1,4 @@ Joey Hess is the author of git-annex. The [VCS-home mailing list](http://lists.madduck.net/listinfo/vcs-home) -is a good place to discuss it. +is a good place to discuss it, or use the [[forum]]. diff --git a/doc/index.mdwn b/doc/index.mdwn index 439e2da09f..d863a9ede9 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -21,6 +21,7 @@ To get a feel for it, see the [[walkthrough]]. * [[news]] * [[bugs]] * [[todo]] +* [[forum]] * [[contact]] """]] From 8542ca49fe724d46fc10609fa424afbb2e12ed95 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 14 Nov 2010 18:09:44 +0000 Subject: [PATCH 0509/2835] --- doc/forum/git-annex_on_OSX.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/forum/git-annex_on_OSX.mdwn diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn new file mode 100644 index 0000000000..85a8413bf9 --- /dev/null +++ b/doc/forum/git-annex_on_OSX.mdwn @@ -0,0 +1,19 @@ +
+sudo port install haskell-platform git-core ossp-uuid md5sha1sum
+
+[waits forever…]
+[finished]
+[realizes missingh isn't working in MacPorts]
+
+sudo cabal  update
+sudo cabal install missingh
+sudo cabal install utf8-string
+
+git clone  git://git.kitenet.net/git-annex
+
+cd git-annex
+make
+sudo install git-annex /usr/local/bin/ # makefile is weird
+
+ +Originally posted by Jon at --[[Joey]] From 1277cd7669445bf12e7ae59982fb2d07cd323409 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 14 Nov 2010 18:12:08 +0000 Subject: [PATCH 0510/2835] --- doc/users/joey.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/users/joey.mdwn diff --git a/doc/users/joey.mdwn b/doc/users/joey.mdwn new file mode 100644 index 0000000000..306e1cc768 --- /dev/null +++ b/doc/users/joey.mdwn @@ -0,0 +1,2 @@ +Joey Hess + From 10f30cf6383167e72ced5c6138ee6857b3fd63eb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:14:27 -0400 Subject: [PATCH 0511/2835] add --- doc/users.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/users.mdwn diff --git a/doc/users.mdwn b/doc/users.mdwn new file mode 100644 index 0000000000..b9bab48ecf --- /dev/null +++ b/doc/users.mdwn @@ -0,0 +1,9 @@ +Users of this wiki, feel free to create a subpage of this one and talk +about yourself on it, within reason. You can link to it to sign your +comments. + +List of users +============= +[[!inline pages="users/* and !users/*/* and !*/Discussion" +feeds=no archive=yes sort=title template=titlepage +rootpage="users" postformtext="Add yourself as an git-annex user:"]] From 0e55d6a907a39c3b7239268261edc2d5b5f55caf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:44:24 -0400 Subject: [PATCH 0512/2835] move stuff out of Core --- Command/Init.hs | 40 +++++++++++++++++++++++- Core.hs | 81 +------------------------------------------------ Upgrade.hs | 63 ++++++++++++++++++++++++++++++++++++++ git-annex.hs | 3 +- 4 files changed, 105 insertions(+), 82 deletions(-) create mode 100644 Upgrade.hs diff --git a/Command/Init.hs b/Command/Init.hs index 8110948a41..c928647a50 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -9,14 +9,15 @@ module Command.Init where import Control.Monad.State (liftIO) import Control.Monad (when) +import System.Directory import Command import qualified Annex -import Core import qualified GitRepo as Git import UUID import Version import Messages +import Locations seek :: [SubCmdSeek] seek = [withString start] @@ -46,3 +47,40 @@ cleanup = do liftIO $ Git.run g ["add", logfile] liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] return True + +{- configure git to use union merge driver on state files, if it is not + - already -} +gitAttributes :: Git.Repo -> IO () +gitAttributes repo = do + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + when (all (/= attrLine) (lines content)) $ do + appendFile attributes $ attrLine ++ "\n" + commit + where + attrLine = stateLoc ++ "*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] + +{- set up a git pre-commit hook, if one is not already present -} +gitPreCommitHook :: Git.Repo -> IO () +gitPreCommitHook repo = do + let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ + "/hooks/pre-commit" + exists <- doesFileExist hook + if (exists) + then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" + else do + writeFile hook $ "#!/bin/sh\n" ++ + "# automatically configured by git-annex\n" ++ + "git annex pre-commit .\n" + p <- getPermissions hook + setPermissions hook $ p {executable = True} diff --git a/Core.hs b/Core.hs index 9faaada56f..2928dc06df 100644 --- a/Core.hs +++ b/Core.hs @@ -26,7 +26,6 @@ import qualified Annex import qualified Backend import Utility import Messages -import Version {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). @@ -46,11 +45,10 @@ tryRun' state errnum (a:as) = do tryRun' _ errnum [] = when (errnum > 0) $ error $ show errnum ++ " failed" -{- Sets up a git repo for git-annex. -} +{- Actions to perform each time ran. -} startup :: Annex Bool startup = do prepUUID - autoUpgrade return True {- When git-annex is done, it runs this. -} @@ -71,43 +69,6 @@ shutdown = do return True -{- configure git to use union merge driver on state files, if it is not - - already -} -gitAttributes :: Git.Repo -> IO () -gitAttributes repo = do - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - commit - else do - content <- readFile attributes - when (all (/= attrLine) (lines content)) $ do - appendFile attributes $ attrLine ++ "\n" - commit - where - attrLine = stateLoc ++ "*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] - -{- set up a git pre-commit hook, if one is not already present -} -gitPreCommitHook :: Git.Repo -> IO () -gitPreCommitHook repo = do - let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ - "/hooks/pre-commit" - exists <- doesFileExist hook - if (exists) - then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" - else do - writeFile hook $ "#!/bin/sh\n" ++ - "# automatically configured by git-annex\n" ++ - "git annex pre-commit .\n" - p <- getPermissions hook - setPermissions hook $ p {executable = True} - {- Checks if a given key is currently present in the annexLocation. -} inAnnex :: Key -> Annex Bool inAnnex key = do @@ -237,43 +198,3 @@ getKeysReferenced = do files <- liftIO $ Git.inRepo g $ Git.workTree g keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs - -{- Uses the annex.version git config setting to automate upgrades. -} -autoUpgrade :: Annex () -autoUpgrade = do - version <- getVersion - case version of - Just "0" -> upgradeFrom0 - Nothing -> return () -- repo not initted yet, no version - Just v | v == currentVersion -> return () - Just _ -> error "this version of git-annex is too old for this git repository!" - -upgradeFrom0 :: Annex () -upgradeFrom0 = do - showSideAction "Upgrading object directory layout..." - g <- Annex.gitRepo - - -- do the reorganisation of the files - let olddir = annexDir g - keys <- getKeysPresent' olddir - _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys - - -- update the symlinks to the files - files <- liftIO $ Git.inRepo g $ Git.workTree g - fixlinks files - Annex.queueRun - - setVersion - - where - fixlinks [] = return () - fixlinks (f:fs) = do - r <- Backend.lookupFile f - case r of - Nothing -> return () - Just (k, _) -> do - link <- calcGitLink f k - liftIO $ removeFile f - liftIO $ createSymbolicLink link f - Annex.queue "add" ["--"] f - fixlinks fs diff --git a/Upgrade.hs b/Upgrade.hs new file mode 100644 index 0000000000..d64d5287d1 --- /dev/null +++ b/Upgrade.hs @@ -0,0 +1,63 @@ +{- git-annex upgrade support + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade where + +import System.Directory +import Control.Monad.State (liftIO) +import System.Posix.Files + +import Core +import Types +import Locations +import qualified GitRepo as Git +import qualified Annex +import qualified Backend +import Messages +import Version + +{- Uses the annex.version git config setting to automate upgrades. -} +upgrade :: Annex Bool +upgrade = do + version <- getVersion + case version of + Just "0" -> upgradeFrom0 + Nothing -> return True -- repo not initted yet, no version + Just v | v == currentVersion -> return True + Just _ -> error "this version of git-annex is too old for this git repository!" + +upgradeFrom0 :: Annex Bool +upgradeFrom0 = do + showSideAction "Upgrading object directory layout..." + g <- Annex.gitRepo + + -- do the reorganisation of the files + let olddir = annexDir g + keys <- getKeysPresent' olddir + _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys + + -- update the symlinks to the files + files <- liftIO $ Git.inRepo g $ Git.workTree g + fixlinks files + Annex.queueRun + + setVersion + + return True + + where + fixlinks [] = return () + fixlinks (f:fs) = do + r <- Backend.lookupFile f + case r of + Nothing -> return () + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" ["--"] f + fixlinks fs diff --git a/git-annex.hs b/git-annex.hs index 098ccac2d4..d111156f01 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -9,6 +9,7 @@ import System.Environment import qualified Annex import Core +import Upgrade import CmdLine import qualified GitRepo as Git import BackendList @@ -19,4 +20,4 @@ main = do gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends (configure, actions) <- parseCmd args state - tryRun state $ [startup] ++ configure ++ actions ++ [shutdown] + tryRun state $ [startup, upgrade] ++ configure ++ actions ++ [shutdown] From 27ffa0e86f7c5537888563cdda77be45b2700478 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:58:42 -0400 Subject: [PATCH 0513/2835] comment --- Command.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command.hs b/Command.hs index 60d82d09fb..09c935b3bf 100644 --- a/Command.hs +++ b/Command.hs @@ -158,7 +158,7 @@ withAll w a params = do w a [Git.workTree g] else w a params -{- Provides a default parameter to a with search. -} +{- Provides a default parameter to act on if none is specified. -} withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings withDefault d w a params = do if null params From 0b57b68918cc9ba84196df188c76bfb92eb1b549 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 17:56:12 -0400 Subject: [PATCH 0514/2835] deal with OSX case brokenness HELLO, it's 2010.. POSIX calling.. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c94fbbc8f2..77e7a5edb8 100644 --- a/Makefile +++ b/Makefile @@ -32,4 +32,4 @@ clean: rm -rf build git-annex git-annex.1 test rm -rf doc/.ikiwiki html -.PHONY: git-annex test +.PHONY: git-annex test install From e87d77964445bb7045d4a82350ec9410457d62a8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 17:56:49 -0400 Subject: [PATCH 0515/2835] should work now --- doc/forum/git-annex_on_OSX.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn index 85a8413bf9..44c275ff1c 100644 --- a/doc/forum/git-annex_on_OSX.mdwn +++ b/doc/forum/git-annex_on_OSX.mdwn @@ -13,7 +13,7 @@ git clone git://git.kitenet.net/git-annex cd git-annex make -sudo install git-annex /usr/local/bin/ # makefile is weird +sudo make install Originally posted by Jon at --[[Joey]] From a51b364a354a3817298a9b0dfef4016850c175de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 13:00:43 -0400 Subject: [PATCH 0516/2835] trim exports --- LocationLog.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 9a9dad1333..1ddbf3c0bb 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -23,9 +23,7 @@ module LocationLog ( LogStatus(..), logChange, - keyLocations, - logFile, - readLog + keyLocations ) where import Data.Time.Clock.POSIX From 0e1e32cb6b6f5fcd9e535d171c639f3c1663a6ce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 13:13:01 -0400 Subject: [PATCH 0517/2835] problem --- doc/todo/branching.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index 81f230a92a..ebcc56aa62 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -46,6 +46,7 @@ Let's use one branch per uuid, named git-annex/$UUID. - TODO: What will need to be done to git to make it push/pull these new branches? - A given repo only ever writes to its UUID branch. So no conflicts. + - **problem**: git annex move needs to update log info for other repos! - (BTW, UUIDs probably don't compress well, and this reduces the bloat of having them repeated lots of times in the tree.) - Per UUID branches mean that if it wants to find a file's location @@ -57,7 +58,7 @@ can cache location log changes and write them all to locationlog in a single git operation on shutdown. - TODO: what if it's ctrl-c'd with changes pending? Perhaps it should - collect them to ,git/annex/locationlog, and inject that file on shutdown? + collect them to .git/annex/locationlog, and inject that file on shutdown? - This will be less overhead than the current staging of all the log files. The log is not appended to, so in git we have a series of commits each of From 40ae21c895b095409ae9af3c44ab9a6b63ea10bb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 14:00:28 -0400 Subject: [PATCH 0518/2835] this is looking more and more problimatic --- doc/todo/branching.mdwn | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index ebcc56aa62..c9707c6f9d 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -52,6 +52,11 @@ Let's use one branch per uuid, named git-annex/$UUID. - Per UUID branches mean that if it wants to find a file's location amoung configured remotes, it can examine only their branches, if desired. +- It's important that the per-repo branches propigate beyond immediate + remotes. If there is a central bare repo, that means push --all. Without + one, it means that when repo B pulls from A, and then C pulls from B, + C needs to get A's branch -- which means that B should have a tracking + branch for A's branch. In the branch, only one file is needed. Call it locationlog. git-annex can cache location log changes and write them all to locationlog in @@ -73,3 +78,18 @@ with a indication of the presense/absense of the key is found. - It could get pretty slow when digging deeper. - Only 3 places in git-annex will be affected by any slowdown: move --from, get and drop. + +## alternate + +As above, but use a single git-annex branch, and keep the per-UUID +info in their own log files. Hope that git can auto-merge as long as +each observing repo only writes to its own files. (Well, it can, but for +non-fast-forward merges, the git-annex branch would need to be checked out, +which is problimatic.) + +Use filenames like: + + / + +That allows one repo to record another's state when doing a +`move`. From c305fb2bf318e72ea4f905a38733d93e9a81cc1d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 15:21:11 -0400 Subject: [PATCH 0519/2835] thought --- doc/todo/branching.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index c9707c6f9d..b7441c6e4e 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -93,3 +93,15 @@ Use filenames like: That allows one repo to record another's state when doing a `move`. + +## outside the box approach + +If the problem is limited to only that the `.git-annex/` files make +branching difficult (and not to the related problem that commits to them +and having them in the tree are sorta annoying), then a simple approach +would be to have git-annex look in other branches for location log info +too. + +The problem would then be that any locationlog lookup would need to look in +all other branches (any branch could have more current info after all), +which could get expensive. From 9dc43d25995b085745c16cc31ad11106a74a9973 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 16:35:06 -0400 Subject: [PATCH 0520/2835] unused: New subcommand, finds unused data (the global part of fsck). --- CmdLine.hs | 4 +++ Command/Fsck.hs | 52 ++---------------------------------- Command/Unused.hs | 66 ++++++++++++++++++++++++++++++++++++++++++++++ debian/changelog | 1 + doc/git-annex.mdwn | 5 ++++ 5 files changed, 78 insertions(+), 50 deletions(-) create mode 100644 Command/Unused.hs diff --git a/CmdLine.hs b/CmdLine.hs index caf7279463..cc163fff52 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -25,6 +25,7 @@ import qualified Command.SetKey import qualified Command.Fix import qualified Command.Init import qualified Command.Fsck +import qualified Command.Unused import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit @@ -62,6 +63,8 @@ subCmds = "fix up symlinks to point to annexed content" , SubCommand "fsck" maybepath Command.Fsck.seek "check for problems" + , SubCommand "unused" nothing Command.Unused.seek + "look for unused file content" , SubCommand "find" maybepath Command.Find.seek "lists available files" ] @@ -70,6 +73,7 @@ subCmds = maybepath = "[PATH ...]" key = "KEY ..." desc = "DESCRIPTION" + nothing = "" -- Each dashed command-line option results in generation of an action -- in the Annex monad that performs the necessary setting. diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 5b731a6968..02b66d01ad 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -14,55 +14,7 @@ import Types import Core import Messages import qualified Command.FsckFile +import qualified Command.Unused seek :: [SubCmdSeek] -seek = [withNothing start, withAll withFilesInGit Command.FsckFile.start] - -{- Checks the whole annex for problems, only if specific files were not - - specified. -} -start :: SubCmdStartNothing -start = do - showStart "fsck" "" - return $ Just perform - -perform :: SubCmdPerform -perform = do - ok <- checkUnused - if ok - then return $ Just $ return True - else return Nothing - -checkUnused :: Annex Bool -checkUnused = do - showNote "checking for unused data..." - unused <- unusedKeys - if (null unused) - then return True - else do - showLongNote $ w unused - return False - where - w u = unlines $ [ - "Some annexed data is no longer pointed to by any files in the repository.", - "If this data is no longer needed, it can be removed using git-annex dropkey:" - ] ++ map (\k -> " " ++ show k) u - -{- Finds keys whose content is present, but that do not seem to be used - - by any files in the git repo. -} -unusedKeys :: Annex [Key] -unusedKeys = do - present <- getKeysPresent - referenced <- getKeysReferenced - - -- Constructing a single map, of the set that tends to be smaller, - -- appears more efficient in both memory and CPU than constructing - -- and taking the M.difference of two maps. - let present_m = existsMap present - let unused_m = remove referenced present_m - return $ M.keys unused_m - where - remove [] m = m - remove (x:xs) m = remove xs $ M.delete x m - -existsMap :: Ord k => [k] -> M.Map k Int -existsMap l = M.fromList $ map (\k -> (k, 1)) l +seek = [withNothing Command.Unused.start, withAll withFilesInGit Command.FsckFile.start] diff --git a/Command/Unused.hs b/Command/Unused.hs new file mode 100644 index 0000000000..ed3de5d575 --- /dev/null +++ b/Command/Unused.hs @@ -0,0 +1,66 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unused where + +import qualified Data.Map as M + +import Command +import Types +import Core +import Messages + +seek :: [SubCmdSeek] +seek = [withNothing start] + +{- Finds unused content in the annex. -} +start :: SubCmdStartNothing +start = do + showStart "unused" "" + return $ Just perform + +perform :: SubCmdPerform +perform = do + ok <- checkUnused + if ok + then return $ Just $ return True + else return Nothing + +checkUnused :: Annex Bool +checkUnused = do + showNote "checking for unused data..." + unused <- unusedKeys + if (null unused) + then return True + else do + showLongNote $ w unused + return False + where + w u = unlines $ [ + "Some annexed data is no longer pointed to by any files in the repository.", + "If this data is no longer needed, it can be removed using git-annex dropkey:" + ] ++ map (\k -> " " ++ show k) u + +{- Finds keys whose content is present, but that do not seem to be used + - by any files in the git repo. -} +unusedKeys :: Annex [Key] +unusedKeys = do + present <- getKeysPresent + referenced <- getKeysReferenced + + -- Constructing a single map, of the set that tends to be smaller, + -- appears more efficient in both memory and CPU than constructing + -- and taking the M.difference of two maps. + let present_m = existsMap present + let unused_m = remove referenced present_m + return $ M.keys unused_m + where + remove [] m = m + remove (x:xs) m = remove xs $ M.delete x m + +existsMap :: Ord k => [k] -> M.Map k Int +existsMap l = M.fromList $ map (\k -> (k, 1)) l diff --git a/debian/changelog b/debian/changelog index 07eceae6cd..dcdbe15e27 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.07) UNRELEASED; urgency=low * find: New subcommand. + * unused: New subcommand, finds unused data (the global part of fsck). -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index c496ca0d8c..a522534da8 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -166,6 +166,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. With parameters, only the specified files are checked. +* unused + + Checks the annex for data that is not used by any files currently + in the annex, and prints a report. + * find [path ...] Outputs a list of annexed files whose content is currently present. From 748a7475bb99e1127dc12bb2cc9d5653e4648200 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:04:19 -0400 Subject: [PATCH 0521/2835] dropunused --- Backend.hs | 13 +++++++--- CmdLine.hs | 4 +++ Command.hs | 2 ++ Command/Fsck.hs | 5 ---- Command/Unused.hs | 24 +++++++++++++---- Locations.hs | 9 +++++-- debian/changelog | 2 ++ doc/git-annex.mdwn | 61 ++++++++++++++++++++++++-------------------- doc/walkthrough.mdwn | 10 +++++--- 9 files changed, 84 insertions(+), 46 deletions(-) diff --git a/Backend.hs b/Backend.hs index 14af56bbfa..2f0f71d749 100644 --- a/Backend.hs +++ b/Backend.hs @@ -25,7 +25,8 @@ module Backend ( hasKey, fsckKey, lookupFile, - chooseBackends + chooseBackends, + keyBackend ) where import Control.Monad.State @@ -111,8 +112,8 @@ removeKey backend key = (Internals.removeKey backend) key {- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool hasKey key = do - bs <- Annex.supportedBackends - (Internals.hasKey (lookupBackendName bs $ backendName key)) key + backend <- keyBackend key + (Internals.hasKey backend) key {- Checks a key's backend for problems. -} fsckKey :: Backend -> Key -> Annex Bool @@ -154,3 +155,9 @@ chooseBackends fs = do bs <- Annex.supportedBackends pairs <- liftIO $ Git.checkAttr g "git-annex-backend" fs return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs + +{- Returns the backend to use for a key. -} +keyBackend :: Key -> Annex Backend +keyBackend key = do + bs <- Annex.supportedBackends + return $ lookupBackendName bs $ backendName key diff --git a/CmdLine.hs b/CmdLine.hs index cc163fff52..35e889d7df 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -26,6 +26,7 @@ import qualified Command.Fix import qualified Command.Init import qualified Command.Fsck import qualified Command.Unused +import qualified Command.DropUnused import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit @@ -65,6 +66,8 @@ subCmds = "check for problems" , SubCommand "unused" nothing Command.Unused.seek "look for unused file content" + , SubCommand "dropunused" number Command.DropUnused.seek + "drop unused file content" , SubCommand "find" maybepath Command.Find.seek "lists available files" ] @@ -73,6 +76,7 @@ subCmds = maybepath = "[PATH ...]" key = "KEY ..." desc = "DESCRIPTION" + number = "NUMBER ..." nothing = "" -- Each dashed command-line option results in generation of an action diff --git a/Command.hs b/Command.hs index 09c935b3bf..c6d2d0d5d4 100644 --- a/Command.hs +++ b/Command.hs @@ -129,6 +129,8 @@ backendPairs a files = do return $ map a pairs withString :: SubCmdSeekStrings withString a params = return [a $ unwords params] +withStrings :: SubCmdSeekStrings +withStrings a params = return $ map a params withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 02b66d01ad..a72d753fa0 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -7,12 +7,7 @@ module Command.Fsck where -import qualified Data.Map as M - import Command -import Types -import Core -import Messages import qualified Command.FsckFile import qualified Command.Unused diff --git a/Command/Unused.hs b/Command/Unused.hs index ed3de5d575..7a34d393cd 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -7,12 +7,15 @@ module Command.Unused where +import Control.Monad.State (liftIO) import qualified Data.Map as M import Command import Types import Core import Messages +import Locations +import qualified Annex seek :: [SubCmdSeek] seek = [withNothing start] @@ -37,13 +40,24 @@ checkUnused = do if (null unused) then return True else do - showLongNote $ w unused + let list = number 1 unused + g <- Annex.gitRepo + liftIO $ writeFile (annexUnusedLog g) $ unlines $ + map (\(n, k) -> show n ++ " " ++ show k) list + showLongNote $ w list return False where - w u = unlines $ [ - "Some annexed data is no longer pointed to by any files in the repository.", - "If this data is no longer needed, it can be removed using git-annex dropkey:" - ] ++ map (\k -> " " ++ show k) u + w u = unlines $ + ["Some annexed data is no longer pointed to by any files in the repository:", + " NUMBER KEY"] + ++ (map (\(n, k) -> " " ++ (pad 6 $ show n) ++ " " ++ show k) u) ++ + ["(To see where data was previously used, try: git log --stat -S'KEY')", + "(To remove unwanted data: git-annex dropunused NUMBER)"] + pad n s = s ++ replicate (n - length s) ' ' + +number :: Integer -> [a] -> [(Integer, a)] +number _ [] = [] +number n (x:xs) = (n, x):(number (n+1) xs) {- Finds keys whose content is present, but that do not seem to be used - by any files in the git repo. -} diff --git a/Locations.hs b/Locations.hs index c3bab285d4..24ccc75c6d 100644 --- a/Locations.hs +++ b/Locations.hs @@ -14,6 +14,7 @@ module Locations ( annexLocationRelative, annexTmpLocation, annexBadLocation, + annexUnusedLog, annexDir, annexObjectDir, @@ -56,14 +57,18 @@ annexDir r = Git.workTree r ++ "/.git/annex" annexObjectDir :: Git.Repo -> FilePath annexObjectDir r = annexDir r ++ "/objects" -{- .git-annex/tmp is used for temp files -} +{- .git-annex/tmp/ is used for temp files -} annexTmpLocation :: Git.Repo -> FilePath annexTmpLocation r = annexDir r ++ "/tmp/" -{- .git-annex/bad is used for bad files found during fsck -} +{- .git-annex/bad/ is used for bad files found during fsck -} annexBadLocation :: Git.Repo -> FilePath annexBadLocation r = annexDir r ++ "/bad/" +{- .git/annex/unused is used to number possibly unused keys -} +annexUnusedLog :: Git.Repo -> FilePath +annexUnusedLog r = annexDir r ++ "/unused" + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid diff --git a/debian/changelog b/debian/changelog index dcdbe15e27..cffa7bb261 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,8 @@ git-annex (0.07) UNRELEASED; urgency=low * find: New subcommand. * unused: New subcommand, finds unused data (the global part of fsck). + * dropunused: New subcommand, provides for easy dropping of unused keys + by number, as listed by unused subcommand. -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index a522534da8..618ddf2039 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -112,6 +112,32 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Use this to undo an unlock command if you don't want to modify the files, or have made modifications you want to discard. +* fsck [path ...] + + With no parameters, this subcommand checks the whole annex for consistency, + and warns about any problems found. + + With parameters, only the specified files are checked. + +* unused + + Checks the annex for data that is not used by any files currently + in the annex, and prints a numbered list of the data. + + (This is run as part of `git annex fsck`.) + +* dropunused [number ...] + + Drops the data corresponding to the numbers, as listed by the last + `git annex unused` or `git annex fsck` + +* find [path ...] + + Outputs a list of annexed files whose content is currently present. + + With no parameters, defaults to finding all files in the current directory + and its subdirectories. + * unannex [path ...] Use this to undo an accidental add command. This is not the command you @@ -159,25 +185,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git annex setkey --key=1287765018:3 /tmp/file -* fsck [path ...] - - With no parameters, this subcommand checks the whole annex for consistency, - and warns about any problems found. - - With parameters, only the specified files are checked. - -* unused - - Checks the annex for data that is not used by any files currently - in the annex, and prints a report. - -* find [path ...] - - Outputs a list of annexed files whose content is currently present. - - With no parameters, defaults to finding all files in the current directory - and its subdirectories. - # OPTIONS * --force @@ -194,14 +201,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Enable verbose logging. -* --backend=name - - Specifies the key-value backend to use when adding a file. - -* --key=name - - Specifies a key to operate on, for use with the addkey subcommand. - * --from=repository Specifies a repository that content will be retrieved from. @@ -212,6 +211,14 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a git repository that content will be sent to. It should be specified using the name of a configured git remote. +* --backend=name + + Specifies which key-value backend to use. + +* --key=name + + Specifies a key to operate on. + # CONFIGURATION Like other git commands, git-annex is configured via `git-config`. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 9a6ee220c2..1e07053aec 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -287,7 +287,7 @@ setting is satisfied for all files, and it warns about any dangling values in `.git/annex/objects/`. # git annex fsck - fsck (checking for unused data...) ok + unused (checking for unused data...) ok fsck my_cool_big_file (checksum...) ok ...... @@ -304,10 +304,12 @@ Fsck never deletes possibly bad data; instead it will be moved to might say about a badly messed up annex: # git annex fsck - fsck (checking for unused data...) + unused (checking for unused data...) Some annexed data is no longer pointed to by any files in the repository. - If this data is no longer needed, it can be removed using git-annex dropkey: - WORM:1289672605:3:file + NUMBER KEY + 1 WORM:1289672605:3:file + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused NUMBER) failed fsck my_cool_big_file (checksum...) Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 From 3a4e9398a1652b664fb17dc2072f4b85966dcb61 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:06:21 -0400 Subject: [PATCH 0522/2835] add --- Command/DropUnused.hs | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 Command/DropUnused.hs diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs new file mode 100644 index 0000000000..e8b8d43c4c --- /dev/null +++ b/Command/DropUnused.hs @@ -0,0 +1,46 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DropUnused where + +import Control.Monad.State (liftIO) +import qualified Data.Map as M + +import Command +import Types +import Messages +import Locations +import qualified Annex +import qualified Command.Drop +import Backend + +seek :: [SubCmdSeek] +seek = [withStrings start] + +{- Drops unused content by number. -} +start :: SubCmdStartString +start s = do + m <- readUnusedLog + case M.lookup s m of + Nothing -> return Nothing + Just key -> do + showStart "dropunused" s + backend <- keyBackend key + -- force drop, even if this is the only copy + Annex.flagChange "force" $ FlagBool True + return $ Just $ Command.Drop.perform key backend + +readUnusedLog :: Annex (M.Map String Key) +readUnusedLog = do + g <- Annex.gitRepo + l <- liftIO $ readFile (annexUnusedLog g) + return $ M.fromList $ map parse $ lines l + where + parse line = (head ws, tokey $ unwords $ tail ws) + where + ws = words line + tokey s = read s :: Key From 354be7a00b76b1ee961fa31ba81905e1d8c172bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:13:20 -0400 Subject: [PATCH 0523/2835] on dropunused and unused --- debian/changelog | 4 ++-- doc/walkthrough.mdwn | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index cffa7bb261..b6a426d1ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,9 @@ git-annex (0.07) UNRELEASED; urgency=low * find: New subcommand. - * unused: New subcommand, finds unused data (the global part of fsck). + * unused: New subcommand, finds unused data. (Split out from fsck.) * dropunused: New subcommand, provides for easy dropping of unused keys - by number, as listed by unused subcommand. + by number, as listed by the unused subcommand. -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 1e07053aec..b6f322c499 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -277,6 +277,34 @@ add something like this to `.gitattributes`: * git-annex-backend=SHA1 +## unused data + +It's possible for data to accumulate in the annex that no files point to +nymore. One way it can happen is if you `git rm` a file without +first calling `git annex drop`. And, when you modify an annexed file, the old +content of the file remains in the annex. + +This might be historical data you want to preserve, so git-annex defaults to +preserving it. So from time to time, you may want to check for such data and +eliminate it to save space. + + # git annex unused + unused (checking for unused data...) + Some annexed data is no longer pointed to by any files in the repository. + NUMBER KEY + 1 WORM:1289672605:3:file + 2 WORM:1289672605:14:file + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused NUMBER) + failed + +After running `git annex unused`, you can follow the instructions to examine +the history of files that used the data, and if you decide you don't need that +data anymore, you can easily remove it: + + # git annex dropunused 1 + dropunused 1 ok + ## fsck: verifying your data You can use the fsck subcommand to check for problems in your data. From d6f7ee7db46208d132f305ab651b0bf082be348e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:19:13 -0400 Subject: [PATCH 0524/2835] clearify --- doc/git-annex.mdwn | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 618ddf2039..42f80a3dc2 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -171,19 +171,30 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * dropkey [key ...] - Drops the annexed data for the specified keys from this repository. + This plumbing-level command drops the annexed data for the specified + keys from this repository. This can be used to drop content for arbitrary keys, which do not need to have a file in the git repository pointing at them. -* setkey file - - Sets the annxed data for a key to the content of the specified file, - and then removes the file. + A backend will typically need to be specified with --backend. If none + is specified, the first configured backend is used. Example: - git annex setkey --key=1287765018:3 /tmp/file + git annex dropkey --backend=SHA1 7da006579dd64330eb2456001fd01948430572f2 + +* setkey file + + This plumbing-level command sets the annxed data for a key to the content of + the specified file, and then removes the file. + + A backend will typically need to be specified with --backend. If none + is specified, the first configured backend is used. + + Example: + + git annex setkey --backend=WORM --key=1287765018:3 /tmp/file # OPTIONS From 11096c200f4050c1e4cdb4fbab9410055c6e1c02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:22:50 -0400 Subject: [PATCH 0525/2835] fsck no longer runs unused --- Command/Fsck.hs | 20 +++++++++++++++++--- doc/git-annex.mdwn | 4 +--- doc/walkthrough.mdwn | 12 ++---------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index a72d753fa0..07621ad4ef 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -8,8 +8,22 @@ module Command.Fsck where import Command -import qualified Command.FsckFile -import qualified Command.Unused +import qualified Backend +import Types +import Messages seek :: [SubCmdSeek] -seek = [withNothing Command.Unused.start, withAll withFilesInGit Command.FsckFile.start] +seek = [withFilesInGit start] + +{- Checks a file's backend data for problems. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + showStart "fsck" file + return $ Just $ perform key backend + +perform :: Key -> Backend -> SubCmdPerform +perform key backend = do + success <- Backend.fsckKey backend key + if (success) + then return $ Just $ return True + else return Nothing diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 42f80a3dc2..3df835eac5 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -124,12 +124,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Checks the annex for data that is not used by any files currently in the annex, and prints a numbered list of the data. - (This is run as part of `git annex fsck`.) - * dropunused [number ...] Drops the data corresponding to the numbers, as listed by the last - `git annex unused` or `git annex fsck` + `git annex unused` * find [path ...] diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index b6f322c499..a1a8882425 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -311,8 +311,7 @@ You can use the fsck subcommand to check for problems in your data. What can be checked depends on the [[backend|backends]] you've used to store the data. For example, when you use the SHA1 backend, fsck will verify that the checksums of your files are good. Fsck also checks that the annex.numcopies -setting is satisfied for all files, and it warns about any dangling values -in `.git/annex/objects/`. +setting is satisfied for all files. # git annex fsck unused (checking for unused data...) ok @@ -332,13 +331,6 @@ Fsck never deletes possibly bad data; instead it will be moved to might say about a badly messed up annex: # git annex fsck - unused (checking for unused data...) - Some annexed data is no longer pointed to by any files in the repository. - NUMBER KEY - 1 WORM:1289672605:3:file - (To see where data was previously used, try: git log --stat -S'KEY') - (To remove unwanted data: git-annex dropunused NUMBER) - failed fsck my_cool_big_file (checksum...) Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 ** No known copies of the file exist! @@ -346,4 +338,4 @@ might say about a badly messed up annex: fsck important_file Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. failed - git-annex: 3 failed + git-annex: 2 failed From a5e7f5329f4d9a80a34302e3707f0bd42b91a441 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:24:29 -0400 Subject: [PATCH 0526/2835] fix --- Command/Fsck.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 07621ad4ef..6291d5ba3f 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -13,7 +13,7 @@ import Types import Messages seek :: [SubCmdSeek] -seek = [withFilesInGit start] +seek = [withAll withFilesInGit start] {- Checks a file's backend data for problems. -} start :: SubCmdStartString From 0893820812c9cc10287c861dccbdae3c287cd7cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:37:49 -0400 Subject: [PATCH 0527/2835] fsck: Print warnings to stderr; --quiet can now be used to only see problems. --- Backend/File.hs | 4 ++-- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- Command/Unused.hs | 6 ++---- Messages.hs | 4 +++- debian/changelog | 2 ++ doc/walkthrough.mdwn | 10 +++++----- 7 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 8351778568..c67fb3ce33 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -175,12 +175,12 @@ checkKeyNumCopies key = do let present = length remotes + if inannex then 1 else 0 if (present < needed) then do - showLongNote $ note present needed + warning $ note present needed return False else return True where note 0 _ = "** No known copies of the file exist!" note present needed = "Only " ++ show present ++ " of " ++ show needed ++ - " copies exist. " ++ + " copies of "++show key++" exist. " ++ "Run git annex get somewhere else to back it up." diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 9e9000ba93..46667c9cda 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -57,5 +57,5 @@ checkKeySHA1 key = do then return True else do dest <- moveBad key - showLongNote $ "Bad file content; moved to "++dest + warning $ "Bad file content; moved to "++dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 3749899968..4e2177fed0 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -66,5 +66,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - showLongNote $ "Bad file size; moved to "++dest + warning $ "Bad file size; moved to "++dest return False diff --git a/Command/Unused.hs b/Command/Unused.hs index 7a34d393cd..ae189550ce 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -28,10 +28,8 @@ start = do perform :: SubCmdPerform perform = do - ok <- checkUnused - if ok - then return $ Just $ return True - else return Nothing + _ <- checkUnused + return $ Just $ return True checkUnused :: Annex Bool checkUnused = do diff --git a/Messages.hs b/Messages.hs index ed4f3b90a1..7dafa8284e 100644 --- a/Messages.hs +++ b/Messages.hs @@ -54,4 +54,6 @@ showErr :: (Show a) => a -> Annex () showErr e = warning $ show e warning :: String -> Annex () -warning s = liftIO $ hPutStrLn stderr $ "git-annex: " ++ s +warning s = do + verbose $ liftIO $ putStr "\n" + liftIO $ hPutStrLn stderr $ "git-annex: " ++ s diff --git a/debian/changelog b/debian/changelog index b6a426d1ff..a6d95ab69e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,8 @@ git-annex (0.07) UNRELEASED; urgency=low * unused: New subcommand, finds unused data. (Split out from fsck.) * dropunused: New subcommand, provides for easy dropping of unused keys by number, as listed by the unused subcommand. + * fsck: Print warnings to stderr; --quiet can now be used to only see + problems. -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index a1a8882425..281f460506 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -296,7 +296,7 @@ eliminate it to save space. 2 WORM:1289672605:14:file (To see where data was previously used, try: git log --stat -S'KEY') (To remove unwanted data: git-annex dropunused NUMBER) - failed + ok After running `git annex unused`, you can follow the instructions to examine the history of files that used the data, and if you decide you don't need that @@ -316,7 +316,7 @@ setting is satisfied for all files. # git annex fsck unused (checking for unused data...) ok fsck my_cool_big_file (checksum...) ok - ...... + ... You can also specifiy the files to check. This is particularly useful if you're using sha1 and don't want to spend a long time checksumming everything. @@ -332,10 +332,10 @@ might say about a badly messed up annex: # git annex fsck fsck my_cool_big_file (checksum...) - Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 - ** No known copies of the file exist! + git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 + git-annex: ** No known copies of the file exist! failed fsck important_file - Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. + git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. failed git-annex: 2 failed From 625972d2b3340b6a41d1d47f631c80d6021025dc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:38:41 -0400 Subject: [PATCH 0528/2835] this is kinda weird, but it avoids blank lines after warnings --- Messages.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Messages.hs b/Messages.hs index 7dafa8284e..f9d12c2221 100644 --- a/Messages.hs +++ b/Messages.hs @@ -56,4 +56,4 @@ showErr e = warning $ show e warning :: String -> Annex () warning s = do verbose $ liftIO $ putStr "\n" - liftIO $ hPutStrLn stderr $ "git-annex: " ++ s + liftIO $ hPutStr stderr $ "git-annex: " ++ s ++ " " From 38a75aa26ff70f4ebe5a88eb009d512ff7211163 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:42:32 -0400 Subject: [PATCH 0529/2835] close --- doc/todo/gitrm.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/todo/gitrm.mdwn b/doc/todo/gitrm.mdwn index d771aa32a7..e41c334623 100644 --- a/doc/todo/gitrm.mdwn +++ b/doc/todo/gitrm.mdwn @@ -1,2 +1,5 @@ how to handle git rm file? (should try to drop keys that have no referring file, if it seems safe..) + +[[done]] -- I think that git annex unused and dropunused are the best +solution to this. From 2b7203c5d2a5a22832778d3b3b16bcd8fd7242f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:43:59 -0400 Subject: [PATCH 0530/2835] releasing version 0.07 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index a6d95ab69e..252d9ae45b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.07) UNRELEASED; urgency=low +git-annex (0.07) unstable; urgency=low * find: New subcommand. * unused: New subcommand, finds unused data. (Split out from fsck.) @@ -7,7 +7,7 @@ git-annex (0.07) UNRELEASED; urgency=low * fsck: Print warnings to stderr; --quiet can now be used to only see problems. - -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 + -- Joey Hess Mon, 15 Nov 2010 18:41:50 -0400 git-annex (0.06) unstable; urgency=low From d792024e37895b25f8a00916c97b5cd133582eee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:44:06 -0400 Subject: [PATCH 0531/2835] add news item for git-annex 0.07 --- doc/news/version_0.02.mdwn | 18 ------------------ doc/news/version_0.07.mdwn | 8 ++++++++ 2 files changed, 8 insertions(+), 18 deletions(-) delete mode 100644 doc/news/version_0.02.mdwn create mode 100644 doc/news/version_0.07.mdwn diff --git a/doc/news/version_0.02.mdwn b/doc/news/version_0.02.mdwn deleted file mode 100644 index 689124ab4e..0000000000 --- a/doc/news/version_0.02.mdwn +++ /dev/null @@ -1,18 +0,0 @@ -git-annex 0.02 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Can scp annexed files from remote hosts, and check remote hosts for - file content when dropping files. - * New move subcommand, that makes it easy to move file contents from - or to a remote. - * New fromkey subcommand, for registering urls, etc. - * git-annex init will now set up a pre-commit hook that fixes up symlinks - before they are committed, to ensure that moving symlinks around does not - break them. - * More intelligent and fast staging of modified files; git add coalescing. - * Add remote.annex-ignore git config setting to allow completly disabling - a given remote. - * --from/--to can be used to control the remote repository that git-annex - uses. - * --quiet can be used to avoid verbose output - * New plumbing-level dropkey and addkey subcommands. - * Lots of bug fixes."""]] \ No newline at end of file diff --git a/doc/news/version_0.07.mdwn b/doc/news/version_0.07.mdwn new file mode 100644 index 0000000000..29b4736b92 --- /dev/null +++ b/doc/news/version_0.07.mdwn @@ -0,0 +1,8 @@ +git-annex 0.07 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * find: New subcommand. + * unused: New subcommand, finds unused data. (Split out from fsck.) + * dropunused: New subcommand, provides for easy dropping of unused keys + by number, as listed by the unused subcommand. + * fsck: Print warnings to stderr; --quiet can now be used to only see + problems."""]] \ No newline at end of file From 4dfd134ccb51931e09326d2bd53f2e337638c1ad Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 16 Nov 2010 18:00:42 -0400 Subject: [PATCH 0532/2835] flattr button --- doc/index.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index d863a9ede9..5cd6b45494 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -23,6 +23,10 @@ To get a feel for it, see the [[walkthrough]]. * [[todo]] * [[forum]] * [[contact]] + + + """]] ## sample use cases From 535755cf12a063ae19dca65d79614913084e499a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 16 Nov 2010 18:46:55 -0400 Subject: [PATCH 0533/2835] remove --- doc/news/prerelease.mdwn | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 doc/news/prerelease.mdwn diff --git a/doc/news/prerelease.mdwn b/doc/news/prerelease.mdwn deleted file mode 100644 index 3a1227ff0a..0000000000 --- a/doc/news/prerelease.mdwn +++ /dev/null @@ -1,9 +0,0 @@ -git-annex is not yet formally released. If you [[download]] it from git, -everything described on this web site should already work, and I hope work -well, with these exceptions: - -* The SHA1 backend is incomplete. - -I reserve the right to change its data formats before the first release, -though I already have enough drives using it that I'd probably have to -write a transition tool anyway! --[[Joey]] From ab55689164a2fa29af61bce692cfe5df2739b790 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 17 Nov 2010 13:46:50 -0400 Subject: [PATCH 0534/2835] simplifiy --- Utility.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Utility.hs b/Utility.hs index 0053c687bb..33db4bb086 100644 --- a/Utility.hs +++ b/Utility.hs @@ -39,8 +39,7 @@ parentDir dir = then slash ++ (join s $ take ((length dirs) - 1) dirs) else "" where - dirs = filter (\x -> length x > 0) $ - split s dir + dirs = filter (\x -> not $ null x) $ split s dir slash = if (not $ isAbsolute dir) then "" else s s = [pathSeparator] From 5c7d1b027916ce3fc207329f926041d2bcad3bcd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 17 Nov 2010 13:55:38 -0400 Subject: [PATCH 0535/2835] Fix `git annex add ../foo` (when ran in a subdir of the repo). There was no reason for Git.relative to be used here. --- Backend.hs | 14 ++++++-------- debian/changelog | 6 ++++++ doc/bugs/dotdot_problem.mdwn | 2 ++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Backend.hs b/Backend.hs index 2f0f71d749..f24347ca8d 100644 --- a/Backend.hs +++ b/Backend.hs @@ -79,17 +79,15 @@ maybeLookupBackendName bs s = {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Maybe Backend -> Annex (Maybe (Key, Backend)) storeFileKey file trybackend = do - g <- Annex.gitRepo - let relfile = Git.relative g file bs <- list let bs' = case trybackend of Nothing -> bs Just backend -> backend:bs - storeFileKey' bs' file relfile -storeFileKey' :: [Backend] -> FilePath -> FilePath -> Annex (Maybe (Key, Backend)) -storeFileKey' [] _ _ = return Nothing -storeFileKey' (b:bs) file relfile = do - result <- (Internals.getKey b) relfile + storeFileKey' bs' file +storeFileKey' :: [Backend] -> FilePath -> Annex (Maybe (Key, Backend)) +storeFileKey' [] _ = return Nothing +storeFileKey' (b:bs) file = do + result <- (Internals.getKey b) file case result of Nothing -> nextbackend Just key -> do @@ -98,7 +96,7 @@ storeFileKey' (b:bs) file relfile = do then nextbackend else return $ Just (key, b) where - nextbackend = storeFileKey' bs file relfile + nextbackend = storeFileKey' bs file {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} diff --git a/debian/changelog b/debian/changelog index 252d9ae45b..2a8871c30e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.08) UNRELEASED; urgency=low + + * Fix `git annex add ../foo` (when ran in a subdir of the repo). + + -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 + git-annex (0.07) unstable; urgency=low * find: New subcommand. diff --git a/doc/bugs/dotdot_problem.mdwn b/doc/bugs/dotdot_problem.mdwn index 9d247a9c09..cbefd5dae5 100644 --- a/doc/bugs/dotdot_problem.mdwn +++ b/doc/bugs/dotdot_problem.mdwn @@ -1,2 +1,4 @@ cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) + +[[done]] From 54513c69baffa40f2fcce42eb8651fdd98e05277 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 13:30:42 -0400 Subject: [PATCH 0536/2835] Add configure step to build process. * configure: Check to see if cp -a can be used. * configure: Check to see if cp --reflink=auto can be used. --- .gitignore | 2 ++ Makefile | 8 +++-- configure.hs | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ debian/changelog | 3 ++ 4 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 configure.hs diff --git a/.gitignore b/.gitignore index 6956c49dd2..a4cac10f43 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ build/* test +configure +SysConfig.hs git-annex git-annex.1 doc/.ikiwiki diff --git a/Makefile b/Makefile index 77e7a5edb8..685044146f 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,11 @@ all: git-annex docs ghcmake=ghc -Wall -odir build -hidir build -O2 --make -git-annex: +SysConfig.hs: + $(ghcmake) configure + ./configure + +git-annex: SysConfig.hs $(ghcmake) git-annex install: @@ -29,7 +33,7 @@ docs: --disable-plugin=smiley clean: - rm -rf build git-annex git-annex.1 test + rm -rf build git-annex git-annex.1 test configure SysConfig.hs rm -rf doc/.ikiwiki html .PHONY: git-annex test install diff --git a/configure.hs b/configure.hs new file mode 100644 index 0000000000..fa07be3ab5 --- /dev/null +++ b/configure.hs @@ -0,0 +1,78 @@ +{- Checks system configuration and generates SysConfig.hs. + -} + +import System.IO +import System.Cmd +import System.Exit +import System.Directory + +type Test = IO Bool +data TestDesc = TestDesc String String Test +data Config = Config String Bool + +tests :: [TestDesc] +tests = [ + TestDesc "cp -a" "cp_a" cp_a + , TestDesc "cp --reflink" "cp_reflink" cp_reflink + ] + +tmpDir :: String +tmpDir = "tmp" + +testFile :: String +testFile = tmpDir ++ "/testfile" + +quiet :: String -> String +quiet s = s ++ " 2>/dev/null" + +cp_a :: Test +cp_a = testCmd $ quiet $ "cp -a " ++ testFile ++ " " ++ testFile ++ ".new" + +cp_reflink :: Test +cp_reflink = testCmd $ quiet $ "cp --reflink=auto " ++ testFile ++ " " ++ testFile ++ ".new" + +testCmd :: String -> Test +testCmd c = do + ret <- system c + return $ ret == ExitSuccess + +testStart :: String -> IO () +testStart s = do + putStr $ " checking " ++ s ++ "..." + hFlush stdout + +testEnd :: Bool -> IO () +testEnd r = putStrLn $ " " ++ (show r) + +writeSysConfig :: [Config] -> IO () +writeSysConfig config = do + writeFile "SysConfig.hs" $ unlines $ header ++ vars config ++ footer + where + header = [ + "{- Automatically generated by configure. -}" + , "module SysConfig where" + ] + footer = [] + vars [] = [] + vars (c:cs) = showvar c ++ vars cs + showvar (Config name val) = [ + name ++ " :: Bool" + , name ++ " = " ++ show val + ] + +runTests :: [TestDesc] -> IO [Config] +runTests [] = return [] +runTests ((TestDesc tname key t):ts) = do + testStart tname + val <- t + testEnd val + rest <- runTests ts + return $ (Config key val):rest + +main :: IO () +main = do + createDirectoryIfMissing True tmpDir + writeFile testFile "test file contents" + config <- runTests tests + removeDirectoryRecursive tmpDir + writeSysConfig config diff --git a/debian/changelog b/debian/changelog index 2a8871c30e..e6549c1f08 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,9 @@ git-annex (0.08) UNRELEASED; urgency=low * Fix `git annex add ../foo` (when ran in a subdir of the repo). + * Add configure step to build process. + * configure: Check to see if cp -a can be used. + * configure: Check to see if cp --reflink=auto can be used. -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 From 161823d6eaff2adb7b99475b0edfe819fde11be1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 13:48:28 -0400 Subject: [PATCH 0537/2835] Only use cp -a if it is supported, falling back to cp -p or plain cp. * cp --reflink=auto is used if supported, and will make git annex unlock much faster on filesystems like btrfs that support copy of write. --- Command/Unlock.hs | 7 ++++--- CopyFile.hs | 24 ++++++++++++++++++++++++ Makefile | 2 +- Remotes.hs | 7 ++++--- configure.hs | 18 +++++++++--------- debian/changelog | 6 ++++-- doc/todo/use_cp_reflink.mdwn | 2 ++ 7 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 CopyFile.hs diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 3ff3023b24..34fde819cb 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -8,7 +8,7 @@ module Command.Unlock where import Control.Monad.State (liftIO) -import System.Directory +import System.Directory hiding (copyFile) import Command import qualified Annex @@ -17,6 +17,7 @@ import Messages import Locations import Utility import Core +import CopyFile seek :: [SubCmdSeek] seek = [withFilesInGit start] @@ -34,9 +35,9 @@ perform dest key = do let src = annexLocation g key liftIO $ removeFile dest showNote "copying..." - ok <- liftIO $ boolSystem "cp" ["-p", src, dest] + ok <- liftIO $ copyFile src dest if ok then do liftIO $ allowWrite dest return $ Just $ return True - else error "cp failed!" + else error "copy failed!" diff --git a/CopyFile.hs b/CopyFile.hs new file mode 100644 index 0000000000..39f1b01833 --- /dev/null +++ b/CopyFile.hs @@ -0,0 +1,24 @@ +{- git-annex file copying + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CopyFile (copyFile) where + +import Utility +import qualified SysConfig + +{- The cp command is used, because I hate reinventing the wheel, + - and because this allows easy access to features like cp --reflink. -} +copyFile :: FilePath -> FilePath -> IO Bool +copyFile src dest = boolSystem "cp" opts + where + opts = if (SysConfig.cp_reflink_auto) + then ["--reflink=auto", src, dest] + else if (SysConfig.cp_a) + then ["-a", src, dest] + else if (SysConfig.cp_p) + then ["-p", src, dest] + else [src, dest] diff --git a/Makefile b/Makefile index 685044146f..17918f9567 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: git-annex docs ghcmake=ghc -Wall -odir build -hidir build -O2 --make -SysConfig.hs: +SysConfig.hs: configure.hs $(ghcmake) configure ./configure diff --git a/Remotes.hs b/Remotes.hs index 7aad6c2a06..bf5ede5729 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -23,7 +23,7 @@ import Control.Monad.State (liftIO) import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils -import System.Directory +import System.Directory hiding (copyFile) import System.Posix.Directory import List import Monad (when, unless) @@ -37,6 +37,7 @@ import UUID import Utility import qualified Core import Messages +import CopyFile {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -204,7 +205,7 @@ copyFromRemote r key file = do then getssh else error "copying from non-ssh repo not supported" where - getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] + getlocal = liftIO $ copyFile keyloc file getssh = scp r [sshLocation r keyloc, file] keyloc = annexLocation r key @@ -219,7 +220,7 @@ copyToRemote r key file = do then putssh keyloc else error "copying to non-ssh repo not supported" where - putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] + putlocal src = liftIO $ copyFile src file putssh src = scp r [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath diff --git a/configure.hs b/configure.hs index fa07be3ab5..56daf583a6 100644 --- a/configure.hs +++ b/configure.hs @@ -1,5 +1,4 @@ -{- Checks system configuration and generates SysConfig.hs. - -} +{- Checks system configuration and generates SysConfig.hs. -} import System.IO import System.Cmd @@ -12,8 +11,9 @@ data Config = Config String Bool tests :: [TestDesc] tests = [ - TestDesc "cp -a" "cp_a" cp_a - , TestDesc "cp --reflink" "cp_reflink" cp_reflink + TestDesc "cp -a" "cp_a" $ testCp "-a" + , TestDesc "cp -p" "cp_p" $ testCp "-p" + , TestDesc "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" ] tmpDir :: String @@ -25,11 +25,9 @@ testFile = tmpDir ++ "/testfile" quiet :: String -> String quiet s = s ++ " 2>/dev/null" -cp_a :: Test -cp_a = testCmd $ quiet $ "cp -a " ++ testFile ++ " " ++ testFile ++ ".new" - -cp_reflink :: Test -cp_reflink = testCmd $ quiet $ "cp --reflink=auto " ++ testFile ++ " " ++ testFile ++ ".new" +testCp :: String -> Test +testCp option = testCmd $ quiet $ "cp " ++ option ++ " " ++ testFile ++ + " " ++ testFile ++ ".new" testCmd :: String -> Test testCmd c = do @@ -51,6 +49,7 @@ writeSysConfig config = do header = [ "{- Automatically generated by configure. -}" , "module SysConfig where" + , "" ] footer = [] vars [] = [] @@ -58,6 +57,7 @@ writeSysConfig config = do showvar (Config name val) = [ name ++ " :: Bool" , name ++ " = " ++ show val + , "" ] runTests :: [TestDesc] -> IO [Config] diff --git a/debian/changelog b/debian/changelog index e6549c1f08..21685ba4d3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,8 +2,10 @@ git-annex (0.08) UNRELEASED; urgency=low * Fix `git annex add ../foo` (when ran in a subdir of the repo). * Add configure step to build process. - * configure: Check to see if cp -a can be used. - * configure: Check to see if cp --reflink=auto can be used. + * Only use cp -a if it is supported, falling back to cp -p or plain cp + as needed for portability. + * cp --reflink=auto is used if supported, and will make git annex unlock + much faster on filesystems like btrfs that support copy of write. -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 diff --git a/doc/todo/use_cp_reflink.mdwn b/doc/todo/use_cp_reflink.mdwn index d7974928ce..39518abf18 100644 --- a/doc/todo/use_cp_reflink.mdwn +++ b/doc/todo/use_cp_reflink.mdwn @@ -3,3 +3,5 @@ The unlock command needs to copy a file, and it would be great to use this: O(1) overhead on BTRFS. Needs coreutils 7.6; and remember that git-annex may be used on systems without coreutils.. + +[[done]] From 09964033777ea206db0ff0a27f7928184c70c879 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:07:22 -0400 Subject: [PATCH 0538/2835] tweak --- configure.hs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/configure.hs b/configure.hs index 56daf583a6..d9ce60a14a 100644 --- a/configure.hs +++ b/configure.hs @@ -9,6 +9,12 @@ type Test = IO Bool data TestDesc = TestDesc String String Test data Config = Config String Bool +instance Show Config where + show (Config key value) = unlines $ [ + key ++ " :: Bool" + , key ++ " = " ++ show value + ] + tests :: [TestDesc] tests = [ TestDesc "cp -a" "cp_a" $ testCp "-a" @@ -44,7 +50,7 @@ testEnd r = putStrLn $ " " ++ (show r) writeSysConfig :: [Config] -> IO () writeSysConfig config = do - writeFile "SysConfig.hs" $ unlines $ header ++ vars config ++ footer + writeFile "SysConfig.hs" $ unlines $ header ++ map show config ++ footer where header = [ "{- Automatically generated by configure. -}" @@ -52,13 +58,6 @@ writeSysConfig config = do , "" ] footer = [] - vars [] = [] - vars (c:cs) = showvar c ++ vars cs - showvar (Config name val) = [ - name ++ " :: Bool" - , name ++ " = " ++ show val - , "" - ] runTests :: [TestDesc] -> IO [Config] runTests [] = return [] From b6ecfc916ed46c3d84717ab5097f4731c986c2fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:08:46 -0400 Subject: [PATCH 0539/2835] tweak --- test.hs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test.hs b/test.hs index 2faf579a20..9a6e05a97c 100644 --- a/test.hs +++ b/test.hs @@ -1,5 +1,3 @@ --- TODO find a test harness that is actually in Debian and use it. - import Test.HUnit import Test.HUnit.Tools From fbd1cbf2c915bdb18a09264b356eed55752bb757 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:11:18 -0400 Subject: [PATCH 0540/2835] tweak --- configure.hs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/configure.hs b/configure.hs index d9ce60a14a..b9ea2344fe 100644 --- a/configure.hs +++ b/configure.hs @@ -49,9 +49,9 @@ testEnd :: Bool -> IO () testEnd r = putStrLn $ " " ++ (show r) writeSysConfig :: [Config] -> IO () -writeSysConfig config = do - writeFile "SysConfig.hs" $ unlines $ header ++ map show config ++ footer +writeSysConfig config = writeFile "SysConfig.hs" body where + body = unlines $ header ++ map show config ++ footer header = [ "{- Automatically generated by configure. -}" , "module SysConfig where" @@ -68,10 +68,18 @@ runTests ((TestDesc tname key t):ts) = do rest <- runTests ts return $ (Config key val):rest -main :: IO () -main = do +setup :: IO () +setup = do createDirectoryIfMissing True tmpDir writeFile testFile "test file contents" - config <- runTests tests + +cleanup :: IO () +cleanup = do removeDirectoryRecursive tmpDir + +main :: IO () +main = do + setup + config <- runTests tests writeSysConfig config + cleanup From da794cce7d4630a99d95e39d516485834892c286 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:12:40 -0400 Subject: [PATCH 0541/2835] typo --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 21685ba4d3..7b7ff55ae1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,7 +5,7 @@ git-annex (0.08) UNRELEASED; urgency=low * Only use cp -a if it is supported, falling back to cp -p or plain cp as needed for portability. * cp --reflink=auto is used if supported, and will make git annex unlock - much faster on filesystems like btrfs that support copy of write. + much faster on filesystems like btrfs that support copy on write. -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 From f3f3bc6cae0f907fe023708d2c298c1219b0b603 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:29:14 -0400 Subject: [PATCH 0542/2835] mention git-bigfiles --- doc/not.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index affcb57f11..d965ab86f8 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -11,6 +11,10 @@ too slow, or its strict mirroring of everything to both places too limiting, then git-annex could be a useful alternative. +* git-annex is more than just a workaround for git limitations that might + eventually be fixed by efforts like + [git-bigfiles](http://caca.zoy.org/wiki/git-bigfiles). + * git-annex is not some flaky script that was quickly thrown together. I wrote it in Haskell because I wanted it to be solid and to compile down to a binary. From 68183b47212b7819b3fc0e7715c37ec38e4a9750 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 15:15:40 -0400 Subject: [PATCH 0543/2835] add tests for uuid and xargs -0 --- configure.hs | 13 ++++++++++++- debian/control | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/configure.hs b/configure.hs index b9ea2344fe..4d0624a422 100644 --- a/configure.hs +++ b/configure.hs @@ -20,6 +20,8 @@ tests = [ TestDesc "cp -a" "cp_a" $ testCp "-a" , TestDesc "cp -p" "cp_p" $ testCp "-p" , TestDesc "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" + , TestDesc "uuid" "uuid" $ requireCommand "uuid" "uuid" + , TestDesc "xargs -0" "xargs_0" $ requireCommand "xargs -0" "xargs -0 String -quiet s = s ++ " 2>/dev/null" +quiet s = s ++ " >/dev/null 2>&1" + +requireCommand :: String -> String -> Test +requireCommand command cmdline = do + ret <- testCmd $ quiet cmdline + if (ret) + then return True + else do + testEnd False + error $ "** the " ++ command ++ " command is required to use git-annex" testCp :: String -> Test testCp option = testCmd $ quiet $ "cp " ++ option ++ " " ++ testFile ++ diff --git a/debian/control b/debian/control index 3fba367427..0318dafa4e 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki, uuid Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From d93e877a3e63501b951e0358d9177c639fd1306a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 21 Nov 2010 13:47:44 -0400 Subject: [PATCH 0544/2835] releasing version 0.08 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7b7ff55ae1..66c4bc94da 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.08) UNRELEASED; urgency=low +git-annex (0.08) unstable; urgency=low * Fix `git annex add ../foo` (when ran in a subdir of the repo). * Add configure step to build process. @@ -7,7 +7,7 @@ git-annex (0.08) UNRELEASED; urgency=low * cp --reflink=auto is used if supported, and will make git annex unlock much faster on filesystems like btrfs that support copy on write. - -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 + -- Joey Hess Sun, 21 Nov 2010 13:45:44 -0400 git-annex (0.07) unstable; urgency=low From fd11b5a3e5355a68f182a60eda7916d57e141366 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 21 Nov 2010 13:47:52 -0400 Subject: [PATCH 0545/2835] add news item for git-annex 0.08 --- doc/news/version_0.03.mdwn | 16 ---------------- doc/news/version_0.08.mdwn | 8 ++++++++ 2 files changed, 8 insertions(+), 16 deletions(-) delete mode 100644 doc/news/version_0.03.mdwn create mode 100644 doc/news/version_0.08.mdwn diff --git a/doc/news/version_0.03.mdwn b/doc/news/version_0.03.mdwn deleted file mode 100644 index b22aad91af..0000000000 --- a/doc/news/version_0.03.mdwn +++ /dev/null @@ -1,16 +0,0 @@ -git-annex 0.03 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix support for file:// remotes. - * Add --verbose - * Fix SIGINT handling. - * Fix handling of files with unusual characters in their name. - * Fixed memory leak; git-annex no longer reads the whole file list - from git before starting, and will be much faster with large repos. - * Fix crash on unknown symlinks. - * Added remote.annex-scp-options and remote.annex-ssh-options. - * The backends to use when adding different sets of files can be configured - via gitattributes. - * In .gitattributes, the git-annex-backend attribute can be set to the - names of backends to use when adding different types of files. - * Add fsck subcommand. (For now it only finds unused key contents in the - annex.)"""]] \ No newline at end of file diff --git a/doc/news/version_0.08.mdwn b/doc/news/version_0.08.mdwn new file mode 100644 index 0000000000..8ef7ad7f31 --- /dev/null +++ b/doc/news/version_0.08.mdwn @@ -0,0 +1,8 @@ +git-annex 0.08 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Fix `git annex add ../foo` (when ran in a subdir of the repo). + * Add configure step to build process. + * Only use cp -a if it is supported, falling back to cp -p or plain cp + as needed for portability. + * cp --reflink=auto is used if supported, and will make git annex unlock + much faster on filesystems like btrfs that support copy on write."""]] \ No newline at end of file From 57adb0347bf4eb71ab846a2947680a20263449a2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 22 Nov 2010 15:46:57 -0400 Subject: [PATCH 0546/2835] hlint tweaks Remotes.hs next, and also Backend/* and Command/* --- Backend.hs | 8 ++++---- Command.hs | 26 +++++++++++--------------- CopyFile.hs | 6 +++--- Core.hs | 12 ++++++------ GitQueue.hs | 4 ++-- GitRepo.hs | 28 ++++++++++++++-------------- LocationLog.hs | 20 ++++++++++---------- Locations.hs | 6 +++--- Messages.hs | 9 +++------ TypeInternals.hs | 6 +++--- UUID.hs | 25 ++++++++++++------------- configure.hs | 6 +++--- 12 files changed, 74 insertions(+), 82 deletions(-) diff --git a/Backend.hs b/Backend.hs index f24347ca8d..01a7298b4e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -45,21 +45,21 @@ import Messages list :: Annex [Backend] list = do l <- Annex.backends -- list is cached here - if (not $ null l) + if not $ null l then return l else do bs <- Annex.supportedBackends g <- Annex.gitRepo let defaults = parseBackendList bs $ Git.configGet g "annex.backends" "" backendflag <- Annex.flagGet "backend" - let l' = if (not $ null backendflag) + let l' = if not $ null backendflag then (lookupBackendName bs backendflag):defaults else defaults Annex.backendsChange l' return l' where parseBackendList bs s = - if (null s) + if null s then bs else map (lookupBackendName bs) $ words s @@ -71,7 +71,7 @@ lookupBackendName bs s = Nothing -> error $ "unknown backend " ++ s maybeLookupBackendName :: [Backend] -> String -> Maybe Backend maybeLookupBackendName bs s = - if ((length matches) /= 1) + if 1 /= length matches then Nothing else Just $ head matches where matches = filter (\b -> s == Internals.name b) bs diff --git a/Command.hs b/Command.hs index c6d2d0d5d4..4d10a9e7f1 100644 --- a/Command.hs +++ b/Command.hs @@ -64,17 +64,17 @@ prepSubCmd SubCommand { subcmdseek = seek } state params = do doSubCmd :: SubCmdStart -> SubCmdCleanup doSubCmd start = do s <- start - case (s) of + case s of Nothing -> return True Just perform -> do p <- perform - case (p) of + case p of Nothing -> do showEndFail return False Just cleanup -> do c <- cleanup - if (c) + if c then do showEndOk return True @@ -85,14 +85,14 @@ doSubCmd start = do notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) notAnnexed file a = do r <- Backend.lookupFile file - case (r) of + case r of Just _ -> return Nothing Nothing -> a isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) isAnnexed file a = do r <- Backend.lookupFile file - case (r) of + case r of Just v -> a v Nothing -> return Nothing @@ -153,19 +153,15 @@ withNothing _ _ = return [] {- Default to acting on all files matching the seek action if - none are specified. -} withAll :: SubCmdSeekStrings -> SubCmdSeekStrings -withAll w a params = do - if null params - then do - g <- Annex.gitRepo - w a [Git.workTree g] - else w a params +withAll w a [] = do + g <- Annex.gitRepo + w a [Git.workTree g] +withAll w a p = w a p {- Provides a default parameter to act on if none is specified. -} withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings -withDefault d w a params = do - if null params - then w a [d] - else w a params +withDefault d w a [] = w a [d] +withDefault _ w a p = w a p {- filter out files from the state directory -} notState :: FilePath -> Bool diff --git a/CopyFile.hs b/CopyFile.hs index 39f1b01833..8bd07dc35a 100644 --- a/CopyFile.hs +++ b/CopyFile.hs @@ -15,10 +15,10 @@ import qualified SysConfig copyFile :: FilePath -> FilePath -> IO Bool copyFile src dest = boolSystem "cp" opts where - opts = if (SysConfig.cp_reflink_auto) + opts = if SysConfig.cp_reflink_auto then ["--reflink=auto", src, dest] - else if (SysConfig.cp_a) + else if SysConfig.cp_a then ["-a", src, dest] - else if (SysConfig.cp_p) + else if SysConfig.cp_p then ["-p", src, dest] else [src, dest] diff --git a/Core.hs b/Core.hs index 2928dc06df..6e0ddd65ff 100644 --- a/Core.hs +++ b/Core.hs @@ -36,7 +36,7 @@ tryRun state actions = tryRun' state 0 actions tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () tryRun' state errnum (a:as) = do result <- try $ Annex.run state a - case (result) of + case result of Left err -> do Annex.eval state $ showErr err tryRun' state (errnum + 1) as @@ -64,7 +64,7 @@ shutdown = do g <- Annex.gitRepo let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp - when (exists) $ liftIO $ removeDirectoryRecursive tmp + when exists $ liftIO $ removeDirectoryRecursive tmp liftIO $ createDirectoryIfMissing True tmp return True @@ -81,7 +81,7 @@ calcGitLink :: FilePath -> Key -> Annex FilePath calcGitLink file key = do g <- Annex.gitRepo cwd <- liftIO $ getCurrentDirectory - let absfile = case (absNormPath cwd file) of + let absfile = case absNormPath cwd file of Just f -> f Nothing -> error $ "unable to normalize " ++ file return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++ @@ -104,7 +104,7 @@ getViaTmp key action = do let tmp = annexTmpLocation g ++ keyFile key liftIO $ createDirectoryIfMissing True (parentDir tmp) success <- action tmp - if (success) + if success then do moveAnnex key tmp logStatus key ValuePresent @@ -125,7 +125,7 @@ preventWrite f = unsetFileMode f writebits allowWrite :: FilePath -> IO () allowWrite f = do s <- getFileStatus f - setFileMode f $ (fileMode s) `unionFileModes` ownerWriteMode + setFileMode f $ fileMode s `unionFileModes` ownerWriteMode {- Moves a file into .git/annex/objects/ -} moveAnnex :: Key -> FilePath -> Annex () @@ -188,7 +188,7 @@ getKeysPresent' dir = do where present d = do s <- getFileStatus $ dir ++ "/" ++ d ++ "/" - ++ (takeFileName d) + ++ takeFileName d return $ isRegularFile s {- List of keys referenced by symlinks in the git repo. -} diff --git a/GitQueue.hs b/GitQueue.hs index 2d44a8f108..d8ba861366 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -45,7 +45,7 @@ add queue subcommand params file = M.insertWith (++) action [file] queue {- Runs a queue on a git repository. -} run :: Git.Repo -> Queue -> IO () run repo queue = do - _ <- mapM (\(k, v) -> runAction repo k v) $ M.toList queue + _ <- mapM (uncurry $ runAction repo) $ M.toList queue return () {- Runs an Action on a list of files in a git repository. @@ -56,6 +56,6 @@ runAction repo action files = do unless (null files) runxargs where runxargs = pOpen WriteToPipe "xargs" ("-0":gitcmd) feedxargs - gitcmd = ["git"] ++ Git.gitCommandLine repo + gitcmd = "git" : Git.gitCommandLine repo (getSubcommand action:getParams action) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index fa78e51222..f50b3fb2e9 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -127,19 +127,19 @@ repoIsSsh _ = False assertLocal :: Repo -> a -> a assertLocal repo action = - if (not $ repoIsUrl repo) + if not $ repoIsUrl repo then action else error $ "acting on URL git repo " ++ repoDescribe repo ++ " not supported" assertUrl :: Repo -> a -> a assertUrl repo action = - if (repoIsUrl repo) + if repoIsUrl repo then action else error $ "acting on local git repo " ++ repoDescribe repo ++ " not supported" assertSsh :: Repo -> a -> a assertSsh repo action = - if (repoIsSsh repo) + if repoIsSsh repo then action else error $ "unsupported url in repo " ++ repoDescribe repo bare :: Repo -> Bool @@ -199,14 +199,14 @@ urlPath repo = assertUrl repo $ error "internal" gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++d++"/"++(gitDir repo), "--work-tree="++d] ++ params + ["--git-dir=" ++ d ++ "/" ++ gitDir repo, "--work-tree=" ++ d] ++ params gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo, throwing an error if it fails. -} run :: Repo -> [String] -> IO () run repo params = assertLocal repo $ do ok <- boolSystem "git" (gitCommandLine repo params) - unless (ok) $ error $ "git " ++ show params ++ " failed" + unless ok $ error $ "git " ++ show params ++ " failed" {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String @@ -290,7 +290,7 @@ configRead repo sshopts = assertSsh repo $ do params = case sshopts of Nothing -> [urlHost repo, command] Just l -> l ++ [urlHost repo, command] - command = "cd " ++ (shellEscape $ urlPath repo) ++ + command = "cd " ++ shellEscape (urlPath repo) ++ " && git config --list" hConfigRead :: Repo -> Handle -> IO Repo hConfigRead repo h = do @@ -308,8 +308,8 @@ configRemotes repo = map construct remotepairs where remotepairs = Map.toList $ filterremotes $ config repo filterremotes = Map.filterWithKey (\k _ -> isremote k) - isremote k = (startswith "remote." k) && (endswith ".url" k) - remotename k = (split "." k) !! 1 + isremote k = startswith "remote." k && endswith ".url" k + remotename k = split "." k !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } gen v | isURI v = repoFromUrl v | otherwise = repoFromPath v @@ -319,7 +319,7 @@ configParse :: String -> Map.Map String String configParse s = Map.fromList $ map pair $ lines s where pair l = (key l, val l) - key l = (keyval l) !! 0 + key l = head $ keyval l val l = join sep $ drop 1 $ keyval l keyval l = split sep l :: [String] sep = "=" @@ -377,7 +377,7 @@ decodeGitFile f@(c:s) alloctal = isOctDigit n1 && isOctDigit n2 && isOctDigit n3 - fromoctal = [chr $ readoctal (n1:n2:n3:[])] + fromoctal = [chr $ readoctal [n1, n2, n3]] readoctal o = read $ "0o" ++ o :: Int -- \C is used for a few special characters decode (x:nc:rest) @@ -395,9 +395,9 @@ decodeGitFile f@(c:s) {- Should not need to use this, except for testing decodeGitFile. -} encodeGitFile :: FilePath -> String -encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" +encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\"" where - e c = "\\" ++ [c] + e c = '\\' : [c] echar '\a' = e 'a' echar '\b' = e 'b' echar '\f' = e 'f' @@ -413,7 +413,7 @@ encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" | ord x > 0x7E = e_num x -- high ascii | otherwise = [x] -- printable ascii where - showoctal i = "\\" ++ (printf "%03o" i) + showoctal i = '\\' : printf "%03o" i e_num c = showoctal $ ord c -- unicode character is decomposed to -- Word8s and each is shown in octal @@ -423,7 +423,7 @@ encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" {- for quickcheck -} prop_idempotent_deencode :: String -> Bool -prop_idempotent_deencode s = s == (decodeGitFile $ encodeGitFile s) +prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s) {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo diff --git a/LocationLog.hs b/LocationLog.hs index 1ddbf3c0bb..7497d865bb 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -66,8 +66,8 @@ instance Read LogLine where -- read without an exception being thrown. -- Such lines have a status of Undefined. readsPrec _ string = - if (length w == 3) - then case (pdate) of + if length w == 3 + then case pdate of Just v -> good v Nothing -> bad else bad @@ -75,15 +75,16 @@ instance Read LogLine where w = words string s = read $ w !! 1 u = w !! 2 - pdate = (parseTime defaultTimeLocale "%s%Qs" $ w !! 0) :: Maybe UTCTime + pdate :: Maybe UTCTime + pdate = parseTime defaultTimeLocale "%s%Qs" $ head w good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s u - bad = ret $ LogLine (0) Undefined "" + bad = ret $ LogLine 0 Undefined "" ret v = [(v, "")] {- Log a change in the presence of a key's value in a repository, - and returns the filename of the logfile. -} -logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO (FilePath) +logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath logChange repo key u s = do line <- logNow s u ls <- readLog logfile @@ -101,10 +102,9 @@ readLog file = do then do s <- readFile file -- filter out any unparsable lines - return $ filter (\l -> (status l) /= Undefined ) + return $ filter (\l -> status l /= Undefined ) $ map read $ lines s - else do - return [] + else return [] {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () @@ -124,7 +124,7 @@ logNow s u = do {- Returns the filename of the log file for a given key. -} logFile :: Git.Repo -> Key -> String logFile repo key = - (gitStateDir repo) ++ (Git.relative repo (keyFile key)) ++ ".log" + gitStateDir repo ++ Git.relative repo (keyFile key) ++ ".log" {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} @@ -152,7 +152,7 @@ compactLog' m (l:ls) = compactLog' (mapLog m l) ls - information about a repo than the other logs in the map -} mapLog :: LogMap -> LogLine -> LogMap mapLog m l = - if (better) + if better then Map.insert u l m else m where diff --git a/Locations.hs b/Locations.hs index 24ccc75c6d..6c541e937c 100644 --- a/Locations.hs +++ b/Locations.hs @@ -31,12 +31,12 @@ import qualified GitRepo as Git stateLoc :: String stateLoc = ".git-annex/" gitStateDir :: Git.Repo -> FilePath -gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc +gitStateDir repo = Git.workTree repo ++ "/" ++ stateLoc {- Annexed file's absolute location. -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = - (Git.workTree r) ++ "/" ++ (annexLocationRelative key) + Git.workTree r ++ "/" ++ annexLocationRelative key {- Annexed file's location relative to git's working tree. - @@ -90,5 +90,5 @@ fileKey file = read $ {- for quickcheck -} prop_idempotent_fileKey :: String -> Bool -prop_idempotent_fileKey s = k == (fileKey $ keyFile k) +prop_idempotent_fileKey s = k == fileKey (keyFile k) where k = read $ "test:" ++ s diff --git a/Messages.hs b/Messages.hs index f9d12c2221..e1cf1539a6 100644 --- a/Messages.hs +++ b/Messages.hs @@ -37,17 +37,14 @@ showProgress :: Annex () showProgress = verbose $ liftIO $ putStr "\n" showLongNote :: String -> Annex () -showLongNote s = verbose $ do - liftIO $ putStr $ "\n" ++ indented +showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indented where indented = join "\n" $ map (\l -> " " ++ l) $ lines s showEndOk :: Annex () -showEndOk = verbose $ do - liftIO $ putStrLn "ok" +showEndOk = verbose $ liftIO $ putStrLn "ok" showEndFail :: Annex () -showEndFail = verbose $ do - liftIO $ putStrLn "\nfailed" +showEndFail = verbose $ liftIO $ putStrLn "\nfailed" {- Exception pretty-printing. -} showErr :: (Show a) => a -> Annex () diff --git a/TypeInternals.hs b/TypeInternals.hs index 3078224b15..bcef4ee0a8 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -51,10 +51,10 @@ instance Show Key where show (Key (b, k)) = b ++ ":" ++ k instance Read Key where - readsPrec _ s = [((Key (b,k)) ,"")] + readsPrec _ s = [(Key (b,k), "")] where l = split ":" s - b = l !! 0 + b = head l k = join ":" $ drop 1 l backendName :: Key -> BackendName @@ -81,4 +81,4 @@ data Backend = Backend { } instance Show Backend where - show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" + show backend = "Backend { name =\"" ++ name backend ++ "\" }" diff --git a/UUID.hs b/UUID.hs index 0f8a2173ef..41a35327d9 100644 --- a/UUID.hs +++ b/UUID.hs @@ -20,8 +20,8 @@ module UUID ( ) where import Control.Monad.State -import Maybe -import List +import Data.Maybe +import Data.List import System.Cmd.Utils import System.IO import System.Directory @@ -57,7 +57,7 @@ getUUID r = do let c = cached g let u = uncached - if (c /= u && u /= "") + if c /= u && u /= "" then do updatecache g u return u @@ -66,7 +66,7 @@ getUUID r = do uncached = Git.configGet r "annex.uuid" "" cached g = Git.configGet g cachekey "" updatecache g u = when (g /= r) $ Annex.setConfig cachekey u - cachekey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" + cachekey = "remote." ++ Git.repoRemoteName r ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -79,8 +79,7 @@ prepUUID = do {- Filters a list of repos to ones that have listed UUIDs. -} reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] -reposByUUID repos uuids = do - filterM match repos +reposByUUID repos uuids = filterM match repos where match r = do u <- getUUID r @@ -90,11 +89,11 @@ reposByUUID repos uuids = do prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = do m <- uuidMap - return $ unwords $ map (\u -> "\t"++(prettify m u)++"\n") uuids + return $ unwords $ map (\u -> "\t" ++ prettify m u ++ "\n") uuids where prettify m u = - if (not $ null $ findlog m u) - then u ++ " -- " ++ (findlog m u) + if not $ null $ findlog m u + then u ++ " -- " ++ findlog m u else u findlog m u = M.findWithDefault "" u m @@ -117,11 +116,11 @@ uuidMap :: Annex (M.Map UUID String) uuidMap = do logfile <- uuidLog s <- liftIO $ catch (readFile logfile) ignoreerror - return $ M.fromList $ map (\l -> pair l) $ lines s + return $ M.fromList $ map pair $ lines s where pair l = - if (1 < (length $ words l)) - then ((words l) !! 0, unwords $ drop 1 $ words l) + if 1 < length (words l) + then (head $ words l, unwords $ drop 1 $ words l) else ("", "") ignoreerror _ = return "" @@ -129,4 +128,4 @@ uuidMap = do uuidLog :: Annex String uuidLog = do g <- Annex.gitRepo - return $ (gitStateDir g) ++ "uuid.log" + return $ gitStateDir g ++ "uuid.log" diff --git a/configure.hs b/configure.hs index 4d0624a422..7d3fc01271 100644 --- a/configure.hs +++ b/configure.hs @@ -10,7 +10,7 @@ data TestDesc = TestDesc String String Test data Config = Config String Bool instance Show Config where - show (Config key value) = unlines $ [ + show (Config key value) = unlines [ key ++ " :: Bool" , key ++ " = " ++ show value ] @@ -36,7 +36,7 @@ quiet s = s ++ " >/dev/null 2>&1" requireCommand :: String -> String -> Test requireCommand command cmdline = do ret <- testCmd $ quiet cmdline - if (ret) + if ret then return True else do testEnd False @@ -57,7 +57,7 @@ testStart s = do hFlush stdout testEnd :: Bool -> IO () -testEnd r = putStrLn $ " " ++ (show r) +testEnd r = putStrLn $ " " ++ show r writeSysConfig :: [Config] -> IO () writeSysConfig config = writeFile "SysConfig.hs" body From eeae91024285c85a7e77b1b44e501a63bced7154 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 22 Nov 2010 17:51:55 -0400 Subject: [PATCH 0547/2835] finished hlinting --- Backend.hs | 2 +- Backend/File.hs | 78 ++++++++++++++++------------------- Backend/SHA1.hs | 10 ++--- Backend/WORM.hs | 10 ++--- Command/Add.hs | 4 +- Command/Drop.hs | 6 +-- Command/DropKey.hs | 6 +-- Command/Find.hs | 2 +- Command/Fix.hs | 2 +- Command/FromKey.hs | 4 +- Command/Fsck.hs | 2 +- Command/FsckFile.hs | 2 +- Command/Get.hs | 4 +- Command/Init.hs | 10 ++--- Command/Move.hs | 25 ++++++----- Command/PreCommit.hs | 2 +- Command/SetKey.hs | 2 +- Command/Unannex.hs | 2 +- Command/Unused.hs | 8 ++-- Core.hs | 2 +- Remotes.hs | 98 +++++++++++++++++++++----------------------- Utility.hs | 18 ++++---- Version.hs | 4 +- 23 files changed, 144 insertions(+), 159 deletions(-) diff --git a/Backend.hs b/Backend.hs index 01a7298b4e..d5d8efa03e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -30,7 +30,7 @@ module Backend ( ) where import Control.Monad.State -import IO (try) +import System.IO.Error (try) import System.FilePath import System.Posix.Files diff --git a/Backend/File.hs b/Backend/File.hs index c67fb3ce33..c0fc469921 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -34,7 +34,7 @@ backend = Backend { storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, removeKey = checkRemoveKey, - hasKey = checkKeyFile, + hasKey = inAnnex, fsckKey = mustProvide } @@ -42,19 +42,15 @@ mustProvide :: a mustProvide = error "must provide this field" {- Storing a key is a no-op. -} -dummyStore :: FilePath -> Key -> Annex (Bool) +dummyStore :: FilePath -> Key -> Annex Bool dummyStore _ _ = return True -{- Just check if the .git/annex/ file for the key exists. -} -checkKeyFile :: Key -> Annex Bool -checkKeyFile k = inAnnex k - {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} -copyKeyFile :: Key -> FilePath -> Annex (Bool) +copyKeyFile :: Key -> FilePath -> Annex Bool copyKeyFile key file = do remotes <- Remotes.keyPossibilities key - if (null remotes) + if null remotes then do showNote "not available" showLocations key @@ -68,76 +64,72 @@ copyKeyFile key file = do return False trycopy full (r:rs) = do probablythere <- probablyPresent r - if (probablythere) + if probablythere then do - showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." + showNote $ "copying from " ++ Git.repoDescribe r ++ "..." copied <- Remotes.copyFromRemote r key file - if (copied) + if copied then return True else trycopy full rs else trycopy full rs - probablyPresent r = do - -- This check is to avoid an ugly message if a - -- remote is a drive that is not mounted. - -- Avoid checking inAnnex for ssh remotes because - -- that is unnecessarily slow, and the locationlog - -- should be trusted. (If the ssh remote is down - -- or really lacks the file, it's ok to show - -- an ugly message before going on to the next - -- remote.) - if (not $ Git.repoIsUrl r) + -- This check is to avoid an ugly message if a remote is a + -- drive that is not mounted. Avoid checking inAnnex for ssh + -- remotes because that is unnecessarily slow, and the + -- locationlog should be trusted. (If the ssh remote is down + -- or really lacks the file, it's ok to show an ugly message + -- before going on to the next remote.) + probablyPresent r = + if not $ Git.repoIsUrl r then liftIO $ doesFileExist $ annexLocation r key else return True {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an - error if not. -} -checkRemoveKey :: Key -> Annex (Bool) +checkRemoveKey :: Key -> Annex Bool checkRemoveKey key = do force <- Annex.flagIsSet "force" - if (force) + if force then return True else do remotes <- Remotes.keyPossibilities key numcopies <- getNumCopies - if (numcopies > length remotes) + if numcopies > length remotes then notEnoughCopies numcopies (length remotes) [] else findcopies numcopies 0 remotes [] where - findcopies need have [] bad = - if (have >= need) - then return True - else notEnoughCopies need have bad - findcopies need have (r:rs) bad = do - if (have >= need) - then return True - else do - haskey <- Remotes.inAnnex r key - case (haskey) of - Right True -> findcopies need (have+1) rs bad - Right False -> findcopies need have rs bad - Left _ -> findcopies need have rs (r:bad) + findcopies need have [] bad + | have >= need = return True + | otherwise = notEnoughCopies need have bad + findcopies need have (r:rs) bad + | have >= need = return True + | otherwise = do + haskey <- Remotes.inAnnex r key + case haskey of + Right True -> findcopies need (have+1) rs bad + Right False -> findcopies need have rs bad + Left _ -> findcopies need have rs (r:bad) notEnoughCopies need have bad = do unsafe showLongNote $ "Could only verify the existence of " ++ - (show have) ++ " out of " ++ (show need) ++ + show have ++ " out of " ++ show need ++ " necessary copies" showTriedRemotes bad showLocations key hint return False unsafe = showNote "unsafe" - hint = showLongNote $ "(Use --force to override this check, or adjust annex.numcopies.)" + hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)" showLocations :: Key -> Annex () showLocations key = do g <- Annex.gitRepo u <- getUUID g uuids <- liftIO $ keyLocations g key - let uuidsf = filter (\v -> v /= u) uuids + let uuidsf = filter (/= u) uuids ppuuids <- prettyPrintUUIDs uuidsf - if (null uuidsf) + if null uuidsf then showLongNote $ "No other repository is known to contain the file." else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids @@ -145,7 +137,7 @@ showTriedRemotes :: [Git.Repo] -> Annex () showTriedRemotes [] = return () showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ - (Remotes.list remotes) + Remotes.list remotes getNumCopies :: Annex Int getNumCopies = do @@ -173,7 +165,7 @@ checkKeyNumCopies key = do remotes <- Remotes.keyPossibilities key inannex <- inAnnex key let present = length remotes + if inannex then 1 else 0 - if (present < needed) + if present < needed then do warning $ note present needed return False diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 46667c9cda..68f7f683b5 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -33,15 +33,15 @@ sha1 file = do liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do line <- hGetLine h let bits = split " " line - if (null bits) + if null bits then error "sha1sum parse error" - else return $ bits !! 0 + else return $ head bits -- A key is a sha1 of its contents. keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do s <- sha1 file - return $ Just $ Key ((name backend), s) + return $ Just $ Key (name backend, s) -- A key's sha1 is checked during fsck. checkKeySHA1 :: Key -> Annex Bool @@ -49,11 +49,11 @@ checkKeySHA1 key = do g <- Annex.gitRepo let file = annexLocation g key present <- liftIO $ doesFileExist file - if (not present) + if not present then return True else do s <- sha1 file - if (s == keyName key) + if s == keyName key then return True else do dest <- moveBad key diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 4e2177fed0..e9d8c42855 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -37,11 +37,11 @@ backend = Backend.File.backend { keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do stat <- liftIO $ getFileStatus file - return $ Just $ Key ((name backend), key stat) + return $ Just $ Key (name backend, key stat) where key stat = uniqueid stat ++ sep ++ base - uniqueid stat = (show $ modificationTime stat) ++ sep ++ - (show $ fileSize stat) + uniqueid stat = show (modificationTime stat) ++ sep ++ + show (fileSize stat) base = takeFileName file sep = ":" @@ -58,11 +58,11 @@ checkKeySize key = do g <- Annex.gitRepo let file = annexLocation g key present <- liftIO $ doesFileExist file - if (not present) + if not present then return True else do s <- liftIO $ getFileStatus file - if (fileSize s == keySize key) + if fileSize s == keySize key then return True else do dest <- moveBad key diff --git a/Command/Add.hs b/Command/Add.hs index 586807b530..cf32a8d641 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -28,7 +28,7 @@ seek = [withFilesNotInGit start, withFilesUnlocked start] start :: SubCmdStartBackendFile start pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file - if ((isSymbolicLink s) || (not $ isRegularFile s)) + if (isSymbolicLink s) || (not $ isRegularFile s) then return Nothing else do showStart "add" file @@ -37,7 +37,7 @@ start pair@(file, _) = notAnnexed file $ do perform :: (FilePath, Maybe Backend) -> SubCmdPerform perform (file, backend) = do stored <- Backend.storeFileKey file backend - case (stored) of + case stored of Nothing -> return Nothing Just (key, _) -> return $ Just $ cleanup file key diff --git a/Command/Drop.hs b/Command/Drop.hs index 1e73d8b821..fbe66f5841 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -24,7 +24,7 @@ seek = [withFilesInGit start] start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key - if (not inbackend) + if not inbackend then return Nothing else do showStart "drop" file @@ -33,13 +33,13 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: Key -> Backend -> SubCmdPerform perform key backend = do success <- Backend.removeKey backend key - if (success) + if success then return $ Just $ cleanup key else return Nothing cleanup :: Key -> SubCmdCleanup cleanup key = do inannex <- inAnnex key - when (inannex) $ removeAnnex key + when inannex $ removeAnnex key logStatus key ValueMissing return True diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 34010481dd..aa72e1bbd6 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -22,12 +22,12 @@ seek = [withKeys start] start :: SubCmdStartString start keyname = do backends <- Backend.list - let key = genKey (backends !! 0) keyname + let key = genKey (head backends) keyname present <- inAnnex key force <- Annex.flagIsSet "force" - if (not present) + if not present then return Nothing - else if (not force) + else if not force then error "dropkey is can cause data loss; use --force if you're sure you want to do this" else do showStart "dropkey" keyname diff --git a/Command/Find.hs b/Command/Find.hs index db0589feae..7b3c8c4636 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -20,5 +20,5 @@ seek = [withDefault "." withFilesInGit start] start :: SubCmdStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key - when (exists) $ liftIO $ putStrLn file + when exists $ liftIO $ putStrLn file return Nothing diff --git a/Command/Fix.hs b/Command/Fix.hs index 323aca95e3..33630031f6 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -25,7 +25,7 @@ start :: SubCmdStartString start file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file - if (link == l) + if link == l then return Nothing else do showStart "fix" file diff --git a/Command/FromKey.hs b/Command/FromKey.hs index f25de23a2c..eb9ad5e514 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -29,10 +29,10 @@ start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list - let key = genKey (backends !! 0) keyname + let key = genKey (head backends) keyname inbackend <- Backend.hasKey key - unless (inbackend) $ error $ + unless inbackend $ error $ "key ("++keyname++") is not present in backend" showStart "fromkey" file return $ Just $ perform file key diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 6291d5ba3f..dc01688015 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -24,6 +24,6 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: Key -> Backend -> SubCmdPerform perform key backend = do success <- Backend.fsckKey backend key - if (success) + if success then return $ Just $ return True else return Nothing diff --git a/Command/FsckFile.hs b/Command/FsckFile.hs index c74e94e62f..e7c3d49158 100644 --- a/Command/FsckFile.hs +++ b/Command/FsckFile.hs @@ -24,6 +24,6 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: Key -> Backend -> SubCmdPerform perform key backend = do success <- Backend.fsckKey backend key - if (success) + if success then return $ Just $ return True else return Nothing diff --git a/Command/Get.hs b/Command/Get.hs index 13d1375377..628ed62935 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -20,7 +20,7 @@ seek = [withFilesInGit start] start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key - if (inannex) + if inannex then return Nothing else do showStart "get" file @@ -29,7 +29,7 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: Key -> Backend -> SubCmdPerform perform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) - if (ok) + if ok then return $ Just $ return True -- no cleanup needed else return Nothing diff --git a/Command/Init.hs b/Command/Init.hs index c928647a50..eb5c58696f 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -25,8 +25,8 @@ seek = [withString start] {- Stores description for the repository etc. -} start :: SubCmdStartString start description = do - when (null description) $ error $ - "please specify a description of this repository\n" + when (null description) $ + error "please specify a description of this repository\n" showStart "init" description return $ Just $ perform description @@ -38,7 +38,7 @@ perform description = do setVersion liftIO $ gitAttributes g liftIO $ gitPreCommitHook g - return $ Just $ cleanup + return $ Just cleanup cleanup :: SubCmdCleanup cleanup = do @@ -53,7 +53,7 @@ cleanup = do gitAttributes :: Git.Repo -> IO () gitAttributes repo = do exists <- doesFileExist attributes - if (not exists) + if not exists then do writeFile attributes $ attrLine ++ "\n" commit @@ -76,7 +76,7 @@ gitPreCommitHook repo = do let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit" exists <- doesFileExist hook - if (exists) + if exists then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" else do writeFile hook $ "#!/bin/sh\n" ++ diff --git a/Command/Move.hs b/Command/Move.hs index 7f8f40737d..c18054c904 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,8 +7,7 @@ module Command.Move where -import Control.Monad.State (liftIO) -import Monad (when) +import Control.Monad.State (liftIO, when) import Command import qualified Command.Drop @@ -53,7 +52,7 @@ start file = do moveToStart :: SubCmdStartString moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key - if (not ishere) + if not ishere then return Nothing -- not here, so nothing to do else do showStart "move" file @@ -68,10 +67,10 @@ moveToPerform key = do showNote $ show err return Nothing Right False -> do - showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." - let tmpfile = (annexTmpLocation remote) ++ (keyFile key) + showNote $ "moving to " ++ Git.repoDescribe remote ++ "..." + let tmpfile = annexTmpLocation remote ++ keyFile key ok <- Remotes.copyToRemote remote key tmpfile - if (ok) + if ok then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ Command.Drop.cleanup key @@ -79,7 +78,7 @@ moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", - "--backend=" ++ (backendName key), + "--backend=" ++ backendName key, "--key=" ++ keyName key, tmpfile] if ok @@ -104,7 +103,7 @@ moveFromStart :: SubCmdStartString moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key - if (null $ filter (\r -> Remotes.same r remote) l) + if null $ filter (\r -> Remotes.same r remote) l then return Nothing else do showStart "move" file @@ -113,18 +112,18 @@ moveFromPerform :: Key -> SubCmdPerform moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key - if (ishere) + if ishere then return $ Just $ moveFromCleanup remote key else do - showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." - ok <- getViaTmp key (Remotes.copyFromRemote remote key) - if (ok) + showNote $ "moving from " ++ Git.repoDescribe remote ++ "..." + ok <- getViaTmp key $ Remotes.copyFromRemote remote key + if ok then return $ Just $ moveFromCleanup remote key else return Nothing -- fail moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup moveFromCleanup remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", - "--backend=" ++ (backendName key), + "--backend=" ++ backendName key, keyName key] when ok $ do -- Record locally that the key is not on the remote. diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index a15510bd93..d4e5c04b9c 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -28,7 +28,7 @@ start file = return $ Just $ perform file perform :: FilePath -> SubCmdPerform perform file = do pairs <- Backend.chooseBackends [file] - ok <- doSubCmd $ Command.Add.start $ pairs !! 0 + ok <- doSubCmd $ Command.Add.start $ head pairs if ok then return $ Just $ cleanup file else error $ "failed to add " ++ file ++ "; canceling commit" diff --git a/Command/SetKey.hs b/Command/SetKey.hs index e8d407b832..685872f73d 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -28,7 +28,7 @@ start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list - let key = genKey (backends !! 0) keyname + let key = genKey (head backends) keyname showStart "setkey" file return $ Just $ perform file key perform :: FilePath -> Key -> SubCmdPerform diff --git a/Command/Unannex.hs b/Command/Unannex.hs index e85e8486f5..90ae550581 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -34,7 +34,7 @@ perform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True ok <- Backend.removeKey backend key - if (ok) + if ok then return $ Just $ cleanup file key else return Nothing diff --git a/Command/Unused.hs b/Command/Unused.hs index ae189550ce..de34ceae96 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -35,7 +35,7 @@ checkUnused :: Annex Bool checkUnused = do showNote "checking for unused data..." unused <- unusedKeys - if (null unused) + if null unused then return True else do let list = number 1 unused @@ -48,9 +48,10 @@ checkUnused = do w u = unlines $ ["Some annexed data is no longer pointed to by any files in the repository:", " NUMBER KEY"] - ++ (map (\(n, k) -> " " ++ (pad 6 $ show n) ++ " " ++ show k) u) ++ + ++ map cols u ++ ["(To see where data was previously used, try: git log --stat -S'KEY')", "(To remove unwanted data: git-annex dropunused NUMBER)"] + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k pad n s = s ++ replicate (n - length s) ' ' number :: Integer -> [a] -> [(Integer, a)] @@ -71,8 +72,7 @@ unusedKeys = do let unused_m = remove referenced present_m return $ M.keys unused_m where - remove [] m = m - remove (x:xs) m = remove xs $ M.delete x m + remove a b = foldl (flip M.delete) b a existsMap :: Ord k => [k] -> M.Map k Int existsMap l = M.fromList $ map (\k -> (k, 1)) l diff --git a/Core.hs b/Core.hs index 6e0ddd65ff..b61d186666 100644 --- a/Core.hs +++ b/Core.hs @@ -7,7 +7,7 @@ module Core where -import IO (try) +import System.IO.Error (try) import System.Directory import Control.Monad.State (liftIO) import System.Path diff --git a/Remotes.hs b/Remotes.hs index bf5ede5729..cb8081d740 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -17,16 +17,14 @@ module Remotes ( runCmd ) where -import IO (bracket_) -import Control.Exception.Extensible hiding (bracket_) +import Control.Exception.Extensible import Control.Monad.State (liftIO) -import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils import System.Directory hiding (copyFile) import System.Posix.Directory -import List -import Monad (when, unless) +import Data.List +import Control.Monad (when, unless, filterM) import Types import qualified GitRepo as Git @@ -55,7 +53,7 @@ keyPossibilities key = do -- But, reading the config of remotes can be expensive, so make -- sure we only do it once per git-annex run. remotesread <- Annex.flagIsSet "remotesread" - if (remotesread) + if remotesread then reposByUUID allremotes uuids else do -- We assume that it's cheap to read the config @@ -65,11 +63,11 @@ keyPossibilities key = do let cheap = filter (not . Git.repoIsUrl) allremotes let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive - unless (null doexpensive) $ do + unless (null doexpensive) $ showNote $ "getting UUID for " ++ - (list doexpensive) ++ "..." + list doexpensive ++ "..." let todo = cheap ++ doexpensive - if (not $ null todo) + if not $ null todo then do _ <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True @@ -84,10 +82,9 @@ keyPossibilities key = do - If the remote cannot be accessed, returns a Left error. -} inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) -inAnnex r key = do - if (not $ Git.repoIsUrl r) - then liftIO $ ((try checklocal)::IO (Either IOException Bool)) - else checkremote +inAnnex r key = if Git.repoIsUrl r + then checkremote + else liftIO (try checklocal ::IO (Either IOException Bool)) where checklocal = do -- run a local check by making an Annex monad @@ -112,12 +109,12 @@ reposByCost :: [Git.Repo] -> Annex [Git.Repo] reposByCost l = do notignored <- filterM repoNotIgnored l costpairs <- mapM costpair notignored - return $ fst $ unzip $ sortBy bycost $ costpairs + return $ fst $ unzip $ sortBy cmpcost costpairs where costpair r = do cost <- repoCost r return (r, cost) - bycost (_, c1) (_, c2) = compare c1 c2 + cmpcost (_, c1) (_, c2) = compare c1 c2 {- Calculates cost for a repo. - @@ -127,9 +124,9 @@ reposByCost l = do repoCost :: Git.Repo -> Annex Int repoCost r = do cost <- repoConfig r "cost" "" - if (not $ null cost) + if not $ null cost then return $ read cost - else if (Git.repoIsUrl r) + else if Git.repoIsUrl r then return 200 else return 100 @@ -141,13 +138,12 @@ repoNotIgnored r = do ignored <- repoConfig r "ignore" "false" fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" - let name = if (not $ null fromName) then fromName else toName - if (not $ null name) + let name = if null fromName then toName else fromName + if not $ null name then return $ match name - else return $ not $ isIgnored ignored + else return $ not $ Git.configTrue ignored where match name = name == Git.repoRemoteName r - isIgnored ignored = Git.configTrue ignored {- Checks if two repos are the same, by comparing their remote names. -} same :: Git.Repo -> Git.Repo -> Bool @@ -158,14 +154,14 @@ commandLineRemote :: Annex Git.Repo commandLineRemote = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" - let name = if (not $ null fromName) then fromName else toName + let name = if null fromName then toName else fromName when (null name) $ error "no remote specified" g <- Annex.gitRepo let match = filter (\r -> name == Git.repoRemoteName r) $ Git.remotes g when (null match) $ error $ "there is no git remote named \"" ++ name ++ "\"" - return $ match !! 0 + return $ head match {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the @@ -174,12 +170,12 @@ commandLineRemote = do tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r = do sshoptions <- repoConfig r "ssh-options" "" - if (Map.null $ Git.configMap r) + if Map.null $ Git.configMap r then do -- configRead can fail due to IO error or -- for other reasons; catch all possible exceptions - result <- liftIO $ (try (Git.configRead r $ Just $ words sshoptions)::IO (Either SomeException (Git.Repo))) - case (result) of + result <- liftIO (try (Git.configRead r $ Just $ words sshoptions)::IO (Either SomeException Git.Repo)) + case result of Left _ -> return $ Left r Right r' -> do g <- Annex.gitRepo @@ -192,18 +188,16 @@ tryGitConfigRead r = do where exchange [] _ = [] exchange (old:ls) new = - if (Git.repoRemoteName old == Git.repoRemoteName new) - then new:(exchange ls new) - else old:(exchange ls new) + if Git.repoRemoteName old == Git.repoRemoteName new + then new : exchange ls new + else old : exchange ls new {- Tries to copy a key's content from a remote to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool -copyFromRemote r key file = do - if (not $ Git.repoIsUrl r) - then getlocal - else if (Git.repoIsSsh r) - then getssh - else error "copying from non-ssh repo not supported" +copyFromRemote r key file + | not $ Git.repoIsUrl r = getlocal + | Git.repoIsSsh r = getssh + | otherwise = error "copying from non-ssh repo not supported" where getlocal = liftIO $ copyFile keyloc file getssh = scp r [sshLocation r keyloc, file] @@ -214,9 +208,9 @@ copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyToRemote r key file = do g <- Annex.gitRepo let keyloc = annexLocation g key - if (not $ Git.repoIsUrl r) + if not $ Git.repoIsUrl r then putlocal keyloc - else if (Git.repoIsSsh r) + else if Git.repoIsSsh r then putssh keyloc else error "copying to non-ssh repo not supported" where @@ -224,7 +218,7 @@ copyToRemote r key file = do putssh src = scp r [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath -sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file +sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file {- Runs scp against a specified remote. (Honors annex-scp-options.) -} scp :: Git.Repo -> [String] -> Annex Bool @@ -238,21 +232,21 @@ scp r params = do runCmd :: Git.Repo -> String -> [String] -> Annex Bool runCmd r command params = do sshoptions <- repoConfig r "ssh-options" "" - if (not $ Git.repoIsUrl r) + if not $ Git.repoIsUrl r then do - cwd <- liftIO $ getCurrentDirectory - liftIO $ bracket_ (changeWorkingDirectory (Git.workTree r)) - (\_ -> changeWorkingDirectory cwd) $ - boolSystem command params - else if (Git.repoIsSsh r) - then do - liftIO $ boolSystem "ssh" $ - (words sshoptions) ++ - [Git.urlHost r, "cd " ++ - (shellEscape $ Git.workTree r) ++ - " && " ++ (shellEscape command) ++ " " ++ - (unwords $ map shellEscape params)] + cwd <- liftIO getCurrentDirectory + liftIO $ bracket_ + (changeWorkingDirectory (Git.workTree r)) + (changeWorkingDirectory cwd) + (boolSystem command params) + else if Git.repoIsSsh r + then liftIO $ boolSystem "ssh" $ + words sshoptions ++ [Git.urlHost r, sshcmd] else error "running command in non-ssh repo not supported" + where + sshcmd = "cd " ++ shellEscape (Git.workTree r) ++ + " && " ++ shellEscape command ++ " " ++ + unwords (map shellEscape params) {- Looks up a per-remote config option in git config. - Failing that, tries looking for a global config option. -} @@ -262,5 +256,5 @@ repoConfig r key def = do let def' = Git.configGet g global def return $ Git.configGet g local def' where - local = "remote." ++ (Git.repoRemoteName r) ++ ".annex-" ++ key + local = "remote." ++ Git.repoRemoteName r ++ ".annex-" ++ key global = "annex." ++ key diff --git a/Utility.hs b/Utility.hs index 33db4bb086..2bea6e8755 100644 --- a/Utility.hs +++ b/Utility.hs @@ -35,12 +35,12 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s {- Returns the parent directory of a path. Parent of / is "" -} parentDir :: String -> String parentDir dir = - if (not $ null dirs) - then slash ++ (join s $ take ((length dirs) - 1) dirs) + if not $ null dirs + then slash ++ join s (take (length dirs - 1) dirs) else "" where - dirs = filter (\x -> not $ null x) $ split s dir - slash = if (not $ isAbsolute dir) then "" else s + dirs = filter (not . null) $ split s dir + slash = if isAbsolute dir then s else "" s = [pathSeparator] {- Constructs a relative path from the CWD to a directory. @@ -58,7 +58,7 @@ relPathCwdToDir dir = do where -- absolute, normalized form of the directory absnorm cwd = - case (absNormPath cwd dir) of + case absNormPath cwd dir of Just d -> d Nothing -> error $ "unable to normalize " ++ dir @@ -70,7 +70,7 @@ relPathCwdToDir dir = do -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = - if (not $ null path) + if not $ null path then addTrailingPathSeparator path else "" where @@ -80,8 +80,8 @@ relPathDirToDir from to = common = map fst $ filter same $ zip pfrom pto same (c,d) = c == d uncommon = drop numcommon pto - dotdots = take ((length pfrom) - numcommon) $ repeat ".." - numcommon = length $ common + dotdots = replicate (length pfrom - numcommon) ".." + numcommon = length common path = join s $ dotdots ++ uncommon {- Run a system command, and returns True or False @@ -124,4 +124,4 @@ shellEscape f = "'" ++ escaped ++ "'" unsetFileMode :: FilePath -> FileMode -> IO () unsetFileMode f m = do s <- getFileStatus f - setFileMode f $ (fileMode s) `intersectFileModes` (complement m) + setFileMode f $ fileMode s `intersectFileModes` complement m diff --git a/Version.hs b/Version.hs index ce39c0c1b1..8394701201 100644 --- a/Version.hs +++ b/Version.hs @@ -25,13 +25,13 @@ getVersion :: Annex (Maybe String) getVersion = do g <- Annex.gitRepo let v = Git.configGet g versionField "" - if (not $ null v) + if not $ null v then return $ Just v else do -- version 0 was not recorded in .git/config; -- such a repo should have an annexDir d <- liftIO $ doesDirectoryExist $ annexDir g - if (d) + if d then return $ Just "0" else return Nothing -- no version yet From e97d13e29b18c4522395996299651334cb221519 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:02:53 -0400 Subject: [PATCH 0548/2835] Add copy subcommand. --- CmdLine.hs | 5 ++- Command/Move.hs | 107 ++++++++++++++++++++++++--------------------- debian/changelog | 6 +++ doc/git-annex.mdwn | 8 ++++ 4 files changed, 74 insertions(+), 52 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 35e889d7df..837420786f 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -18,6 +18,7 @@ import qualified Command.Add import qualified Command.Unannex import qualified Command.Drop import qualified Command.Move +import qualified Command.Copy import qualified Command.Get import qualified Command.FromKey import qualified Command.DropKey @@ -41,7 +42,9 @@ subCmds = , SubCommand "drop" path Command.Drop.seek "indicate content of files not currently wanted" , SubCommand "move" path Command.Move.seek - "transfer content of files to/from another repository" + "move content of files to/from another repository" + , SubCommand "copy" path Command.Copy.seek + "copy content of files to/from another repository" , SubCommand "unlock" path Command.Unlock.seek "unlock files for modification" , SubCommand "edit" path Command.Unlock.seek diff --git a/Command/Move.hs b/Command/Move.hs index c18054c904..cb6525919e 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,7 +7,7 @@ module Command.Move where -import Control.Monad.State (liftIO, when) +import Control.Monad.State (liftIO) import Command import qualified Command.Drop @@ -22,43 +22,55 @@ import UUID import Messages seek :: [SubCmdSeek] -seek = [withFilesInGit start] +seek = [withFilesInGit $ start True] -{- Move a file either --to or --from a repository. +{- Move (or copy) a file either --to or --from a repository. - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -start :: SubCmdStartString -start file = do +start :: Bool -> SubCmdStartString +start move file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" case (fromName, toName) of ("", "") -> error "specify either --from or --to" - ("", _) -> moveToStart file - (_ , "") -> moveFromStart file + ("", _) -> toStart move file + (_ , "") -> fromStart move file (_ , _) -> error "only one of --from or --to can be specified" -{- Moves the content of an annexed file to another repository, - - removing it from the current repository, and updates locationlog - - information on both. +showAction :: Bool -> FilePath -> Annex () +showAction True file = showStart "move" file +showAction False file = showStart "copy" file + +remoteHasKey :: Git.Repo -> Key -> Bool -> Annex () +remoteHasKey remote key present = do + g <- Annex.gitRepo + remoteuuid <- getUUID remote + logfile <- liftIO $ logChange g key remoteuuid status + Annex.queue "add" ["--"] logfile + where + status = if present then ValuePresent else ValueMissing + +{- Moves (or copies) the content of an annexed file to another repository, + - and updates locationlog information on both. - - - If the destination already has the content, it is still removed - - from the current repository. + - When moving, if the destination already has the content, it is + - still removed from the current repository. - - Note that unlike drop, this does not honor annex.numcopies. - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -moveToStart :: SubCmdStartString -moveToStart file = isAnnexed file $ \(key, _) -> do +toStart :: Bool -> SubCmdStartString +toStart move file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if not ishere then return Nothing -- not here, so nothing to do else do - showStart "move" file - return $ Just $ moveToPerform key -moveToPerform :: Key -> SubCmdPerform -moveToPerform key = do + showAction move file + return $ Just $ toPerform move key +toPerform :: Bool -> Key -> SubCmdPerform +toPerform move key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key @@ -67,15 +79,15 @@ moveToPerform key = do showNote $ show err return Nothing Right False -> do - showNote $ "moving to " ++ Git.repoDescribe remote ++ "..." + showNote $ "to " ++ Git.repoDescribe remote ++ "..." let tmpfile = annexTmpLocation remote ++ keyFile key ok <- Remotes.copyToRemote remote key tmpfile if ok - then return $ Just $ moveToCleanup remote key tmpfile + then return $ Just $ toCleanup move remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ Command.Drop.cleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup -moveToCleanup remote key tmpfile = do +toCleanup :: Bool -> Git.Repo -> Key -> FilePath -> SubCmdCleanup +toCleanup move remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", "--backend=" ++ backendName key, @@ -83,52 +95,45 @@ moveToCleanup remote key tmpfile = do tmpfile] if ok then do - -- Record that the key is present on the remote. - g <- Annex.gitRepo - remoteuuid <- getUUID remote - logfile <- liftIO $ logChange g key remoteuuid ValuePresent - Annex.queue "add" ["--"] logfile - -- Cleanup on the local side is the same as done for the - -- drop subcommand. - Command.Drop.cleanup key + remoteHasKey remote key True + if move + then Command.Drop.cleanup key + else return True else return False -{- Moves the content of an annexed file from another repository to the current - - repository and updates locationlog information on both. +{- Moves (or copies) the content of an annexed file from another repository + - to the current repository and updates locationlog information on both. - - If the current repository already has the content, it is still removed - - from the other repository. + - from the other repository when moving. -} -moveFromStart :: SubCmdStartString -moveFromStart file = isAnnexed file $ \(key, _) -> do +fromStart :: Bool -> SubCmdStartString +fromStart move file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key if null $ filter (\r -> Remotes.same r remote) l then return Nothing else do - showStart "move" file - return $ Just $ moveFromPerform key -moveFromPerform :: Key -> SubCmdPerform -moveFromPerform key = do + showAction move file + return $ Just $ fromPerform move key +fromPerform :: Bool -> Key -> SubCmdPerform +fromPerform move key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key if ishere - then return $ Just $ moveFromCleanup remote key + then return $ Just $ fromCleanup move remote key else do - showNote $ "moving from " ++ Git.repoDescribe remote ++ "..." + showNote $ "from " ++ Git.repoDescribe remote ++ "..." ok <- getViaTmp key $ Remotes.copyFromRemote remote key if ok - then return $ Just $ moveFromCleanup remote key + then return $ Just $ fromCleanup move remote key else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup -moveFromCleanup remote key = do - ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", +fromCleanup :: Bool -> Git.Repo -> Key -> SubCmdCleanup +fromCleanup True remote key = do + ok <- Remotes.runCmd remote "git-annex" + ["dropkey", "--quiet", "--force", "--backend=" ++ backendName key, keyName key] - when ok $ do - -- Record locally that the key is not on the remote. - remoteuuid <- getUUID remote - g <- Annex.gitRepo - logfile <- liftIO $ logChange g key remoteuuid ValueMissing - Annex.queue "add" ["--"] logfile + remoteHasKey remote key False return ok +fromCleanup False _ _ = return True diff --git a/debian/changelog b/debian/changelog index 66c4bc94da..6e5e7ef488 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.09) UNRELEASED; urgency=low + + * Add copy subcommand. + + -- Joey Hess Sat, 27 Nov 2010 16:58:33 -0400 + git-annex (0.08) unstable; urgency=low * Fix `git annex add ../foo` (when ran in a subdir of the repo). diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3df835eac5..27393de503 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -102,6 +102,14 @@ Many git-annex subcommands will stage changes for later `git commit` by you. When used with the --from option, moves the content of annexed files from the specified repository to the current one. +* copy [path ...] + + When used with the --to option, copies the content of annexed files from + the current repository to the specified one. + + When used with the --from option, copies the content of annexed files + from the specified repository to the current one. + * init description Initializes git-annex with a description of the git repository, From e0518a4adc250da1ad088c4362016627e1effb08 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:07:22 -0400 Subject: [PATCH 0549/2835] Fix bug in setkey subcommand triggered by move --to. --- Command/SetKey.hs | 5 ++++- debian/changelog | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 685872f73d..55472ccaed 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -35,7 +35,10 @@ perform :: FilePath -> Key -> SubCmdPerform perform file key = do -- the file might be on a different filesystem, so mv is used -- rather than simply calling moveToObjectDir key file - ok <- getViaTmp key $ \dest -> liftIO $ boolSystem "mv" [file, dest] + ok <- getViaTmp key $ \dest -> do + if dest /= file + then liftIO $ boolSystem "mv" [file, dest] + else return True if ok then return $ Just $ cleanup key else error "mv failed!" diff --git a/debian/changelog b/debian/changelog index 6e5e7ef488..75f4f9f0ae 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.09) UNRELEASED; urgency=low * Add copy subcommand. + * Fix bug in setkey subcommand triggered by move --to. -- Joey Hess Sat, 27 Nov 2010 16:58:33 -0400 From 6dfae19b1b1535869fb4fb5ad2aba6ada24ddeef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:09:22 -0400 Subject: [PATCH 0550/2835] add --- Command/Copy.hs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 Command/Copy.hs diff --git a/Command/Copy.hs b/Command/Copy.hs new file mode 100644 index 0000000000..aa55731d9c --- /dev/null +++ b/Command/Copy.hs @@ -0,0 +1,15 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Copy where + +import Command +import qualified Command.Move + +-- A copy is just a move that does not delete the source file. +seek :: [SubCmdSeek] +seek = [withFilesInGit $ Command.Move.start False] From 1493601982f438691985deb8bf0f279f6928fb6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:17:14 -0400 Subject: [PATCH 0551/2835] releasing version 0.09 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 75f4f9f0ae..8ab14d64c5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,9 @@ -git-annex (0.09) UNRELEASED; urgency=low +git-annex (0.09) unstable; urgency=low * Add copy subcommand. * Fix bug in setkey subcommand triggered by move --to. - -- Joey Hess Sat, 27 Nov 2010 16:58:33 -0400 + -- Joey Hess Sat, 27 Nov 2010 17:14:59 -0400 git-annex (0.08) unstable; urgency=low From f8d463b1b76fb5865dfffd5ab4a1205595c1a2b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:17:25 -0400 Subject: [PATCH 0552/2835] add news item for git-annex 0.09 --- doc/news/version_0.04.mdwn | 23 ----------------------- doc/news/version_0.09.mdwn | 4 ++++ 2 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 doc/news/version_0.04.mdwn create mode 100644 doc/news/version_0.09.mdwn diff --git a/doc/news/version_0.04.mdwn b/doc/news/version_0.04.mdwn deleted file mode 100644 index c5a9fd3ead..0000000000 --- a/doc/news/version_0.04.mdwn +++ /dev/null @@ -1,23 +0,0 @@ -git-annex 0.04 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Add unlock subcommand, which replaces the symlink with a copy of - the file's content in preparation of changing it. The "edit" subcommand - is an alias for unlock. - * Add lock subcommand. - * Unlocked files will now automatically be added back into the annex when - committed (and the updated symlink committed), by some magic in the - pre-commit hook. - * The SHA1 backend is now fully usable. - * Add annex.version, which will be used to automate upgrades - between incompatible versions. - * Reorganised the layout of .git/annex/ - * The new layout will be automatically upgraded to the first time - git-annex is used in a repository with the old layout. - * Note that git-annex 0.04 cannot transfer content from old repositories - that have not yet been upgraded. - * Annexed file contents are now made unwritable and put in unwriteable - directories, to avoid them accidentially being removed or modified. - (Thanks Josh Triplett for the idea.) - * Add build dep on libghc6-testpack-dev. Closes: #[603016](http://bugs.debian.org/603016) - * Avoid using runghc to run test suite as it is not available on all - architectures. Closes: #[603006](http://bugs.debian.org/603006)"""]] \ No newline at end of file diff --git a/doc/news/version_0.09.mdwn b/doc/news/version_0.09.mdwn new file mode 100644 index 0000000000..fff8249f77 --- /dev/null +++ b/doc/news/version_0.09.mdwn @@ -0,0 +1,4 @@ +git-annex 0.09 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Add copy subcommand. + * Fix bug in setkey subcommand triggered by move --to."""]] \ No newline at end of file From 1f9ce9e9a5f6d7eaf149f42de559cb9830c7f28e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:31:20 -0400 Subject: [PATCH 0553/2835] tweak --- configure.hs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/configure.hs b/configure.hs index 7d3fc01271..92a051c80e 100644 --- a/configure.hs +++ b/configure.hs @@ -6,7 +6,7 @@ import System.Exit import System.Directory type Test = IO Bool -data TestDesc = TestDesc String String Test +data TestCase = TestCase String String Test data Config = Config String Bool instance Show Config where @@ -15,13 +15,13 @@ instance Show Config where , key ++ " = " ++ show value ] -tests :: [TestDesc] +tests :: [TestCase] tests = [ - TestDesc "cp -a" "cp_a" $ testCp "-a" - , TestDesc "cp -p" "cp_p" $ testCp "-p" - , TestDesc "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" - , TestDesc "uuid" "uuid" $ requireCommand "uuid" "uuid" - , TestDesc "xargs -0" "xargs_0" $ requireCommand "xargs -0" "xargs -0 IO [Config] +runTests :: [TestCase] -> IO [Config] runTests [] = return [] -runTests ((TestDesc tname key t):ts) = do +runTests ((TestCase tname key t):ts) = do testStart tname val <- t testEnd val From 92e5d28ca83d057a3d8f5d7d30806642de699172 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 14:19:43 -0400 Subject: [PATCH 0554/2835] precommit: Optimise to avoid calling git-check-attr more than once. --- Command.hs | 32 ++++++++++++++++---------------- Command/Add.hs | 2 +- Command/PreCommit.hs | 14 ++++++-------- Command/Unlock.hs | 1 - debian/changelog | 6 ++++++ 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/Command.hs b/Command.hs index 4d10a9e7f1..7f3063abb9 100644 --- a/Command.hs +++ b/Command.hs @@ -41,8 +41,9 @@ type SubCmdCleanup = Annex Bool - functions. -} type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek type SubCmdStartString = String -> SubCmdStart +type BackendFile = (FilePath, Maybe Backend) type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek -type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart +type SubCmdStartBackendFile = BackendFile -> SubCmdStart type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek type SubCmdStartNothing = SubCmdStart @@ -116,17 +117,6 @@ withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ mapM (Git.notInRepo repo) params backendPairs a $ filter notState $ foldl (++) [] newfiles -withFilesUnlocked :: SubCmdSeekBackendFiles -withFilesUnlocked a params = do - -- unlocked files have changed type from a symlink to a regular file - repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params - unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - backendPairs a $ filter notState unlockedfiles -backendPairs :: SubCmdSeekBackendFiles -backendPairs a files = do - pairs <- Backend.chooseBackends files - return $ map a pairs withString :: SubCmdSeekStrings withString a params = return [a $ unwords params] withStrings :: SubCmdSeekStrings @@ -136,12 +126,17 @@ withFilesToBeCommitted a params = do repo <- Annex.gitRepo tocommit <- liftIO $ mapM (Git.stagedFiles repo) params return $ map a $ filter notState $ foldl (++) [] tocommit -withUnlockedFilesToBeCommitted :: SubCmdSeekStrings -withUnlockedFilesToBeCommitted a params = do +withFilesUnlocked :: SubCmdSeekBackendFiles +withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles +withFilesUnlockedToBeCommitted :: SubCmdSeekBackendFiles +withFilesUnlockedToBeCommitted = withFilesUnlocked' Git.typeChangedStagedFiles +withFilesUnlocked' :: (Git.Repo -> FilePath -> IO [FilePath]) -> SubCmdSeekBackendFiles +withFilesUnlocked' typechanged a params = do + -- unlocked files have changed type from a symlink to a regular file repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (Git.typeChangedStagedFiles repo) params + typechangedfiles <- liftIO $ mapM (typechanged repo) params unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - return $ map a $ filter notState unlockedfiles + backendPairs a $ filter notState unlockedfiles withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings @@ -150,6 +145,11 @@ withNothing :: SubCmdSeekNothing withNothing a [] = return [a] withNothing _ _ = return [] +backendPairs :: SubCmdSeekBackendFiles +backendPairs a files = do + pairs <- Backend.chooseBackends files + return $ map a pairs + {- Default to acting on all files matching the seek action if - none are specified. -} withAll :: SubCmdSeekStrings -> SubCmdSeekStrings diff --git a/Command/Add.hs b/Command/Add.hs index cf32a8d641..d141448a3a 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -34,7 +34,7 @@ start pair@(file, _) = notAnnexed file $ do showStart "add" file return $ Just $ perform pair -perform :: (FilePath, Maybe Backend) -> SubCmdPerform +perform :: BackendFile -> SubCmdPerform perform (file, backend) = do stored <- Backend.storeFileKey file backend case stored of diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index d4e5c04b9c..513d5d43f7 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -11,7 +11,6 @@ import Control.Monad.State (liftIO) import Command import qualified Annex -import qualified Backend import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix @@ -20,15 +19,14 @@ import qualified Command.Fix - And, it needs to inject unlocked files into the annex. -} seek :: [SubCmdSeek] seek = [withFilesToBeCommitted Command.Fix.start, - withUnlockedFilesToBeCommitted start] + withFilesUnlockedToBeCommitted start] -start :: SubCmdStartString -start file = return $ Just $ perform file +start :: SubCmdStartBackendFile +start pair = return $ Just $ perform pair -perform :: FilePath -> SubCmdPerform -perform file = do - pairs <- Backend.chooseBackends [file] - ok <- doSubCmd $ Command.Add.start $ head pairs +perform :: BackendFile -> SubCmdPerform +perform pair@(file, _) = do + ok <- doSubCmd $ Command.Add.start pair if ok then return $ Just $ cleanup file else error $ "failed to add " ++ file ++ "; canceling commit" diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 34fde819cb..ff22fa84b3 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -15,7 +15,6 @@ import qualified Annex import Types import Messages import Locations -import Utility import Core import CopyFile diff --git a/debian/changelog b/debian/changelog index 8ab14d64c5..808087dadc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.10) UNRELEASED; urgency=low + + * precommit: Optimise to avoid calling git-check-attr more than once. + + -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 + git-annex (0.09) unstable; urgency=low * Add copy subcommand. From 653ad35a9f728ed5b3e9b557cdfb15a19b4afe16 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 15:28:20 -0400 Subject: [PATCH 0555/2835] In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files. --- Backend.hs | 8 ++++---- Backend/File.hs | 25 +++++++++++++------------ Backend/URL.hs | 10 ++++++++-- Command.hs | 16 +++++++++++++--- Command/Drop.hs | 17 ++++++++++------- Command/DropUnused.hs | 4 +--- Command/Fsck.hs | 17 ++++++++++------- Command/FsckFile.hs | 29 ----------------------------- Command/Unannex.hs | 3 +-- TypeInternals.hs | 9 ++++++--- Utility.hs | 9 ++++++++- debian/changelog | 2 ++ doc/copies.mdwn | 7 +++++-- doc/git-annex.mdwn | 6 ++++++ 14 files changed, 87 insertions(+), 75 deletions(-) delete mode 100644 Command/FsckFile.hs diff --git a/Backend.hs b/Backend.hs index d5d8efa03e..8e17f253f4 100644 --- a/Backend.hs +++ b/Backend.hs @@ -104,8 +104,8 @@ retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} -removeKey :: Backend -> Key -> Annex Bool -removeKey backend key = (Internals.removeKey backend) key +removeKey :: Backend -> Key -> Maybe Int -> Annex Bool +removeKey backend key numcopies = (Internals.removeKey backend) key numcopies {- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool @@ -114,8 +114,8 @@ hasKey key = do (Internals.hasKey backend) key {- Checks a key's backend for problems. -} -fsckKey :: Backend -> Key -> Annex Bool -fsckKey backend key = (Internals.fsckKey backend) key +fsckKey :: Backend -> Key -> Maybe Int -> Annex Bool +fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/Backend/File.hs b/Backend/File.hs index c0fc469921..5984348b36 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -86,14 +86,14 @@ copyKeyFile key file = do {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an - error if not. -} -checkRemoveKey :: Key -> Annex Bool -checkRemoveKey key = do +checkRemoveKey :: Key -> Maybe Int -> Annex Bool +checkRemoveKey key numcopiesM = do force <- Annex.flagIsSet "force" - if force + if force || numcopiesM == Just 0 then return True else do remotes <- Remotes.keyPossibilities key - numcopies <- getNumCopies + numcopies <- getNumCopies numcopiesM if numcopies > length remotes then notEnoughCopies numcopies (length remotes) [] else findcopies numcopies 0 remotes [] @@ -139,8 +139,9 @@ showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ Remotes.list remotes -getNumCopies :: Annex Int -getNumCopies = do +getNumCopies :: Maybe Int -> Annex Int +getNumCopies (Just n) = return n +getNumCopies Nothing = do g <- Annex.gitRepo return $ read $ Git.configGet g config "1" where @@ -153,15 +154,15 @@ getNumCopies = do - The passed action is first run to allow backends deriving this one - to do their own checks. -} -checkKey :: (Key -> Annex Bool) -> Key -> Annex Bool -checkKey a key = do +checkKey :: (Key -> Annex Bool) -> Key -> Maybe Int -> Annex Bool +checkKey a key numcopies = do a_ok <- a key - copies_ok <- checkKeyNumCopies key + copies_ok <- checkKeyNumCopies key numcopies return $ a_ok && copies_ok -checkKeyNumCopies :: Key -> Annex Bool -checkKeyNumCopies key = do - needed <- getNumCopies +checkKeyNumCopies :: Key -> Maybe Int -> Annex Bool +checkKeyNumCopies key numcopies = do + needed <- getNumCopies numcopies remotes <- Remotes.keyPossibilities key inannex <- inAnnex key let present = length remotes + if inannex then 1 else 0 diff --git a/Backend/URL.hs b/Backend/URL.hs index b38ea71c96..3eb7376e01 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -22,11 +22,11 @@ backend = Backend { retrieveKeyFile = downloadUrl, -- allow keys to be removed; presumably they can always be -- downloaded again - removeKey = dummyOk, + removeKey = dummyRemove, -- similarly, keys are always assumed to be out there on the web hasKey = dummyOk, -- and nothing needed to fsck - fsckKey = dummyOk + fsckKey = dummyFsck } -- cannot generate url from filename @@ -37,6 +37,12 @@ keyValue _ = return Nothing dummyStore :: FilePath -> Key -> Annex Bool dummyStore _ _ = return False +dummyRemove :: Key -> Maybe a -> Annex Bool +dummyRemove _ _ = return False + +dummyFsck :: Key -> Maybe a -> Annex Bool +dummyFsck _ _ = return True + dummyOk :: Key -> Annex Bool dummyOk _ = return True diff --git a/Command.hs b/Command.hs index 7f3063abb9..059b6e435e 100644 --- a/Command.hs +++ b/Command.hs @@ -44,6 +44,9 @@ type SubCmdStartString = String -> SubCmdStart type BackendFile = (FilePath, Maybe Backend) type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = BackendFile -> SubCmdStart +type AttrFile = (FilePath, String) +type SubCmdSeekAttrFiles = SubCmdStartAttrFile -> SubCmdSeek +type SubCmdStartAttrFile = AttrFile -> SubCmdStart type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek type SubCmdStartNothing = SubCmdStart @@ -104,6 +107,13 @@ withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params return $ map a $ filter notState $ foldl (++) [] files +withAttrFilesInGit :: String -> SubCmdSeekAttrFiles +withAttrFilesInGit attr a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.inRepo repo) params + pairs <- liftIO $ Git.checkAttr repo attr $ + filter notState $ foldl (++) [] files + return $ map a pairs withFilesMissing :: SubCmdSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params @@ -152,21 +162,21 @@ backendPairs a files = do {- Default to acting on all files matching the seek action if - none are specified. -} -withAll :: SubCmdSeekStrings -> SubCmdSeekStrings +withAll :: (a -> SubCmdSeek) -> a -> SubCmdSeek withAll w a [] = do g <- Annex.gitRepo w a [Git.workTree g] withAll w a p = w a p {- Provides a default parameter to act on if none is specified. -} -withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings +withDefault :: String-> (a -> SubCmdSeek) -> (a -> SubCmdSeek) withDefault d w a [] = w a [d] withDefault _ w a p = w a p {- filter out files from the state directory -} notState :: FilePath -> Bool notState f = stateLoc /= take (length stateLoc) f - + {- filter out symlinks -} notSymlink :: FilePath -> IO Bool notSymlink f = do diff --git a/Command/Drop.hs b/Command/Drop.hs index fbe66f5841..168aa92bd6 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -15,24 +15,27 @@ import LocationLog import Types import Core import Messages +import Utility seek :: [SubCmdSeek] -seek = [withFilesInGit start] +seek = [withAttrFilesInGit "git-annex-numcopies" start] {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -start :: SubCmdStartString -start file = isAnnexed file $ \(key, backend) -> do +start :: SubCmdStartAttrFile +start (file, attr) = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if not inbackend then return Nothing else do showStart "drop" file - return $ Just $ perform key backend + return $ Just $ perform key backend numcopies + where + numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> SubCmdPerform -perform key backend = do - success <- Backend.removeKey backend key +perform :: Key -> Backend -> Maybe Int -> SubCmdPerform +perform key backend numcopies = do + success <- Backend.removeKey backend key numcopies if success then return $ Just $ cleanup key else return Nothing diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index e8b8d43c4c..016a9faa73 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -30,9 +30,7 @@ start s = do Just key -> do showStart "dropunused" s backend <- keyBackend key - -- force drop, even if this is the only copy - Annex.flagChange "force" $ FlagBool True - return $ Just $ Command.Drop.perform key backend + return $ Just $ Command.Drop.perform key backend (Just 0) readUnusedLog :: Annex (M.Map String Key) readUnusedLog = do diff --git a/Command/Fsck.hs b/Command/Fsck.hs index dc01688015..4341f85cd5 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -11,19 +11,22 @@ import Command import qualified Backend import Types import Messages +import Utility seek :: [SubCmdSeek] -seek = [withAll withFilesInGit start] +seek = [withAll (withAttrFilesInGit "git-annex-numcopies") start] {- Checks a file's backend data for problems. -} -start :: SubCmdStartString -start file = isAnnexed file $ \(key, backend) -> do +start :: SubCmdStartAttrFile +start (file, attr) = isAnnexed file $ \(key, backend) -> do showStart "fsck" file - return $ Just $ perform key backend + return $ Just $ perform key backend numcopies + where + numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> SubCmdPerform -perform key backend = do - success <- Backend.fsckKey backend key +perform :: Key -> Backend -> Maybe Int -> SubCmdPerform +perform key backend numcopies = do + success <- Backend.fsckKey backend key numcopies if success then return $ Just $ return True else return Nothing diff --git a/Command/FsckFile.hs b/Command/FsckFile.hs deleted file mode 100644 index e7c3d49158..0000000000 --- a/Command/FsckFile.hs +++ /dev/null @@ -1,29 +0,0 @@ -{- git-annex command - - - - Copyright 2010 Joey Hess - - - - Licensed under the GNU GPL version 3 or higher. - -} - -module Command.FsckFile where - -import Command -import qualified Backend -import Types -import Messages - -seek :: [SubCmdSeek] -seek = [withFilesInGit start] - -{- Checks a file's backend data for problems. -} -start :: SubCmdStartString -start file = isAnnexed file $ \(key, backend) -> do - showStart "fsck" file - return $ Just $ perform key backend - -perform :: Key -> Backend -> SubCmdPerform -perform key backend = do - success <- Backend.fsckKey backend key - if success - then return $ Just $ return True - else return Nothing diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 90ae550581..9580fc5e7c 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -32,8 +32,7 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: FilePath -> Key -> Backend -> SubCmdPerform perform file key backend = do -- force backend to always remove - Annex.flagChange "force" $ FlagBool True - ok <- Backend.removeKey backend key + ok <- Backend.removeKey backend key (Just 0) if ok then return $ Just $ cleanup file key else return Nothing diff --git a/TypeInternals.hs b/TypeInternals.hs index bcef4ee0a8..9acc06bb33 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -72,12 +72,15 @@ data Backend = Backend { storeFileKey :: FilePath -> Key -> Annex Bool, -- retrieves a key's contents to a file retrieveKeyFile :: Key -> FilePath -> Annex Bool, - -- removes a key - removeKey :: Key -> Annex Bool, + -- removes a key, optionally checking that enough copies are stored + -- elsewhere + removeKey :: Key -> Maybe Int -> Annex Bool, -- checks if a backend is storing the content of a key hasKey :: Key -> Annex Bool, -- called during fsck to check a key - fsckKey :: Key -> Annex Bool + -- (second parameter may be the number of copies that there should + -- be of the key) + fsckKey :: Key -> Maybe Int -> Annex Bool } instance Show Backend where diff --git a/Utility.hs b/Utility.hs index 2bea6e8755..882492a2d9 100644 --- a/Utility.hs +++ b/Utility.hs @@ -12,7 +12,8 @@ module Utility ( relPathDirToDir, boolSystem, shellEscape, - unsetFileMode + unsetFileMode, + readMaybe ) where import System.IO @@ -125,3 +126,9 @@ unsetFileMode :: FilePath -> FileMode -> IO () unsetFileMode f m = do s <- getFileStatus f setFileMode f $ fileMode s `intersectFileModes` complement m + +{- Attempts to read a value from a String. -} +readMaybe :: (Read a) => String -> Maybe a +readMaybe s = case reads s of + ((x,_):_) -> Just x + _ -> Nothing diff --git a/debian/changelog b/debian/changelog index 808087dadc..213895c5f7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.10) UNRELEASED; urgency=low * precommit: Optimise to avoid calling git-check-attr more than once. + * In .gitattributes, the git-annex-numcopies attribute can be used + to control the number of copies to retain of different types of files. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 diff --git a/doc/copies.mdwn b/doc/copies.mdwn index aec10ab7af..f647ea622d 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -3,8 +3,11 @@ your git repository's `.git` directory, not in some external data store. It's important that data not get lost by an ill-considered `git annex drop` command. So, then using those backends, git-annex can be configured to try -to keep N copies of a file's content available across all repositories. By -default, N is 1; it is configured by annex.numcopies. +to keep N copies of a file's content available across all repositories. + +By default, N is 1; it is configured by annex.numcopies. This default +can be overridden on a per-file-type basis by the git-annex-numcopies +setting in the `.gitattributes` file. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 27393de503..65cce8cc29 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -275,6 +275,12 @@ but the SHA1 backend for ogg files: * git-annex-backend=WORM *.ogg git-annex-backend=SHA1 +The numcopies setting can also be configured on a per-file-type basis via +the `git-annex-numcopies` attribute. For example, this makes two copies +be needed for ogg files: + + *.ogg git-annex-numcopies=2 + # FILES These files are used, in your git repository: From 52ec6e748d5ef8350e9788d0f19dc5c3d609ab86 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 16:31:20 -0400 Subject: [PATCH 0556/2835] do not need to use Git.relative here (it is a no-op in this case anyway) --- LocationLog.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LocationLog.hs b/LocationLog.hs index 7497d865bb..6d52f4bdd1 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -124,7 +124,7 @@ logNow s u = do {- Returns the filename of the log file for a given key. -} logFile :: Git.Repo -> Key -> String logFile repo key = - gitStateDir repo ++ Git.relative repo (keyFile key) ++ ".log" + gitStateDir repo ++ keyFile key ++ ".log" {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} From abf084f628a8c5f5a3685dbb2826739e3e38541e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:17:18 -0400 Subject: [PATCH 0557/2835] Bugfix: Always correctly handle gitattributes when in a subdirectory of the repository. --- GitRepo.hs | 26 ++++++++++++++++++-------- Utility.hs | 20 +++++++++++--------- debian/changelog | 2 ++ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index f50b3fb2e9..539acecb7e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -171,15 +171,17 @@ workTree (Repo { location = Dir d }) = d {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} -relative :: Repo -> String -> String -relative repo@(Repo { location = Dir d }) file = drop (length absrepo) absfile +relative :: Repo -> FilePath -> IO FilePath +relative repo@(Repo { location = Dir d }) file = do + cwd <- getCurrentDirectory + return $ drop (length absrepo) (absfile cwd) where -- normalize both repo and file, so that repo -- will be substring of file absrepo = case (absNormPath "/" d) of Just f -> f ++ "/" Nothing -> error $ "bad repo" ++ repoDescribe repo - absfile = case (secureAbsNormPath absrepo file) of + absfile c = case (secureAbsNormPath c file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo relative repo _ = assertLocal repo $ error "internal" @@ -333,18 +335,26 @@ configGet repo key defaultValue = configMap :: Repo -> Map.Map String String configMap repo = config repo -{- Looks up a gitattributes value for each file in a list. -} +{- Efficiently looks up a gitattributes value for each file in a list. -} checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] checkAttr repo attr files = do - (_, s) <- pipeBoth "git" params files0 - return $ map topair $ lines s + -- git check-attr wants files that are absolute (or relative to the + -- top of the repo). But we're passed files relative to the current + -- directory. Convert to absolute, and then convert the filenames + -- in its output back to relative. + absfiles <- mapM absPath files + (_, s) <- pipeBoth "git" params $ join "\0" absfiles + cwd <- getCurrentDirectory + return $ map (topair $ cwd++"/") $ lines s -- XXX handle is left open, this is ok for git-annex, but may need -- to be cleaned up for other uses. where params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] - files0 = join "\0" files - topair l = (file, value) + topair cwd l = (relfile, value) where + relfile + | startswith cwd file = drop (length cwd) file + | otherwise = file file = decodeGitFile $ join sep $ take end bits value = bits !! end end = length bits - 1 diff --git a/Utility.hs b/Utility.hs index 882492a2d9..0f7ce42aa6 100644 --- a/Utility.hs +++ b/Utility.hs @@ -8,6 +8,7 @@ module Utility ( hGetContentsStrict, parentDir, + absPath, relPathCwdToDir, relPathDirToDir, boolSystem, @@ -44,24 +45,25 @@ parentDir dir = slash = if isAbsolute dir then s else "" s = [pathSeparator] +{- Converts a filename into a normalized, absolute path. -} +absPath :: FilePath -> IO FilePath +absPath file = do + cwd <- getCurrentDirectory + case absNormPath cwd file of + Just f -> return f + Nothing -> error $ "unable to normalize " ++ file + {- Constructs a relative path from the CWD to a directory. - - For example, assuming CWD is /tmp/foo/bar: - relPathCwdToDir "/tmp/foo" == "../" - relPathCwdToDir "/tmp/foo/bar" == "" - - relPathCwdToDir "/tmp/foo/bar" == "" -} relPathCwdToDir :: FilePath -> IO FilePath relPathCwdToDir dir = do cwd <- getCurrentDirectory - let absdir = absnorm cwd - return $ relPathDirToDir cwd absdir - where - -- absolute, normalized form of the directory - absnorm cwd = - case absNormPath cwd dir of - Just d -> d - Nothing -> error $ "unable to normalize " ++ dir + a <- absPath dir + return $ relPathDirToDir cwd a {- Constructs a relative path from one directory to another. - diff --git a/debian/changelog b/debian/changelog index 213895c5f7..34e1757dfd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,8 @@ git-annex (0.10) UNRELEASED; urgency=low * precommit: Optimise to avoid calling git-check-attr more than once. * In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files. + * Bugfix: Always correctly handle gitattributes when in a subdirectory of + the repository. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 From 949e4abc56df1fa23d426f7f86d726ad119fbf54 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:26:15 -0400 Subject: [PATCH 0558/2835] fsck: Fix warning about not enough copies of a file, when locations are known, but are not available in currently configured remotes. --- Backend/File.hs | 6 +++--- debian/changelog | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 5984348b36..e3225a8b85 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -163,9 +163,9 @@ checkKey a key numcopies = do checkKeyNumCopies :: Key -> Maybe Int -> Annex Bool checkKeyNumCopies key numcopies = do needed <- getNumCopies numcopies - remotes <- Remotes.keyPossibilities key - inannex <- inAnnex key - let present = length remotes + if inannex then 1 else 0 + g <- Annex.gitRepo + locations <- liftIO $ keyLocations g key + let present = length locations if present < needed then do warning $ note present needed diff --git a/debian/changelog b/debian/changelog index 34e1757dfd..ad9c00f536 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ git-annex (0.10) UNRELEASED; urgency=low to control the number of copies to retain of different types of files. * Bugfix: Always correctly handle gitattributes when in a subdirectory of the repository. + * fsck: Fix warning about not enough copies of a file, when locations + are known, but are not available in currently configured remotes. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 From 7e82d420d80572af2b9f047039ecbafbb2d79e8e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:32:06 -0400 Subject: [PATCH 0559/2835] missing \n in -q mode --- Messages.hs | 2 +- debian/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Messages.hs b/Messages.hs index e1cf1539a6..2934de4287 100644 --- a/Messages.hs +++ b/Messages.hs @@ -53,4 +53,4 @@ showErr e = warning $ show e warning :: String -> Annex () warning s = do verbose $ liftIO $ putStr "\n" - liftIO $ hPutStr stderr $ "git-annex: " ++ s ++ " " + liftIO $ hPutStrLn stderr $ "git-annex: " ++ s ++ " " diff --git a/debian/changelog b/debian/changelog index ad9c00f536..76ee103d0f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,12 +1,12 @@ git-annex (0.10) UNRELEASED; urgency=low - * precommit: Optimise to avoid calling git-check-attr more than once. * In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files. * Bugfix: Always correctly handle gitattributes when in a subdirectory of the repository. * fsck: Fix warning about not enough copies of a file, when locations are known, but are not available in currently configured remotes. + * precommit: Optimise to avoid calling git-check-attr more than once. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 From fe4f1aae4b6c5fe3527f3e2462efac6e337f4978 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:33:01 -0400 Subject: [PATCH 0560/2835] include key in message --- Backend/File.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend/File.hs b/Backend/File.hs index e3225a8b85..d09e09f56e 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -172,7 +172,7 @@ checkKeyNumCopies key numcopies = do return False else return True where - note 0 _ = "** No known copies of the file exist!" + note 0 _ = "** No known copies of "++show key++" exist!" note present needed = "Only " ++ show present ++ " of " ++ show needed ++ " copies of "++show key++" exist. " ++ From dabfc455c678b073dadda41a4029493716e29c84 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:34:31 -0400 Subject: [PATCH 0561/2835] clarify --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 76ee103d0f..3f66190e59 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,8 @@ git-annex (0.10) UNRELEASED; urgency=low * In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files. * Bugfix: Always correctly handle gitattributes when in a subdirectory of - the repository. + the repository. (Had worked ok for ones like "*.mp3", but failed for + ones like "dir/*".) * fsck: Fix warning about not enough copies of a file, when locations are known, but are not available in currently configured remotes. * precommit: Optimise to avoid calling git-check-attr more than once. From 1d78dc2c8db2550b530a87cf893c2445162e4ea1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:54:42 -0400 Subject: [PATCH 0562/2835] update docs --- doc/copies.mdwn | 2 +- doc/git-annex.mdwn | 12 ++++++------ doc/walkthrough.mdwn | 28 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/doc/copies.mdwn b/doc/copies.mdwn index f647ea622d..a89376a5e4 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -7,7 +7,7 @@ to keep N copies of a file's content available across all repositories. By default, N is 1; it is configured by annex.numcopies. This default can be overridden on a per-file-type basis by the git-annex-numcopies -setting in the `.gitattributes` file. +setting in `.gitattributes` files. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 65cce8cc29..09282e7213 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -238,7 +238,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. # CONFIGURATION -Like other git commands, git-annex is configured via `git-config`. +Like other git commands, git-annex is configured via `.git/config`. Here are all the supported configuration settings. * `annex.uuid` -- a unique UUID for this repository (automatically set) @@ -267,7 +267,7 @@ Here are all the supported configuration settings. between versions. The backend used when adding a new file to the annex can be configured -on a per-file-type basis via the `.gitattributes` file. In the file, +on a per-file-type basis via `.gitattributes` files. In the file, the `git-annex-backend` attribute can be set to the name of the backend to use. For example, this here's how to use the WORM backend by default, but the SHA1 backend for ogg files: @@ -276,14 +276,14 @@ but the SHA1 backend for ogg files: *.ogg git-annex-backend=SHA1 The numcopies setting can also be configured on a per-file-type basis via -the `git-annex-numcopies` attribute. For example, this makes two copies -be needed for ogg files: +the `git-annex-numcopies` attribute in `.gitattributes` files. +For example, this makes two copies be needed for wav files: - *.ogg git-annex-numcopies=2 + *.wav git-annex-numcopies=2 # FILES -These files are used, in your git repository: +These files are used by git-annex, in your git repository: `.git/annex/objects/` contains the annexed file contents that are currently available. Annexed files in your git repository symlink to that content. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 281f460506..9d4d2ce594 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -339,3 +339,31 @@ might say about a badly messed up annex: git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. failed git-annex: 2 failed + +## backups + +git-annex can be configured to require more than one copy of a file exists, +as a simple backup for your data. This is controled by the "numcopies" +setting, which defaults to 1 copy. Let's change that to require 2 copies, +and send a copy of every file to a USB drive. + + # echo "* git-annex-numcopies=2" >> .gitattributes + # git annex copy . --to usbdrive + +Now when we try to `git annex drop` a file, it will verify that it +knows of 2 other repositories that have a copy before removing its +content from the current repository. + +You can also vary the number of copies needed, depending on the file name. +So, if you want 3 copies of all your flac files, but only 1 copy of oggs: + + # echo "*.ogg git-annex-numcopies=1" >> .gitattributes + # echo "*.flac git-annex-numcopies=3" >> .gitattributes + +Or, you might want to make a directory for important stuff, and configure +it so anything put in there is backed up more thoroughly: + + # mkdir important_stuff + # echo "* git-annex-numcopies=3" > important_stuff/.gitattributes + +For more details about the numcopies setting, see [[copies]]. From dc54214404dbe6e9d602a8a14c08411ca3eaceda Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:56:02 -0400 Subject: [PATCH 0563/2835] update --- doc/copies.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/copies.mdwn b/doc/copies.mdwn index a89376a5e4..f157893ce6 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -30,4 +30,6 @@ to both USB and Server. Note that different repositories can be configured with different values of N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1. +copies falling to 1, when USB and Server have N=1. To avoid this, +configure it in `.gitattributes`, which is shared between repositories +using git. From 9d82e815ff5307187d195fd2529420ef7970c412 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 18:55:49 -0400 Subject: [PATCH 0564/2835] change name of numcopies attribute --- Command/Drop.hs | 2 +- Command/Fsck.hs | 2 +- debian/changelog | 2 +- doc/copies.mdwn | 2 +- doc/git-annex.mdwn | 4 ++-- doc/walkthrough.mdwn | 10 +++++----- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Command/Drop.hs b/Command/Drop.hs index 168aa92bd6..7c4fbea602 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -18,7 +18,7 @@ import Messages import Utility seek :: [SubCmdSeek] -seek = [withAttrFilesInGit "git-annex-numcopies" start] +seek = [withAttrFilesInGit "annex.numcopies" start] {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 4341f85cd5..9acecfce67 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -14,7 +14,7 @@ import Messages import Utility seek :: [SubCmdSeek] -seek = [withAll (withAttrFilesInGit "git-annex-numcopies") start] +seek = [withAll (withAttrFilesInGit "annex.numcopies") start] {- Checks a file's backend data for problems. -} start :: SubCmdStartAttrFile diff --git a/debian/changelog b/debian/changelog index 3f66190e59..767583844e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,6 @@ git-annex (0.10) UNRELEASED; urgency=low - * In .gitattributes, the git-annex-numcopies attribute can be used + * In .gitattributes, the annex.numcopies attribute can be used to control the number of copies to retain of different types of files. * Bugfix: Always correctly handle gitattributes when in a subdirectory of the repository. (Had worked ok for ones like "*.mp3", but failed for diff --git a/doc/copies.mdwn b/doc/copies.mdwn index f157893ce6..165e54b340 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -6,7 +6,7 @@ command. So, then using those backends, git-annex can be configured to try to keep N copies of a file's content available across all repositories. By default, N is 1; it is configured by annex.numcopies. This default -can be overridden on a per-file-type basis by the git-annex-numcopies +can be overridden on a per-file-type basis by the annex.numcopies setting in `.gitattributes` files. `git annex drop` attempts to check with other git remotes, to check that N diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 09282e7213..a2af9d990e 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -276,10 +276,10 @@ but the SHA1 backend for ogg files: *.ogg git-annex-backend=SHA1 The numcopies setting can also be configured on a per-file-type basis via -the `git-annex-numcopies` attribute in `.gitattributes` files. +the `annex.numcopies` attribute in `.gitattributes` files. For example, this makes two copies be needed for wav files: - *.wav git-annex-numcopies=2 + *.wav annex.numcopies=2 # FILES diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 9d4d2ce594..6a1c688b34 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -343,11 +343,11 @@ might say about a badly messed up annex: ## backups git-annex can be configured to require more than one copy of a file exists, -as a simple backup for your data. This is controled by the "numcopies" +as a simple backup for your data. This is controled by the "annex.numcopies" setting, which defaults to 1 copy. Let's change that to require 2 copies, and send a copy of every file to a USB drive. - # echo "* git-annex-numcopies=2" >> .gitattributes + # echo "* annex.numcopies=2" >> .gitattributes # git annex copy . --to usbdrive Now when we try to `git annex drop` a file, it will verify that it @@ -357,13 +357,13 @@ content from the current repository. You can also vary the number of copies needed, depending on the file name. So, if you want 3 copies of all your flac files, but only 1 copy of oggs: - # echo "*.ogg git-annex-numcopies=1" >> .gitattributes - # echo "*.flac git-annex-numcopies=3" >> .gitattributes + # echo "*.ogg annex.numcopies=1" >> .gitattributes + # echo "*.flac annex.numcopies=3" >> .gitattributes Or, you might want to make a directory for important stuff, and configure it so anything put in there is backed up more thoroughly: # mkdir important_stuff - # echo "* git-annex-numcopies=3" > important_stuff/.gitattributes + # echo "* annex.numcopies=3" > important_stuff/.gitattributes For more details about the numcopies setting, see [[copies]]. From ca32c7859bb675c70433f23d7f5b4fb5f569d704 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 18:58:03 -0400 Subject: [PATCH 0565/2835] The git-annex-backend attribute has been renamed to annex.backend. --- Backend.hs | 2 +- debian/changelog | 1 + doc/backends.mdwn | 6 +++--- doc/git-annex.mdwn | 6 +++--- doc/walkthrough.mdwn | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Backend.hs b/Backend.hs index 8e17f253f4..8142e4707a 100644 --- a/Backend.hs +++ b/Backend.hs @@ -151,7 +151,7 @@ chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe Backend)] chooseBackends fs = do g <- Annex.gitRepo bs <- Annex.supportedBackends - pairs <- liftIO $ Git.checkAttr g "git-annex-backend" fs + pairs <- liftIO $ Git.checkAttr g "annex.backend" fs return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs {- Returns the backend to use for a key. -} diff --git a/debian/changelog b/debian/changelog index 767583844e..fcd768bf7a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.10) UNRELEASED; urgency=low * fsck: Fix warning about not enough copies of a file, when locations are known, but are not available in currently configured remotes. * precommit: Optimise to avoid calling git-check-attr more than once. + * The git-annex-backend attribute has been renamed to annex.backend. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 diff --git a/doc/backends.mdwn b/doc/backends.mdwn index be4eac7232..3e605e4b15 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -26,12 +26,12 @@ git-annex should use. The first one listed will be used by default when new files are added. For finer control of what backend is used when adding different types of -files, the `.gitattributes` file can be used. The `git-annex-backend` +files, the `.gitattributes` file can be used. The `annex.backend` attribute can be set to the name of the backend to use for matching files. For example, to use the SHA1 backend for sound files, which tend to be smallish and might be modified over time, you could set in `.gitattributes`: - *.mp3 git-annex-backend=SHA1 - *.ogg git-annex-backend=SHA1 + *.mp3 annex.backend=SHA1 + *.ogg annex.backend=SHA1 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index a2af9d990e..64ca6a5497 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -268,12 +268,12 @@ Here are all the supported configuration settings. The backend used when adding a new file to the annex can be configured on a per-file-type basis via `.gitattributes` files. In the file, -the `git-annex-backend` attribute can be set to the name of the backend to +the `annex.backend` attribute can be set to the name of the backend to use. For example, this here's how to use the WORM backend by default, but the SHA1 backend for ogg files: - * git-annex-backend=WORM - *.ogg git-annex-backend=SHA1 + * annex.backend=WORM + *.ogg annex.backend=SHA1 The numcopies setting can also be configured on a per-file-type basis via the `annex.numcopies` attribute in `.gitattributes` files. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 6a1c688b34..2aa2478b32 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -275,7 +275,7 @@ files when they're added to the annex, and this can slow things down significantly for really big files. To make SHA1 the detault, just add something like this to `.gitattributes`: - * git-annex-backend=SHA1 + * annex.backend=SHA1 ## unused data From 0643b7f4c66831177173654b2617e2ce479cce63 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 19:32:00 -0400 Subject: [PATCH 0566/2835] releasing version 0.10 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index fcd768bf7a..b87f835b80 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.10) UNRELEASED; urgency=low +git-annex (0.10) unstable; urgency=low * In .gitattributes, the annex.numcopies attribute can be used to control the number of copies to retain of different types of files. @@ -10,7 +10,7 @@ git-annex (0.10) UNRELEASED; urgency=low * precommit: Optimise to avoid calling git-check-attr more than once. * The git-annex-backend attribute has been renamed to annex.backend. - -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 + -- Joey Hess Sun, 28 Nov 2010 19:28:05 -0400 git-annex (0.09) unstable; urgency=low From d5bb2da688fc85d748e0b1f8e3d97a6d0c0a6cb3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 19:32:47 -0400 Subject: [PATCH 0567/2835] add news item for git-annex 0.10 --- doc/news/version_0.05.mdwn | 10 ---------- doc/news/version_0.10.mdwn | 11 +++++++++++ 2 files changed, 11 insertions(+), 10 deletions(-) delete mode 100644 doc/news/version_0.05.mdwn create mode 100644 doc/news/version_0.10.mdwn diff --git a/doc/news/version_0.05.mdwn b/doc/news/version_0.05.mdwn deleted file mode 100644 index 88c7391c13..0000000000 --- a/doc/news/version_0.05.mdwn +++ /dev/null @@ -1,10 +0,0 @@ -git-annex 0.05 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Optimize both pre-commit and lock subcommands to not call git diff - on every file being committed/locked. - (This actually also works around a bug in ghc, that caused - git-annex 0.04 pre-commit to sometimes corrupt filename being read - from git ls-files and fail. - The excessive number of calls made by pre-commit exposed the ghc bug. - Thanks Josh Triplett for the debugging.) - * Build with -O2."""]] \ No newline at end of file diff --git a/doc/news/version_0.10.mdwn b/doc/news/version_0.10.mdwn new file mode 100644 index 0000000000..450b7d272f --- /dev/null +++ b/doc/news/version_0.10.mdwn @@ -0,0 +1,11 @@ +git-annex 0.10 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * In .gitattributes, the annex.numcopies attribute can be used + to control the number of copies to retain of different types of files. + * Bugfix: Always correctly handle gitattributes when in a subdirectory of + the repository. (Had worked ok for ones like "*.mp3", but failed for + ones like "dir/*".) + * fsck: Fix warning about not enough copies of a file, when locations + are known, but are not available in currently configured remotes. + * precommit: Optimise to avoid calling git-check-attr more than once. + * The git-annex-backend attribute has been renamed to annex.backend."""]] \ No newline at end of file From ade43d8fab2b42846a06201f03e4a03255ef8fce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 1 Dec 2010 19:19:36 -0400 Subject: [PATCH 0568/2835] tweak --- BackendList.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BackendList.hs b/BackendList.hs index 59ec026d91..d1180d22f9 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -11,7 +11,7 @@ module BackendList (allBackends) where import qualified Backend.WORM import qualified Backend.SHA1 import qualified Backend.URL -import TypeInternals +import Types allBackends :: [Backend] allBackends = From 37941184f99896a459fd889071e47ed2fa5ceaa6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 16:55:21 -0400 Subject: [PATCH 0569/2835] Rsync will now be used to resume interrupted/failed partial file transfers from a remote. --- Remotes.hs | 20 +++++++++++++++++--- configure.hs | 11 ++++++----- debian/changelog | 7 +++++++ debian/control | 4 ++-- doc/install.mdwn | 1 + doc/todo/rsync.mdwn | 2 ++ 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index cb8081d740..6bb67216bf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -36,6 +36,7 @@ import Utility import qualified Core import Messages import CopyFile +import qualified SysConfig {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -199,9 +200,13 @@ copyFromRemote r key file | Git.repoIsSsh r = getssh | otherwise = error "copying from non-ssh repo not supported" where - getlocal = liftIO $ copyFile keyloc file - getssh = scp r [sshLocation r keyloc, file] keyloc = annexLocation r key + getlocal = liftIO $ copyFile keyloc file + getssh = do + exists <- liftIO $ doesFileExist file + if exists && SysConfig.rsync + then rsync r [sshLocation r keyloc, file] + else scp r [sshLocation r keyloc, file] {- Tries to copy a key's content to a file on a remote. -} copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool @@ -224,9 +229,18 @@ sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file scp :: Git.Repo -> [String] -> Annex Bool scp r params = do scpoptions <- repoConfig r "scp-options" "" - showProgress -- make way for scp progress bar + showProgress -- make way for progress bar liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params +{- Runs rsync against a specified remote, resuming any interrupted file + - transfer. (Honors annex-rsync-options.) -} +rsync :: Git.Repo -> [String] -> Annex Bool +rsync r params = do + rsyncoptions <- repoConfig r "rsync-options" "" + showProgress -- make way for progress bar + liftIO $ boolSystem "rsync" $ ["--progress", "-a", "--inplace"] ++ + words rsyncoptions ++ params + {- Runs a command in a remote, using ssh if necessary. - (Honors annex-ssh-options.) -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool diff --git a/configure.hs b/configure.hs index 92a051c80e..2334385a38 100644 --- a/configure.hs +++ b/configure.hs @@ -20,8 +20,9 @@ tests = [ TestCase "cp -a" "cp_a" $ testCp "-a" , TestCase "cp -p" "cp_p" $ testCp "-p" , TestCase "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" - , TestCase "uuid" "uuid" $ requireCommand "uuid" "uuid" - , TestCase "xargs -0" "xargs_0" $ requireCommand "xargs -0" "xargs -0 /dev/null" ] tmpDir :: String @@ -33,14 +34,14 @@ testFile = tmpDir ++ "/testfile" quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" -requireCommand :: String -> String -> Test -requireCommand command cmdline = do +requireCmd :: String -> String -> Test +requireCmd c cmdline = do ret <- testCmd $ quiet cmdline if ret then return True else do testEnd False - error $ "** the " ++ command ++ " command is required to use git-annex" + error $ "** the " ++ c ++ " command is required to use git-annex" testCp :: String -> Test testCp option = testCmd $ quiet $ "cp " ++ option ++ " " ++ testFile ++ diff --git a/debian/changelog b/debian/changelog index b87f835b80..0bdfbf76ac 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (0.11) UNRELEASED; urgency=low + + * Rsync will now be used to resume interrupted/failed partial file + transfers from a remote. + + -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 + git-annex (0.10) unstable; urgency=low * In .gitattributes, the annex.numcopies attribute can be used diff --git a/debian/control b/debian/control index 0318dafa4e..88dcab9a3a 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki, uuid +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex @@ -10,7 +10,7 @@ Homepage: http://git-annex.branchable.com/ Package: git-annex Architecture: any Section: utils -Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client +Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/install.mdwn b/doc/install.mdwn index ec9ea92c7f..bca6eb0190 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,6 +5,7 @@ To build and use git-annex, you will need: * MissingH: * `uuid`: * `xargs`: +* `rsync` (optional but recommended) * Then just [[download]] git-annex and run: `make; make install` ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation, diff --git a/doc/todo/rsync.mdwn b/doc/todo/rsync.mdwn index 75e0175c86..3353f19c43 100644 --- a/doc/todo/rsync.mdwn +++ b/doc/todo/rsync.mdwn @@ -1,2 +1,4 @@ Transferring a file from a ssh:// remote should use rsync to allow resuming of a prior transfer. + +[[done]] From adad12d3374102790cab0cfbb9a5f0721ff88fc9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 16:57:02 -0400 Subject: [PATCH 0570/2835] update --- doc/install.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index bca6eb0190..7da45c85b4 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,7 +5,7 @@ To build and use git-annex, you will need: * MissingH: * `uuid`: * `xargs`: -* `rsync` (optional but recommended) +* `rsync`: (optional but recommended) * Then just [[download]] git-annex and run: `make; make install` ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation, From b9320ee1d53bfe72b0fbf7e08c927f5b45bbc5c9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:45:28 -0400 Subject: [PATCH 0571/2835] use rsync for all remote file transfers --- Remotes.hs | 45 +++++++++++++++++++++++++-------------------- debian/changelog | 5 +++-- doc/git-annex.mdwn | 10 +++++++--- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 6bb67216bf..6dc5160482 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -202,11 +202,7 @@ copyFromRemote r key file where keyloc = annexLocation r key getlocal = liftIO $ copyFile keyloc file - getssh = do - exists <- liftIO $ doesFileExist file - if exists && SysConfig.rsync - then rsync r [sshLocation r keyloc, file] - else scp r [sshLocation r keyloc, file] + getssh = remoteCopyFile r (sshLocation r keyloc) file {- Tries to copy a key's content to a file on a remote. -} copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool @@ -220,26 +216,35 @@ copyToRemote r key file = do else error "copying to non-ssh repo not supported" where putlocal src = liftIO $ copyFile src file - putssh src = scp r [src, sshLocation r file] + putssh src = remoteCopyFile r src (sshLocation r file) sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file -{- Runs scp against a specified remote. (Honors annex-scp-options.) -} -scp :: Git.Repo -> [String] -> Annex Bool -scp r params = do - scpoptions <- repoConfig r "scp-options" "" +{- Copys a file from or to a remote, using rsync (when available) or scp. -} +remoteCopyFile :: Git.Repo -> String -> String -> Annex Bool +remoteCopyFile r src dest = do showProgress -- make way for progress bar - liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params - -{- Runs rsync against a specified remote, resuming any interrupted file - - transfer. (Honors annex-rsync-options.) -} -rsync :: Git.Repo -> [String] -> Annex Bool -rsync r params = do - rsyncoptions <- repoConfig r "rsync-options" "" - showProgress -- make way for progress bar - liftIO $ boolSystem "rsync" $ ["--progress", "-a", "--inplace"] ++ - words rsyncoptions ++ params + o <- repoConfig r configopt "" + res <- liftIO $ boolSystem cmd $ options ++ words o ++ [src, dest] + if res + then return res + else do + when rsync $ + showLongNote "run git annex again to resume file transfer" + return res + where + cmd + | rsync = "rsync" + | otherwise = "scp" + configopt + | rsync = "rsync-options" + | otherwise = "scp-options" + options + -- inplace makes rsync resume partial files + | rsync = ["-p", "--progress", "--inplace"] + | otherwise = ["-p"] + rsync = SysConfig.rsync {- Runs a command in a remote, using ssh if necessary. - (Honors annex-ssh-options.) -} diff --git a/debian/changelog b/debian/changelog index 0bdfbf76ac..5a9d332bcb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,8 @@ git-annex (0.11) UNRELEASED; urgency=low - * Rsync will now be used to resume interrupted/failed partial file - transfers from a remote. + * If available, rsync will be used for file transfers from remote + repositories. This allows resuming interrupted transfers. + * Added remote.annex-rsync-options. -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 64ca6a5497..63bc11eb72 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -258,11 +258,15 @@ Here are all the supported configuration settings. here. * `remote..annex-scp-options` -- Options to use when using scp to or from this repository. For example, to force ipv6, and limit - the bandwidth to 100Kbit/s, set it to "-6 -l 100" + the bandwidth to 1000Kbit/s, set it to "-6 -l 1000" * `remote..annex-ssh-options` -- Options to use when using ssh to talk to this repository. -* `annex.scp-options` and `annex.ssh-options` -- Default scp and ssh - options to use if a remote does not have specific options. +* `remote..annex-rsync-options` -- Options to use when using rsync + to or from this repository. For example, to force ipv6, and limit + the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" +* `annex.scp-options`, `annex.ssh-options`, `annex.rsync-options` -- + Default scp, ssh, and rsync options to use if a remote does not have + specific options. * `annex.version` -- Automatically maintained, and used to automate upgrades between versions. From 2fba1ba40d9c3f07f36e68515ceb1031e7983421 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:51:02 -0400 Subject: [PATCH 0572/2835] Avoid deleting temp files when rsync fails. --- Core.hs | 25 ++++++++++++++----------- debian/changelog | 1 + git-annex.hs | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Core.hs b/Core.hs index b61d186666..8cdb063c79 100644 --- a/Core.hs +++ b/Core.hs @@ -29,7 +29,7 @@ import Messages {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). - - Propigates an overall error status at the end. + - Runs shutdown and propigates an overall error status at the end. -} tryRun :: AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions @@ -42,7 +42,8 @@ tryRun' state errnum (a:as) = do tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' _ errnum [] = +tryRun' state errnum [] = do + _ <- try $ Annex.run state $ shutdown errnum when (errnum > 0) $ error $ show errnum ++ " failed" {- Actions to perform each time ran. -} @@ -52,20 +53,22 @@ startup = do return True {- When git-annex is done, it runs this. -} -shutdown :: Annex Bool -shutdown = do +shutdown :: Integer -> Annex Bool +shutdown errnum = do q <- Annex.queueGet unless (q == GitQueue.empty) $ do showSideAction "Recording state in git..." Annex.queueRun - -- clean up any files left in the temp directory, but leave - -- the tmp directory itself - g <- Annex.gitRepo - let tmp = annexTmpLocation g - exists <- liftIO $ doesDirectoryExist tmp - when exists $ liftIO $ removeDirectoryRecursive tmp - liftIO $ createDirectoryIfMissing True tmp + -- If nothing failed, clean up any files left in the temp directory, + -- but leave the directory itself. If something failed, temp files + -- are left behind to allow resuming on re-run. + when (errnum == 0) $ do + g <- Annex.gitRepo + let tmp = annexTmpLocation g + exists <- liftIO $ doesDirectoryExist tmp + when exists $ liftIO $ removeDirectoryRecursive tmp + liftIO $ createDirectoryIfMissing True tmp return True diff --git a/debian/changelog b/debian/changelog index 5a9d332bcb..fb2d2aae22 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.11) UNRELEASED; urgency=low * If available, rsync will be used for file transfers from remote repositories. This allows resuming interrupted transfers. * Added remote.annex-rsync-options. + * Avoid deleting temp files when rsync fails. -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 diff --git a/git-annex.hs b/git-annex.hs index d111156f01..417d335e16 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -20,4 +20,4 @@ main = do gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends (configure, actions) <- parseCmd args state - tryRun state $ [startup, upgrade] ++ configure ++ actions ++ [shutdown] + tryRun state $ [startup, upgrade] ++ configure ++ actions From 07648e2daa29fe8d275605c1873efc7912ee8499 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:52:23 -0400 Subject: [PATCH 0573/2835] remove note that looked ugly with resume message --- Backend/File.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Backend/File.hs b/Backend/File.hs index d09e09f56e..afbe38ac0f 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -58,7 +58,6 @@ copyKeyFile key file = do else trycopy remotes remotes where trycopy full [] = do - showNote "not available" showTriedRemotes full showLocations key return False From ece7481faad5ce5f040b684c4bd02a37f50a5ff0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:54:08 -0400 Subject: [PATCH 0574/2835] reword --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index 6dc5160482..cf49b624b4 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -231,7 +231,7 @@ remoteCopyFile r src dest = do then return res else do when rsync $ - showLongNote "run git annex again to resume file transfer" + showLongNote "rsync failed -- run git annex again to resume file transfer" return res where cmd From d1b5ef9565cfc5991a473985a36fe00c9384ece5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:56:25 -0400 Subject: [PATCH 0575/2835] update for rsync --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 2aa2478b32..9c8106d513 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -198,7 +198,7 @@ to clone the laptop's annex to it: # cd ~/annex # git annex init "my desktop" -Now you can get files and they will be transferred by `scp`: +Now you can get files and they will be transferred (using `rsync` or `scp`): # git annex get my_cool_big_file get my_cool_big_file (getting UUID for origin...) (copying from origin...) From 83a87a522903e18a16ae19e1b741ab4e1f2b95a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 21:07:16 -0400 Subject: [PATCH 0576/2835] Improve detection of version 0 repos. --- Version.hs | 6 ++++-- debian/changelog | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Version.hs b/Version.hs index 8394701201..fc1ce3d7ec 100644 --- a/Version.hs +++ b/Version.hs @@ -29,9 +29,11 @@ getVersion = do then return $ Just v else do -- version 0 was not recorded in .git/config; - -- such a repo should have an annexDir + -- such a repo should have an annexDir but no + -- annexObjectDir d <- liftIO $ doesDirectoryExist $ annexDir g - if d + o <- liftIO $ doesDirectoryExist $ annexObjectDir g + if d && not o then return $ Just "0" else return Nothing -- no version yet diff --git a/debian/changelog b/debian/changelog index fb2d2aae22..9e378400c6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.11) UNRELEASED; urgency=low repositories. This allows resuming interrupted transfers. * Added remote.annex-rsync-options. * Avoid deleting temp files when rsync fails. + * Improve detection of version 0 repos. -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 From 71d60eb87a492a9561f7142cc897ff5961c21e25 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 21:26:37 -0400 Subject: [PATCH 0577/2835] robustness fix don't crash if an object directory does not contain a file --- Core.hs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core.hs b/Core.hs index 8cdb063c79..e3702044e8 100644 --- a/Core.hs +++ b/Core.hs @@ -190,9 +190,11 @@ getKeysPresent' dir = do return $ map fileKey files where present d = do - s <- getFileStatus $ dir ++ "/" ++ d ++ "/" - ++ takeFileName d - return $ isRegularFile s + result <- try $ + getFileStatus $ dir ++ "/" ++ d ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False {- List of keys referenced by symlinks in the git repo. -} getKeysReferenced :: Annex [Key] From 6f932a0963341b235f262c43982cc138f443f2de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 21:37:55 -0400 Subject: [PATCH 0578/2835] avoid building news page when building doc wiki for package --- Makefile | 3 ++- doc/news.mdwn | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 17918f9567..ad143e7b69 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,8 @@ docs: $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ --underlaydir=/dev/null --disable-plugin=shortcut \ - --disable-plugin=smiley + --disable-plugin=smiley \ + --exclude='news/.*' clean: rm -rf build git-annex git-annex.1 test configure SysConfig.hs diff --git a/doc/news.mdwn b/doc/news.mdwn index d0ff1ca2c1..3d9ece3bbb 100644 --- a/doc/news.mdwn +++ b/doc/news.mdwn @@ -1,5 +1,11 @@ +[[!if test="news/*" then=""" This is where announcements of new releases, features, and other news is posted. git-annex users are recommended to subscribe to this page's RSS feed. [[!inline pages="./news/* and !*/Discussion" rootpage="news" show="30"]] + +""" +else=""" +(Please see the changelog.) +"""]] From 57305570eb5fce88c743ca4f7ff127c7ef582310 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Dec 2010 00:33:41 -0400 Subject: [PATCH 0579/2835] Add uninit subcommand. Closes: #605749 --- CmdLine.hs | 3 +++ Command/Init.hs | 41 ++++++++++++++++++++------------ Command/Uninit.hs | 59 ++++++++++++++++++++++++++++++++++++++++++++++ Utility.hs | 5 ++++ debian/changelog | 1 + doc/git-annex.mdwn | 6 +++++ 6 files changed, 100 insertions(+), 15 deletions(-) create mode 100644 Command/Uninit.hs diff --git a/CmdLine.hs b/CmdLine.hs index 837420786f..0903cc1fb7 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -32,6 +32,7 @@ import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit import qualified Command.Find +import qualified Command.Uninit subCmds :: [SubCommand] subCmds = @@ -55,6 +56,8 @@ subCmds = "initialize git-annex with repository description" , SubCommand "unannex" path Command.Unannex.seek "undo accidential add command" + , SubCommand "uninit" path Command.Uninit.seek + "de-initialize git-annex and clean out repository" , SubCommand "pre-commit" path Command.PreCommit.seek "run by git pre-commit hook" , SubCommand "fromkey" key Command.FromKey.seek diff --git a/Command/Init.hs b/Command/Init.hs index eb5c58696f..e19849ba3b 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -18,6 +18,7 @@ import UUID import Version import Messages import Locations +import Types seek :: [SubCmdSeek] seek = [withString start] @@ -36,8 +37,8 @@ perform description = do u <- getUUID g describeUUID u description setVersion - liftIO $ gitAttributes g - liftIO $ gitPreCommitHook g + liftIO $ gitAttributesWrite g + gitPreCommitHookWrite g return $ Just cleanup cleanup :: SubCmdCleanup @@ -50,8 +51,8 @@ cleanup = do {- configure git to use union merge driver on state files, if it is not - already -} -gitAttributes :: Git.Repo -> IO () -gitAttributes repo = do +gitAttributesWrite :: Git.Repo -> IO () +gitAttributesWrite repo = do exists <- doesFileExist attributes if not exists then do @@ -63,24 +64,34 @@ gitAttributes repo = do appendFile attributes $ attrLine ++ "\n" commit where - attrLine = stateLoc ++ "*.log merge=union" attributes = Git.attributes repo commit = do Git.run repo ["add", attributes] Git.run repo ["commit", "-m", "git-annex setup", attributes] +attrLine :: String +attrLine = stateLoc ++ "*.log merge=union" + {- set up a git pre-commit hook, if one is not already present -} -gitPreCommitHook :: Git.Repo -> IO () -gitPreCommitHook repo = do - let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ - "/hooks/pre-commit" - exists <- doesFileExist hook +gitPreCommitHookWrite :: Git.Repo -> Annex () +gitPreCommitHookWrite repo = do + exists <- liftIO $ doesFileExist hook if exists - then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" - else do - writeFile hook $ "#!/bin/sh\n" ++ - "# automatically configured by git-annex\n" ++ - "git annex pre-commit .\n" + then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" + else liftIO $ do + writeFile hook preCommitScript p <- getPermissions hook setPermissions hook $ p {executable = True} + where + hook = preCommitHook repo + +preCommitHook :: Git.Repo -> FilePath +preCommitHook repo = + Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit" + +preCommitScript :: String +preCommitScript = + "#!/bin/sh\n" ++ + "# automatically configured by git-annex\n" ++ + "git annex pre-commit .\n" diff --git a/Command/Uninit.hs b/Command/Uninit.hs new file mode 100644 index 0000000000..fcb77a92b0 --- /dev/null +++ b/Command/Uninit.hs @@ -0,0 +1,59 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Uninit where + +import Control.Monad.State (liftIO) +import Control.Monad (when) +import System.Directory + +import Command +import Messages +import Types +import Utility +import qualified GitRepo as Git +import qualified Annex +import qualified Command.Unannex +import qualified Command.Init + +seek :: [SubCmdSeek] +seek = [withAll withFilesInGit Command.Unannex.start, withNothing start] + +start :: SubCmdStartNothing +start = do + showStart "uninit" "" + return $ Just $ perform + +perform :: SubCmdPerform +perform = do + g <- Annex.gitRepo + + gitPreCommitHookUnWrite g + liftIO $ gitAttributesUnWrite g + + return $ Just $ return True + +gitPreCommitHookUnWrite :: Git.Repo -> Annex () +gitPreCommitHookUnWrite repo = do + let hook = Command.Init.preCommitHook repo + hookexists <- liftIO $ doesFileExist hook + when hookexists $ do + c <- liftIO $ readFile hook + if c == Command.Init.preCommitScript + then liftIO $ removeFile hook + else warning $ "pre-commit hook (" ++ hook ++ + ") contents modified; not deleting." ++ + " Edit it to remove call to git annex." + +gitAttributesUnWrite :: Git.Repo -> IO () +gitAttributesUnWrite repo = do + let attributes = Git.attributes repo + attrexists <- doesFileExist attributes + when attrexists $ do + c <- readFileStrict attributes + writeFile attributes $ unlines $ + filter (/= Command.Init.attrLine) $ lines c diff --git a/Utility.hs b/Utility.hs index 0f7ce42aa6..2447c95a0b 100644 --- a/Utility.hs +++ b/Utility.hs @@ -7,6 +7,7 @@ module Utility ( hGetContentsStrict, + readFileStrict, parentDir, absPath, relPathCwdToDir, @@ -34,6 +35,10 @@ import Foreign (complement) hGetContentsStrict :: Handle -> IO String hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s +{- A version of readFile that is not lazy. -} +readFileStrict :: FilePath -> IO String +readFileStrict f = readFile f >>= \s -> length s `seq` return s + {- Returns the parent directory of a path. Parent of / is "" -} parentDir :: String -> String parentDir dir = diff --git a/debian/changelog b/debian/changelog index 9e378400c6..b5ef4b5a03 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ git-annex (0.11) UNRELEASED; urgency=low * Added remote.annex-rsync-options. * Avoid deleting temp files when rsync fails. * Improve detection of version 0 repos. + * Add uninit subcommand. Closes: #605749 -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 63bc11eb72..66e8bbaa8b 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -151,6 +151,12 @@ Many git-annex subcommands will stage changes for later `git commit` by you. any more. In that case you should use `git annex drop` instead, and you can also `git rm` the file. +* uninit + + Use this to stop using git annex. It will unannex every file in the + repository, and remove all of git-annex's other data, leaving you with a + git repository plus the previously annexed files. + * fix [path ...] Fixes up symlinks that have become broken to again point to annexed content. From b34da73ec2eb97001b4d82e214b5a517a8d8d8c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 4 Dec 2010 17:29:28 -0400 Subject: [PATCH 0580/2835] releasing version 0.11 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index b5ef4b5a03..5ecf942013 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.11) UNRELEASED; urgency=low +git-annex (0.11) unstable; urgency=low * If available, rsync will be used for file transfers from remote repositories. This allows resuming interrupted transfers. @@ -7,7 +7,7 @@ git-annex (0.11) UNRELEASED; urgency=low * Improve detection of version 0 repos. * Add uninit subcommand. Closes: #605749 - -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 + -- Joey Hess Sat, 04 Dec 2010 17:27:42 -0400 git-annex (0.10) unstable; urgency=low From 627a3014376f83d613c448da929231bb9d866435 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 4 Dec 2010 17:29:36 -0400 Subject: [PATCH 0581/2835] add news item for git-annex 0.11 --- doc/news/version_0.06.mdwn | 8 -------- doc/news/version_0.11.mdwn | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-) delete mode 100644 doc/news/version_0.06.mdwn create mode 100644 doc/news/version_0.11.mdwn diff --git a/doc/news/version_0.06.mdwn b/doc/news/version_0.06.mdwn deleted file mode 100644 index 39689232a7..0000000000 --- a/doc/news/version_0.06.mdwn +++ /dev/null @@ -1,8 +0,0 @@ -git-annex 0.06 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * fsck: Check if annex.numcopies is satisfied. - * fsck: Verify the sha1 of files when the SHA1 backend is used. - * fsck: Verify the size of files when the WORM backend is used. - * fsck: Allow specifying individual files if fscking everything - is not desired. - * fsck: Fix bug, introduced in 0.04, in detection of unused data."""]] \ No newline at end of file diff --git a/doc/news/version_0.11.mdwn b/doc/news/version_0.11.mdwn new file mode 100644 index 0000000000..03c4d7d54d --- /dev/null +++ b/doc/news/version_0.11.mdwn @@ -0,0 +1,8 @@ +git-annex 0.11 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * If available, rsync will be used for file transfers from remote + repositories. This allows resuming interrupted transfers. + * Added remote.annex-rsync-options. + * Avoid deleting temp files when rsync fails. + * Improve detection of version 0 repos. + * Add uninit subcommand. Closes: #[605749](http://bugs.debian.org/605749)"""]] \ No newline at end of file From 2099407d8aa1b1e94f29de0d9094ccfa6e05e471 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 8 Dec 2010 14:07:49 -0400 Subject: [PATCH 0582/2835] Add --exclude option to exclude files from processing. Required some lifting so flags are evaled in the Annex monad before file filtering. --- CmdLine.hs | 22 ++++++++++------------ Command.hs | 47 +++++++++++++++++++++++++++++++++------------- debian/changelog | 6 ++++++ doc/git-annex.mdwn | 5 +++++ git-annex.hs | 4 ++-- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 0903cc1fb7..cb164a6ab2 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -9,6 +9,7 @@ module CmdLine (parseCmd) where import System.Console.GetOpt import Control.Monad (when) +import Control.Monad.State (liftIO) import qualified Annex import Types @@ -103,6 +104,8 @@ options = [ "specify to where to transfer content" , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storestring "exclude") "GLOB") + "skip files matching the glob pattern" ] where storebool n b = Annex.flagChange n $ FlagBool b @@ -125,22 +128,17 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = replicate (n - length s) ' ' -{- Parses command line and returns two lists of actions to be - - run in the Annex monad. The first actions configure it - - according to command line options, while the second actions - - handle subcommands. -} -parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) -parseCmd argv state = do - (flags, params) <- getopt +{- Parses command line, stores configure flags, and returns a + - list of actions to be run in the Annex monad. -} +parseCmd :: [String] -> Annex [Annex Bool] +parseCmd argv = do + (flags, params) <- liftIO $ getopt when (null params) $ error usage case lookupCmd (head params) of [] -> error usage [subcommand] -> do - actions <- prepSubCmd subcommand state (drop 1 params) - let configactions = map (\flag -> do - flag - return True) flags - return (configactions, actions) + _ <- sequence flags + prepSubCmd subcommand (drop 1 params) _ -> error "internal error: multiple matching subcommands" where getopt = case getOpt Permute options argv of diff --git a/Command.hs b/Command.hs index 059b6e435e..8edea7622e 100644 --- a/Command.hs +++ b/Command.hs @@ -11,6 +11,8 @@ import Control.Monad.State (liftIO) import System.Directory import System.Posix.Files import Control.Monad (filterM) +import System.Path.WildMatch +import Text.Regex import Types import qualified Backend @@ -59,9 +61,9 @@ data SubCommand = SubCommand { {- Prepares a list of actions to run to perform a subcommand, based on - the parameters passed to it. -} -prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdseek = seek } state params = do - lists <- Annex.eval state $ mapM (\s -> s params) seek +prepSubCmd :: SubCommand -> [String] -> Annex [Annex Bool] +prepSubCmd SubCommand { subcmdseek = seek } params = do + lists <- mapM (\s -> s params) seek return $ map doSubCmd $ foldl (++) [] lists {- Runs a subcommand through the start, perform and cleanup stages -} @@ -106,18 +108,20 @@ withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params - return $ map a $ filter notState $ foldl (++) [] files + files' <- filterFiles $ foldl (++) [] files + return $ map a files' withAttrFilesInGit :: String -> SubCmdSeekAttrFiles withAttrFilesInGit attr a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params - pairs <- liftIO $ Git.checkAttr repo attr $ - filter notState $ foldl (++) [] files + files' <- filterFiles $ foldl (++) [] files + pairs <- liftIO $ Git.checkAttr repo attr files' return $ map a pairs withFilesMissing :: SubCmdSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params - return $ map a $ filter notState files + files' <- filterFiles files + return $ map a files' where missing f = do e <- doesFileExist f @@ -126,7 +130,8 @@ withFilesNotInGit :: SubCmdSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ mapM (Git.notInRepo repo) params - backendPairs a $ filter notState $ foldl (++) [] newfiles + newfiles' <- filterFiles $ foldl (++) [] newfiles + backendPairs a newfiles' withString :: SubCmdSeekStrings withString a params = return [a $ unwords params] withStrings :: SubCmdSeekStrings @@ -135,7 +140,8 @@ withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo tocommit <- liftIO $ mapM (Git.stagedFiles repo) params - return $ map a $ filter notState $ foldl (++) [] tocommit + tocommit' <- filterFiles $ foldl (++) [] tocommit + return $ map a tocommit' withFilesUnlocked :: SubCmdSeekBackendFiles withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles withFilesUnlockedToBeCommitted :: SubCmdSeekBackendFiles @@ -146,7 +152,8 @@ withFilesUnlocked' typechanged a params = do repo <- Annex.gitRepo typechangedfiles <- liftIO $ mapM (typechanged repo) params unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - backendPairs a $ filter notState unlockedfiles + unlockedfiles' <- filterFiles unlockedfiles + backendPairs a unlockedfiles' withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings @@ -173,9 +180,23 @@ withDefault :: String-> (a -> SubCmdSeek) -> (a -> SubCmdSeek) withDefault d w a [] = w a [d] withDefault _ w a p = w a p -{- filter out files from the state directory -} -notState :: FilePath -> Bool -notState f = stateLoc /= take (length stateLoc) f +{- Filter out files from the state directory, and those matching the + - exclude glob pattern, if it was specified. -} +filterFiles :: [FilePath] -> Annex [FilePath] +filterFiles l = do + let l' = filter notState l + exclude <- Annex.flagGet "exclude" + if null exclude + then return l' + else do + let regexp = mkRegex $ "^" ++ wildToRegex exclude + return $ filter (notExcluded regexp) l' + where + notState f = stateLoc /= take stateLocLen f + stateLocLen = length stateLoc + notExcluded r f = case matchRegex r f of + Nothing -> True + Just _ -> False {- filter out symlinks -} notSymlink :: FilePath -> IO Bool diff --git a/debian/changelog b/debian/changelog index 5ecf942013..4b8fb1050b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.12) UNRELEASED; urgency=low + + * Add --exclude option to exclude files from processing. + + -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 + git-annex (0.11) unstable; urgency=low * If available, rsync will be used for file transfers from remote diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 66e8bbaa8b..f6dc2fe5b3 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -234,6 +234,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a git repository that content will be sent to. It should be specified using the name of a configured git remote. +* --exclude=glob + + Skips files matching the glob pattern. The glob is matched relative to + the current directory. + * --backend=name Specifies which key-value backend to use. diff --git a/git-annex.hs b/git-annex.hs index 417d335e16..1173ab9139 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -19,5 +19,5 @@ main = do args <- getArgs gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends - (configure, actions) <- parseCmd args state - tryRun state $ [startup, upgrade] ++ configure ++ actions + (actions, state') <- Annex.run state $ parseCmd args + tryRun state' $ [startup, upgrade] ++ actions From f8851aad66974691f1fef9d118eceb7ec909c8f4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 8 Dec 2010 14:48:10 -0400 Subject: [PATCH 0583/2835] use Text.Regex.PCRE.Light.Char8 rather than Text.Regexp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Text.Regexp does not think that á matches . -- seems to be a unicode problem. --- Command.hs | 6 +++--- debian/control | 2 +- doc/install.mdwn | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Command.hs b/Command.hs index 8edea7622e..4ba6ea4b36 100644 --- a/Command.hs +++ b/Command.hs @@ -12,7 +12,7 @@ import System.Directory import System.Posix.Files import Control.Monad (filterM) import System.Path.WildMatch -import Text.Regex +import Text.Regex.PCRE.Light.Char8 import Types import qualified Backend @@ -189,12 +189,12 @@ filterFiles l = do if null exclude then return l' else do - let regexp = mkRegex $ "^" ++ wildToRegex exclude + let regexp = compile ("^" ++ wildToRegex exclude) [] return $ filter (notExcluded regexp) l' where notState f = stateLoc /= take stateLocLen f stateLocLen = length stateLoc - notExcluded r f = case matchRegex r f of + notExcluded r f = case match r f [] of Nothing -> True Just _ -> False diff --git a/debian/control b/debian/control index 88dcab9a3a..d90041f77e 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex diff --git a/doc/install.mdwn b/doc/install.mdwn index 7da45c85b4..1cff4462e0 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -3,6 +3,7 @@ To build and use git-annex, you will need: * `git`: * The Haskell Platform: * MissingH: +* pcre-light: * `uuid`: * `xargs`: * `rsync`: (optional but recommended) From 939475a6ba50c401398cb624a426a2124fe773b2 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmfO7UdOn_TB4WD13ZBKMTuHW44MBJdgiI" Date: Thu, 9 Dec 2010 00:10:03 +0000 Subject: [PATCH 0584/2835] --- doc/bugs/conflicting_haskell_packages.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/conflicting_haskell_packages.mdwn diff --git a/doc/bugs/conflicting_haskell_packages.mdwn b/doc/bugs/conflicting_haskell_packages.mdwn new file mode 100644 index 0000000000..58e25539d3 --- /dev/null +++ b/doc/bugs/conflicting_haskell_packages.mdwn @@ -0,0 +1,3 @@ +The compilation command should states which packages are used and avoid the default mechnasim that automatically search for them. + +This can be done by the flags -hide-packages and then -package foo From 7b2ed635d3215452b903de20a93bc9f1641d6554 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 11:32:02 -0400 Subject: [PATCH 0585/2835] typo correction via LWN --- doc/use_case/Alice.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index c42eb3a74c..1dc456d731 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -13,6 +13,6 @@ podcasts, videos, and games, first letting git-annex copy them from her USB drive to the netbook (this saves battery power). When she's done, she tells git-annex which to keep and which to remove. -They're all removed from her netbook to save space, and Alice knowns +They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back to her server. From 668ccafc93005d02b7881bc88bc9e556bec64a89 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 11:35:07 -0400 Subject: [PATCH 0586/2835] add pointer to lwn article --- doc/news/LWN_article.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/news/LWN_article.mdwn diff --git a/doc/news/LWN_article.mdwn b/doc/news/LWN_article.mdwn new file mode 100644 index 0000000000..c1c0c40472 --- /dev/null +++ b/doc/news/LWN_article.mdwn @@ -0,0 +1,2 @@ +[Linux Weekly News](http://lwn.net/) has a nice +[article on git-annex](http://lwn.net/Articles/418337/) in it this week. From 7c46439de0ff6f099fad29ad09217a402f88607e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 11:36:10 -0400 Subject: [PATCH 0587/2835] add news to sidebar --- doc/index.mdwn | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 5cd6b45494..cbce1260e4 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -18,12 +18,15 @@ To get a feel for it, see the [[walkthrough]]. * **[[download]]** * [[install]] -* [[news]] * [[bugs]] * [[todo]] * [[forum]] * [[contact]] +[[News]]: + +[[!inlines pages="news/* and !*/discussion" archive=yes show=3]] + Flattr this From 136877264a192d315a28b1778962b0d8c7a65be6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 11:36:38 -0400 Subject: [PATCH 0588/2835] typo --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index cbce1260e4..595db91e01 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -25,7 +25,7 @@ To get a feel for it, see the [[walkthrough]]. [[News]]: -[[!inlines pages="news/* and !*/discussion" archive=yes show=3]] +[[!inline pages="news/* and !*/discussion" archive=yes show=3]] Date: Thu, 9 Dec 2010 11:37:21 -0400 Subject: [PATCH 0589/2835] no feeds --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 595db91e01..a279e3e5fb 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -25,7 +25,7 @@ To get a feel for it, see the [[walkthrough]]. [[News]]: -[[!inline pages="news/* and !*/discussion" archive=yes show=3]] +[[!inline pages="news/* and !*/discussion" archive=yes show=3 feeds=no]] Date: Thu, 9 Dec 2010 11:44:00 -0400 Subject: [PATCH 0590/2835] update --- doc/download.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/download.mdwn b/doc/download.mdwn index ccf21c091b..6c56c8e25a 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,6 +1,7 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] -Users of Debian unstable can `apt-get install git-annex` +Users of Debian unstable/testing and Ubuntu natty can +`apt-get install git-annex` Next: [[install]] From 1f22c9ea38b85b2e3e4d25bb56887e55e1c51f40 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 13:45:16 -0400 Subject: [PATCH 0591/2835] response --- doc/bugs/conflicting_haskell_packages.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/bugs/conflicting_haskell_packages.mdwn b/doc/bugs/conflicting_haskell_packages.mdwn index 58e25539d3..6a619de90f 100644 --- a/doc/bugs/conflicting_haskell_packages.mdwn +++ b/doc/bugs/conflicting_haskell_packages.mdwn @@ -1,3 +1,8 @@ The compilation command should states which packages are used and avoid the default mechnasim that automatically search for them. This can be done by the flags -hide-packages and then -package foo + +> My ghc does not have a `--hide-packages` option. +> +> Could you just show the build problem that you are suggesting I work +> around? --[[Joey]] From cdf040e81656b5e8034e4fd2d5e0583c367dff12 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 10 Dec 2010 17:27:02 -0400 Subject: [PATCH 0592/2835] mwdn2man: Fix a bug in newline supression. Closes: #606578 --- debian/changelog | 1 + mdwn2man | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 4b8fb1050b..323cd2e75e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.12) UNRELEASED; urgency=low * Add --exclude option to exclude files from processing. + * mwdn2man: Fix a bug in newline supression. Closes: #606578 -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 diff --git a/mdwn2man b/mdwn2man index c212539454..ad6d3c6026 100755 --- a/mdwn2man +++ b/mdwn2man @@ -14,7 +14,7 @@ while (<>) { s/^#\s/.SH /; <>; # blank; } - s/^ +//; + s/^[ \n]+//; s/^\t/ /; s/-/\\-/g; s/^Warning:.*//g; From eef009fa2a94fe8d2934d314898839f7a864c7ee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 10 Dec 2010 17:30:13 -0400 Subject: [PATCH 0593/2835] formatting --- doc/git-annex.mdwn | 63 +++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index f6dc2fe5b3..71a4889ac7 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -252,34 +252,63 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Like other git commands, git-annex is configured via `.git/config`. Here are all the supported configuration settings. -* `annex.uuid` -- a unique UUID for this repository (automatically set) -* `annex.numcopies` -- number of copies of files to keep across all - repositories (default: 1) -* `annex.backends` -- space-separated list of names of - the key-value backends to use. The first listed is used to store - new files by default. (default: "WORM SHA1 URL") -* `remote..annex-cost` -- When determining which repository to +* `annex.uuid` + + A unique UUID for this repository (automatically set). + +* `annex.numcopies` + + Number of copies of files to keep across all repositories. (default: 1) + +* `annex.backends` + + Space-separated list of names of the key-value backends to use. + The first listed is used to store new files by default. + (default: "WORM SHA1 URL") + +* `remote..annex-cost` + + When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 100 for local repositories, and 200 for remote repositories. -* `remote..annex-ignore` -- If set to `true`, prevents git-annex + +* `remote..annex-ignore` + + If set to `true`, prevents git-annex from ever using this remote. This is, for example, useful if the remote is a bare repository, which git-annex does not currently support. -* `remote..annex-uuid` -- git-annex caches UUIDs of repositories - here. -* `remote..annex-scp-options` -- Options to use when using scp + +* `remote..annex-uuid` + + git-annex caches UUIDs of repositories here. + +* `remote..annex-scp-options` + + Options to use when using scp to or from this repository. For example, to force ipv6, and limit the bandwidth to 1000Kbit/s, set it to "-6 -l 1000" -* `remote..annex-ssh-options` -- Options to use when using ssh - to talk to this repository. -* `remote..annex-rsync-options` -- Options to use when using rsync + +* `remote..annex-ssh-options` + + Options to use when using ssh to talk to this repository. + +* `remote..annex-rsync-options` + + Options to use when using rsync to or from this repository. For example, to force ipv6, and limit the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" -* `annex.scp-options`, `annex.ssh-options`, `annex.rsync-options` -- + +* `annex.scp-options`, `annex.ssh-options`, `annex.rsync-options` + Default scp, ssh, and rsync options to use if a remote does not have specific options. -* `annex.version` -- Automatically maintained, and used to automate upgrades - between versions. + +* `annex.version` + + Automatically maintained, and used to automate upgrades between versions. + +# CONFIGURATION VIA .gitattributes The backend used when adding a new file to the annex can be configured on a per-file-type basis via `.gitattributes` files. In the file, From 77e52a52dd2f77e5acc0643673169b4e1d2550f9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 11 Dec 2010 17:14:54 -0400 Subject: [PATCH 0594/2835] Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 --- Command.hs | 4 +++- debian/changelog | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Command.hs b/Command.hs index 4ba6ea4b36..69b9dee9f9 100644 --- a/Command.hs +++ b/Command.hs @@ -151,7 +151,9 @@ withFilesUnlocked' typechanged a params = do -- unlocked files have changed type from a symlink to a regular file repo <- Annex.gitRepo typechangedfiles <- liftIO $ mapM (typechanged repo) params - unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + unlockedfiles <- liftIO $ filterM notSymlink $ + map (\f -> Git.workTree repo ++ "/" ++ f) $ + foldl (++) [] typechangedfiles unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' withKeys :: SubCmdSeekStrings diff --git a/debian/changelog b/debian/changelog index 323cd2e75e..6bc3c788a6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.12) UNRELEASED; urgency=low * Add --exclude option to exclude files from processing. * mwdn2man: Fix a bug in newline supression. Closes: #606578 + * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 From 10484a69b8a7617fea5ce7ba95e31fe1a9eff7b4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 11 Dec 2010 17:29:50 -0400 Subject: [PATCH 0595/2835] Makefile: Add PREFIX variable. --- Makefile | 15 ++++++++------- debian/changelog | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ad143e7b69..b5f5a720be 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,18 @@ +PREFIX=/usr +GHCMAKE=ghc -Wall -odir build -hidir build -O2 --make + all: git-annex docs -ghcmake=ghc -Wall -odir build -hidir build -O2 --make - SysConfig.hs: configure.hs - $(ghcmake) configure + $(GHCMAKE) configure ./configure git-annex: SysConfig.hs - $(ghcmake) git-annex + $(GHCMAKE) git-annex install: - install -d $(DESTDIR)/usr/bin - install git-annex $(DESTDIR)/usr/bin + install -d $(DESTDIR)$(PREFIX)/bin + install git-annex $(DESTDIR)$(PREFIX)/bin # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. @@ -22,7 +23,7 @@ IKIWIKI=ikiwiki endif test: - $(ghcmake) test + $(GHCMAKE) test ./test docs: diff --git a/debian/changelog b/debian/changelog index 6bc3c788a6..fe15dfdf86 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.12) UNRELEASED; urgency=low * Add --exclude option to exclude files from processing. * mwdn2man: Fix a bug in newline supression. Closes: #606578 * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 + * Makefile: Add PREFIX variable. -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 From 3a252efd9d679a85cca959537af03cbe474167b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 11 Dec 2010 17:37:24 -0400 Subject: [PATCH 0596/2835] releasing version 0.12 --- Makefile | 8 ++++---- debian/changelog | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b5f5a720be..db34294f7b 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,10 @@ install: install -d $(DESTDIR)$(PREFIX)/bin install git-annex $(DESTDIR)$(PREFIX)/bin +test: + $(GHCMAKE) test + ./test + # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. ifeq ($(shell which ikiwiki),) @@ -22,10 +26,6 @@ else IKIWIKI=ikiwiki endif -test: - $(GHCMAKE) test - ./test - docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ diff --git a/debian/changelog b/debian/changelog index fe15dfdf86..c66a536d5b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,11 +1,11 @@ -git-annex (0.12) UNRELEASED; urgency=low +git-annex (0.12) unstable; urgency=low * Add --exclude option to exclude files from processing. * mwdn2man: Fix a bug in newline supression. Closes: #606578 * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 * Makefile: Add PREFIX variable. - -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 + -- Joey Hess Sat, 11 Dec 2010 17:32:00 -0400 git-annex (0.11) unstable; urgency=low From f8b59360e386c725dc5e6961beaa78621585d1e1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 11 Dec 2010 17:37:40 -0400 Subject: [PATCH 0597/2835] add news item for git-annex 0.12 --- doc/news/version_0.07.mdwn | 8 -------- doc/news/version_0.12.mdwn | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 doc/news/version_0.07.mdwn create mode 100644 doc/news/version_0.12.mdwn diff --git a/doc/news/version_0.07.mdwn b/doc/news/version_0.07.mdwn deleted file mode 100644 index 29b4736b92..0000000000 --- a/doc/news/version_0.07.mdwn +++ /dev/null @@ -1,8 +0,0 @@ -git-annex 0.07 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * find: New subcommand. - * unused: New subcommand, finds unused data. (Split out from fsck.) - * dropunused: New subcommand, provides for easy dropping of unused keys - by number, as listed by the unused subcommand. - * fsck: Print warnings to stderr; --quiet can now be used to only see - problems."""]] \ No newline at end of file diff --git a/doc/news/version_0.12.mdwn b/doc/news/version_0.12.mdwn new file mode 100644 index 0000000000..e7fccd1b36 --- /dev/null +++ b/doc/news/version_0.12.mdwn @@ -0,0 +1,6 @@ +git-annex 0.12 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Add --exclude option to exclude files from processing. + * mwdn2man: Fix a bug in newline supression. Closes: #[606578](http://bugs.debian.org/606578) + * Bugfix to git annex add of an unlocked file in a subdir. Closes: #[606579](http://bugs.debian.org/606579) + * Makefile: Add PREFIX variable."""]] \ No newline at end of file From 98ad5402d9b095b07df82f088cf0a19c8298ca0c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 12 Dec 2010 13:15:16 -0400 Subject: [PATCH 0598/2835] Makefile: Install man page and html (when built). --- Makefile | 16 ++++++++++++---- debian/changelog | 6 ++++++ debian/docs | 1 - 3 files changed, 18 insertions(+), 5 deletions(-) delete mode 100644 debian/docs diff --git a/Makefile b/Makefile index db34294f7b..5a39356432 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,27 @@ PREFIX=/usr GHCMAKE=ghc -Wall -odir build -hidir build -O2 --make -all: git-annex docs +all: git-annex git-annex.1 docs SysConfig.hs: configure.hs $(GHCMAKE) configure ./configure +git-annex.1: + ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 + git-annex: SysConfig.hs $(GHCMAKE) git-annex -install: +install: all install -d $(DESTDIR)$(PREFIX)/bin install git-annex $(DESTDIR)$(PREFIX)/bin + install -d $(DESTDIR)$(PREFIX)/share/man/man1 + install -m 0644 git-annex.1 $(DESTDIR)$(PREFIX)/share/man/man1 + install -d $(DESTDIR)$(PREFIX)/share/doc/git-annex + if [ -d html ]; then \ + rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \ + fi test: $(GHCMAKE) test @@ -26,8 +35,7 @@ else IKIWIKI=ikiwiki endif -docs: - ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 +docs: git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ --underlaydir=/dev/null --disable-plugin=shortcut \ diff --git a/debian/changelog b/debian/changelog index c66a536d5b..7e08b07310 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.13) UNRELEASED; urgency=low + + * Makefile: Install man page and html (when built). + + -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 + git-annex (0.12) unstable; urgency=low * Add --exclude option to exclude files from processing. diff --git a/debian/docs b/debian/docs deleted file mode 100644 index 1936cc1d44..0000000000 --- a/debian/docs +++ /dev/null @@ -1 +0,0 @@ -html From b805b9ae66d279c4fee20c433e6c218f29103978 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 12 Dec 2010 13:16:34 -0400 Subject: [PATCH 0599/2835] Makefile: Add GHCOPTS variable. --- Makefile | 3 ++- debian/changelog | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5a39356432..d5f1f652f1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ PREFIX=/usr -GHCMAKE=ghc -Wall -odir build -hidir build -O2 --make +GHCOPTS=-O2 -Wall +GHCMAKE=ghc -odir build -hidir build $(GHCOPTS) --make all: git-annex git-annex.1 docs diff --git a/debian/changelog b/debian/changelog index 7e08b07310..5d088f1bdd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). + * Makefile: Add GHCOPTS variable. -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From 67c5036579a4c4af99711cba45ffffb032262d60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 12 Dec 2010 13:17:53 -0400 Subject: [PATCH 0600/2835] Makefile: Add GHCFLAGS variable. --- Makefile | 4 ++-- debian/changelog | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d5f1f652f1..c338427df4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr -GHCOPTS=-O2 -Wall -GHCMAKE=ghc -odir build -hidir build $(GHCOPTS) --make +GHCFLAGS=-O2 -Wall +GHCMAKE=ghc -odir build -hidir build $(GHCFLAGS) --make all: git-annex git-annex.1 docs diff --git a/debian/changelog b/debian/changelog index 5d088f1bdd..19ff29652d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). - * Makefile: Add GHCOPTS variable. + * Makefile: Add GHCFLAGS variable. -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From 0210628263f5a9da4f7ceb6b359108174ac01182 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 13 Dec 2010 11:35:00 -0400 Subject: [PATCH 0601/2835] Fix upgrade from 0.03. --- Upgrade.hs | 18 +++++++++++++++++- debian/changelog | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Upgrade.hs b/Upgrade.hs index d64d5287d1..52ecfa07d7 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -7,9 +7,12 @@ module Upgrade where +import System.IO.Error (try) import System.Directory import Control.Monad.State (liftIO) +import Control.Monad (filterM) import System.Posix.Files +import System.FilePath import Core import Types @@ -37,7 +40,7 @@ upgradeFrom0 = do -- do the reorganisation of the files let olddir = annexDir g - keys <- getKeysPresent' olddir + keys <- getKeysPresent0' olddir _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys -- update the symlinks to the files @@ -61,3 +64,16 @@ upgradeFrom0 = do liftIO $ createSymbolicLink link f Annex.queue "add" ["--"] f fixlinks fs + +getKeysPresent0' :: FilePath -> Annex [Key] +getKeysPresent0' dir = do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey files + where + present d = do + result <- try $ + getFileStatus $ dir ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False diff --git a/debian/changelog b/debian/changelog index 19ff29652d..7ac893c13c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). * Makefile: Add GHCFLAGS variable. + * Fix upgrade from 0.03. -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From 5ec3cea05961e4d9fc3484746860415126db5365 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 Dec 2010 11:37:11 -0400 Subject: [PATCH 0602/2835] Support remotes using git+ssh:// as protocol. Closes: #607056 --- GitRepo.hs | 1 + debian/changelog | 1 + 2 files changed, 2 insertions(+) diff --git a/GitRepo.hs b/GitRepo.hs index 539acecb7e..3cbeae192d 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -122,6 +122,7 @@ repoIsUrl _ = False repoIsSsh :: Repo -> Bool repoIsSsh Repo { location = Url url } | uriScheme url == "ssh:" = True + | uriScheme url == "git+ssh:" = True | otherwise = False repoIsSsh _ = False diff --git a/debian/changelog b/debian/changelog index 7ac893c13c..c9f47273f0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). * Makefile: Add GHCFLAGS variable. * Fix upgrade from 0.03. + * Support remotes using git+ssh:// as protocol. Closes: #607056 -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From 5d4052d0e05d391d2ebd167b528133ade726eb3b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 Dec 2010 12:46:09 -0400 Subject: [PATCH 0603/2835] Support remotes using git+ssh and ssh+git as protocol. Closes: #607056 --- GitRepo.hs | 2 ++ debian/changelog | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 3cbeae192d..1ce6b8aecf 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -122,7 +122,9 @@ repoIsUrl _ = False repoIsSsh :: Repo -> Bool repoIsSsh Repo { location = Url url } | uriScheme url == "ssh:" = True + -- git treats these the same as ssh | uriScheme url == "git+ssh:" = True + | uriScheme url == "ssh+git:" = True | otherwise = False repoIsSsh _ = False diff --git a/debian/changelog b/debian/changelog index c9f47273f0..582c05d2c5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,8 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). * Makefile: Add GHCFLAGS variable. * Fix upgrade from 0.03. - * Support remotes using git+ssh:// as protocol. Closes: #607056 + * Support remotes using git+ssh and ssh+git as protocol. + Closes: #607056 -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From dc25c7030a1b7888440ea58b8774073c2c28eb4e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 Dec 2010 13:12:24 -0400 Subject: [PATCH 0604/2835] releasing version 0.13 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 582c05d2c5..af63601ba8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.13) UNRELEASED; urgency=low +git-annex (0.13) unstable; urgency=low * Makefile: Install man page and html (when built). * Makefile: Add GHCFLAGS variable. @@ -6,7 +6,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Support remotes using git+ssh and ssh+git as protocol. Closes: #607056 - -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 + -- Joey Hess Tue, 14 Dec 2010 13:05:10 -0400 git-annex (0.12) unstable; urgency=low From ada2a925195152cde1961eee4b3529f72b51aef8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 Dec 2010 13:13:20 -0400 Subject: [PATCH 0605/2835] add news item for git-annex 0.13 --- doc/news/version_0.08.mdwn | 8 -------- doc/news/version_0.13.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 doc/news/version_0.08.mdwn create mode 100644 doc/news/version_0.13.mdwn diff --git a/doc/news/version_0.08.mdwn b/doc/news/version_0.08.mdwn deleted file mode 100644 index 8ef7ad7f31..0000000000 --- a/doc/news/version_0.08.mdwn +++ /dev/null @@ -1,8 +0,0 @@ -git-annex 0.08 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix `git annex add ../foo` (when ran in a subdir of the repo). - * Add configure step to build process. - * Only use cp -a if it is supported, falling back to cp -p or plain cp - as needed for portability. - * cp --reflink=auto is used if supported, and will make git annex unlock - much faster on filesystems like btrfs that support copy on write."""]] \ No newline at end of file diff --git a/doc/news/version_0.13.mdwn b/doc/news/version_0.13.mdwn new file mode 100644 index 0000000000..d4de64d3e2 --- /dev/null +++ b/doc/news/version_0.13.mdwn @@ -0,0 +1,7 @@ +git-annex 0.13 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Makefile: Install man page and html (when built). + * Makefile: Add GHCFLAGS variable. + * Fix upgrade from 0.03. + * Support remotes using git+ssh and ssh+git as protocol. + Closes: #[607056](http://bugs.debian.org/607056)"""]] \ No newline at end of file From d0cba98c5b1297b2bebbf2d75eb4960e3ac07779 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Mon, 20 Dec 2010 13:20:00 +0000 Subject: [PATCH 0606/2835] misleading error message on empty repo --- ...git_annex_unused_failes_on_empty_repository.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn diff --git a/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn new file mode 100644 index 0000000000..9be98104a0 --- /dev/null +++ b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn @@ -0,0 +1,13 @@ +[[!meta title="`git annex unused` fails on empty repository"]] + +The ``git annex unused`` command fails on a git-annex repository, if there are no objects yet: + + $ git annex unused + unused (checking for unused data...) + git-annex: /tmp/annextest/other_annex/.git/annex/objects: getDirectoryContents: does not exist (No such file or directory) + git-annex: 1 failed + $ + +This can give a user (especially one that wants to try out simple commands with his newly created repo) the impression that something is wrong, while it is not. I'd expect the program either to show the same message ``git annex unused`` shows when everything is ok (since it is, or should be). + +This can be a bug in the ``unused`` subcommand (that fails to accept the absence of an objects directory) or in the ``init`` subcommand (that fails to create it). From 11cf7136ed0fd88ac62cb6d8d16c5eb4c915b392 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Mon, 20 Dec 2010 15:48:58 +0000 Subject: [PATCH 0607/2835] suggestion for out-of-branch location tracking --- doc/forum/working_without_git-annex_commits.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/forum/working_without_git-annex_commits.mdwn diff --git a/doc/forum/working_without_git-annex_commits.mdwn b/doc/forum/working_without_git-annex_commits.mdwn new file mode 100644 index 0000000000..88c2804c5c --- /dev/null +++ b/doc/forum/working_without_git-annex_commits.mdwn @@ -0,0 +1,7 @@ +Is it possible to use git-annex without having [[location tracking]] commits in the style of "got a video I want to rewatch on the plane" or "freed up space" in the main tree? + +I consider these changes to be volatile, and irrelevant to the archive history. While they are unproblematic when it comes to merging, they make the commit tree rather complicated, especially with multiple users (as opposed to a single user managing his files on an external disk, a server and his laptop). Some users might even want to contribute to a shared repository without reporting on what they checked out. + +As a minimal solution, I configured a repository to ``.gitignore`` ``.git-annex/*:*.log``, but even when using modes that do not require that information (``git annex copy --from`` instead of ``git annex get``), that failes when git-annex tried to git-add ignored files. + +A more elaborate solution might be to keep location tracking information in a branch on its own (as suggested in [[todo/branching]]), keeping the main tree clean of such commits. A stealth user could then configure that branch to never be pushed. (Alternatively, if git-annex respects .gitignore and doesn't try to check in changes on ignored files, he could locally ``.gitignore`` ``.git-annex/*:*.log``.) From ea70873b112c9a1a4471bbd263001975e69e6bdf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 20 Dec 2010 14:54:03 -0400 Subject: [PATCH 0608/2835] response --- doc/forum/working_without_git-annex_commits.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/forum/working_without_git-annex_commits.mdwn b/doc/forum/working_without_git-annex_commits.mdwn index 88c2804c5c..00a61f3f0f 100644 --- a/doc/forum/working_without_git-annex_commits.mdwn +++ b/doc/forum/working_without_git-annex_commits.mdwn @@ -5,3 +5,16 @@ I consider these changes to be volatile, and irrelevant to the archive history. As a minimal solution, I configured a repository to ``.gitignore`` ``.git-annex/*:*.log``, but even when using modes that do not require that information (``git annex copy --from`` instead of ``git annex get``), that failes when git-annex tried to git-add ignored files. A more elaborate solution might be to keep location tracking information in a branch on its own (as suggested in [[todo/branching]]), keeping the main tree clean of such commits. A stealth user could then configure that branch to never be pushed. (Alternatively, if git-annex respects .gitignore and doesn't try to check in changes on ignored files, he could locally ``.gitignore`` ``.git-annex/*:*.log``.) + +> A stealth user can simply avoid pushing, and so keep their repository +> in a forked state, that can still pull changes from origin. +> +> Beyond that, [[todo/branching]] is the best solution. +> +> I don't think that gitignoring the log files is a good plan, because +> if the files are left modified and uncommitted, git will not be able to +> merge other changes it pulls. The automerging of log files only works +> if any local changes to them have been committed. +> +> It would be possible to add a knob that +> simply blocks all local modifications to the log files. --[[Joey]] From c4a357d5d13e6c2f09ee98ee6ef8b18104e5d167 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 20 Dec 2010 14:57:43 -0400 Subject: [PATCH 0609/2835] Bugfix to git annex unused in a repository with nothing yet annexed. --- Core.hs | 10 +++++++--- debian/changelog | 6 ++++++ .../git_annex_unused_failes_on_empty_repository.mdwn | 2 ++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Core.hs b/Core.hs index e3702044e8..9da4f3f2ec 100644 --- a/Core.hs +++ b/Core.hs @@ -185,9 +185,13 @@ getKeysPresent = do getKeysPresent' $ annexObjectDir g getKeysPresent' :: FilePath -> Annex [Key] getKeysPresent' dir = do - contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM present contents - return $ map fileKey files + exists <- liftIO $ doesDirectoryExist dir + if (not exists) + then return [] + else do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey files where present d = do result <- try $ diff --git a/debian/changelog b/debian/changelog index af63601ba8..70d375f30e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.14) UNRELEASED; urgency=low + + * Bugfix to git annex unused in a repository with nothing yet annexed. + + -- Joey Hess Mon, 20 Dec 2010 14:54:49 -0400 + git-annex (0.13) unstable; urgency=low * Makefile: Install man page and html (when built). diff --git a/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn index 9be98104a0..05aa695727 100644 --- a/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn +++ b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn @@ -11,3 +11,5 @@ The ``git annex unused`` command fails on a git-annex repository, if there are n This can give a user (especially one that wants to try out simple commands with his newly created repo) the impression that something is wrong, while it is not. I'd expect the program either to show the same message ``git annex unused`` shows when everything is ok (since it is, or should be). This can be a bug in the ``unused`` subcommand (that fails to accept the absence of an objects directory) or in the ``init`` subcommand (that fails to create it). + +> [[fixed|done]] --[[Joey]] From eedebb0057b4d0fc7d0d13214d8f5e3890034374 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 20 Dec 2010 15:01:04 -0400 Subject: [PATCH 0610/2835] Support upgrading from a v0 annex with nothing in it. --- Upgrade.hs | 10 +++++++--- debian/changelog | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Upgrade.hs b/Upgrade.hs index 52ecfa07d7..204e2d555f 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -67,9 +67,13 @@ upgradeFrom0 = do getKeysPresent0' :: FilePath -> Annex [Key] getKeysPresent0' dir = do - contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM present contents - return $ map fileKey files + exists <- liftIO $ doesDirectoryExist dir + if (not exists) + then return [] + else do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey files where present d = do result <- try $ diff --git a/debian/changelog b/debian/changelog index 70d375f30e..ae6c3f0469 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.14) UNRELEASED; urgency=low * Bugfix to git annex unused in a repository with nothing yet annexed. + * Support upgrading from a v0 annex with nothing in it. -- Joey Hess Mon, 20 Dec 2010 14:54:49 -0400 From 61b7f3dea3b54ead64cd5e957da28c83e7c25c85 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 22 Dec 2010 11:52:16 +0000 Subject: [PATCH 0611/2835] but report on umlaut handling --- doc/bugs/problems_with_utf8_names.mdwn | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/bugs/problems_with_utf8_names.mdwn diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn new file mode 100644 index 0000000000..521f92405e --- /dev/null +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -0,0 +1,20 @@ +There are problems with displaying filenames in UTF8 encoding, as shown here: + + $ echo $LANG + en_GB.UTF-8 + $ git init + $ git annex init test + [...] + $ touch "Umlaut Ü.txt" + $ git annex add Uml* + add Umlaut Ã.txt ok + (Recording state in git...) + $ find -name U\* | hexdump -C + 00000000 2e 2f 55 6d 6c 61 75 74 20 c3 9c 2e 74 78 74 0a |./Umlaut ...txt.| + 00000010 + $ git annex find | hexdump -C + 00000000 55 6d 6c 61 75 74 20 c3 83 c2 9c 2e 74 78 74 0a |Umlaut .....txt.| + 00000010 + $ + +It looks like the common latin1-to-UTF8 encoding. Functionality other than otuput seems not to be affected. From 346c7a025780e011cdfdbb7eb27281f808a8de92 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Dec 2010 21:53:32 -0400 Subject: [PATCH 0612/2835] Avoid multiple calls to git ls-files when passed eg, "*". --- Command.hs | 23 +++++++++++------------ Core.hs | 2 +- GitRepo.hs | 46 ++++++++++++++++++++++------------------------ Upgrade.hs | 2 +- debian/changelog | 1 + 5 files changed, 36 insertions(+), 38 deletions(-) diff --git a/Command.hs b/Command.hs index 69b9dee9f9..e30904d0f7 100644 --- a/Command.hs +++ b/Command.hs @@ -107,14 +107,14 @@ isAnnexed file a = do withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.inRepo repo) params - files' <- filterFiles $ foldl (++) [] files + files <- liftIO $ Git.inRepo repo params + files' <- filterFiles files return $ map a files' withAttrFilesInGit :: String -> SubCmdSeekAttrFiles withAttrFilesInGit attr a params = do repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.inRepo repo) params - files' <- filterFiles $ foldl (++) [] files + files <- liftIO $ Git.inRepo repo params + files' <- filterFiles files pairs <- liftIO $ Git.checkAttr repo attr files' return $ map a pairs withFilesMissing :: SubCmdSeekStrings @@ -129,8 +129,8 @@ withFilesMissing a params = do withFilesNotInGit :: SubCmdSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo - newfiles <- liftIO $ mapM (Git.notInRepo repo) params - newfiles' <- filterFiles $ foldl (++) [] newfiles + newfiles <- liftIO $ Git.notInRepo repo params + newfiles' <- filterFiles newfiles backendPairs a newfiles' withString :: SubCmdSeekStrings withString a params = return [a $ unwords params] @@ -139,21 +139,20 @@ withStrings a params = return $ map a params withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo - tocommit <- liftIO $ mapM (Git.stagedFiles repo) params - tocommit' <- filterFiles $ foldl (++) [] tocommit + tocommit <- liftIO $ Git.stagedFiles repo params + tocommit' <- filterFiles tocommit return $ map a tocommit' withFilesUnlocked :: SubCmdSeekBackendFiles withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles withFilesUnlockedToBeCommitted :: SubCmdSeekBackendFiles withFilesUnlockedToBeCommitted = withFilesUnlocked' Git.typeChangedStagedFiles -withFilesUnlocked' :: (Git.Repo -> FilePath -> IO [FilePath]) -> SubCmdSeekBackendFiles +withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> SubCmdSeekBackendFiles withFilesUnlocked' typechanged a params = do -- unlocked files have changed type from a symlink to a regular file repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (typechanged repo) params + typechangedfiles <- liftIO $ typechanged repo params unlockedfiles <- liftIO $ filterM notSymlink $ - map (\f -> Git.workTree repo ++ "/" ++ f) $ - foldl (++) [] typechangedfiles + map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' withKeys :: SubCmdSeekStrings diff --git a/Core.hs b/Core.hs index 9da4f3f2ec..d91595a049 100644 --- a/Core.hs +++ b/Core.hs @@ -204,6 +204,6 @@ getKeysPresent' dir = do getKeysReferenced :: Annex [Key] getKeysReferenced = do g <- Annex.gitRepo - files <- liftIO $ Git.inRepo g $ Git.workTree g + files <- liftIO $ Git.inRepo g [Git.workTree g] keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs diff --git a/GitRepo.hs b/GitRepo.hs index 1ce6b8aecf..122b5cc60f 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -230,41 +230,39 @@ hPipeRead :: Repo -> [String] -> IO (PipeHandle, String) hPipeRead repo params = assertLocal repo $ do pipeFrom "git" (gitCommandLine repo params) -{- Passed a location, recursively scans for all files that - - are checked into git at that location. -} -inRepo :: Repo -> FilePath -> IO [FilePath] -inRepo repo l = pipeNullSplit repo - ["ls-files", "--cached", "--exclude-standard", "-z", "--", l] +{- Scans for files that are checked into git at the specified locations. -} +inRepo :: Repo -> [FilePath] -> IO [FilePath] +inRepo repo l = pipeNullSplit repo $ + ["ls-files", "--cached", "--exclude-standard", "-z", "--"] ++ l -{- Passed a location, recursively scans for all files that are not checked - - into git, and not gitignored. -} -notInRepo :: Repo -> FilePath -> IO [FilePath] -notInRepo repo l = pipeNullSplit repo - ["ls-files", "--others", "--exclude-standard", "-z", "--", l] +{- Scans for files at the specified locations that are not checked into git, + - and not gitignored. -} +notInRepo :: Repo -> [FilePath] -> IO [FilePath] +notInRepo repo l = pipeNullSplit repo $ + ["ls-files", "--others", "--exclude-standard", "-z", "--"] ++ l -{- Passed a location, returns a list of the files, staged for - - commit, that are being added, moved, or changed (but not deleted). -} -stagedFiles :: Repo -> FilePath -> IO [FilePath] -stagedFiles repo l = pipeNullSplit repo +{- Returns a list of the files, staged for commit, that are being added, + - moved, or changed (but not deleted), from the specified locations. -} +stagedFiles :: Repo -> [FilePath] -> IO [FilePath] +stagedFiles repo l = pipeNullSplit repo $ ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", - "--", l] + "--"] ++ l -{- Passed a location, returns a list of the files, staged for - - commit, whose type has changed. -} -typeChangedStagedFiles :: Repo -> FilePath -> IO [FilePath] +{- Returns a list of the files in the specified locations that are staged + - for commit, and whose type has changed. -} +typeChangedStagedFiles :: Repo -> [FilePath] -> IO [FilePath] typeChangedStagedFiles repo l = typeChangedFiles' repo l ["--cached"] -{- Passed a location, returns a list of the files whose type has changed. - - Files only staged for commit will not be included. -} -typeChangedFiles :: Repo -> FilePath -> IO [FilePath] +{- Returns a list of the files in the specified locations whose type has + - changed. Files only staged for commit will not be included. -} +typeChangedFiles :: Repo -> [FilePath] -> IO [FilePath] typeChangedFiles repo l = typeChangedFiles' repo l [] -typeChangedFiles' :: Repo -> FilePath -> [String] -> IO [FilePath] +typeChangedFiles' :: Repo -> [FilePath] -> [String] -> IO [FilePath] typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where start = ["diff", "--name-only", "--diff-filter=T", "-z"] - end = ["--", l] - + end = ["--"] ++ l {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} diff --git a/Upgrade.hs b/Upgrade.hs index 204e2d555f..2e1708439b 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -44,7 +44,7 @@ upgradeFrom0 = do _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys -- update the symlinks to the files - files <- liftIO $ Git.inRepo g $ Git.workTree g + files <- liftIO $ Git.inRepo g [Git.workTree g] fixlinks files Annex.queueRun diff --git a/debian/changelog b/debian/changelog index ae6c3f0469..bddd256ae5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.14) UNRELEASED; urgency=low * Bugfix to git annex unused in a repository with nothing yet annexed. * Support upgrading from a v0 annex with nothing in it. + * Avoid multiple calls to git ls-files when passed eg, "*". -- Joey Hess Mon, 20 Dec 2010 14:54:49 -0400 From 80aaa68e91a8c5d4fb5b426ac24d7de384cf9b8a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Dec 2010 17:41:15 -0400 Subject: [PATCH 0613/2835] releasing version 0.14 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index bddd256ae5..f8a3c3129d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,10 @@ -git-annex (0.14) UNRELEASED; urgency=low +git-annex (0.14) unstable; urgency=low * Bugfix to git annex unused in a repository with nothing yet annexed. * Support upgrading from a v0 annex with nothing in it. * Avoid multiple calls to git ls-files when passed eg, "*". - -- Joey Hess Mon, 20 Dec 2010 14:54:49 -0400 + -- Joey Hess Fri, 24 Dec 2010 17:38:48 -0400 git-annex (0.13) unstable; urgency=low From 36662638df5b3b167f20d1b9e772a2059b96da3d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Dec 2010 17:41:39 -0400 Subject: [PATCH 0614/2835] add news item for git-annex 0.14 --- doc/news/version_0.09.mdwn | 4 ---- doc/news/version_0.14.mdwn | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 doc/news/version_0.09.mdwn create mode 100644 doc/news/version_0.14.mdwn diff --git a/doc/news/version_0.09.mdwn b/doc/news/version_0.09.mdwn deleted file mode 100644 index fff8249f77..0000000000 --- a/doc/news/version_0.09.mdwn +++ /dev/null @@ -1,4 +0,0 @@ -git-annex 0.09 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Add copy subcommand. - * Fix bug in setkey subcommand triggered by move --to."""]] \ No newline at end of file diff --git a/doc/news/version_0.14.mdwn b/doc/news/version_0.14.mdwn new file mode 100644 index 0000000000..fc6c42bc19 --- /dev/null +++ b/doc/news/version_0.14.mdwn @@ -0,0 +1,5 @@ +git-annex 0.14 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Bugfix to git annex unused in a repository with nothing yet annexed. + * Support upgrading from a v0 annex with nothing in it. + * Avoid multiple calls to git ls-files when passed eg, "*"."""]] \ No newline at end of file From 904af72b82ee513748672620dd51bae050494a6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Dec 2010 19:25:59 -0400 Subject: [PATCH 0615/2835] better message --- Backend/File.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend/File.hs b/Backend/File.hs index afbe38ac0f..f88bb7c704 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -175,4 +175,4 @@ checkKeyNumCopies key numcopies = do note present needed = "Only " ++ show present ++ " of " ++ show needed ++ " copies of "++show key++" exist. " ++ - "Run git annex get somewhere else to back it up." + "Back it up with git-annex copy." From dd55f21450d5ed9f71b015295c33c0c9bf0060f9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Dec 2010 19:28:02 -0400 Subject: [PATCH 0616/2835] add a newline --- Command/Unused.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Command/Unused.hs b/Command/Unused.hs index de34ceae96..69a16f2547 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -50,7 +50,8 @@ checkUnused = do " NUMBER KEY"] ++ map cols u ++ ["(To see where data was previously used, try: git log --stat -S'KEY')", - "(To remove unwanted data: git-annex dropunused NUMBER)"] + "(To remove unwanted data: git-annex dropunused NUMBER)", + ""] cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k pad n s = s ++ replicate (n - length s) ' ' From 35622738aa2ac80e14c9e6248fb123e70ea49e02 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawk_lgZQnQAGL6OznVti_Dc90BJeiH7Ai4U" Date: Tue, 28 Dec 2010 03:51:55 +0000 Subject: [PATCH 0617/2835] --- doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn new file mode 100644 index 0000000000..5ae8f7a326 --- /dev/null +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -0,0 +1,11 @@ +git-annex does not seem to support all kinds of urls that git does. + +Specifically, if I have ~/bar set up on host foo: + +[remote "foo"] +## this one is not recognized as ssh url at all +# url = foo:bar +## this one makes git-annex try to access '/~/bar' literally +# url = ssh://foo/~/bar +## this one works + url = ssh://foo/home/tv/bar From 1f1d2a2ea7caadb0720427b1a66efb5ffca17c38 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawk_lgZQnQAGL6OznVti_Dc90BJeiH7Ai4U" Date: Tue, 28 Dec 2010 03:53:33 +0000 Subject: [PATCH 0618/2835] Layout. --- .../wishlist:_support_for_more_ssh_urls_.mdwn | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn index 5ae8f7a326..42107e51a3 100644 --- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -2,10 +2,10 @@ git-annex does not seem to support all kinds of urls that git does. Specifically, if I have ~/bar set up on host foo: -[remote "foo"] -## this one is not recognized as ssh url at all -# url = foo:bar -## this one makes git-annex try to access '/~/bar' literally -# url = ssh://foo/~/bar -## this one works - url = ssh://foo/home/tv/bar + [remote "foo"] + ## this one is not recognized as ssh url at all + # url = foo:bar + ## this one makes git-annex try to access '/~/bar' literally + # url = ssh://foo/~/bar + ## this one works + url = ssh://foo/home/tv/bar From 70d68212f5283d27bac00812afdf582b54424e47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 12:55:19 -0400 Subject: [PATCH 0619/2835] let's not use the forum for bug reports --- doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn new file mode 100644 index 0000000000..42107e51a3 --- /dev/null +++ b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn @@ -0,0 +1,11 @@ +git-annex does not seem to support all kinds of urls that git does. + +Specifically, if I have ~/bar set up on host foo: + + [remote "foo"] + ## this one is not recognized as ssh url at all + # url = foo:bar + ## this one makes git-annex try to access '/~/bar' literally + # url = ssh://foo/~/bar + ## this one works + url = ssh://foo/home/tv/bar From 0b3cf9bc682e5dfbc78248aa263c2905fd3e79be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 12:56:06 -0400 Subject: [PATCH 0620/2835] moved to bugs --- doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn index 42107e51a3..3fd6482219 100644 --- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -1,11 +1,4 @@ git-annex does not seem to support all kinds of urls that git does. -Specifically, if I have ~/bar set up on host foo: - - [remote "foo"] - ## this one is not recognized as ssh url at all - # url = foo:bar - ## this one makes git-annex try to access '/~/bar' literally - # url = ssh://foo/~/bar - ## this one works - url = ssh://foo/home/tv/bar +> Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls]]. +> --[[Joey]] From c7344eff81c7de4f303097816ef26cc74146588a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 12:56:43 -0400 Subject: [PATCH 0621/2835] fix --- doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn index 3fd6482219..6ee01d2300 100644 --- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -1,4 +1,4 @@ git-annex does not seem to support all kinds of urls that git does. -> Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls]]. +> Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls_]]. > --[[Joey]] From 022e0c7751db23805169c5429851903a3d482cb2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 13:48:11 -0400 Subject: [PATCH 0622/2835] Support scp-style urls for remotes (host:path). --- GitRepo.hs | 15 ++++++++++++++- debian/changelog | 6 ++++++ .../wishlist:_support_for_more_ssh_urls_.mdwn | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 122b5cc60f..2c2ad7b539 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -60,6 +60,7 @@ import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf +import Data.List (isInfixOf) import Utility @@ -314,8 +315,20 @@ configRemotes repo = map construct remotepairs isremote k = startswith "remote." k && endswith ".url" k remotename k = split "." k !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } - gen v | isURI v = repoFromUrl v + gen v | scpstyle v = repoFromUrl $ scptourl v + | isURI v = repoFromUrl v | otherwise = repoFromPath v + -- git remotes can be written scp style -- [user@]host:dir + -- where dir is relative to the user's home directory. + scpstyle v = isInfixOf ":" v && (not $ isInfixOf "//" v) + scptourl v = "ssh://" ++ host ++ slash dir + where + bits = split ":" v + host = bits !! 0 + dir = join ":" $ drop 1 bits + slash d | d == "" = "/~/" ++ dir + | d !! 0 == '/' = dir + | otherwise = "/~/" ++ dir {- Parses git config --list output into a config map. -} configParse :: String -> Map.Map String String diff --git a/debian/changelog b/debian/changelog index f8a3c3129d..dba343deff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.15) UNRELEASED; urgency=low + + * Support scp-style urls for remotes (host:path). + + -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 + git-annex (0.14) unstable; urgency=low * Bugfix to git annex unused in a repository with nothing yet annexed. diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn index 42107e51a3..6156ade054 100644 --- a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn @@ -9,3 +9,12 @@ Specifically, if I have ~/bar set up on host foo: # url = ssh://foo/~/bar ## this one works url = ssh://foo/home/tv/bar + +> scp-style is now supported. + +> `~` expansions (for the user's home, or other users) +> are somewhat tricky to support as they require running +> code on the remote to lookup homedirs. If git-annex grows a +> `git annex shell` that is run on the remote side +> (something I am considering for other reasons), it +> could handle the expansions there. --[[Joey]] From 6c58a58393a1d6d2257cb9e72e534387561943d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 15:28:23 -0400 Subject: [PATCH 0623/2835] details.. --- doc/bugs/problems_with_utf8_names.mdwn | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 521f92405e..30f3495f40 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -18,3 +18,45 @@ There are problems with displaying filenames in UTF8 encoding, as shown here: $ It looks like the common latin1-to-UTF8 encoding. Functionality other than otuput seems not to be affected. + +> Yes, I believe that git-annex is reading filename data from git +> as a stream of char8s, and not decoding unicode in it into logical +> characters. +> Haskell then I guess, tries to unicode encode it when it's output to +> the console. +> This only seems to matter WRT its output to the console; the data +> does not get mangled internally and so it accesses the right files +> under the hood. +> +> I am too new to haskell to really have a handle on how to handle +> unicode and other encodings issues with it. In general, there are three +> valid approaches: --[[Joey]] +> +> 1. Convert all input data to unicode and be unicode clean end-to-end +> internally. Problimatic here since filenames may not necessarily be +> encoded in utf-8 (an archive could have historical filenames using +> varying encodings), and you don't want which files are accessed to +> depend on locale settings. +> 1. Keep input and internal data un-decoded, but decode it when +> outputting a filename (assuming the filename is encoded using the +> user's configured encoding), and allow haskell's output encoding to then +> encode it according to the user's locale configuration. +> 1. Avoid encodings entirely. Mostly what I'm doing now; probably +> could find a way to disable encoding of console output. Then the raw +> filename would be displayed, which should work ok. git-annex does +> not really need to pull apart filenames; they are almost entirely +> opaque blobs. I guess that the `--exclude` option is the exception +> to that, but it is currently not unicode safe anyway. +> One other possible +> issue would be that this could cause problems if git-annex were +> translated. +> +> BTW, for more fun, try unsetting LANG, and then you can see +> stuff like this: + + joey@gnu:~/tmp/aa>git annex add ./Üa + add add add add git-annex: : commitAndReleaseBuffer: invalid + argument (Invalid or incomplete multibyte or wide character) + +> (Add -q to work around this; once it doesn't need to print the filename, +> it can act on it ok!) From aa4f91b2d67b9f7827b02acebfbf4e67ba33bb80 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:17:02 -0400 Subject: [PATCH 0624/2835] Add trust and untrust subcommands, to allow configuring remotes that are trusted to retain files without explicit checking. --- Backend/File.hs | 12 +++++++--- CmdLine.hs | 7 ++++++ Remotes.hs | 6 +++++ UUID.hs | 45 ++++++++++++++++++++++++++++++-------- Utility.hs | 12 +++++++++- debian/changelog | 2 ++ doc/copies.mdwn | 3 ++- doc/git-annex.mdwn | 13 ++++++++++- doc/location_tracking.mdwn | 3 +++ doc/trust.mdwn | 20 +++++++++++++++++ doc/walkthrough.mdwn | 2 +- 11 files changed, 109 insertions(+), 16 deletions(-) create mode 100644 doc/trust.mdwn diff --git a/Backend/File.hs b/Backend/File.hs index f88bb7c704..0b1cdd8e52 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -16,6 +16,7 @@ module Backend.File (backend, checkKey) where import Control.Monad.State import System.Directory +import Data.List (intersect) import TypeInternals import LocationLog @@ -91,11 +92,16 @@ checkRemoveKey key numcopiesM = do if force || numcopiesM == Just 0 then return True else do + g <- Annex.gitRepo + locations <- liftIO $ keyLocations g key + trusted <- getTrusted + let trustedlocations = intersect locations trusted remotes <- Remotes.keyPossibilities key + untrustedremotes <- reposWithoutUUID remotes trusted numcopies <- getNumCopies numcopiesM - if numcopies > length remotes - then notEnoughCopies numcopies (length remotes) [] - else findcopies numcopies 0 remotes [] + if numcopies > length untrustedremotes + then notEnoughCopies numcopies (length untrustedremotes) [] + else findcopies numcopies (length trustedlocations) untrustedremotes [] where findcopies need have [] bad | have >= need = return True diff --git a/CmdLine.hs b/CmdLine.hs index cb164a6ab2..7eab0a7e28 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -34,6 +34,8 @@ import qualified Command.Lock import qualified Command.PreCommit import qualified Command.Find import qualified Command.Uninit +import qualified Command.Trust +import qualified Command.Untrust subCmds :: [SubCommand] subCmds = @@ -61,6 +63,10 @@ subCmds = "de-initialize git-annex and clean out repository" , SubCommand "pre-commit" path Command.PreCommit.seek "run by git pre-commit hook" + , SubCommand "trust" remote Command.Trust.seek + "trust a repository" + , SubCommand "untrust" remote Command.Untrust.seek + "do not trust a repository" , SubCommand "fromkey" key Command.FromKey.seek "adds a file using a specific key" , SubCommand "dropkey" key Command.DropKey.seek @@ -84,6 +90,7 @@ subCmds = key = "KEY ..." desc = "DESCRIPTION" number = "NUMBER ..." + remote = "REMOTE ..." nothing = "" -- Each dashed command-line option results in generation of an action diff --git a/Remotes.hs b/Remotes.hs index cf49b624b4..725348a6a3 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -12,6 +12,7 @@ module Remotes ( inAnnex, same, commandLineRemote, + byName, copyFromRemote, copyToRemote, runCmd @@ -156,6 +157,11 @@ commandLineRemote = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" let name = if null fromName then toName else fromName + byName name + +{- Looks up a remote by name. -} +byName :: String -> Annex Git.Repo +byName name = do when (null name) $ error "no remote specified" g <- Annex.gitRepo let match = filter (\r -> name == Git.repoRemoteName r) $ diff --git a/UUID.hs b/UUID.hs index 41a35327d9..3e43c80612 100644 --- a/UUID.hs +++ b/UUID.hs @@ -14,9 +14,13 @@ module UUID ( prepUUID, genUUID, reposByUUID, + reposWithoutUUID, prettyPrintUUIDs, describeUUID, - uuidLog + uuidLog, + trustLog, + getTrusted, + setTrusted ) where import Control.Monad.State @@ -24,9 +28,7 @@ import Data.Maybe import Data.List import System.Cmd.Utils import System.IO -import System.Directory import qualified Data.Map as M -import System.Posix.Process import qualified GitRepo as Git import Types @@ -85,6 +87,14 @@ reposByUUID repos uuids = filterM match repos u <- getUUID r return $ isJust $ elemIndex u uuids +{- Filters a list of repos to ones that do not have the listed UUIDs. -} +reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] +reposWithoutUUID repos uuids = filterM unmatch repos + where + unmatch r = do + u <- getUUID r + return $ not $ isJust $ elemIndex u uuids + {- Pretty-prints a list of UUIDs -} prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = do @@ -103,11 +113,7 @@ describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m logfile <- uuidLog - pid <- liftIO $ getProcessID - let tmplogfile = logfile ++ ".tmp" ++ show pid - liftIO $ createDirectoryIfMissing True (parentDir logfile) - liftIO $ writeFile tmplogfile $ serialize m' - liftIO $ renameFile tmplogfile logfile + liftIO $ safeWriteFile logfile (serialize m') where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m @@ -125,7 +131,28 @@ uuidMap = do ignoreerror _ = return "" {- Filename of uuid.log. -} -uuidLog :: Annex String +uuidLog :: Annex FilePath uuidLog = do g <- Annex.gitRepo return $ gitStateDir g ++ "uuid.log" + +{- Filename of trust.log. -} +trustLog :: Annex FilePath +trustLog = do + g <- Annex.gitRepo + return $ gitStateDir g ++ "trust.log" + +{- List of trusted UUIDs. -} +getTrusted :: Annex [UUID] +getTrusted = do + logfile <- trustLog + s <- liftIO $ catch (readFile logfile) ignoreerror + return $ map (\l -> head $ words l) $ lines s + where + ignoreerror _ = return "" + +{- Changes the list of trusted UUIDs. -} +setTrusted :: [UUID] -> Annex () +setTrusted u = do + logfile <- trustLog + liftIO $ safeWriteFile logfile $ unlines u diff --git a/Utility.hs b/Utility.hs index 2447c95a0b..3a6c757515 100644 --- a/Utility.hs +++ b/Utility.hs @@ -15,7 +15,8 @@ module Utility ( boolSystem, shellEscape, unsetFileMode, - readMaybe + readMaybe, + safeWriteFile ) where import System.IO @@ -139,3 +140,12 @@ readMaybe :: (Read a) => String -> Maybe a readMaybe s = case reads s of ((x,_):_) -> Just x _ -> Nothing + +{- Writes a file using a temp file that is renamed atomically into place. -} +safeWriteFile :: FilePath -> String -> IO () +safeWriteFile file content = do + pid <- getProcessID + let tmpfile = file ++ ".tmp" ++ show pid + createDirectoryIfMissing True (parentDir file) + writeFile tmpfile content + renameFile tmpfile file diff --git a/debian/changelog b/debian/changelog index dba343deff..9be8b37fd0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.15) UNRELEASED; urgency=low * Support scp-style urls for remotes (host:path). + * Add trust and untrust subcommands, to allow configuring remotes + that are trusted to retain files without explicit checking. -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 diff --git a/doc/copies.mdwn b/doc/copies.mdwn index 165e54b340..5b3cbf5154 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -11,7 +11,8 @@ setting in `.gitattributes` files. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have -it, it will retain the file content to avoid data loss. +it, it will retain the file content to avoid data loss. Note that +[[trusted_remotes|trust]] are not explicitly checked. For example, consider three repositories: Server, Laptop, and USB. Both Server and USB have a copy of a file, and N=1. If on Laptop, you `git annex get diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 71a4889ac7..caef49d977 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -171,6 +171,15 @@ Many git-annex subcommands will stage changes for later `git commit` by you. This is meant to be called from git's pre-commit hook. `git annex init` automatically creates a pre-commit hook using this. +* trust [repository ...] + + Records that a repository is [[trusted]] to not unexpectedly lose content. + Use with care. + +* untrust [repository ...] + + Undoes a trust command. + * fromkey file This can be used to maually set up a file to link to a specified key @@ -333,7 +342,9 @@ These files are used by git-annex, in your git repository: available. Annexed files in your git repository symlink to that content. `.git-annex/uuid.log` is used to map between repository UUID and -decscriptions. You may edit it. +decscriptions. + +`.git-annex/trust.log` is used to list the UUIDs of trusted repositories. `.git-annex/*.log` is where git-annex records its content tracking information. These files should be committed to git. diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn index a7d5c150b1..3791029f8f 100644 --- a/doc/location_tracking.mdwn +++ b/doc/location_tracking.mdwn @@ -26,3 +26,6 @@ descriptions to help you with finding them: Try making some of these repositories available: c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 e1938fee-d95b-11df-96cc-002170d25c55 + +In certian cases you may want to configure git-annex to [[trust]] +that location tracking information is always correct for a repository. diff --git a/doc/trust.mdwn b/doc/trust.mdwn new file mode 100644 index 0000000000..b04a112ecf --- /dev/null +++ b/doc/trust.mdwn @@ -0,0 +1,20 @@ +Normally, git-annex does not fully trust its stored [[location_tracking]] +information. When removing content, it will directly check +that other repositories have [[copies]]. + +Generally that explicit checking is a good idea. Consider that the current +[[location_tracking]] information for a remote may not yet have propigated +out. Or, a remote may have suffered a catastrophic loss of data, or itself +been lost. + +Sometimes though, you may have reasons to trust the location tracking +information for a remote repository. For example, it may be an offline +archival drive, from which you rarely or never remove content. Deciding +when it makes sense to trust the tracking info is up to you. + +One way to handle this is just to use `--force` when a command cannot +access a remote you trust. + +Another option is to configure which remotes you trust with the +`git annex trust` command, or by manually adding the UUIDs of trusted remotes +to `.git-annex/trust.log`. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 9c8106d513..b486a4b1ff 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -134,7 +134,7 @@ you'll see something like this. (Use --force to override this check, or adjust annex.numcopies.) failed -Here you might --force it to drop `important_file` if you trust your backup. +Here you might --force it to drop `important_file` if you [[trust]] your backup. But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. From ee5d81429dcc6c8c2922c4c6696abc17e0440ff6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:19:01 -0400 Subject: [PATCH 0625/2835] tweak --- Backend/File.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 0b1cdd8e52..092cf7e73d 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -95,13 +95,13 @@ checkRemoveKey key numcopiesM = do g <- Annex.gitRepo locations <- liftIO $ keyLocations g key trusted <- getTrusted - let trustedlocations = intersect locations trusted + let trustedcopies = length $ intersect locations trusted remotes <- Remotes.keyPossibilities key untrustedremotes <- reposWithoutUUID remotes trusted numcopies <- getNumCopies numcopiesM if numcopies > length untrustedremotes then notEnoughCopies numcopies (length untrustedremotes) [] - else findcopies numcopies (length trustedlocations) untrustedremotes [] + else findcopies numcopies trustedcopies untrustedremotes [] where findcopies need have [] bad | have >= need = return True From 7329a1b655f7011f09a4519209da748061a014c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:19:52 -0400 Subject: [PATCH 0626/2835] note --- debian/changelog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 9be8b37fd0..bc2901ef74 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.15) UNRELEASED; urgency=low - * Support scp-style urls for remotes (host:path). + * Support scp-style urls for remotes (host:path). Note that + paths relative to the user's home directory, or containing "~" are + not yet supported. * Add trust and untrust subcommands, to allow configuring remotes that are trusted to retain files without explicit checking. From 3714364905a23f3b64c6573a4121bd1cc5acb11e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:44:36 -0400 Subject: [PATCH 0627/2835] design for a git-annex-shell --- .../wishlist:_support_for_more_ssh_urls_.mdwn | 2 +- doc/todo/git-annex-shell.mdwn | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 doc/todo/git-annex-shell.mdwn diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn index 6156ade054..ee3b4a64ad 100644 --- a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn @@ -16,5 +16,5 @@ Specifically, if I have ~/bar set up on host foo: > are somewhat tricky to support as they require running > code on the remote to lookup homedirs. If git-annex grows a > `git annex shell` that is run on the remote side -> (something I am considering for other reasons), it +> (something I am [[considering|todo/git-annex-shell]] for other reasons), it > could handle the expansions there. --[[Joey]] diff --git a/doc/todo/git-annex-shell.mdwn b/doc/todo/git-annex-shell.mdwn new file mode 100644 index 0000000000..a5ff7d6fd7 --- /dev/null +++ b/doc/todo/git-annex-shell.mdwn @@ -0,0 +1,42 @@ +I've been considering adding a `git-annex-shell` command. This would +be similar to `git-shell` (and in fact would pass unknown commands off to +`git-shell`). + +## Reasons + +* Allows locking down an account to only be able to use git-annex (and + git). +* Avoids needing to construct complex shell commands to run on the remote + system. (Mostly already avoided by the plumbing level commands.) +* Could possibly allow multiple things to be done with one ssh connection + in future. +* Allows expanding `~` and `~user` in repopath on the remote system. + +## Design + +`git-annex-shell -c ` + +### options + +Need at least `--quiet`, `--backend`, `--key`, `--force` + +### commands + +* `configlist repopath` + + Returns `git config --list`, for use by `tryGitConfigRead`. + + May filter the listed config to only the options git-annex really needs, + to prevent info disclosure. + +* `inannex repopath key ...` + + Checks if the keys are in the annex; shell exits zero if so. + +* `dropkey repopath key ... ` + + Same as `git annex dropkey`, and taking the same dashed options. + +* `setkey repopath tmpfile` + + Same as `git annex setkey`, and taking the same dashed options. From 1f20277ec7f23fa638422987d312f02f6ae1a5ee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:44:55 -0400 Subject: [PATCH 0628/2835] forgot to add these --- Command/Trust.hs | 40 ++++++++++++++++++++++++++++++++++++++++ Command/Untrust.hs | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 Command/Trust.hs create mode 100644 Command/Untrust.hs diff --git a/Command/Trust.hs b/Command/Trust.hs new file mode 100644 index 0000000000..8060ee66f9 --- /dev/null +++ b/Command/Trust.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Trust where + +import Control.Monad.State (liftIO) +import Control.Monad (unless) + +import Command +import qualified Annex +import qualified GitRepo as Git +import qualified Remotes +import UUID +import Messages + +seek :: [SubCmdSeek] +seek = [withString start] + +{- Marks a remote as trusted. -} +start :: SubCmdStartString +start name = do + r <- Remotes.byName name + showStart "trust" name + return $ Just $ perform r + +perform :: Git.Repo -> SubCmdPerform +perform repo = do + uuid <- getUUID repo + trusted <- getTrusted + unless (elem uuid trusted) $ do + setTrusted $ uuid:trusted + g <- Annex.gitRepo + logfile <- trustLog + liftIO $ Git.run g ["add", logfile] + liftIO $ Git.run g ["commit", "-m", "git annex untrust", logfile] + return $ Just $ return True diff --git a/Command/Untrust.hs b/Command/Untrust.hs new file mode 100644 index 0000000000..5ed8de2451 --- /dev/null +++ b/Command/Untrust.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Untrust where + +import Control.Monad.State (liftIO) +import Control.Monad (when) + +import Command +import qualified Annex +import qualified GitRepo as Git +import qualified Remotes +import UUID +import Messages + +seek :: [SubCmdSeek] +seek = [withString start] + +{- Marks a remote as not trusted. -} +start :: SubCmdStartString +start name = do + r <- Remotes.byName name + showStart "untrust" name + return $ Just $ perform r + +perform :: Git.Repo -> SubCmdPerform +perform repo = do + uuid <- getUUID repo + trusted <- getTrusted + when (elem uuid trusted) $ do + setTrusted $ filter (\u -> u /= uuid) trusted + g <- Annex.gitRepo + logfile <- trustLog + liftIO $ Git.run g ["add", logfile] + liftIO $ Git.run g ["commit", "-m", "git annex untrust", logfile] + return $ Just $ return True From 39d0bcb79332426ac9e20f15614a8eef57a8e7d2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:46:38 -0400 Subject: [PATCH 0629/2835] things to do still --- doc/todo/git-annex-shell.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/todo/git-annex-shell.mdwn b/doc/todo/git-annex-shell.mdwn index a5ff7d6fd7..47db0c1ca7 100644 --- a/doc/todo/git-annex-shell.mdwn +++ b/doc/todo/git-annex-shell.mdwn @@ -40,3 +40,9 @@ Need at least `--quiet`, `--backend`, `--key`, `--force` * `setkey repopath tmpfile` Same as `git annex setkey`, and taking the same dashed options. + +### TODO + +* To be usable as a locked down shell, needs a way to launch the + rsync server, for file receiving. Safely? +* Also needs a way to support receiving files by scp. From d475aac37544d9442fe6ca6af5e949fd48d3bc96 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:21:38 -0400 Subject: [PATCH 0630/2835] refactor --- Backend/File.hs | 17 ++++++----------- Command/Move.hs | 4 ++-- Remotes.hs | 29 +++++++++++++++++++++-------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 092cf7e73d..f9a3dbfcbf 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -16,7 +16,6 @@ module Backend.File (backend, checkKey) where import Control.Monad.State import System.Directory -import Data.List (intersect) import TypeInternals import LocationLog @@ -50,7 +49,8 @@ dummyStore _ _ = return True - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex Bool copyKeyFile key file = do - remotes <- Remotes.keyPossibilities key + (trusted, untrusted) <- Remotes.keyPossibilities key + let remotes = trusted ++ untrusted if null remotes then do showNote "not available" @@ -92,16 +92,11 @@ checkRemoveKey key numcopiesM = do if force || numcopiesM == Just 0 then return True else do - g <- Annex.gitRepo - locations <- liftIO $ keyLocations g key - trusted <- getTrusted - let trustedcopies = length $ intersect locations trusted - remotes <- Remotes.keyPossibilities key - untrustedremotes <- reposWithoutUUID remotes trusted + (trusted, untrusted) <- Remotes.keyPossibilities key numcopies <- getNumCopies numcopiesM - if numcopies > length untrustedremotes - then notEnoughCopies numcopies (length untrustedremotes) [] - else findcopies numcopies trustedcopies untrustedremotes [] + if numcopies > length untrusted + then notEnoughCopies numcopies (length untrusted) [] + else findcopies numcopies (length trusted) untrusted [] where findcopies need have [] bad | have >= need = return True diff --git a/Command/Move.hs b/Command/Move.hs index cb6525919e..61242955e3 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -110,8 +110,8 @@ toCleanup move remote key tmpfile = do fromStart :: Bool -> SubCmdStartString fromStart move file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote - l <- Remotes.keyPossibilities key - if null $ filter (\r -> Remotes.same r remote) l + (trusted, untrusted) <- Remotes.keyPossibilities key + if null $ filter (\r -> Remotes.same r remote) (trusted ++ untrusted) then return Nothing else do showAction move file diff --git a/Remotes.hs b/Remotes.hs index 725348a6a3..3905315508 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -24,7 +24,7 @@ import qualified Data.Map as Map import Data.String.Utils import System.Directory hiding (copyFile) import System.Posix.Directory -import Data.List +import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) import Types @@ -43,11 +43,11 @@ import qualified SysConfig list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes -{- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -keyPossibilities :: Key -> Annex [Git.Repo] +{- Cost ordered lists of remotes that the LocationLog indicate may have a key. + - The first list is of remotes that are trusted to have the key; the + - second is of untrusted remotes that may have the key. -} +keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo]) keyPossibilities key = do - g <- Annex.gitRepo - uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost -- To determine if a remote has a key, its UUID needs to be known. -- The locally cached UUIDs of remotes can fall out of date if @@ -56,7 +56,7 @@ keyPossibilities key = do -- sure we only do it once per git-annex run. remotesread <- Annex.flagIsSet "remotesread" if remotesread - then reposByUUID allremotes uuids + then partition allremotes else do -- We assume that it's cheap to read the config -- of non-URL remotes, so that is done each time. @@ -74,11 +74,24 @@ keyPossibilities key = do _ <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True keyPossibilities key - else reposByUUID allremotes uuids + else partition allremotes where cachedUUID r = do u <- getUUID r - return $ null u + return $ null u + partition allremotes = do + g <- Annex.gitRepo + validuuids <- liftIO $ keyLocations g key + alltrusted <- getTrusted + -- get uuids trusted to have the key + -- note that validuuids is assumed to not have dups + let validtrusted = intersect validuuids alltrusted + -- remotes that match uuids that have the key + validremotes <- reposByUUID allremotes validuuids + -- partition out the trusted and untrusted remotes + trustedremotes <- reposByUUID validremotes validtrusted + untrustedremotes <- reposWithoutUUID validremotes alltrusted + return (trustedremotes, untrustedremotes) {- Checks if a given remote has the content for a key inAnnex. - If the remote cannot be accessed, returns a Left error. From 885f7048d5598409476016be3b46c1427532ede3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:31:25 -0400 Subject: [PATCH 0631/2835] Fix bug in numcopies handling when a repoisitory has multiple remotes that point to the same repository. --- Backend/File.hs | 29 +++++++++++++++++------------ debian/changelog | 2 ++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index f9a3dbfcbf..4773ade9d0 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -94,26 +94,31 @@ checkRemoveKey key numcopiesM = do else do (trusted, untrusted) <- Remotes.keyPossibilities key numcopies <- getNumCopies numcopiesM + trusteduuids <- mapM getUUID trusted if numcopies > length untrusted then notEnoughCopies numcopies (length untrusted) [] - else findcopies numcopies (length trusted) untrusted [] + else findcopies numcopies trusteduuids untrusted [] where findcopies need have [] bad - | have >= need = return True - | otherwise = notEnoughCopies need have bad + | length have >= need = return True + | otherwise = notEnoughCopies need (length have) bad findcopies need have (r:rs) bad - | have >= need = return True - | otherwise = do - haskey <- Remotes.inAnnex r key - case haskey of - Right True -> findcopies need (have+1) rs bad - Right False -> findcopies need have rs bad - Left _ -> findcopies need have rs (r:bad) - notEnoughCopies need have bad = do + | length have >= need = return True + | otherwise = do + u <- getUUID r + if not $ elem u have + then do + haskey <- Remotes.inAnnex r key + case haskey of + Right True -> findcopies need (u:have) rs bad + Right False -> findcopies need have rs bad + Left _ -> findcopies need have rs (r:bad) + else findcopies need have rs bad + notEnoughCopies need numhave bad = do unsafe showLongNote $ "Could only verify the existence of " ++ - show have ++ " out of " ++ show need ++ + show numhave ++ " out of " ++ show need ++ " necessary copies" showTriedRemotes bad showLocations key diff --git a/debian/changelog b/debian/changelog index bc2901ef74..3707995160 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ git-annex (0.15) UNRELEASED; urgency=low not yet supported. * Add trust and untrust subcommands, to allow configuring remotes that are trusted to retain files without explicit checking. + * Fix bug in numcopies handling when a repoisitory has multiple remotes + that point to the same repository. -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 From 1448d8b42dbc8c29477ab75bbc1e5a33d1eae3ba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:41:27 -0400 Subject: [PATCH 0632/2835] improve list of remotes in error message --- Backend/File.hs | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 4773ade9d0..918ba4716a 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -54,13 +54,13 @@ copyKeyFile key file = do if null remotes then do showNote "not available" - showLocations key + showLocations key [] return False else trycopy remotes remotes where trycopy full [] = do showTriedRemotes full - showLocations key + showLocations key [] return False trycopy full (r:rs) = do probablythere <- probablyPresent r @@ -95,44 +95,40 @@ checkRemoveKey key numcopiesM = do (trusted, untrusted) <- Remotes.keyPossibilities key numcopies <- getNumCopies numcopiesM trusteduuids <- mapM getUUID trusted - if numcopies > length untrusted - then notEnoughCopies numcopies (length untrusted) [] - else findcopies numcopies trusteduuids untrusted [] + findcopies numcopies trusteduuids untrusted [] where findcopies need have [] bad | length have >= need = return True - | otherwise = notEnoughCopies need (length have) bad + | otherwise = notEnoughCopies need have bad findcopies need have (r:rs) bad | length have >= need = return True | otherwise = do u <- getUUID r - if not $ elem u have - then do - haskey <- Remotes.inAnnex r key - case haskey of - Right True -> findcopies need (u:have) rs bad - Right False -> findcopies need have rs bad - Left _ -> findcopies need have rs (r:bad) - else findcopies need have rs bad - notEnoughCopies need numhave bad = do + let dup = elem u have + haskey <- Remotes.inAnnex r key + case (dup, haskey) of + (False, Right True) -> findcopies need (u:have) rs bad + (False, Left _) -> findcopies need have rs (r:bad) + _ -> findcopies need have rs bad + notEnoughCopies need have bad = do unsafe showLongNote $ "Could only verify the existence of " ++ - show numhave ++ " out of " ++ show need ++ + show (length have) ++ " out of " ++ show need ++ " necessary copies" showTriedRemotes bad - showLocations key + showLocations key have hint return False unsafe = showNote "unsafe" hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)" -showLocations :: Key -> Annex () -showLocations key = do +showLocations :: Key -> [UUID] -> Annex () +showLocations key exclude = do g <- Annex.gitRepo u <- getUUID g uuids <- liftIO $ keyLocations g key - let uuidsf = filter (/= u) uuids + let uuidsf = filter (\l -> l /= u && (not $ elem l exclude)) uuids ppuuids <- prettyPrintUUIDs uuidsf if null uuidsf then showLongNote $ "No other repository is known to contain the file." From ef85e699babd2aabdc64e048422456ac68244ef2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:48:55 -0400 Subject: [PATCH 0633/2835] simplify --- UUID.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/UUID.hs b/UUID.hs index 3e43c80612..21f2b202e6 100644 --- a/UUID.hs +++ b/UUID.hs @@ -24,8 +24,6 @@ module UUID ( ) where import Control.Monad.State -import Data.Maybe -import Data.List import System.Cmd.Utils import System.IO import qualified Data.Map as M @@ -85,7 +83,7 @@ reposByUUID repos uuids = filterM match repos where match r = do u <- getUUID r - return $ isJust $ elemIndex u uuids + return $ elem u uuids {- Filters a list of repos to ones that do not have the listed UUIDs. -} reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] @@ -93,7 +91,7 @@ reposWithoutUUID repos uuids = filterM unmatch repos where unmatch r = do u <- getUUID r - return $ not $ isJust $ elemIndex u uuids + return $ not $ elem u uuids {- Pretty-prints a list of UUIDs -} prettyPrintUUIDs :: [UUID] -> Annex String From e64ffc212e394d814ab73a32f750acbccb43dd1f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:58:44 -0400 Subject: [PATCH 0634/2835] support trusted repositories that are not configured as remotes --- Backend/File.hs | 7 +++---- Command/Move.hs | 2 +- Remotes.hs | 26 ++++++++++++++++---------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 918ba4716a..ee73150211 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -49,7 +49,7 @@ dummyStore _ _ = return True - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex Bool copyKeyFile key file = do - (trusted, untrusted) <- Remotes.keyPossibilities key + (trusted, untrusted, _) <- Remotes.keyPossibilities key let remotes = trusted ++ untrusted if null remotes then do @@ -92,10 +92,9 @@ checkRemoveKey key numcopiesM = do if force || numcopiesM == Just 0 then return True else do - (trusted, untrusted) <- Remotes.keyPossibilities key + (_, untrusted, have) <- Remotes.keyPossibilities key numcopies <- getNumCopies numcopiesM - trusteduuids <- mapM getUUID trusted - findcopies numcopies trusteduuids untrusted [] + findcopies numcopies have untrusted [] where findcopies need have [] bad | length have >= need = return True diff --git a/Command/Move.hs b/Command/Move.hs index 61242955e3..eb223f5abf 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -110,7 +110,7 @@ toCleanup move remote key tmpfile = do fromStart :: Bool -> SubCmdStartString fromStart move file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote - (trusted, untrusted) <- Remotes.keyPossibilities key + (trusted, untrusted, _) <- Remotes.keyPossibilities key if null $ filter (\r -> Remotes.same r remote) (trusted ++ untrusted) then return Nothing else do diff --git a/Remotes.hs b/Remotes.hs index 3905315508..99c0930adf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -44,9 +44,15 @@ list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes {- Cost ordered lists of remotes that the LocationLog indicate may have a key. - - The first list is of remotes that are trusted to have the key; the - - second is of untrusted remotes that may have the key. -} -keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo]) + - + - The first list is of remotes that are trusted to have the key. + - + - The second is of untrusted remotes that may have the key. + - + - Also returns a list of all UUIDs that are trusted to have the key + - (some may not have configured remotes). + -} +keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID]) keyPossibilities key = do allremotes <- remotesByCost -- To determine if a remote has a key, its UUID needs to be known. @@ -79,19 +85,19 @@ keyPossibilities key = do cachedUUID r = do u <- getUUID r return $ null u - partition allremotes = do + partition remotes = do g <- Annex.gitRepo validuuids <- liftIO $ keyLocations g key - alltrusted <- getTrusted + trusted <- getTrusted -- get uuids trusted to have the key -- note that validuuids is assumed to not have dups - let validtrusted = intersect validuuids alltrusted + let validtrusteduuids = intersect validuuids trusted -- remotes that match uuids that have the key - validremotes <- reposByUUID allremotes validuuids + validremotes <- reposByUUID remotes validuuids -- partition out the trusted and untrusted remotes - trustedremotes <- reposByUUID validremotes validtrusted - untrustedremotes <- reposWithoutUUID validremotes alltrusted - return (trustedremotes, untrustedremotes) + trustedremotes <- reposByUUID validremotes validtrusteduuids + untrustedremotes <- reposWithoutUUID validremotes trusted + return (trustedremotes, untrustedremotes, validtrusteduuids) {- Checks if a given remote has the content for a key inAnnex. - If the remote cannot be accessed, returns a Left error. From 25ffa04c5a116dc6ff4266dd738a93f03e7e6017 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 17:00:14 -0400 Subject: [PATCH 0635/2835] wording --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 3707995160..76e16af2ce 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,10 +3,10 @@ git-annex (0.15) UNRELEASED; urgency=low * Support scp-style urls for remotes (host:path). Note that paths relative to the user's home directory, or containing "~" are not yet supported. - * Add trust and untrust subcommands, to allow configuring remotes + * Add trust and untrust subcommands, to allow configuring repositories that are trusted to retain files without explicit checking. - * Fix bug in numcopies handling when a repoisitory has multiple remotes - that point to the same repository. + * Fix bug in numcopies handling when multiple remotes pointed to the + same repository. -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 From de2f6137d3f38f822b0bfbf534e8f785459ea3cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 17:01:56 -0400 Subject: [PATCH 0636/2835] wording --- doc/copies.mdwn | 2 +- doc/trust.mdwn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/copies.mdwn b/doc/copies.mdwn index 5b3cbf5154..98f315081f 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -12,7 +12,7 @@ setting in `.gitattributes` files. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have it, it will retain the file content to avoid data loss. Note that -[[trusted_remotes|trust]] are not explicitly checked. +[[trusted_repositories|trust]] are not explicitly checked. For example, consider three repositories: Server, Laptop, and USB. Both Server and USB have a copy of a file, and N=1. If on Laptop, you `git annex get diff --git a/doc/trust.mdwn b/doc/trust.mdwn index b04a112ecf..6ce43f30eb 100644 --- a/doc/trust.mdwn +++ b/doc/trust.mdwn @@ -1,6 +1,6 @@ Normally, git-annex does not fully trust its stored [[location_tracking]] information. When removing content, it will directly check -that other repositories have [[copies]]. +that other repositories have enough [[copies]]. Generally that explicit checking is a good idea. Consider that the current [[location_tracking]] information for a remote may not yet have propigated From 5b8f8ced0b6c3b22dcb5425862b9218685fbe394 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 19:09:02 -0400 Subject: [PATCH 0637/2835] don't include the current repo in trusted uuid list --- Remotes.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 99c0930adf..053814d66d 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -49,7 +49,7 @@ list remotes = join ", " $ map Git.repoDescribe remotes - - The second is of untrusted remotes that may have the key. - - - Also returns a list of all UUIDs that are trusted to have the key + - Also returns a list of UUIDs that are trusted to have the key - (some may not have configured remotes). -} keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID]) @@ -87,11 +87,13 @@ keyPossibilities key = do return $ null u partition remotes = do g <- Annex.gitRepo + u <- getUUID g validuuids <- liftIO $ keyLocations g key trusted <- getTrusted -- get uuids trusted to have the key -- note that validuuids is assumed to not have dups - let validtrusteduuids = intersect validuuids trusted + let validtrusteduuids = filter (/= u) $ + intersect validuuids trusted -- remotes that match uuids that have the key validremotes <- reposByUUID remotes validuuids -- partition out the trusted and untrusted remotes From 36b73714d04ad478c0f06d82cf10b24706cd8c84 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 19:21:13 -0400 Subject: [PATCH 0638/2835] better filtering out of current repo --- Remotes.hs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 053814d66d..78ab010cef 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -88,12 +88,14 @@ keyPossibilities key = do partition remotes = do g <- Annex.gitRepo u <- getUUID g - validuuids <- liftIO $ keyLocations g key trusted <- getTrusted + -- get uuids of other repositories that are + -- believed to have the key + uuids <- liftIO $ keyLocations g key + let validuuids = filter (/= u) uuids -- get uuids trusted to have the key -- note that validuuids is assumed to not have dups - let validtrusteduuids = filter (/= u) $ - intersect validuuids trusted + let validtrusteduuids = intersect validuuids trusted -- remotes that match uuids that have the key validremotes <- reposByUUID remotes validuuids -- partition out the trusted and untrusted remotes From 88edf83b5c0a0be2b7776dcc6a0d16676a716d8a Mon Sep 17 00:00:00 2001 From: praet Date: Wed, 29 Dec 2010 23:47:46 +0000 Subject: [PATCH 0639/2835] --- doc/todo/add_--exclude_option_to_git_annex_find.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/todo/add_--exclude_option_to_git_annex_find.mdwn diff --git a/doc/todo/add_--exclude_option_to_git_annex_find.mdwn b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn new file mode 100644 index 0000000000..3441d262be --- /dev/null +++ b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn @@ -0,0 +1 @@ +Seems pretty self-explanatory. From 80bad5c43507d1a3c79d473287a18b041cfd1ed1 Mon Sep 17 00:00:00 2001 From: praet Date: Thu, 30 Dec 2010 00:01:02 +0000 Subject: [PATCH 0640/2835] --- ...e_repo_description_and__47__or_UUID_in_commit_message.mdwn | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn new file mode 100644 index 0000000000..9802738d12 --- /dev/null +++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn @@ -0,0 +1,4 @@ +Would help alot when having to add large(ish) amounts of remotes. + +Maybe detect this kind of commit message and ask user whether to automatically add them? See [[auto_remotes]]: +> Question: When should git-annex update the remote.log? (If not just on init.) Whenever it reads in a repo's remotes? From f6db527af8f4c65d5e8f26abb56408303dc48050 Mon Sep 17 00:00:00 2001 From: praet Date: Thu, 30 Dec 2010 00:12:44 +0000 Subject: [PATCH 0641/2835] --- doc/todo/auto_remotes/discussion.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/todo/auto_remotes/discussion.mdwn diff --git a/doc/todo/auto_remotes/discussion.mdwn b/doc/todo/auto_remotes/discussion.mdwn new file mode 100644 index 0000000000..691741cab8 --- /dev/null +++ b/doc/todo/auto_remotes/discussion.mdwn @@ -0,0 +1,2 @@ +Remotes log should probably be stored in ".git/annex/remote.log" +instead of ".git-annex/remote.log" to prevent leaking credentials. From c74347a50879401f4661a16f059b3fc467c28989 Mon Sep 17 00:00:00 2001 From: praet Date: Thu, 30 Dec 2010 00:21:00 +0000 Subject: [PATCH 0642/2835] --- doc/walkthrough.mdwn | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index b486a4b1ff..3e437e8f34 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -275,7 +275,11 @@ files when they're added to the annex, and this can slow things down significantly for really big files. To make SHA1 the detault, just add something like this to `.gitattributes`: - * annex.backend=SHA1 +> annex.backends=SHA1 + +or run: + +> $ git config annex.backends SHA1 ## unused data From 829c8b76605ad3c52eb6147202cc9b867d58cf1b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 20:28:34 -0400 Subject: [PATCH 0643/2835] clarify --- doc/todo/auto_remotes/discussion.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/todo/auto_remotes/discussion.mdwn b/doc/todo/auto_remotes/discussion.mdwn index 691741cab8..b9e1522a8f 100644 --- a/doc/todo/auto_remotes/discussion.mdwn +++ b/doc/todo/auto_remotes/discussion.mdwn @@ -1,2 +1,7 @@ Remotes log should probably be stored in ".git/annex/remote.log" instead of ".git-annex/remote.log" to prevent leaking credentials. + +> The idea is to distribute the info between repositories, which is +> why it'd go in `.git-annex`. Of course that does mean that repository +> location information would be included, and if that'd not desirable +> this feature would need to be turned off. --[[Joey]] From 03e579060b11dcca6115426e7a4255a7ded944e2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 20:30:03 -0400 Subject: [PATCH 0644/2835] already exists --- doc/todo/add_--exclude_option_to_git_annex_find.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/todo/add_--exclude_option_to_git_annex_find.mdwn b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn index 3441d262be..a797e97f58 100644 --- a/doc/todo/add_--exclude_option_to_git_annex_find.mdwn +++ b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn @@ -1 +1,4 @@ Seems pretty self-explanatory. + +> This was already implemented, the --exclude option can be used +> for find as well as most any other subcommand. --[[Joey]] [[done]] From a91e31cef3598f622dd73e95151f96eeaedba327 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 20:33:55 -0400 Subject: [PATCH 0645/2835] response --- ...epo_description_and__47__or_UUID_in_commit_message.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn index 9802738d12..9ca61bff55 100644 --- a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn +++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn @@ -2,3 +2,10 @@ Would help alot when having to add large(ish) amounts of remotes. Maybe detect this kind of commit message and ask user whether to automatically add them? See [[auto_remotes]]: > Question: When should git-annex update the remote.log? (If not just on init.) Whenever it reads in a repo's remotes? + +---- + +I'm not sure that the above suggestion is going down a path that really +makes sense. If you want a list of repository UUIDs and descriptions, +it's there in machine-usable form in `.git-annex/uuid.log`, there is no +need to try to pull this info out of git commit messages. --[[Joey]] From 14d59b40fb4f3a4c9a89266fecae91a0daf08088 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 20:35:51 -0400 Subject: [PATCH 0646/2835] Revert walkthrough annex.backend change This reverts commit c74347a50879401f4661a16f059b3fc467c28989. I'm afraid that change was just all wrong. * The gitattributes setting is `annex.backend`. Not `annex.backends`. * `gitattributes` files need to start with a glob, which this commit removed. * git config's `annex.backends` is NOT the same as the gitattributes setting, and is not a knob that we should be encouraging users tweak. --- doc/walkthrough.mdwn | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 3e437e8f34..b486a4b1ff 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -275,11 +275,7 @@ files when they're added to the annex, and this can slow things down significantly for really big files. To make SHA1 the detault, just add something like this to `.gitattributes`: -> annex.backends=SHA1 - -or run: - -> $ git config annex.backends SHA1 + * annex.backend=SHA1 ## unused data From 6a5be9d53cad9ee2988c6d54001f387dfe1f2716 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 14:19:16 -0400 Subject: [PATCH 0647/2835] rename some stuff and prepare to break out more into Command/* --- Annex.hs | 4 +- CmdLine.hs | 69 +++++++++++++------------- Command.hs | 110 ++++++++++++++++++++++++------------------ Command/Add.hs | 11 +++-- Command/Copy.hs | 2 +- Command/Drop.hs | 8 +-- Command/DropKey.hs | 8 +-- Command/DropUnused.hs | 4 +- Command/Find.hs | 4 +- Command/Fix.hs | 8 +-- Command/FromKey.hs | 8 +-- Command/Fsck.hs | 6 +-- Command/Get.hs | 6 +-- Command/Init.hs | 8 +-- Command/Lock.hs | 6 +-- Command/Move.hs | 16 +++--- Command/PreCommit.hs | 10 ++-- Command/SetKey.hs | 8 +-- Command/Trust.hs | 6 +-- Command/Unannex.hs | 8 +-- Command/Uninit.hs | 6 +-- Command/Unlock.hs | 6 +-- Command/Untrust.hs | 6 +-- Command/Unused.hs | 6 +-- 24 files changed, 176 insertions(+), 158 deletions(-) diff --git a/Annex.hs b/Annex.hs index af761051dc..6e5198e8ef 100644 --- a/Annex.hs +++ b/Annex.hs @@ -110,10 +110,10 @@ flagGet name = do {- Adds a git command to the queue. -} queue :: String -> [String] -> FilePath -> Annex () -queue subcommand params file = do +queue command params file = do state <- get let q = Internals.repoqueue state - put state { Internals.repoqueue = GitQueue.add q subcommand params file } + put state { Internals.repoqueue = GitQueue.add q command params file } {- Returns the queue. -} queueGet :: Annex GitQueue.Queue diff --git a/CmdLine.hs b/CmdLine.hs index 7eab0a7e28..40ce4b1215 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -37,51 +37,50 @@ import qualified Command.Uninit import qualified Command.Trust import qualified Command.Untrust -subCmds :: [SubCommand] -subCmds = - [ SubCommand "add" path Command.Add.seek - "add files to annex" - , SubCommand "get" path Command.Get.seek +cmds :: [Command] +cmds = + [ Command.Add.command + , Command "get" path Command.Get.seek "make content of annexed files available" - , SubCommand "drop" path Command.Drop.seek + , Command "drop" path Command.Drop.seek "indicate content of files not currently wanted" - , SubCommand "move" path Command.Move.seek + , Command "move" path Command.Move.seek "move content of files to/from another repository" - , SubCommand "copy" path Command.Copy.seek + , Command "copy" path Command.Copy.seek "copy content of files to/from another repository" - , SubCommand "unlock" path Command.Unlock.seek + , Command "unlock" path Command.Unlock.seek "unlock files for modification" - , SubCommand "edit" path Command.Unlock.seek + , Command "edit" path Command.Unlock.seek "same as unlock" - , SubCommand "lock" path Command.Lock.seek + , Command "lock" path Command.Lock.seek "undo unlock command" - , SubCommand "init" desc Command.Init.seek + , Command "init" desc Command.Init.seek "initialize git-annex with repository description" - , SubCommand "unannex" path Command.Unannex.seek + , Command "unannex" path Command.Unannex.seek "undo accidential add command" - , SubCommand "uninit" path Command.Uninit.seek + , Command "uninit" path Command.Uninit.seek "de-initialize git-annex and clean out repository" - , SubCommand "pre-commit" path Command.PreCommit.seek + , Command "pre-commit" path Command.PreCommit.seek "run by git pre-commit hook" - , SubCommand "trust" remote Command.Trust.seek + , Command "trust" remote Command.Trust.seek "trust a repository" - , SubCommand "untrust" remote Command.Untrust.seek + , Command "untrust" remote Command.Untrust.seek "do not trust a repository" - , SubCommand "fromkey" key Command.FromKey.seek + , Command "fromkey" key Command.FromKey.seek "adds a file using a specific key" - , SubCommand "dropkey" key Command.DropKey.seek + , Command "dropkey" key Command.DropKey.seek "drops annexed content for specified keys" - , SubCommand "setkey" key Command.SetKey.seek + , Command "setkey" key Command.SetKey.seek "sets annexed content for a key using a temp file" - , SubCommand "fix" path Command.Fix.seek + , Command "fix" path Command.Fix.seek "fix up symlinks to point to annexed content" - , SubCommand "fsck" maybepath Command.Fsck.seek + , Command "fsck" maybepath Command.Fsck.seek "check for problems" - , SubCommand "unused" nothing Command.Unused.seek + , Command "unused" nothing Command.Unused.seek "look for unused file content" - , SubCommand "dropunused" number Command.DropUnused.seek + , Command "dropunused" number Command.DropUnused.seek "drop unused file content" - , SubCommand "find" maybepath Command.Find.seek + , Command "find" maybepath Command.Find.seek "lists available files" ] where @@ -125,13 +124,13 @@ header = "Usage: git-annex subcommand [option ..]" usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where - cmddescs = unlines $ map (indent . showcmd) subCmds + cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = - subcmdname c ++ - pad 11 (subcmdname c) ++ - subcmdparams c ++ - pad 13 (subcmdparams c) ++ - subcmddesc c + cmdname c ++ + pad 11 (cmdname c) ++ + cmdparams c ++ + pad 13 (cmdparams c) ++ + cmddesc c indent l = " " ++ l pad n s = replicate (n - length s) ' ' @@ -143,12 +142,12 @@ parseCmd argv = do when (null params) $ error usage case lookupCmd (head params) of [] -> error usage - [subcommand] -> do + [command] -> do _ <- sequence flags - prepSubCmd subcommand (drop 1 params) - _ -> error "internal error: multiple matching subcommands" + prepCmd command (drop 1 params) + _ -> error "internal error: multiple matching commands" where getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) - lookupCmd cmd = filter (\c -> cmd == subcmdname c) subCmds + lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds diff --git a/Command.hs b/Command.hs index e30904d0f7..2144da353b 100644 --- a/Command.hs +++ b/Command.hs @@ -21,54 +21,54 @@ import qualified Annex import qualified GitRepo as Git import Locations -{- A subcommand runs in four stages. +{- A command runs in four stages. - - - 0. The seek stage takes the parameters passed to the subcommand, + - 0. The seek stage takes the parameters passed to the command, - looks through the repo to find the ones that are relevant - - to that subcommand (ie, new files to add), and generates + - to that command (ie, new files to add), and generates - a list of start stage actions. -} -type SubCmdSeek = [String] -> Annex [SubCmdStart] +type CommandSeek = [String] -> Annex [CommandStart] {- 1. The start stage is run before anything is printed about the - - subcommand, is passed some input, and can early abort it + - command, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and - should not modify Annex state. -} -type SubCmdStart = Annex (Maybe SubCmdPerform) -{- 2. The perform stage is run after a message is printed about the subcommand +type CommandStart = Annex (Maybe CommandPerform) +{- 2. The perform stage is run after a message is printed about the command - being run, and it should be where the bulk of the work happens. -} -type SubCmdPerform = Annex (Maybe SubCmdCleanup) +type CommandPerform = Annex (Maybe CommandCleanup) {- 3. The cleanup stage is run only if the perform stage succeeds, and it - - returns the overall success/fail of the subcommand. -} -type SubCmdCleanup = Annex Bool -{- Some helper functions are used to build up SubCmdSeek and SubCmdStart + - returns the overall success/fail of the command. -} +type CommandCleanup = Annex Bool +{- Some helper functions are used to build up CommandSeek and CommandStart - functions. -} -type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek -type SubCmdStartString = String -> SubCmdStart +type CommandSeekStrings = CommandStartString -> CommandSeek +type CommandStartString = String -> CommandStart type BackendFile = (FilePath, Maybe Backend) -type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek -type SubCmdStartBackendFile = BackendFile -> SubCmdStart +type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek +type CommandStartBackendFile = BackendFile -> CommandStart type AttrFile = (FilePath, String) -type SubCmdSeekAttrFiles = SubCmdStartAttrFile -> SubCmdSeek -type SubCmdStartAttrFile = AttrFile -> SubCmdStart -type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek -type SubCmdStartNothing = SubCmdStart +type CommandSeekAttrFiles = CommandStartAttrFile -> CommandSeek +type CommandStartAttrFile = AttrFile -> CommandStart +type CommandSeekNothing = CommandStart -> CommandSeek +type CommandStartNothing = CommandStart -data SubCommand = SubCommand { - subcmdname :: String, - subcmdparams :: String, - subcmdseek :: [SubCmdSeek], - subcmddesc :: String +data Command = Command { + cmdname :: String, + cmdparams :: String, + cmdseek :: [CommandSeek], + cmddesc :: String } -{- Prepares a list of actions to run to perform a subcommand, based on +{- Prepares a list of actions to run to perform a command, based on - the parameters passed to it. -} -prepSubCmd :: SubCommand -> [String] -> Annex [Annex Bool] -prepSubCmd SubCommand { subcmdseek = seek } params = do +prepCmd :: Command -> [String] -> Annex [Annex Bool] +prepCmd Command { cmdseek = seek } params = do lists <- mapM (\s -> s params) seek - return $ map doSubCmd $ foldl (++) [] lists + return $ map doCommand $ foldl (++) [] lists -{- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdStart -> SubCmdCleanup -doSubCmd start = do +{- Runs a command through the start, perform and cleanup stages -} +doCommand :: CommandStart -> CommandCleanup +doCommand start = do s <- start case s of Nothing -> return True @@ -104,20 +104,20 @@ isAnnexed file a = do {- These functions find appropriate files or other things based on a user's parameters, and run a specified action on them. -} -withFilesInGit :: SubCmdSeekStrings +withFilesInGit :: CommandSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ Git.inRepo repo params files' <- filterFiles files return $ map a files' -withAttrFilesInGit :: String -> SubCmdSeekAttrFiles +withAttrFilesInGit :: String -> CommandSeekAttrFiles withAttrFilesInGit attr a params = do repo <- Annex.gitRepo files <- liftIO $ Git.inRepo repo params files' <- filterFiles files pairs <- liftIO $ Git.checkAttr repo attr files' return $ map a pairs -withFilesMissing :: SubCmdSeekStrings +withFilesMissing :: CommandSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params files' <- filterFiles files @@ -126,27 +126,27 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e -withFilesNotInGit :: SubCmdSeekBackendFiles +withFilesNotInGit :: CommandSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ Git.notInRepo repo params newfiles' <- filterFiles newfiles backendPairs a newfiles' -withString :: SubCmdSeekStrings +withString :: CommandSeekStrings withString a params = return [a $ unwords params] -withStrings :: SubCmdSeekStrings +withStrings :: CommandSeekStrings withStrings a params = return $ map a params -withFilesToBeCommitted :: SubCmdSeekStrings +withFilesToBeCommitted :: CommandSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo tocommit <- liftIO $ Git.stagedFiles repo params tocommit' <- filterFiles tocommit return $ map a tocommit' -withFilesUnlocked :: SubCmdSeekBackendFiles +withFilesUnlocked :: CommandSeekBackendFiles withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles -withFilesUnlockedToBeCommitted :: SubCmdSeekBackendFiles +withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles withFilesUnlockedToBeCommitted = withFilesUnlocked' Git.typeChangedStagedFiles -withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> SubCmdSeekBackendFiles +withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBackendFiles withFilesUnlocked' typechanged a params = do -- unlocked files have changed type from a symlink to a regular file repo <- Annex.gitRepo @@ -155,29 +155,29 @@ withFilesUnlocked' typechanged a params = do map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' -withKeys :: SubCmdSeekStrings +withKeys :: CommandSeekStrings withKeys a params = return $ map a params -withTempFile :: SubCmdSeekStrings +withTempFile :: CommandSeekStrings withTempFile a params = return $ map a params -withNothing :: SubCmdSeekNothing +withNothing :: CommandSeekNothing withNothing a [] = return [a] withNothing _ _ = return [] -backendPairs :: SubCmdSeekBackendFiles +backendPairs :: CommandSeekBackendFiles backendPairs a files = do pairs <- Backend.chooseBackends files return $ map a pairs {- Default to acting on all files matching the seek action if - none are specified. -} -withAll :: (a -> SubCmdSeek) -> a -> SubCmdSeek +withAll :: (a -> CommandSeek) -> a -> CommandSeek withAll w a [] = do g <- Annex.gitRepo w a [Git.workTree g] withAll w a p = w a p {- Provides a default parameter to act on if none is specified. -} -withDefault :: String-> (a -> SubCmdSeek) -> (a -> SubCmdSeek) +withDefault :: String-> (a -> CommandSeek) -> (a -> CommandSeek) withDefault d w a [] = w a [d] withDefault _ w a p = w a p @@ -204,3 +204,19 @@ notSymlink :: FilePath -> IO Bool notSymlink f = do s <- liftIO $ getSymbolicLinkStatus f return $ not $ isSymbolicLink s + +{- descriptions of params used in usage message -} +paramPath :: String +paramPath = "PATH ..." +paramMaybePath :: String +paramMaybePath = "[PATH ...]" +paramKey :: String +paramKey = "KEY ..." +paramDesc :: String +paramDesc = "DESCRIPTION" +paramNumber :: String +paramNumber = "NUMBER ..." +paramRemote :: String +paramRemote = "REMOTE ..." +paramNothing :: String +paramNothing = "" diff --git a/Command/Add.hs b/Command/Add.hs index d141448a3a..08a880206b 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -18,14 +18,17 @@ import Types import Core import Messages +command :: Command +command = Command "add" paramPath seek "add files to annex" + {- Add acts on both files not checked into git yet, and unlocked files. -} -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesNotInGit start, withFilesUnlocked start] {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -start :: SubCmdStartBackendFile +start :: CommandStartBackendFile start pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if (isSymbolicLink s) || (not $ isRegularFile s) @@ -34,14 +37,14 @@ start pair@(file, _) = notAnnexed file $ do showStart "add" file return $ Just $ perform pair -perform :: BackendFile -> SubCmdPerform +perform :: BackendFile -> CommandPerform perform (file, backend) = do stored <- Backend.storeFileKey file backend case stored of Nothing -> return Nothing Just (key, _) -> return $ Just $ cleanup file key -cleanup :: FilePath -> Key -> SubCmdCleanup +cleanup :: FilePath -> Key -> CommandCleanup cleanup file key = do moveAnnex key file logStatus key ValuePresent diff --git a/Command/Copy.hs b/Command/Copy.hs index aa55731d9c..873df7ef2e 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -11,5 +11,5 @@ import Command import qualified Command.Move -- A copy is just a move that does not delete the source file. -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit $ Command.Move.start False] diff --git a/Command/Drop.hs b/Command/Drop.hs index 7c4fbea602..3f27405703 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -17,12 +17,12 @@ import Core import Messages import Utility -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withAttrFilesInGit "annex.numcopies" start] {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -start :: SubCmdStartAttrFile +start :: CommandStartAttrFile start (file, attr) = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if not inbackend @@ -33,14 +33,14 @@ start (file, attr) = isAnnexed file $ \(key, backend) -> do where numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> Maybe Int -> SubCmdPerform +perform :: Key -> Backend -> Maybe Int -> CommandPerform perform key backend numcopies = do success <- Backend.removeKey backend key numcopies if success then return $ Just $ cleanup key else return Nothing -cleanup :: Key -> SubCmdCleanup +cleanup :: Key -> CommandCleanup cleanup key = do inannex <- inAnnex key when inannex $ removeAnnex key diff --git a/Command/DropKey.hs b/Command/DropKey.hs index aa72e1bbd6..870e9a7ab1 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -15,11 +15,11 @@ import Types import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withKeys start] {- Drops cached content for a key. -} -start :: SubCmdStartString +start :: CommandStartString start keyname = do backends <- Backend.list let key = genKey (head backends) keyname @@ -33,12 +33,12 @@ start keyname = do showStart "dropkey" keyname return $ Just $ perform key -perform :: Key -> SubCmdPerform +perform :: Key -> CommandPerform perform key = do removeAnnex key return $ Just $ cleanup key -cleanup :: Key -> SubCmdCleanup +cleanup :: Key -> CommandCleanup cleanup key = do logStatus key ValueMissing return True diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 016a9faa73..9984e49f3f 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -18,11 +18,11 @@ import qualified Annex import qualified Command.Drop import Backend -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withStrings start] {- Drops unused content by number. -} -start :: SubCmdStartString +start :: CommandStartString start s = do m <- readUnusedLog case M.lookup s m of diff --git a/Command/Find.hs b/Command/Find.hs index 7b3c8c4636..9927b692d8 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -13,11 +13,11 @@ import Control.Monad.State (liftIO) import Command import Core -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withDefault "." withFilesInGit start] {- Output a list of files. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key when exists $ liftIO $ putStrLn file diff --git a/Command/Fix.hs b/Command/Fix.hs index 33630031f6..accdadd315 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -17,11 +17,11 @@ import Utility import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit start] {- Fixes the symlink to an annexed file. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file @@ -31,14 +31,14 @@ start file = isAnnexed file $ \(key, _) -> do showStart "fix" file return $ Just $ perform file link -perform :: FilePath -> FilePath -> SubCmdPerform +perform :: FilePath -> FilePath -> CommandPerform perform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file return $ Just $ cleanup file -cleanup :: FilePath -> SubCmdCleanup +cleanup :: FilePath -> CommandCleanup cleanup file = do Annex.queue "add" ["--"] file return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs index eb9ad5e514..991428136e 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -20,11 +20,11 @@ import Types import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesMissing start] {- Adds a file pointing at a manually-specified key -} -start :: SubCmdStartString +start :: CommandStartString start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -36,13 +36,13 @@ start file = do "key ("++keyname++") is not present in backend" showStart "fromkey" file return $ Just $ perform file key -perform :: FilePath -> Key -> SubCmdPerform +perform :: FilePath -> Key -> CommandPerform perform file key = do link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ cleanup file -cleanup :: FilePath -> SubCmdCleanup +cleanup :: FilePath -> CommandCleanup cleanup file = do Annex.queue "add" ["--"] file return True diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 9acecfce67..034bdc388b 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -13,18 +13,18 @@ import Types import Messages import Utility -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withAll (withAttrFilesInGit "annex.numcopies") start] {- Checks a file's backend data for problems. -} -start :: SubCmdStartAttrFile +start :: CommandStartAttrFile start (file, attr) = isAnnexed file $ \(key, backend) -> do showStart "fsck" file return $ Just $ perform key backend numcopies where numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> Maybe Int -> SubCmdPerform +perform :: Key -> Backend -> Maybe Int -> CommandPerform perform key backend numcopies = do success <- Backend.fsckKey backend key numcopies if success diff --git a/Command/Get.hs b/Command/Get.hs index 628ed62935..214b689b8a 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -13,11 +13,11 @@ import Types import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit start] {- Gets an annexed file from one of the backends. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if inannex @@ -26,7 +26,7 @@ start file = isAnnexed file $ \(key, backend) -> do showStart "get" file return $ Just $ perform key backend -perform :: Key -> Backend -> SubCmdPerform +perform :: Key -> Backend -> CommandPerform perform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if ok diff --git a/Command/Init.hs b/Command/Init.hs index e19849ba3b..806c34c989 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -20,18 +20,18 @@ import Messages import Locations import Types -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withString start] {- Stores description for the repository etc. -} -start :: SubCmdStartString +start :: CommandStartString start description = do when (null description) $ error "please specify a description of this repository\n" showStart "init" description return $ Just $ perform description -perform :: String -> SubCmdPerform +perform :: String -> CommandPerform perform description = do g <- Annex.gitRepo u <- getUUID g @@ -41,7 +41,7 @@ perform description = do gitPreCommitHookWrite g return $ Just cleanup -cleanup :: SubCmdCleanup +cleanup :: CommandCleanup cleanup = do g <- Annex.gitRepo logfile <- uuidLog diff --git a/Command/Lock.hs b/Command/Lock.hs index 27a030bc22..381162536e 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -15,16 +15,16 @@ import Messages import qualified Annex import qualified GitRepo as Git -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesUnlocked start] {- Undo unlock -} -start :: SubCmdStartBackendFile +start :: CommandStartBackendFile start (file, _) = do showStart "lock" file return $ Just $ perform file -perform :: FilePath -> SubCmdPerform +perform :: FilePath -> CommandPerform perform file = do liftIO $ removeFile file g <- Annex.gitRepo diff --git a/Command/Move.hs b/Command/Move.hs index eb223f5abf..8ba8dbfacc 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -21,14 +21,14 @@ import qualified Remotes import UUID import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit $ start True] {- Move (or copy) a file either --to or --from a repository. - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -start :: Bool -> SubCmdStartString +start :: Bool -> CommandStartString start move file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" @@ -61,7 +61,7 @@ remoteHasKey remote key present = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -toStart :: Bool -> SubCmdStartString +toStart :: Bool -> CommandStartString toStart move file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if not ishere @@ -69,7 +69,7 @@ toStart move file = isAnnexed file $ \(key, _) -> do else do showAction move file return $ Just $ toPerform move key -toPerform :: Bool -> Key -> SubCmdPerform +toPerform :: Bool -> Key -> CommandPerform toPerform move key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote @@ -86,7 +86,7 @@ toPerform move key = do then return $ Just $ toCleanup move remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ Command.Drop.cleanup key -toCleanup :: Bool -> Git.Repo -> Key -> FilePath -> SubCmdCleanup +toCleanup :: Bool -> Git.Repo -> Key -> FilePath -> CommandCleanup toCleanup move remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", @@ -107,7 +107,7 @@ toCleanup move remote key tmpfile = do - If the current repository already has the content, it is still removed - from the other repository when moving. -} -fromStart :: Bool -> SubCmdStartString +fromStart :: Bool -> CommandStartString fromStart move file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote (trusted, untrusted, _) <- Remotes.keyPossibilities key @@ -116,7 +116,7 @@ fromStart move file = isAnnexed file $ \(key, _) -> do else do showAction move file return $ Just $ fromPerform move key -fromPerform :: Bool -> Key -> SubCmdPerform +fromPerform :: Bool -> Key -> CommandPerform fromPerform move key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key @@ -128,7 +128,7 @@ fromPerform move key = do if ok then return $ Just $ fromCleanup move remote key else return Nothing -- fail -fromCleanup :: Bool -> Git.Repo -> Key -> SubCmdCleanup +fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup fromCleanup True remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 513d5d43f7..8d488514a8 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -17,21 +17,21 @@ import qualified Command.Fix {- The pre-commit hook needs to fix symlinks to all files being committed. - And, it needs to inject unlocked files into the annex. -} -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesToBeCommitted Command.Fix.start, withFilesUnlockedToBeCommitted start] -start :: SubCmdStartBackendFile +start :: CommandStartBackendFile start pair = return $ Just $ perform pair -perform :: BackendFile -> SubCmdPerform +perform :: BackendFile -> CommandPerform perform pair@(file, _) = do - ok <- doSubCmd $ Command.Add.start pair + ok <- doCommand $ Command.Add.start pair if ok then return $ Just $ cleanup file else error $ "failed to add " ++ file ++ "; canceling commit" -cleanup :: FilePath -> SubCmdCleanup +cleanup :: FilePath -> CommandCleanup cleanup file = do -- git commit will have staged the file's content; -- drop that and run command queued by Add.state to diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 55472ccaed..4c82de3a5b 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -19,11 +19,11 @@ import Types import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withTempFile start] {- Sets cached content for a key. -} -start :: SubCmdStartString +start :: CommandStartString start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -31,7 +31,7 @@ start file = do let key = genKey (head backends) keyname showStart "setkey" file return $ Just $ perform file key -perform :: FilePath -> Key -> SubCmdPerform +perform :: FilePath -> Key -> CommandPerform perform file key = do -- the file might be on a different filesystem, so mv is used -- rather than simply calling moveToObjectDir key file @@ -43,7 +43,7 @@ perform file key = do then return $ Just $ cleanup key else error "mv failed!" -cleanup :: Key -> SubCmdCleanup +cleanup :: Key -> CommandCleanup cleanup key = do logStatus key ValuePresent return True diff --git a/Command/Trust.hs b/Command/Trust.hs index 8060ee66f9..3c3ec3b7e5 100644 --- a/Command/Trust.hs +++ b/Command/Trust.hs @@ -17,17 +17,17 @@ import qualified Remotes import UUID import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withString start] {- Marks a remote as trusted. -} -start :: SubCmdStartString +start :: CommandStartString start name = do r <- Remotes.byName name showStart "trust" name return $ Just $ perform r -perform :: Git.Repo -> SubCmdPerform +perform :: Git.Repo -> CommandPerform perform repo = do uuid <- getUUID repo trusted <- getTrusted diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 9580fc5e7c..42354b8c49 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -20,16 +20,16 @@ import Core import qualified GitRepo as Git import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit start] {- The unannex subcommand undoes an add. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file return $ Just $ perform file key backend -perform :: FilePath -> Key -> Backend -> SubCmdPerform +perform :: FilePath -> Key -> Backend -> CommandPerform perform file key backend = do -- force backend to always remove ok <- Backend.removeKey backend key (Just 0) @@ -37,7 +37,7 @@ perform file key backend = do then return $ Just $ cleanup file key else return Nothing -cleanup :: FilePath -> Key -> SubCmdCleanup +cleanup :: FilePath -> Key -> CommandCleanup cleanup file key = do g <- Annex.gitRepo diff --git a/Command/Uninit.hs b/Command/Uninit.hs index fcb77a92b0..6001c55cd9 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -20,15 +20,15 @@ import qualified Annex import qualified Command.Unannex import qualified Command.Init -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withAll withFilesInGit Command.Unannex.start, withNothing start] -start :: SubCmdStartNothing +start :: CommandStartNothing start = do showStart "uninit" "" return $ Just $ perform -perform :: SubCmdPerform +perform :: CommandPerform perform = do g <- Annex.gitRepo diff --git a/Command/Unlock.hs b/Command/Unlock.hs index ff22fa84b3..21f34d1dba 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -18,17 +18,17 @@ import Locations import Core import CopyFile -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit start] {- The unlock subcommand replaces the symlink with a copy of the file's - content. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do showStart "unlock" file return $ Just $ perform file key -perform :: FilePath -> Key -> SubCmdPerform +perform :: FilePath -> Key -> CommandPerform perform dest key = do g <- Annex.gitRepo let src = annexLocation g key diff --git a/Command/Untrust.hs b/Command/Untrust.hs index 5ed8de2451..6458040b3f 100644 --- a/Command/Untrust.hs +++ b/Command/Untrust.hs @@ -17,17 +17,17 @@ import qualified Remotes import UUID import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withString start] {- Marks a remote as not trusted. -} -start :: SubCmdStartString +start :: CommandStartString start name = do r <- Remotes.byName name showStart "untrust" name return $ Just $ perform r -perform :: Git.Repo -> SubCmdPerform +perform :: Git.Repo -> CommandPerform perform repo = do uuid <- getUUID repo trusted <- getTrusted diff --git a/Command/Unused.hs b/Command/Unused.hs index 69a16f2547..dba9aa517a 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -17,16 +17,16 @@ import Messages import Locations import qualified Annex -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withNothing start] {- Finds unused content in the annex. -} -start :: SubCmdStartNothing +start :: CommandStartNothing start = do showStart "unused" "" return $ Just perform -perform :: SubCmdPerform +perform :: CommandPerform perform = do _ <- checkUnused return $ Just $ return True From a89a6f21145966e625b811741d8ae972a11d92b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 15:06:26 -0400 Subject: [PATCH 0648/2835] refactor in preparation for adding a git-annex-shell command --- CmdLine.hs | 164 ++++++++++-------------------------------- Command.hs | 20 ++++-- Command/Add.hs | 4 +- Command/Copy.hs | 4 ++ Command/Drop.hs | 4 ++ Command/DropKey.hs | 4 ++ Command/DropUnused.hs | 4 ++ Command/Find.hs | 4 ++ Command/Fix.hs | 4 ++ Command/FromKey.hs | 4 ++ Command/Fsck.hs | 4 ++ Command/Get.hs | 4 ++ Command/Init.hs | 4 ++ Command/Lock.hs | 3 + Command/Move.hs | 4 ++ Command/PreCommit.hs | 3 + Command/SetKey.hs | 4 ++ Command/Trust.hs | 4 ++ Command/Unannex.hs | 3 + Command/Uninit.hs | 4 ++ Command/Unlock.hs | 6 ++ Command/Untrust.hs | 4 ++ Command/Unused.hs | 3 + git-annex.hs | 74 ++++++++++++++++++- 24 files changed, 204 insertions(+), 136 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 40ce4b1215..54c2289c61 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -1,11 +1,16 @@ -{- git-annex command line +{- git-annex command line parsing - - Copyright 2010 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module CmdLine (parseCmd) where +module CmdLine ( + parseCmd, + Option, + storeOptBool, + storeOptString, +) where import System.Console.GetOpt import Control.Monad (when) @@ -13,116 +18,41 @@ import Control.Monad.State (liftIO) import qualified Annex import Types - import Command -import qualified Command.Add -import qualified Command.Unannex -import qualified Command.Drop -import qualified Command.Move -import qualified Command.Copy -import qualified Command.Get -import qualified Command.FromKey -import qualified Command.DropKey -import qualified Command.SetKey -import qualified Command.Fix -import qualified Command.Init -import qualified Command.Fsck -import qualified Command.Unused -import qualified Command.DropUnused -import qualified Command.Unlock -import qualified Command.Lock -import qualified Command.PreCommit -import qualified Command.Find -import qualified Command.Uninit -import qualified Command.Trust -import qualified Command.Untrust -cmds :: [Command] -cmds = - [ Command.Add.command - , Command "get" path Command.Get.seek - "make content of annexed files available" - , Command "drop" path Command.Drop.seek - "indicate content of files not currently wanted" - , Command "move" path Command.Move.seek - "move content of files to/from another repository" - , Command "copy" path Command.Copy.seek - "copy content of files to/from another repository" - , Command "unlock" path Command.Unlock.seek - "unlock files for modification" - , Command "edit" path Command.Unlock.seek - "same as unlock" - , Command "lock" path Command.Lock.seek - "undo unlock command" - , Command "init" desc Command.Init.seek - "initialize git-annex with repository description" - , Command "unannex" path Command.Unannex.seek - "undo accidential add command" - , Command "uninit" path Command.Uninit.seek - "de-initialize git-annex and clean out repository" - , Command "pre-commit" path Command.PreCommit.seek - "run by git pre-commit hook" - , Command "trust" remote Command.Trust.seek - "trust a repository" - , Command "untrust" remote Command.Untrust.seek - "do not trust a repository" - , Command "fromkey" key Command.FromKey.seek - "adds a file using a specific key" - , Command "dropkey" key Command.DropKey.seek - "drops annexed content for specified keys" - , Command "setkey" key Command.SetKey.seek - "sets annexed content for a key using a temp file" - , Command "fix" path Command.Fix.seek - "fix up symlinks to point to annexed content" - , Command "fsck" maybepath Command.Fsck.seek - "check for problems" - , Command "unused" nothing Command.Unused.seek - "look for unused file content" - , Command "dropunused" number Command.DropUnused.seek - "drop unused file content" - , Command "find" maybepath Command.Find.seek - "lists available files" - ] +{- Each dashed command-line option results in generation of an action + - in the Annex monad that performs the necessary setting. + -} +type Option = OptDescr (Annex ()) + +storeOptBool :: FlagName -> Bool -> Annex () +storeOptBool name val = Annex.flagChange name $ FlagBool val +storeOptString :: FlagName -> String -> Annex () +storeOptString name val = Annex.flagChange name $ FlagString val + +{- Parses command line, stores configure flags, and returns a + - list of actions to be run in the Annex monad. -} +parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool] +parseCmd argv header cmds options = do + (flags, params) <- liftIO $ getopt + when (null params) $ error usagemsg + case lookupCmd (head params) of + [] -> error usagemsg + [command] -> do + _ <- sequence flags + prepCmd command (drop 1 params) + _ -> error "internal error: multiple matching commands" where - path = "PATH ..." - maybepath = "[PATH ...]" - key = "KEY ..." - desc = "DESCRIPTION" - number = "NUMBER ..." - remote = "REMOTE ..." - nothing = "" + getopt = case getOpt Permute options argv of + (flags, params, []) -> return (flags, params) + (_, _, errs) -> ioError (userError (concat errs ++ usagemsg)) + lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds + usagemsg = usage header cmds options --- Each dashed command-line option results in generation of an action --- in the Annex monad that performs the necessary setting. -options :: [OptDescr (Annex ())] -options = [ - Option ['f'] ["force"] (NoArg (storebool "force" True)) - "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) - "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storebool "quiet" False)) - "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") - "specify default key-value backend to use" - , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") - "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storestring "exclude") "GLOB") - "skip files matching the glob pattern" - ] - where - storebool n b = Annex.flagChange n $ FlagBool b - storestring n s = Annex.flagChange n $ FlagString s - -header :: String -header = "Usage: git-annex subcommand [option ..]" - -{- Usage message with lists of options and subcommands. -} -usage :: String -usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs +{- Usage message with lists of commands and options. -} +usage :: String -> [Command] -> [Option] -> String +usage header cmds options = + usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = @@ -133,21 +63,3 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs cmddesc c indent l = " " ++ l pad n s = replicate (n - length s) ' ' - -{- Parses command line, stores configure flags, and returns a - - list of actions to be run in the Annex monad. -} -parseCmd :: [String] -> Annex [Annex Bool] -parseCmd argv = do - (flags, params) <- liftIO $ getopt - when (null params) $ error usage - case lookupCmd (head params) of - [] -> error usage - [command] -> do - _ <- sequence flags - prepCmd command (drop 1 params) - _ -> error "internal error: multiple matching commands" - where - getopt = case getOpt Permute options argv of - (flags, params, []) -> return (flags, params) - (_, _, errs) -> ioError (userError (concat errs ++ usage)) - lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds diff --git a/Command.hs b/Command.hs index 2144da353b..690dd20ecf 100644 --- a/Command.hs +++ b/Command.hs @@ -205,18 +205,24 @@ notSymlink f = do s <- liftIO $ getSymbolicLinkStatus f return $ not $ isSymbolicLink s -{- descriptions of params used in usage message -} +{- Descriptions of params used in usage messages. -} +paramRepeating :: String -> String +paramRepeating s = s ++ " ..." +paramOptional :: String -> String +paramOptional s = "[" ++ s ++ "]" paramPath :: String -paramPath = "PATH ..." -paramMaybePath :: String -paramMaybePath = "[PATH ...]" +paramPath = "PATH" paramKey :: String -paramKey = "KEY ..." +paramKey = "KEY" paramDesc :: String paramDesc = "DESCRIPTION" paramNumber :: String -paramNumber = "NUMBER ..." +paramNumber = "NUMBER" paramRemote :: String -paramRemote = "REMOTE ..." +paramRemote = "REMOTE" +paramGlob :: String +paramGlob = "GLOB" +paramName :: String +paramName = "NAME" paramNothing :: String paramNothing = "" diff --git a/Command/Add.hs b/Command/Add.hs index 08a880206b..bc869a67de 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -18,8 +18,8 @@ import Types import Core import Messages -command :: Command -command = Command "add" paramPath seek "add files to annex" +command :: [Command] +command = [Command "add" paramPath seek "add files to annex"] {- Add acts on both files not checked into git yet, and unlocked files. -} seek :: [CommandSeek] diff --git a/Command/Copy.hs b/Command/Copy.hs index 873df7ef2e..93342e11bb 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -10,6 +10,10 @@ module Command.Copy where import Command import qualified Command.Move +command :: [Command] +command = [Command "copy" paramPath seek + "copy content of files to/from another repository"] + -- A copy is just a move that does not delete the source file. seek :: [CommandSeek] seek = [withFilesInGit $ Command.Move.start False] diff --git a/Command/Drop.hs b/Command/Drop.hs index 3f27405703..a425c6138d 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -17,6 +17,10 @@ import Core import Messages import Utility +command :: [Command] +command = [Command "drop" paramPath seek + "indicate content of files not currently wanted"] + seek :: [CommandSeek] seek = [withAttrFilesInGit "annex.numcopies" start] diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 870e9a7ab1..29056139d3 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -15,6 +15,10 @@ import Types import Core import Messages +command :: [Command] +command = [Command "dropkey" (paramRepeating paramKey) seek + "drops annexed content for specified keys"] + seek :: [CommandSeek] seek = [withKeys start] diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 9984e49f3f..ea2ff46eba 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -18,6 +18,10 @@ import qualified Annex import qualified Command.Drop import Backend +command :: [Command] +command = [Command "dropunused" (paramRepeating paramNumber) seek + "drop unused file content"] + seek :: [CommandSeek] seek = [withStrings start] diff --git a/Command/Find.hs b/Command/Find.hs index 9927b692d8..7cb781ce8c 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -13,6 +13,10 @@ import Control.Monad.State (liftIO) import Command import Core +command :: [Command] +command = [Command "find" (paramOptional $ paramRepeating paramPath) seek + "lists available files"] + seek :: [CommandSeek] seek = [withDefault "." withFilesInGit start] diff --git a/Command/Fix.hs b/Command/Fix.hs index accdadd315..8b08a26f6d 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -17,6 +17,10 @@ import Utility import Core import Messages +command :: [Command] +command = [Command "fix" paramPath seek + "fix up symlinks to point to annexed content"] + seek :: [CommandSeek] seek = [withFilesInGit start] diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 991428136e..f1cb717fac 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -20,6 +20,10 @@ import Types import Core import Messages +command :: [Command] +command = [Command "fromkey" (paramRepeating paramKey) seek + "adds a file using a specific key"] + seek :: [CommandSeek] seek = [withFilesMissing start] diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 034bdc388b..d870bd4198 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -13,6 +13,10 @@ import Types import Messages import Utility +command :: [Command] +command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek + "check for problems"] + seek :: [CommandSeek] seek = [withAll (withAttrFilesInGit "annex.numcopies") start] diff --git a/Command/Get.hs b/Command/Get.hs index 214b689b8a..e3668649ef 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -13,6 +13,10 @@ import Types import Core import Messages +command :: [Command] +command = [Command "get" paramPath seek + "make content of annexed files available"] + seek :: [CommandSeek] seek = [withFilesInGit start] diff --git a/Command/Init.hs b/Command/Init.hs index 806c34c989..8ad9f79d70 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -19,6 +19,10 @@ import Version import Messages import Locations import Types + +command :: [Command] +command = [Command "init" paramDesc seek + "initialize git-annex with repository description"] seek :: [CommandSeek] seek = [withString start] diff --git a/Command/Lock.hs b/Command/Lock.hs index 381162536e..00a553e956 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -14,6 +14,9 @@ import Command import Messages import qualified Annex import qualified GitRepo as Git + +command :: [Command] +command = [Command "lock" paramPath seek "undo unlock command"] seek :: [CommandSeek] seek = [withFilesUnlocked start] diff --git a/Command/Move.hs b/Command/Move.hs index 8ba8dbfacc..addeeae8a9 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -20,6 +20,10 @@ import qualified GitRepo as Git import qualified Remotes import UUID import Messages + +command :: [Command] +command = [Command "move" paramPath seek + "move content of files to/from another repository"] seek :: [CommandSeek] seek = [withFilesInGit $ start True] diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 8d488514a8..12e5ed806d 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -15,6 +15,9 @@ import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix +command :: [Command] +command = [Command "pre-commit" paramPath seek "run by git pre-commit hook"] + {- The pre-commit hook needs to fix symlinks to all files being committed. - And, it needs to inject unlocked files into the annex. -} seek :: [CommandSeek] diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 4c82de3a5b..5048d052f0 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -19,6 +19,10 @@ import Types import Core import Messages +command :: [Command] +command = [Command "setkey" (paramRepeating paramKey) seek + "sets annexed content for a key using a temp file"] + seek :: [CommandSeek] seek = [withTempFile start] diff --git a/Command/Trust.hs b/Command/Trust.hs index 3c3ec3b7e5..35ddefe842 100644 --- a/Command/Trust.hs +++ b/Command/Trust.hs @@ -17,6 +17,10 @@ import qualified Remotes import UUID import Messages +command :: [Command] +command = [Command "trust" (paramRepeating paramRemote) seek + "trust a repository"] + seek :: [CommandSeek] seek = [withString start] diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 42354b8c49..288f9da44a 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -20,6 +20,9 @@ import Core import qualified GitRepo as Git import Messages +command :: [Command] +command = [Command "unannex" paramPath seek "undo accidential add command"] + seek :: [CommandSeek] seek = [withFilesInGit start] diff --git a/Command/Uninit.hs b/Command/Uninit.hs index 6001c55cd9..1a4e9b0d71 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -20,6 +20,10 @@ import qualified Annex import qualified Command.Unannex import qualified Command.Init +command :: [Command] +command = [Command "uninit" paramPath seek + "de-initialize git-annex and clean out repository"] + seek :: [CommandSeek] seek = [withAll withFilesInGit Command.Unannex.start, withNothing start] diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 21f34d1dba..0e55585ae3 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -18,6 +18,12 @@ import Locations import Core import CopyFile +command :: [Command] +command = + [ Command "unlock" paramPath seek "unlock files for modification" + , Command "edit" paramPath seek "same as unlock" + ] + seek :: [CommandSeek] seek = [withFilesInGit start] diff --git a/Command/Untrust.hs b/Command/Untrust.hs index 6458040b3f..f49a2e989a 100644 --- a/Command/Untrust.hs +++ b/Command/Untrust.hs @@ -17,6 +17,10 @@ import qualified Remotes import UUID import Messages +command :: [Command] +command = [Command "untrust" (paramRepeating paramRemote) seek + "do not trust a repository"] + seek :: [CommandSeek] seek = [withString start] diff --git a/Command/Unused.hs b/Command/Unused.hs index dba9aa517a..d2dfc9aa3e 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -17,6 +17,9 @@ import Messages import Locations import qualified Annex +command :: [Command] +command = [Command "unused" paramNothing seek "look for unused file content"] + seek :: [CommandSeek] seek = [withNothing start] diff --git a/git-annex.hs b/git-annex.hs index 1173ab9139..31d90e4fc3 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,6 +6,7 @@ -} import System.Environment +import System.Console.GetOpt import qualified Annex import Core @@ -14,10 +15,81 @@ import CmdLine import qualified GitRepo as Git import BackendList +import Command +import qualified Command.Add +import qualified Command.Unannex +import qualified Command.Drop +import qualified Command.Move +import qualified Command.Copy +import qualified Command.Get +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.SetKey +import qualified Command.Fix +import qualified Command.Init +import qualified Command.Fsck +import qualified Command.Unused +import qualified Command.DropUnused +import qualified Command.Unlock +import qualified Command.Lock +import qualified Command.PreCommit +import qualified Command.Find +import qualified Command.Uninit +import qualified Command.Trust +import qualified Command.Untrust + +cmds :: [Command] +cmds = concat + [ Command.Add.command + , Command.Get.command + , Command.Drop.command + , Command.Move.command + , Command.Copy.command + , Command.Unlock.command + , Command.Lock.command + , Command.Init.command + , Command.Unannex.command + , Command.Uninit.command + , Command.PreCommit.command + , Command.Trust.command + , Command.Untrust.command + , Command.FromKey.command + , Command.DropKey.command + , Command.SetKey.command + , Command.Fix.command + , Command.Fsck.command + , Command.Unused.command + , Command.DropUnused.command + , Command.Find.command + ] + +options :: [Option] +options = [ + Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) + "allow actions that may lose annexed data" + , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) + "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) + "allow verbose output" + , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) + "specify default key-value backend to use" + , Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + "skip files matching the glob pattern" + ] + +header :: String +header = "Usage: git-annex subcommand [option ..]" + main :: IO () main = do args <- getArgs gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends - (actions, state') <- Annex.run state $ parseCmd args + (actions, state') <- Annex.run state $ parseCmd args header cmds options tryRun state' $ [startup, upgrade] ++ actions From c274aadabcd1737e03003834dcf1f6bd9a79b0aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 15:12:55 -0400 Subject: [PATCH 0649/2835] remove magic numbers --- CmdLine.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 54c2289c61..3767fc2406 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -57,9 +57,12 @@ usage header cmds options = cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = cmdname c ++ - pad 11 (cmdname c) ++ + pad (commandlen + 1) (cmdname c) ++ cmdparams c ++ - pad 13 (cmdparams c) ++ + pad (commandparamlen + 2) (cmdparams c) ++ cmddesc c indent l = " " ++ l pad n s = replicate (n - length s) ' ' + longest l = foldl max 0 $ map length l + commandlen = longest $ map cmdname cmds + commandparamlen = longest $ map cmdparams cmds From 1c451fe3628f535898e7cf87ccad30270c6d16fb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 15:15:22 -0400 Subject: [PATCH 0650/2835] tweak --- CmdLine.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 3767fc2406..a2645f75f5 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -57,12 +57,10 @@ usage header cmds options = cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = cmdname c ++ - pad (commandlen + 1) (cmdname c) ++ + pad (longest cmdname + 1) (cmdname c) ++ cmdparams c ++ - pad (commandparamlen + 2) (cmdparams c) ++ + pad (longest cmdparams + 2) (cmdparams c) ++ cmddesc c indent l = " " ++ l pad n s = replicate (n - length s) ' ' - longest l = foldl max 0 $ map length l - commandlen = longest $ map cmdname cmds - commandparamlen = longest $ map cmdparams cmds + longest f = foldl max 0 $ map (length . f) cmds From 88ff9e82fc3dcb653b2a116f1c162d98a1f6bdcf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 15:44:15 -0400 Subject: [PATCH 0651/2835] factor out a little more --- CmdLine.hs | 15 +++++++++++++++ Core.hs | 2 +- git-annex.hs | 18 ++---------------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index a2645f75f5..b3dfc984df 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -6,19 +6,25 @@ -} module CmdLine ( + cmdLine, parseCmd, Option, storeOptBool, storeOptString, ) where +import System.Environment import System.Console.GetOpt import Control.Monad (when) import Control.Monad.State (liftIO) import qualified Annex +import qualified GitRepo as Git import Types import Command +import BackendList +import Core +import Upgrade {- Each dashed command-line option results in generation of an action - in the Annex monad that performs the necessary setting. @@ -30,6 +36,15 @@ storeOptBool name val = Annex.flagChange name $ FlagBool val storeOptString :: FlagName -> String -> Annex () storeOptString name val = Annex.flagChange name $ FlagString val +{- It all starts here. -} +cmdLine :: [Command] -> [Option] -> String -> IO () +cmdLine cmds options header = do + args <- getArgs + gitrepo <- Git.repoFromCwd + state <- Annex.new gitrepo allBackends + (actions, state') <- Annex.run state $ parseCmd args header cmds options + tryRun state' $ [startup, upgrade] ++ actions + {- Parses command line, stores configure flags, and returns a - list of actions to be run in the Annex monad. -} parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool] diff --git a/Core.hs b/Core.hs index d91595a049..08e2265920 100644 --- a/Core.hs +++ b/Core.hs @@ -45,7 +45,7 @@ tryRun' state errnum (a:as) = do tryRun' state errnum [] = do _ <- try $ Annex.run state $ shutdown errnum when (errnum > 0) $ error $ show errnum ++ " failed" - + {- Actions to perform each time ran. -} startup :: Annex Bool startup = do diff --git a/git-annex.hs b/git-annex.hs index 31d90e4fc3..b8176befaa 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -5,17 +5,11 @@ - Licensed under the GNU GPL version 3 or higher. -} -import System.Environment import System.Console.GetOpt -import qualified Annex -import Core -import Upgrade import CmdLine -import qualified GitRepo as Git -import BackendList - import Command + import qualified Command.Add import qualified Command.Unannex import qualified Command.Drop @@ -83,13 +77,5 @@ options = [ "skip files matching the glob pattern" ] -header :: String -header = "Usage: git-annex subcommand [option ..]" - main :: IO () -main = do - args <- getArgs - gitrepo <- Git.repoFromCwd - state <- Annex.new gitrepo allBackends - (actions, state') <- Annex.run state $ parseCmd args header cmds options - tryRun state' $ [startup, upgrade] ++ actions +main = cmdLine cmds options "Usage: git-annex subcommand [option ..]" From 7a52b34e0631609d5d862c3ba100cc499b30b5fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 16:52:24 -0400 Subject: [PATCH 0652/2835] add git-annex-shell command This is not yet complete, as it does not allow starting rsync or scp. --- .gitignore | 2 ++ CmdLine.hs | 36 +++++++++-------------- Command/FromKey.hs | 2 +- Makefile | 23 +++++++++------ Options.hs | 44 ++++++++++++++++++++++++++++ doc/git-annex-shell.mdwn | 63 ++++++++++++++++++++++++++++++++++++++++ doc/git-annex.mdwn | 10 +++---- git-annex-shell.hs | 52 +++++++++++++++++++++++++++++++++ git-annex.hs | 27 ++++------------- 9 files changed, 200 insertions(+), 59 deletions(-) create mode 100644 Options.hs create mode 100644 doc/git-annex-shell.mdwn create mode 100644 git-annex-shell.hs diff --git a/.gitignore b/.gitignore index a4cac10f43..d2f4c2b743 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ test configure SysConfig.hs git-annex +git-annex-shell git-annex.1 +git-annex-shell.1 doc/.ikiwiki html diff --git a/CmdLine.hs b/CmdLine.hs index b3dfc984df..34cc22656a 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -6,14 +6,13 @@ -} module CmdLine ( - cmdLine, + dispatch, parseCmd, Option, storeOptBool, storeOptString, ) where -import System.Environment import System.Console.GetOpt import Control.Monad (when) import Control.Monad.State (liftIO) @@ -25,21 +24,11 @@ import Command import BackendList import Core import Upgrade +import Options -{- Each dashed command-line option results in generation of an action - - in the Annex monad that performs the necessary setting. - -} -type Option = OptDescr (Annex ()) - -storeOptBool :: FlagName -> Bool -> Annex () -storeOptBool name val = Annex.flagChange name $ FlagBool val -storeOptString :: FlagName -> String -> Annex () -storeOptString name val = Annex.flagChange name $ FlagString val - -{- It all starts here. -} -cmdLine :: [Command] -> [Option] -> String -> IO () -cmdLine cmds options header = do - args <- getArgs +{- Runs the passed command line. -} +dispatch :: [String] -> [Command] -> [Option] -> String -> IO () +dispatch args cmds options header = do gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends (actions, state') <- Annex.run state $ parseCmd args header cmds options @@ -50,24 +39,27 @@ cmdLine cmds options header = do parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool] parseCmd argv header cmds options = do (flags, params) <- liftIO $ getopt - when (null params) $ error usagemsg + when (null params) $ error $ "missing command" ++ usagemsg case lookupCmd (head params) of - [] -> error usagemsg + [] -> error $ "unknown command" ++ usagemsg [command] -> do _ <- sequence flags prepCmd command (drop 1 params) _ -> error "internal error: multiple matching commands" where getopt = case getOpt Permute options argv of - (flags, params, []) -> return (flags, params) - (_, _, errs) -> ioError (userError (concat errs ++ usagemsg)) + (flags, params, []) -> + return (flags, params) + (_, _, errs) -> + ioError (userError (concat errs ++ usagemsg)) lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds - usagemsg = usage header cmds options + usagemsg = "\n\n" ++ usage header cmds options {- Usage message with lists of commands and options. -} usage :: String -> [Command] -> [Option] -> String usage header cmds options = - usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs + usageInfo (header ++ "\n\nOptions:") options ++ + "\nCommands:\n" ++ cmddescs where cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = diff --git a/Command/FromKey.hs b/Command/FromKey.hs index f1cb717fac..0a13b8c734 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -21,7 +21,7 @@ import Core import Messages command :: [Command] -command = [Command "fromkey" (paramRepeating paramKey) seek +command = [Command "fromkey" paramPath seek "adds a file using a specific key"] seek :: [CommandSeek] diff --git a/Makefile b/Makefile index c338427df4..2f1fd05b91 100644 --- a/Makefile +++ b/Makefile @@ -2,23 +2,28 @@ PREFIX=/usr GHCFLAGS=-O2 -Wall GHCMAKE=ghc -odir build -hidir build $(GHCFLAGS) --make -all: git-annex git-annex.1 docs +bins=git-annex git-annex-shell +mans=git-annex.1 git-annex-shell.1 + +all: $(bins) $(mans) docs SysConfig.hs: configure.hs $(GHCMAKE) configure ./configure +$(bins): SysConfig.hs + $(GHCMAKE) $@ + git-annex.1: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 - -git-annex: SysConfig.hs - $(GHCMAKE) git-annex +git-annex-shell.1: + ./mdwn2man git-annex 1 doc/git-annex-shell.mdwn > git-annex-shell.1 install: all install -d $(DESTDIR)$(PREFIX)/bin - install git-annex $(DESTDIR)$(PREFIX)/bin + install $(bins) $(DESTDIR)$(PREFIX)/bin install -d $(DESTDIR)$(PREFIX)/share/man/man1 - install -m 0644 git-annex.1 $(DESTDIR)$(PREFIX)/share/man/man1 + install -m 0644 $(mans) $(DESTDIR)$(PREFIX)/share/man/man1 install -d $(DESTDIR)$(PREFIX)/share/doc/git-annex if [ -d html ]; then \ rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \ @@ -36,7 +41,7 @@ else IKIWIKI=ikiwiki endif -docs: git-annex.1 +docs: $(mans) $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ --underlaydir=/dev/null --disable-plugin=shortcut \ @@ -44,7 +49,7 @@ docs: git-annex.1 --exclude='news/.*' clean: - rm -rf build git-annex git-annex.1 test configure SysConfig.hs + rm -rf build $(bins) $(mans) test configure SysConfig.hs rm -rf doc/.ikiwiki html -.PHONY: git-annex test install +.PHONY: $(bins) test install diff --git a/Options.hs b/Options.hs new file mode 100644 index 0000000000..684aed97d6 --- /dev/null +++ b/Options.hs @@ -0,0 +1,44 @@ +{- git-annex dashed options + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Options where + +import System.Console.GetOpt + +import qualified Annex +import Types +import Command + +{- Each dashed command-line option results in generation of an action + - in the Annex monad that performs the necessary setting. + -} +type Option = OptDescr (Annex ()) + +storeOptBool :: FlagName -> Bool -> Annex () +storeOptBool name val = Annex.flagChange name $ FlagBool val +storeOptString :: FlagName -> String -> Annex () +storeOptString name val = Annex.flagChange name $ FlagString val + +commonOptions :: [Option] +commonOptions = [ + Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) + "allow actions that may lose annexed data" + , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) + "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) + "allow verbose output" + , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) + "specify default key-value backend to use" + , Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + "skip files matching the glob pattern" + ] diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn new file mode 100644 index 0000000000..34d9c8afef --- /dev/null +++ b/doc/git-annex-shell.mdwn @@ -0,0 +1,63 @@ +# NAME + +git-annex-shell - Restricted login shell for git-annex only SSH access + +# SYNOPSIS + +git-annex-shell -c command [params ...] + +# DESCRIPTION + +git-annex-shell is a restricted shell, similar to git-shell, which +can be used as a login shell for SSH accounts you want to restrict. + +# COMMANDS + +* git-annex fromkey file + + This can be used to maually set up a file to link to a specified key + in the key-value backend. How you determine an existing key in the backend + varies. For the URL backend, the key is just a URL to the content. + + Example: + + git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + +* git-annex dropkey [key ...] + + This drops the annexed data for the specified + keys from this repository. + + This can be used to drop content for arbitrary keys, which do not need + to have a file in the git repository pointing at them. + + A backend will typically need to be specified with --backend. If none + is specified, the first configured backend is used. + +* git-annex setkey file + + This sets the annxed data for a key to the content of + the specified file, and then removes the file. + + A backend will typically need to be specified with --backend. If none + is specified, the first configured backend is used. + +Any other command is passed through to git-shell. + +# OPTIONS + +Same as git-annex or git-shell, depending on the command being run. + +# SEE ALSO + +[[git-annex]](1) + +git-shell(1) + +# AUTHOR + +Joey Hess + + + +Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD). Edit with care diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index caef49d977..8e6ff2c0cc 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -4,7 +4,7 @@ git-annex - manage files with git, without checking their contents in # SYNOPSIS -git annex subcommand [params ...] +git annex command [params ...] # DESCRIPTION @@ -55,13 +55,13 @@ content from the key-value store. # git annex move iso --to=usbdrive move iso/Debian_5.0.iso (moving to usbdrive...) ok -# SUBCOMMANDS +# COMMANDS Like many git commands, git-annex can be passed a path that is either a file or a directory. In the latter case it acts on all relevant files in the directory. -Many git-annex subcommands will stage changes for later `git commit` by you. +Many git-annex commands will stage changes for later `git commit` by you. * add [path ...] @@ -91,7 +91,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * edit [path ...] - This is an alias for the unlock subcommand. May be easier to remember, + This is an alias for the unlock command. May be easier to remember, if you think of this as allowing you to edit an annexed file. * move [path ...] @@ -122,7 +122,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * fsck [path ...] - With no parameters, this subcommand checks the whole annex for consistency, + With no parameters, this command checks the whole annex for consistency, and warns about any problems found. With parameters, only the specified files are checked. diff --git a/git-annex-shell.hs b/git-annex-shell.hs new file mode 100644 index 0000000000..7adb5e7905 --- /dev/null +++ b/git-annex-shell.hs @@ -0,0 +1,52 @@ +{- git-annex-shell main program + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +import System.Console.GetOpt +import System.Environment +import Control.Monad (when) + +import CmdLine +import Command +import Utility +import Options + +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.SetKey + +cmds :: [Command] +cmds = concat + [ Command.FromKey.command + , Command.DropKey.command + , Command.SetKey.command + ] + +options :: [Option] +options = [ Option ['c'] ["command"] (NoArg (storeOptBool "command" True)) + "ignored for compatability with git-shell" + ] ++ commonOptions + +header :: String +header = "Usage:\n" ++ + "\tgit-annex-shell -c git-annex command [option ..]\n" ++ + "\tgit-annex-shell -c shellcommand argument" + +main :: IO () +main = do + args <- getArgs + -- dispatch git-annex commands to builtin versions, + -- and pass everything else to git-shell + case args of + ("git-annex":as) -> builtin as + [] -> builtin [] + _ -> external args + where + builtin l = dispatch l cmds options header + external l = do + ret <- boolSystem "git-shell" l + when (not ret) $ + error "git-shell failed" diff --git a/git-annex.hs b/git-annex.hs index b8176befaa..6c143972a1 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -5,10 +5,11 @@ - Licensed under the GNU GPL version 3 or higher. -} -import System.Console.GetOpt +import System.Environment import CmdLine import Command +import Options import qualified Command.Add import qualified Command.Unannex @@ -57,25 +58,7 @@ cmds = concat , Command.Find.command ] -options :: [Option] -options = [ - Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) - "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) - "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) - "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) - "specify default key-value backend to use" - , Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) - "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) - "skip files matching the glob pattern" - ] - main :: IO () -main = cmdLine cmds options "Usage: git-annex subcommand [option ..]" +main = do + args <- getArgs + dispatch args cmds commonOptions "Usage: git-annex command [option ..]" From a5a302b77d816b189ae5ae55f03b18d2cf6ef45b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 20:08:22 -0400 Subject: [PATCH 0653/2835] git-annex-shell mostly done now, only needs 2 more subcommands --- CmdLine.hs | 9 ++--- Command/ConfigList.hs | 27 +++++++++++++++ Command/InAnnex.hs | 32 ++++++++++++++++++ doc/git-annex-shell.mdwn | 38 +++++++++------------ git-annex-shell.hs | 72 +++++++++++++++++++++++++--------------- git-annex.hs | 7 +++- 6 files changed, 130 insertions(+), 55 deletions(-) create mode 100644 Command/ConfigList.hs create mode 100644 Command/InAnnex.hs diff --git a/CmdLine.hs b/CmdLine.hs index 34cc22656a..fbcfb6405d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -8,9 +8,7 @@ module CmdLine ( dispatch, parseCmd, - Option, - storeOptBool, - storeOptString, + usage, ) where import System.Console.GetOpt @@ -27,9 +25,8 @@ import Upgrade import Options {- Runs the passed command line. -} -dispatch :: [String] -> [Command] -> [Option] -> String -> IO () -dispatch args cmds options header = do - gitrepo <- Git.repoFromCwd +dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO () +dispatch gitrepo args cmds options header = do state <- Annex.new gitrepo allBackends (actions, state') <- Annex.run state $ parseCmd args header cmds options tryRun state' $ [startup, upgrade] ++ actions diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs new file mode 100644 index 0000000000..0d9d789b54 --- /dev/null +++ b/Command/ConfigList.hs @@ -0,0 +1,27 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ConfigList where + +import Control.Monad.State (liftIO) + +import Annex +import Command +import qualified GitRepo as Git + +command :: [Command] +command = [Command "configlist" paramNothing seek + "outputs relevant git configuration"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStartNothing +start = do + g <- Annex.gitRepo + liftIO $ Git.run g ["config", "--list"] + return Nothing diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs new file mode 100644 index 0000000000..d49539513b --- /dev/null +++ b/Command/InAnnex.hs @@ -0,0 +1,32 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.InAnnex where + +import Control.Monad.State (liftIO) +import System.Exit + +import Command +import Types +import Core +import qualified Backend + +command :: [Command] +command = [Command "inannex" (paramRepeating paramKey) seek + "checks if keys are present in the annex"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: CommandStartString +start keyname = do + backends <- Backend.list + let key = genKey (head backends) keyname + present <- inAnnex key + if present + then return Nothing + else liftIO $ exitFailure diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index 34d9c8afef..9f51b6813d 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -4,43 +4,37 @@ git-annex-shell - Restricted login shell for git-annex only SSH access # SYNOPSIS -git-annex-shell -c command [params ...] +git-annex-shell [-c] command [params ...] # DESCRIPTION -git-annex-shell is a restricted shell, similar to git-shell, which -can be used as a login shell for SSH accounts you want to restrict. +git-annex-shell is a restricted shell, similar to git-shell, which +can be used as a login shell for SSH accounts. # COMMANDS -* git-annex fromkey file +* configlist directory - This can be used to maually set up a file to link to a specified key - in the key-value backend. How you determine an existing key in the backend - varies. For the URL backend, the key is just a URL to the content. + This outputs the git configuration, in the same form as + `git config --list` - Example: +* inannex directory [key ...] - git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + This checks if all specified keys are present in the annex, + and exits zero if so. -* git-annex dropkey [key ...] +* dropkey directory [key ...] - This drops the annexed data for the specified - keys from this repository. + This drops the annexed data for the specified keys. - This can be used to drop content for arbitrary keys, which do not need - to have a file in the git repository pointing at them. +* recvkey directory key - A backend will typically need to be specified with --backend. If none - is specified, the first configured backend is used. + This runs rsync in server mode to receive the content of a key, + and stores the content in the annex. -* git-annex setkey file +* sendkey directory key - This sets the annxed data for a key to the content of - the specified file, and then removes the file. - - A backend will typically need to be specified with --backend. If none - is specified, the first configured backend is used. + This runs rsync in server mode to transfer out the content of a key. Any other command is passed through to git-shell. diff --git a/git-annex-shell.hs b/git-annex-shell.hs index 7adb5e7905..492d184469 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -5,48 +5,68 @@ - Licensed under the GNU GPL version 3 or higher. -} -import System.Console.GetOpt import System.Environment import Control.Monad (when) +import qualified GitRepo as Git import CmdLine import Command import Utility import Options -import qualified Command.FromKey +import qualified Command.ConfigList +import qualified Command.InAnnex import qualified Command.DropKey -import qualified Command.SetKey +--import qualified Command.RecvKey +--import qualified Command.SendKey cmds :: [Command] -cmds = concat - [ Command.FromKey.command +cmds = map adddirparam $ concat + [ Command.ConfigList.command + , Command.InAnnex.command , Command.DropKey.command - , Command.SetKey.command +-- , Command.RecvKey.command +-- , Command.SendKey.command ] - -options :: [Option] -options = [ Option ['c'] ["command"] (NoArg (storeOptBool "command" True)) - "ignored for compatability with git-shell" - ] ++ commonOptions + where + adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c } header :: String -header = "Usage:\n" ++ - "\tgit-annex-shell -c git-annex command [option ..]\n" ++ - "\tgit-annex-shell -c shellcommand argument" +header = "Usage: git-annex-shell [-c] command [option ..]" main :: IO () main = do args <- getArgs - -- dispatch git-annex commands to builtin versions, - -- and pass everything else to git-shell - case args of - ("git-annex":as) -> builtin as - [] -> builtin [] - _ -> external args - where - builtin l = dispatch l cmds options header - external l = do - ret <- boolSystem "git-shell" l - when (not ret) $ - error "git-shell failed" + main' args + +main' :: [String] -> IO () +main' [] = failure +-- skip leading -c options, passed by eg, ssh +main' ("-c":p) = main' p +-- Since git-annex explicitly runs git-annex-shell, we will be passed +-- a redundant "git-annex-shell" parameter when we're the user's login shell. +main' ("git-annex-shell":p) = main' p +-- a command can be either a builtin or something to pass to git-shell +main' c@(cmd:dir:params) + | elem cmd builtins = builtin cmd dir params + | otherwise = external c +main' c@(cmd:_) + | elem cmd builtins = failure + | otherwise = external c + +builtins :: [String] +builtins = map cmdname cmds + +builtin :: String -> String -> [String] -> IO () +builtin cmd dir params = do + let gitrepo = Git.repoFromPath dir + dispatch gitrepo (cmd:params) cmds commonOptions header + +external :: [String] -> IO () +external l = do + ret <- boolSystem "git-shell" ("-c":l) + when (not ret) $ + error "git-shell failed" + +failure :: IO () +failure = error $ "bad parameters\n\n" ++ usage header cmds commonOptions diff --git a/git-annex.hs b/git-annex.hs index 6c143972a1..110054fd5a 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -7,6 +7,7 @@ import System.Environment +import qualified GitRepo as Git import CmdLine import Command import Options @@ -58,7 +59,11 @@ cmds = concat , Command.Find.command ] +header :: String +header = "Usage: git-annex command [option ..]" + main :: IO () main = do args <- getArgs - dispatch args cmds commonOptions "Usage: git-annex command [option ..]" + gitrepo <- Git.repoFromCwd + dispatch gitrepo args cmds commonOptions header From f38aa3e83abb251a88362dbaf6e8fbddd477fa53 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 20:31:52 -0400 Subject: [PATCH 0654/2835] unfinished switch to using git-annex-shell --- Command/Move.hs | 4 ++-- Remotes.hs | 40 +++++++++++++++++++--------------------- configure.hs | 2 +- doc/install.mdwn | 2 +- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index addeeae8a9..e872d86fe7 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -134,8 +134,8 @@ fromPerform move key = do else return Nothing -- fail fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup fromCleanup True remote key = do - ok <- Remotes.runCmd remote "git-annex" - ["dropkey", "--quiet", "--force", + ok <- Remotes.onRemote remote "dropkey" + ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] remoteHasKey remote key False diff --git a/Remotes.hs b/Remotes.hs index 78ab010cef..a775f71d4b 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -15,7 +15,8 @@ module Remotes ( byName, copyFromRemote, copyToRemote, - runCmd + runCmd, + onRemote ) where import Control.Exception.Extensible @@ -37,7 +38,6 @@ import Utility import qualified Core import Messages import CopyFile -import qualified SysConfig {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -118,7 +118,8 @@ inAnnex r key = if Git.repoIsUrl r Annex.eval a (Core.inAnnex key) checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- runCmd r "test" ["-e", annexLocation r key] + inannex <- onRemote r "inannex" + ["--backend=" ++ backendName key, keyName key] -- XXX Note that ssh failing and the file not existing -- are not currently differentiated. return $ Right inannex @@ -231,7 +232,7 @@ copyFromRemote r key file where keyloc = annexLocation r key getlocal = liftIO $ copyFile keyloc file - getssh = remoteCopyFile r (sshLocation r keyloc) file + getssh = remoteCopyFile True r (sshLocation r keyloc) file {- Tries to copy a key's content to a file on a remote. -} copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool @@ -245,35 +246,32 @@ copyToRemote r key file = do else error "copying to non-ssh repo not supported" where putlocal src = liftIO $ copyFile src file - putssh src = remoteCopyFile r src (sshLocation r file) + putssh src = remoteCopyFile False r src (sshLocation r file) sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file -{- Copys a file from or to a remote, using rsync (when available) or scp. -} -remoteCopyFile :: Git.Repo -> String -> String -> Annex Bool -remoteCopyFile r src dest = do +{- Copies a file from or to a remote, using rsync (when available) or scp. -} +remoteCopyFile :: Bool -> Git.Repo -> String -> String -> Annex Bool +remoteCopyFile recv r src dest = do showProgress -- make way for progress bar o <- repoConfig r configopt "" res <- liftIO $ boolSystem cmd $ options ++ words o ++ [src, dest] if res then return res else do - when rsync $ - showLongNote "rsync failed -- run git annex again to resume file transfer" + showLongNote "rsync failed -- run git annex again to resume file transfer" return res where - cmd - | rsync = "rsync" - | otherwise = "scp" - configopt - | rsync = "rsync-options" - | otherwise = "scp-options" - options - -- inplace makes rsync resume partial files - | rsync = ["-p", "--progress", "--inplace"] - | otherwise = ["-p"] - rsync = SysConfig.rsync + cmd = "rsync" + configopt= "rsync-options" + -- inplace makes rsync resume partial files + options = ["-p", "--progress", "--inplace"] + +onRemote :: Git.Repo -> String -> [String] -> Annex Bool +onRemote r command params = runCmd r "git-annex-shell" (command:dir:params) + where + dir = Git.workTree r {- Runs a command in a remote, using ssh if necessary. - (Honors annex-ssh-options.) -} diff --git a/configure.hs b/configure.hs index 2334385a38..1abdc8914d 100644 --- a/configure.hs +++ b/configure.hs @@ -22,7 +22,7 @@ tests = [ , TestCase "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" , TestCase "uuid" "uuid" $ requireCmd "uuid" "uuid" , TestCase "xargs -0" "xargs_0" $ requireCmd "xargs -0" "xargs -0 /dev/null" + , TestCase "rsync" "rsync" $ requireCmd "rsync" "rsync --version >/dev/null" ] tmpDir :: String diff --git a/doc/install.mdwn b/doc/install.mdwn index 1cff4462e0..bad1d9f258 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -6,7 +6,7 @@ To build and use git-annex, you will need: * pcre-light: * `uuid`: * `xargs`: -* `rsync`: (optional but recommended) +* `rsync`: * Then just [[download]] git-annex and run: `make; make install` ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation, From 60df4e5728b8af804f06c39ef3b897af12247ceb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 13:39:30 -0400 Subject: [PATCH 0655/2835] git-annex-shell is complete still not used --- Command/RecvKey.hs | 38 ++++++++++++++++++++++++++++++++++++++ Command/SendKey.hs | 38 ++++++++++++++++++++++++++++++++++++++ Options.hs | 20 ++++++-------------- Remotes.hs | 2 +- RsyncFile.hs | 33 +++++++++++++++++++++++++++++++++ git-annex-shell.hs | 8 ++++---- git-annex.hs | 15 ++++++++++++++- 7 files changed, 134 insertions(+), 20 deletions(-) create mode 100644 Command/RecvKey.hs create mode 100644 Command/SendKey.hs create mode 100644 RsyncFile.hs diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs new file mode 100644 index 0000000000..3232010d49 --- /dev/null +++ b/Command/RecvKey.hs @@ -0,0 +1,38 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.RecvKey where + +import Control.Monad (when) +import Control.Monad.State (liftIO) +import System.Exit + +import Command +import Types +import Core +import qualified Backend +import RsyncFile + +command :: [Command] +command = [Command "recvkey" paramKey seek + "runs rsync in server mode to receive content"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: CommandStartString +start keyname = do + backends <- Backend.list + let key = genKey (head backends) keyname + present <- inAnnex key + when present $ + error "key is already present in annex" + + ok <- getViaTmp key (liftIO . rsyncServerReceive) + if ok + then return Nothing + else liftIO exitFailure diff --git a/Command/SendKey.hs b/Command/SendKey.hs new file mode 100644 index 0000000000..0ddc0d23b9 --- /dev/null +++ b/Command/SendKey.hs @@ -0,0 +1,38 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.SendKey where + +import Control.Monad (when) +import Control.Monad.State (liftIO) +import System.Exit + +import Locations +import qualified Annex +import Command +import Types +import Core +import qualified Backend +import RsyncFile + +command :: [Command] +command = [Command "sendkey" paramKey seek + "runs rsync in server mode to send content"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: CommandStartString +start keyname = do + backends <- Backend.list + let key = genKey (head backends) keyname + present <- inAnnex key + g <- Annex.gitRepo + let file = annexLocation g key + when present $ + liftIO $ rsyncServerSend file + liftIO exitFailure diff --git a/Options.hs b/Options.hs index 684aed97d6..5f367c9dd4 100644 --- a/Options.hs +++ b/Options.hs @@ -24,21 +24,13 @@ storeOptString :: FlagName -> String -> Annex () storeOptString name val = Annex.flagChange name $ FlagString val commonOptions :: [Option] -commonOptions = [ - Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) +commonOptions = + [ Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) + , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) + , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) + , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) "specify default key-value backend to use" - , Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) - "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) - "skip files matching the glob pattern" - ] + ] diff --git a/Remotes.hs b/Remotes.hs index a775f71d4b..ca65c99ff0 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -251,7 +251,7 @@ copyToRemote r key file = do sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file -{- Copies a file from or to a remote, using rsync (when available) or scp. -} +{- Copies a file from or to a remote, using rsync. -} remoteCopyFile :: Bool -> Git.Repo -> String -> String -> Annex Bool remoteCopyFile recv r src dest = do showProgress -- make way for progress bar diff --git a/RsyncFile.hs b/RsyncFile.hs new file mode 100644 index 0000000000..14f6dc926b --- /dev/null +++ b/RsyncFile.hs @@ -0,0 +1,33 @@ +{- git-annex file copying with rsync + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module RsyncFile where + +import Utility +import System.Posix.Process + +{- Runs rsync in server mode to send a file, and exits. -} +rsyncServerSend :: FilePath -> IO () +rsyncServerSend file = rsyncExec $ rsyncServerParams ++ ["--sender", file] + +{- Runs rsync in server mode to receive a file. -} +rsyncServerReceive :: FilePath -> IO Bool +rsyncServerReceive file = rsync $ rsyncServerParams ++ [file] + +rsyncServerParams :: [String] +rsyncServerParams = + [ "--server" + , "-p" -- preserve permissions + , "--inplace" -- allow resuming of transfers of big files + , "-e.Lsf", "." -- other options rsync normally uses in server mode + ] + +rsync :: [String] -> IO Bool +rsync params = boolSystem "rsync" params + +rsyncExec :: [String] -> IO () +rsyncExec params = executeFile "rsync" True params Nothing diff --git a/git-annex-shell.hs b/git-annex-shell.hs index 492d184469..8783e7f60a 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -17,16 +17,16 @@ import Options import qualified Command.ConfigList import qualified Command.InAnnex import qualified Command.DropKey ---import qualified Command.RecvKey ---import qualified Command.SendKey +import qualified Command.RecvKey +import qualified Command.SendKey cmds :: [Command] cmds = map adddirparam $ concat [ Command.ConfigList.command , Command.InAnnex.command , Command.DropKey.command --- , Command.RecvKey.command --- , Command.SendKey.command + , Command.RecvKey.command + , Command.SendKey.command ] where adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c } diff --git a/git-annex.hs b/git-annex.hs index 110054fd5a..dff67f9d84 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,6 +6,7 @@ -} import System.Environment +import System.Console.GetOpt import qualified GitRepo as Git import CmdLine @@ -59,6 +60,18 @@ cmds = concat , Command.Find.command ] +options :: [Option] +options = commonOptions ++ + [ Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + "skip files matching the glob pattern" + ] + header :: String header = "Usage: git-annex command [option ..]" @@ -66,4 +79,4 @@ main :: IO () main = do args <- getArgs gitrepo <- Git.repoFromCwd - dispatch gitrepo args cmds commonOptions header + dispatch gitrepo args cmds options header From eac433a84ad397e371300343b7cd30b7741ee023 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 15:46:33 -0400 Subject: [PATCH 0656/2835] use git-annex-shell configlist --- Annex.hs | 4 +-- Command/Move.hs | 7 +++-- GitRepo.hs | 35 +++++++++++-------------- Remotes.hs | 68 ++++++++++++++++++++++++++++++++----------------- 4 files changed, 66 insertions(+), 48 deletions(-) diff --git a/Annex.hs b/Annex.hs index 6e5198e8ef..55f9edb36e 100644 --- a/Annex.hs +++ b/Annex.hs @@ -47,7 +47,7 @@ new gitrepo allbackends = do where prep = do -- read git config and update state - gitrepo' <- liftIO $ Git.configRead gitrepo Nothing + gitrepo' <- liftIO $ Git.configRead gitrepo Annex.gitRepoChange gitrepo' {- performs an action in the Annex monad -} @@ -136,5 +136,5 @@ setConfig key value = do g <- Annex.gitRepo liftIO $ Git.run g ["config", key, value] -- re-read git config and update the repo's state - g' <- liftIO $ Git.configRead g Nothing + g' <- liftIO $ Git.configRead g Annex.gitRepoChange g' diff --git a/Command/Move.hs b/Command/Move.hs index e872d86fe7..4291d221a0 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,6 +7,7 @@ module Command.Move where +import Control.Monad (when) import Control.Monad.State (liftIO) import Command @@ -20,6 +21,7 @@ import qualified GitRepo as Git import qualified Remotes import UUID import Messages +import Utility command :: [Command] command = [Command "move" paramPath seek @@ -134,10 +136,11 @@ fromPerform move key = do else return Nothing -- fail fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup fromCleanup True remote key = do - ok <- Remotes.onRemote remote "dropkey" + ok <- Remotes.onRemote remote boolSystem False "dropkey" ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] - remoteHasKey remote key False + when ok $ + remoteHasKey remote key False return ok fromCleanup False _ _ = return True diff --git a/GitRepo.hs b/GitRepo.hs index 2c2ad7b539..9dfce0d35b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -24,6 +24,8 @@ module GitRepo ( configGet, configMap, configRead, + hConfigRead, + configStore, configTrue, gitCommandLine, run, @@ -141,11 +143,7 @@ assertUrl repo action = then action else error $ "acting on local git repo " ++ repoDescribe repo ++ " not supported" -assertSsh :: Repo -> a -> a -assertSsh repo action = - if repoIsSsh repo - then action - else error $ "unsupported url in repo " ++ repoDescribe repo + bare :: Repo -> Bool bare repo = case Map.lookup "core.bare" $ config repo of Just v -> configTrue v @@ -276,11 +274,9 @@ pipeNullSplit repo params = do where split0 s = filter (not . null) $ split "\0" s -{- Runs git config and populates a repo with its config. - - - - For a ssh repository, a list of ssh options may optionally be specified. -} -configRead :: Repo -> Maybe [String] -> IO Repo -configRead repo@(Repo { location = Dir d }) _ = do +{- Runs git config and populates a repo with its config. -} +configRead :: Repo -> IO Repo +configRead repo@(Repo { location = Dir d }) = do {- Cannot use pipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory @@ -288,19 +284,18 @@ configRead repo@(Repo { location = Dir d }) _ = do (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] $ hConfigRead repo -configRead repo sshopts = assertSsh repo $ do - pOpen ReadFromPipe "ssh" params $ hConfigRead repo - where - params = case sshopts of - Nothing -> [urlHost repo, command] - Just l -> l ++ [urlHost repo, command] - command = "cd " ++ shellEscape (urlPath repo) ++ - " && git config --list" +configRead r = assertLocal r $ error "internal" + +{- Reads git config from a handle and populates a repo with it. -} hConfigRead :: Repo -> Handle -> IO Repo hConfigRead repo h = do val <- hGetContentsStrict h - let r = repo { config = configParse val } - return r { remotes = configRemotes r } + return $ configStore repo val + +{- Parses a git config and returns a version of the repo using it. -} +configStore :: Repo -> String -> Repo +configStore repo s = r { remotes = configRemotes r } + where r = repo { config = configParse s } {- Checks if a string fron git config is a true value. -} configTrue :: String -> Bool diff --git a/Remotes.hs b/Remotes.hs index ca65c99ff0..841fe947fe 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -25,6 +25,7 @@ import qualified Data.Map as Map import Data.String.Utils import System.Directory hiding (copyFile) import System.Posix.Directory +import System.Cmd.Utils import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) @@ -112,16 +113,14 @@ inAnnex r key = if Git.repoIsUrl r else liftIO (try checklocal ::IO (Either IOException Bool)) where checklocal = do - -- run a local check by making an Annex monad - -- using the remote + -- run a local check inexpensively, + -- by making an Annex monad using the remote a <- Annex.new r [] Annex.eval a (Core.inAnnex key) checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- onRemote r "inannex" + inannex <- onRemote r boolSystem False "inannex" ["--backend=" ++ backendName key, keyName key] - -- XXX Note that ssh failing and the file not existing - -- are not currently differentiated. return $ Right inannex {- Cost Ordered list of remotes. -} @@ -199,24 +198,29 @@ byName name = do - config for a specified remote, and updates state. If successful, it - returns the updated git repo. -} tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) -tryGitConfigRead r = do - sshoptions <- repoConfig r "ssh-options" "" - if Map.null $ Git.configMap r - then do - -- configRead can fail due to IO error or - -- for other reasons; catch all possible exceptions - result <- liftIO (try (Git.configRead r $ Just $ words sshoptions)::IO (Either SomeException Git.Repo)) +tryGitConfigRead r + | not $ Map.null $ Git.configMap r = return $ Right r -- already read + | Git.repoIsSsh r = store $ onRemote r pipedconfig r "configlist" [] + | Git.repoIsUrl r = return $ Left r + | otherwise = store $ safely $ Git.configRead r + where + -- Reading config can fail due to IO error or + -- for other reasons; catch all possible exceptions. + safely a = do + result <- liftIO (try (a)::IO (Either SomeException Git.Repo)) case result of - Left _ -> return $ Left r - Right r' -> do - g <- Annex.gitRepo - let l = Git.remotes g - let g' = Git.remotesAdd g $ - exchange l r' - Annex.gitRepoChange g' - return $ Right r' - else return $ Right r -- config already read - where + Left _ -> return r + Right r' -> return r' + pipedconfig cmd params = safely $ + pOpen ReadFromPipe cmd params $ + Git.hConfigRead r + store a = do + r' <- a + g <- Annex.gitRepo + let l = Git.remotes g + let g' = Git.remotesAdd g $ exchange l r' + Annex.gitRepoChange g' + return $ Right r' exchange [] _ = [] exchange (old:ls) new = if Git.repoRemoteName old == Git.repoRemoteName new @@ -268,10 +272,26 @@ remoteCopyFile recv r src dest = do -- inplace makes rsync resume partial files options = ["-p", "--progress", "--inplace"] -onRemote :: Git.Repo -> String -> [String] -> Annex Bool -onRemote r command params = runCmd r "git-annex-shell" (command:dir:params) +{- Uses a supplied function to run a git-annex-shell command on a remote. -} +onRemote + :: Git.Repo + -> (String -> [String] -> IO a) + -> a + -> String + -> [String] + -> Annex a +onRemote r with errorval command params + | not $ Git.repoIsUrl r = liftIO $ with shellcmd shellopts + | Git.repoIsSsh r = do + sshoptions <- repoConfig r "ssh-options" "" + liftIO $ with "ssh" $ + words sshoptions ++ [Git.urlHost r, sshcmd] + | otherwise = return errorval where dir = Git.workTree r + shellcmd = "git-annex-shell" + shellopts = command:dir:params + sshcmd = shellcmd ++ " " ++ unwords (map shellEscape shellopts) {- Runs a command in a remote, using ssh if necessary. - (Honors annex-ssh-options.) -} From 30e0065ab97843f866a7fe095b8a18ee6eb4c321 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 15:52:59 -0400 Subject: [PATCH 0657/2835] tuple makes it clearer --- Command/Move.hs | 2 +- Remotes.hs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index 4291d221a0..d96d36138c 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -136,7 +136,7 @@ fromPerform move key = do else return Nothing -- fail fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup fromCleanup True remote key = do - ok <- Remotes.onRemote remote boolSystem False "dropkey" + ok <- Remotes.onRemote remote (boolSystem, False) "dropkey" ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] diff --git a/Remotes.hs b/Remotes.hs index 841fe947fe..70356de024 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -119,7 +119,7 @@ inAnnex r key = if Git.repoIsUrl r Annex.eval a (Core.inAnnex key) checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- onRemote r boolSystem False "inannex" + inannex <- onRemote r (boolSystem, False) "inannex" ["--backend=" ++ backendName key, keyName key] return $ Right inannex @@ -200,7 +200,7 @@ byName name = do tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r | not $ Map.null $ Git.configMap r = return $ Right r -- already read - | Git.repoIsSsh r = store $ onRemote r pipedconfig r "configlist" [] + | Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] | Git.repoIsUrl r = return $ Left r | otherwise = store $ safely $ Git.configRead r where @@ -275,12 +275,11 @@ remoteCopyFile recv r src dest = do {- Uses a supplied function to run a git-annex-shell command on a remote. -} onRemote :: Git.Repo - -> (String -> [String] -> IO a) - -> a + -> ((String -> [String] -> IO a), a) -> String -> [String] -> Annex a -onRemote r with errorval command params +onRemote r (with, errorval) command params | not $ Git.repoIsUrl r = liftIO $ with shellcmd shellopts | Git.repoIsSsh r = do sshoptions <- repoConfig r "ssh-options" "" From 700aed13cff27f9315df1209e0cd37d5e51f5390 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 19:09:17 -0400 Subject: [PATCH 0658/2835] git-annex-shell now exclusively used for all remote access --- Backend/File.hs | 13 ++-- Command/Move.hs | 33 ++++----- Command/RecvKey.hs | 6 +- Remotes.hs | 124 +++++++++++++++++----------------- RsyncFile.hs | 14 +++- doc/todo/git-annex-shell.mdwn | 37 +--------- 6 files changed, 102 insertions(+), 125 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index ee73150211..9bc5a2aa63 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -65,12 +65,7 @@ copyKeyFile key file = do trycopy full (r:rs) = do probablythere <- probablyPresent r if probablythere - then do - showNote $ "copying from " ++ Git.repoDescribe r ++ "..." - copied <- Remotes.copyFromRemote r key file - if copied - then return True - else trycopy full rs + then docopy r (trycopy full rs) else trycopy full rs -- This check is to avoid an ugly message if a remote is a -- drive that is not mounted. Avoid checking inAnnex for ssh @@ -82,6 +77,12 @@ copyKeyFile key file = do if not $ Git.repoIsUrl r then liftIO $ doesFileExist $ annexLocation r key else return True + docopy r continue = do + showNote $ "copying from " ++ Git.repoDescribe r ++ "..." + copied <- Remotes.copyFromRemote r key file + if copied + then return True + else continue {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an diff --git a/Command/Move.hs b/Command/Move.hs index d96d36138c..fa847e6bab 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,13 +7,11 @@ module Command.Move where -import Control.Monad (when) import Control.Monad.State (liftIO) import Command import qualified Command.Drop import qualified Annex -import Locations import LocationLog import Types import Core @@ -86,26 +84,17 @@ toPerform move key = do return Nothing Right False -> do showNote $ "to " ++ Git.repoDescribe remote ++ "..." - let tmpfile = annexTmpLocation remote ++ keyFile key - ok <- Remotes.copyToRemote remote key tmpfile + ok <- Remotes.copyToRemote remote key if ok - then return $ Just $ toCleanup move remote key tmpfile + then return $ Just $ toCleanup move remote key else return Nothing -- failed Right True -> return $ Just $ Command.Drop.cleanup key -toCleanup :: Bool -> Git.Repo -> Key -> FilePath -> CommandCleanup -toCleanup move remote key tmpfile = do - -- Tell remote to use the transferred content. - ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", - "--backend=" ++ backendName key, - "--key=" ++ keyName key, - tmpfile] - if ok - then do - remoteHasKey remote key True - if move - then Command.Drop.cleanup key - else return True - else return False +toCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup +toCleanup move remote key = do + remoteHasKey remote key True + if move + then Command.Drop.cleanup key + else return True {- Moves (or copies) the content of an annexed file from another repository - to the current repository and updates locationlog information on both. @@ -140,7 +129,9 @@ fromCleanup True remote key = do ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] - when ok $ - remoteHasKey remote key False + -- better safe than sorry: assume the remote dropped the key + -- even if it seemed to fail; the failure could have occurred + -- after it really dropped it + remoteHasKey remote key False return ok fromCleanup False _ _ = return True diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 3232010d49..840b328613 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -34,5 +34,9 @@ start keyname = do ok <- getViaTmp key (liftIO . rsyncServerReceive) if ok - then return Nothing + then do + -- forcibly quit after receiving one key, + -- and shutdown cleanly so queued git commands run + _ <- shutdown 0 + liftIO exitSuccess else liftIO exitFailure diff --git a/Remotes.hs b/Remotes.hs index 70356de024..19d1bfdd37 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -15,7 +15,6 @@ module Remotes ( byName, copyFromRemote, copyToRemote, - runCmd, onRemote ) where @@ -23,11 +22,10 @@ import Control.Exception.Extensible import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils -import System.Directory hiding (copyFile) -import System.Posix.Directory import System.Cmd.Utils import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) +import Data.Maybe import Types import qualified GitRepo as Git @@ -39,6 +37,7 @@ import Utility import qualified Core import Messages import CopyFile +import RsyncFile {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -227,92 +226,95 @@ tryGitConfigRead r then new : exchange ls new else old : exchange ls new -{- Tries to copy a key's content from a remote to a file. -} +{- Tries to copy a key's content from a remote's annex to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file - | not $ Git.repoIsUrl r = getlocal - | Git.repoIsSsh r = getssh + | not $ Git.repoIsUrl r = liftIO $ copyFile (annexLocation r key) file + | Git.repoIsSsh r = rsynchelper r True key file | otherwise = error "copying from non-ssh repo not supported" - where - keyloc = annexLocation r key - getlocal = liftIO $ copyFile keyloc file - getssh = remoteCopyFile True r (sshLocation r keyloc) file -{- Tries to copy a key's content to a file on a remote. -} -copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool -copyToRemote r key file = do - g <- Annex.gitRepo - let keyloc = annexLocation g key - if not $ Git.repoIsUrl r - then putlocal keyloc - else if Git.repoIsSsh r - then putssh keyloc - else error "copying to non-ssh repo not supported" - where - putlocal src = liftIO $ copyFile src file - putssh src = remoteCopyFile False r src (sshLocation r file) +{- Tries to copy a key's content to a remote's annex. -} +copyToRemote :: Git.Repo -> Key -> Annex Bool +copyToRemote r key + | not $ Git.repoIsUrl r = do + g <- Annex.gitRepo + let keysrc = annexLocation g key + let keydest = annexLocation r key + liftIO $ copyFile keysrc keydest + | Git.repoIsSsh r = do + g <- Annex.gitRepo + let keysrc = annexLocation g key + rsynchelper r False key keysrc + | otherwise = error "copying to non-ssh repo not supported" -sshLocation :: Git.Repo -> FilePath -> FilePath -sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file - -{- Copies a file from or to a remote, using rsync. -} -remoteCopyFile :: Bool -> Git.Repo -> String -> String -> Annex Bool -remoteCopyFile recv r src dest = do +rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool) +rsynchelper r sending key file = do showProgress -- make way for progress bar - o <- repoConfig r configopt "" - res <- liftIO $ boolSystem cmd $ options ++ words o ++ [src, dest] + p <- rsyncParams r sending key file + liftIO $ putStrLn $ unwords p + res <- liftIO $ boolSystem "rsync" p if res then return res else do showLongNote "rsync failed -- run git annex again to resume file transfer" return res + +{- Generates rsync parameters that ssh to the remote and asks it + - to either receive or send the key's content. -} +rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [String] +rsyncParams r sending key file = do + -- Note that the command is terminated with "--", because + -- rsync will tack on its own options to this command, + -- and they need to be ignored. + shellcmd <- git_annex_shell r + (if sending then "sendkey" else "recvkey") + ["--backend=" ++ backendName key, keyName key, "--"] + -- Convert the ssh command into rsync command line. + let eparam = rsyncShell $ fromJust shellcmd + o <- repoConfig r "rsync-options" "" + let base = options ++ words o ++ eparam + if sending + then return $ base ++ [dummy, file] + else return $ base ++ [file, dummy] where - cmd = "rsync" - configopt= "rsync-options" -- inplace makes rsync resume partial files options = ["-p", "--progress", "--inplace"] + -- the rsync shell parameter controls where rsync + -- does, so the source/dest parameter can be a dummy value, + -- that just enables remote rsync mode. + dummy = ":" -{- Uses a supplied function to run a git-annex-shell command on a remote. -} +{- Uses a supplied function to run a git-annex-shell command on a remote. + - + - Or, if the remote does not support running remote commands, returns + - a specified error value. -} onRemote :: Git.Repo - -> ((String -> [String] -> IO a), a) + -> (String -> [String] -> IO a, a) -> String -> [String] -> Annex a -onRemote r (with, errorval) command params - | not $ Git.repoIsUrl r = liftIO $ with shellcmd shellopts +onRemote r (with, errorval) command params = do + s <- git_annex_shell r command params + case s of + Just shellcmd -> liftIO $ with (shellcmd !! 0) (tail shellcmd) + Nothing -> return errorval + +{- Generates parameters to run a git-annex-shell command on a remote. -} +git_annex_shell :: Git.Repo -> String -> [String] -> Annex (Maybe [String]) +git_annex_shell r command params + | not $ Git.repoIsUrl r = return $ Just (shellcmd:shellopts) | Git.repoIsSsh r = do sshoptions <- repoConfig r "ssh-options" "" - liftIO $ with "ssh" $ - words sshoptions ++ [Git.urlHost r, sshcmd] - | otherwise = return errorval + return $ Just $ ["ssh"] ++ words sshoptions ++ + [Git.urlHost r, sshcmd] + | otherwise = return Nothing where dir = Git.workTree r shellcmd = "git-annex-shell" shellopts = command:dir:params sshcmd = shellcmd ++ " " ++ unwords (map shellEscape shellopts) -{- Runs a command in a remote, using ssh if necessary. - - (Honors annex-ssh-options.) -} -runCmd :: Git.Repo -> String -> [String] -> Annex Bool -runCmd r command params = do - sshoptions <- repoConfig r "ssh-options" "" - if not $ Git.repoIsUrl r - then do - cwd <- liftIO getCurrentDirectory - liftIO $ bracket_ - (changeWorkingDirectory (Git.workTree r)) - (changeWorkingDirectory cwd) - (boolSystem command params) - else if Git.repoIsSsh r - then liftIO $ boolSystem "ssh" $ - words sshoptions ++ [Git.urlHost r, sshcmd] - else error "running command in non-ssh repo not supported" - where - sshcmd = "cd " ++ shellEscape (Git.workTree r) ++ - " && " ++ shellEscape command ++ " " ++ - unwords (map shellEscape params) - {- Looks up a per-remote config option in git config. - Failing that, tries looking for a global config option. -} repoConfig :: Git.Repo -> String -> String -> Annex String diff --git a/RsyncFile.hs b/RsyncFile.hs index 14f6dc926b..274e66151b 100644 --- a/RsyncFile.hs +++ b/RsyncFile.hs @@ -7,8 +7,20 @@ module RsyncFile where -import Utility import System.Posix.Process +import Data.String.Utils + +import Utility + +{- Generates parameters to make rsync use a specified command as its remote + - shell. -} +rsyncShell :: [String] -> [String] +rsyncShell command = ["-e", unwords $ map escape command] + where + {- rsync requires some weird, non-shell like quoting in + - here. A doubled single quote inside the single quoted + - string is a single quote. -} + escape s = "'" ++ (join "''" $ split "'" s) ++ "'" {- Runs rsync in server mode to send a file, and exits. -} rsyncServerSend :: FilePath -> IO () diff --git a/doc/todo/git-annex-shell.mdwn b/doc/todo/git-annex-shell.mdwn index 47db0c1ca7..a9e3b43ede 100644 --- a/doc/todo/git-annex-shell.mdwn +++ b/doc/todo/git-annex-shell.mdwn @@ -1,3 +1,5 @@ +[[done]] + I've been considering adding a `git-annex-shell` command. This would be similar to `git-shell` (and in fact would pass unknown commands off to `git-shell`). @@ -11,38 +13,3 @@ be similar to `git-shell` (and in fact would pass unknown commands off to * Could possibly allow multiple things to be done with one ssh connection in future. * Allows expanding `~` and `~user` in repopath on the remote system. - -## Design - -`git-annex-shell -c ` - -### options - -Need at least `--quiet`, `--backend`, `--key`, `--force` - -### commands - -* `configlist repopath` - - Returns `git config --list`, for use by `tryGitConfigRead`. - - May filter the listed config to only the options git-annex really needs, - to prevent info disclosure. - -* `inannex repopath key ...` - - Checks if the keys are in the annex; shell exits zero if so. - -* `dropkey repopath key ... ` - - Same as `git annex dropkey`, and taking the same dashed options. - -* `setkey repopath tmpfile` - - Same as `git annex setkey`, and taking the same dashed options. - -### TODO - -* To be usable as a locked down shell, needs a way to launch the - rsync server, for file receiving. Safely? -* Also needs a way to support receiving files by scp. From f48658d4eeaa76062f412d4fec9cdfcecd7b0e9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 19:11:18 -0400 Subject: [PATCH 0659/2835] Now rsync is exclusively used for copying files to and from remotes. scp is not longer supported. --- debian/changelog | 2 ++ doc/git-annex.mdwn | 10 ++-------- doc/walkthrough.mdwn | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/debian/changelog b/debian/changelog index 76e16af2ce..5b3f2ebfd5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,8 @@ git-annex (0.15) UNRELEASED; urgency=low that are trusted to retain files without explicit checking. * Fix bug in numcopies handling when multiple remotes pointed to the same repository. + * Now rsync is exclusively used for copying files to and from remotes. + scp is not longer supported. -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 8e6ff2c0cc..e99be4e409 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -292,12 +292,6 @@ Here are all the supported configuration settings. git-annex caches UUIDs of repositories here. -* `remote..annex-scp-options` - - Options to use when using scp - to or from this repository. For example, to force ipv6, and limit - the bandwidth to 1000Kbit/s, set it to "-6 -l 1000" - * `remote..annex-ssh-options` Options to use when using ssh to talk to this repository. @@ -308,9 +302,9 @@ Here are all the supported configuration settings. to or from this repository. For example, to force ipv6, and limit the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" -* `annex.scp-options`, `annex.ssh-options`, `annex.rsync-options` +* `annex.ssh-options`, `annex.rsync-options` - Default scp, ssh, and rsync options to use if a remote does not have + Default ssh and rsync options to use if a remote does not have specific options. * `annex.version` diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index b486a4b1ff..c2ae583f02 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -198,7 +198,7 @@ to clone the laptop's annex to it: # cd ~/annex # git annex init "my desktop" -Now you can get files and they will be transferred (using `rsync` or `scp`): +Now you can get files and they will be transferred (using `rsync`): # git annex get my_cool_big_file get my_cool_big_file (getting UUID for origin...) (copying from origin...) From e6af35d20667221074219c8e91655c719c0f8b4a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 19:19:26 -0400 Subject: [PATCH 0660/2835] update --- debian/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/debian/changelog b/debian/changelog index 5b3f2ebfd5..d74eb8e369 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,11 @@ git-annex (0.15) UNRELEASED; urgency=low that are trusted to retain files without explicit checking. * Fix bug in numcopies handling when multiple remotes pointed to the same repository. + * Introduce the git-annex-shell command. It's now possible to make + a user have it as a restricted login shell, similar to git-shell. + * Note that git-annex will always use git-annex-shell when accessing + a ssh remote, so all of your remotes need to be upgraded to this + version of git-annex at the same time. * Now rsync is exclusively used for copying files to and from remotes. scp is not longer supported. From e153a116bb45eac409a7d4b0a07c5ba10634bd36 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 19:27:34 -0400 Subject: [PATCH 0661/2835] remove debug --- Remotes.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index 19d1bfdd37..297fa8d391 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -251,7 +251,6 @@ rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool) rsynchelper r sending key file = do showProgress -- make way for progress bar p <- rsyncParams r sending key file - liftIO $ putStrLn $ unwords p res <- liftIO $ boolSystem "rsync" p if res then return res From 5c29bb3b7c280b3e2db26dbcb38f063430f731d6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 20:33:43 -0400 Subject: [PATCH 0662/2835] git-annex-shell can now be used as a login shell --- Utility.hs | 29 ++++++++++++++++++++++++++++- git-annex-shell.hs | 20 ++++++++++++++------ test.hs | 5 ++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/Utility.hs b/Utility.hs index 3a6c757515..4ab5f09302 100644 --- a/Utility.hs +++ b/Utility.hs @@ -14,9 +14,13 @@ module Utility ( relPathDirToDir, boolSystem, shellEscape, + shellUnEscape, unsetFileMode, readMaybe, - safeWriteFile + safeWriteFile, + + prop_idempotent_shellescape, + prop_idempotent_shellescape_multiword ) where import System.IO @@ -128,6 +132,29 @@ shellEscape f = "'" ++ escaped ++ "'" -- replace ' with '"'"' escaped = join "'\"'\"'" $ split "'" f +{- Unescapes a set of shellEscaped words or filenames. -} +shellUnEscape :: String -> [String] +shellUnEscape [] = [] +shellUnEscape s = word:(shellUnEscape rest) + where + (word, rest) = findword "" s + findword w [] = (w, "") + findword w (c:cs) + | c == ' ' = (w, cs) + | c == '\'' = inquote c w cs + | c == '"' = inquote c w cs + | otherwise = findword (w++[c]) cs + inquote _ w [] = (w, "") + inquote q w (c:cs) + | c == q = findword w cs + | otherwise = inquote q (w++[c]) cs + +{- For quickcheck. -} +prop_idempotent_shellescape :: String -> Bool +prop_idempotent_shellescape s = [s] == (shellUnEscape $ shellEscape s) +prop_idempotent_shellescape_multiword :: [String] -> Bool +prop_idempotent_shellescape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) + {- Removes a FileMode from a file. - For example, call with otherWriteMode to chmod o-w -} unsetFileMode :: FilePath -> FileMode -> IO () diff --git a/git-annex-shell.hs b/git-annex-shell.hs index 8783e7f60a..251acf6132 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -7,6 +7,7 @@ import System.Environment import Control.Monad (when) +import Data.List import qualified GitRepo as Git import CmdLine @@ -43,14 +44,14 @@ main' :: [String] -> IO () main' [] = failure -- skip leading -c options, passed by eg, ssh main' ("-c":p) = main' p --- Since git-annex explicitly runs git-annex-shell, we will be passed --- a redundant "git-annex-shell" parameter when we're the user's login shell. -main' ("git-annex-shell":p) = main' p -- a command can be either a builtin or something to pass to git-shell main' c@(cmd:dir:params) | elem cmd builtins = builtin cmd dir params | otherwise = external c main' c@(cmd:_) + -- Handle the case of being the user's login shell. It will be passed + -- a single string containing all the real parameters. + | isPrefixOf "git-annex-shell " cmd = main' $ drop 1 $ shellUnEscape cmd | elem cmd builtins = failure | otherwise = external c @@ -60,13 +61,20 @@ builtins = map cmdname cmds builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do let gitrepo = Git.repoFromPath dir - dispatch gitrepo (cmd:params) cmds commonOptions header + dispatch gitrepo (cmd:(filterparams params)) cmds commonOptions header external :: [String] -> IO () -external l = do - ret <- boolSystem "git-shell" ("-c":l) +external params = do + ret <- boolSystem "git-shell" ("-c":(filterparams params)) when (not ret) $ error "git-shell failed" +-- Drop all args after "--". +-- These tend to be passed by rsync and not useful. +filterparams :: [String] -> [String] +filterparams [] = [] +filterparams ("--":_) = [] +filterparams (a:as) = a:filterparams as + failure :: IO () failure = error $ "bad parameters\n\n" ++ usage header cmds commonOptions diff --git a/test.hs b/test.hs index 9a6e05a97c..9d64e92607 100644 --- a/test.hs +++ b/test.hs @@ -3,11 +3,14 @@ import Test.HUnit.Tools import GitRepo import Locations +import Utility alltests :: [Test] alltests = [ qctest "prop_idempotent_deencode" prop_idempotent_deencode, - qctest "prop_idempotent_fileKey" prop_idempotent_fileKey + qctest "prop_idempotent_fileKey" prop_idempotent_fileKey, + qctest "prop_idempotent_shellescape" prop_idempotent_shellescape, + qctest "prop_idempotent_shellescape_multiword" prop_idempotent_shellescape_multiword ] main :: IO (Counts, Int) From ed593f1f3f810ae9e456dc233273da23608cea82 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 20:38:50 -0400 Subject: [PATCH 0663/2835] git-annex-shell makes this more tractable --- doc/bugs/bare_git_repos.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn index e24d7a7ee3..67917db8a6 100644 --- a/doc/bugs/bare_git_repos.mdwn +++ b/doc/bugs/bare_git_repos.mdwn @@ -12,3 +12,15 @@ However, that is not currently supported. Problems include: * `.git-annex/` needs to have state recorded to it and committed, and that is not possible with a bare repo. (If [[todo/branching]] were done, that might be fixed.) + +---- + +Update: Now that git-annex-shell is used for accessing remote repos, +it would be possible to add smarts about bare repos there, and avoid +some of the above problems. Probably only the state recording problem +remains. + +A possible other approach to the state recording repo is to not +record state changes on the remote in that case. Git-annex already +records remote state changes locally whenever it modifies the state of a +remote. --[[Joey]] From 14fe13dc2b0f39857eeccdc36483d0f4a0d81fdc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 21:22:03 -0400 Subject: [PATCH 0664/2835] support ssh urls containing "~", and relative user:path --- GitRepo.hs | 36 ++++++++++++++++++- debian/changelog | 5 ++- .../wishlist:_support_for_more_ssh_urls_.mdwn | 4 ++- git-annex-shell.hs | 3 +- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 9dfce0d35b..07f243b661 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -43,6 +43,7 @@ module GitRepo ( encodeGitFile, typeChangedFiles, typeChangedStagedFiles, + absDir, prop_idempotent_deencode ) where @@ -50,6 +51,7 @@ module GitRepo ( import Control.Monad (unless) import System.Directory import System.Posix.Directory +import System.Posix.User import System.Path import System.Cmd.Utils import IO (bracket_) @@ -62,7 +64,7 @@ import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf -import Data.List (isInfixOf) +import Data.List (isInfixOf, isPrefixOf) import Utility @@ -444,6 +446,38 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\"" prop_idempotent_deencode :: String -> Bool prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s) +{- Git ssh remotes can have a directory that is specified relative + - to a home directory. This converts such a directory to an absolute path. + - Note that it has to run on the remote system. + -} +absDir :: String -> IO String +absDir d + | isPrefixOf "/" d = expandt d + | otherwise = do + h <- myhomedir + return $ h ++ d + where + homedir u = (homeDirectory u) ++ "/" + myhomedir = do + uid <- getEffectiveUserID + u <- getUserEntryForID uid + return $ homedir u + expandt [] = return "" + expandt ('/':'~':'/':cs) = do + h <- myhomedir + return $ h ++ cs + expandt ('/':'~':cs) = do + let (name, rest) = findname "" cs + u <- getUserEntryForName name + return $ homedir u ++ rest + expandt (c:cs) = do + v <- expandt cs + return (c:v) + findname n [] = (n, "") + findname n (c:cs) + | c == '/' = (n, cs) + | otherwise = findname (n++[c]) cs + {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo repoFromCwd = do diff --git a/debian/changelog b/debian/changelog index d74eb8e369..b5d0058cc8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,7 @@ git-annex (0.15) UNRELEASED; urgency=low - * Support scp-style urls for remotes (host:path). Note that - paths relative to the user's home directory, or containing "~" are - not yet supported. + * Support scp-style urls for remotes (host:path). + * Support ssh urls containing "~". * Add trust and untrust subcommands, to allow configuring repositories that are trusted to retain files without explicit checking. * Fix bug in numcopies handling when multiple remotes pointed to the diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn index ee3b4a64ad..55b8120a75 100644 --- a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn @@ -17,4 +17,6 @@ Specifically, if I have ~/bar set up on host foo: > code on the remote to lookup homedirs. If git-annex grows a > `git annex shell` that is run on the remote side > (something I am [[considering|todo/git-annex-shell]] for other reasons), it -> could handle the expansions there. --[[Joey]] +> could handle the expansions there. --[[Joey]] + +> Update: Now `~` expansions are supported. [[done]] diff --git a/git-annex-shell.hs b/git-annex-shell.hs index 251acf6132..78dd777904 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -60,7 +60,8 @@ builtins = map cmdname cmds builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do - let gitrepo = Git.repoFromPath dir + dir' <- Git.absDir dir + let gitrepo = Git.repoFromPath dir' dispatch gitrepo (cmd:(filterparams params)) cmds commonOptions header external :: [String] -> IO () From 3902b05b253ffd7a39ccebf996928228bc97732f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 22:22:50 -0400 Subject: [PATCH 0665/2835] releasing version 0.15 --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index b5d0058cc8..12a2efaffa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.15) UNRELEASED; urgency=low +git-annex (0.15) unstable; urgency=low * Support scp-style urls for remotes (host:path). * Support ssh urls containing "~". @@ -12,9 +12,9 @@ git-annex (0.15) UNRELEASED; urgency=low a ssh remote, so all of your remotes need to be upgraded to this version of git-annex at the same time. * Now rsync is exclusively used for copying files to and from remotes. - scp is not longer supported. + scp is not longer supported. - -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 + -- Joey Hess Fri, 31 Dec 2010 22:00:52 -0400 git-annex (0.14) unstable; urgency=low From 2419d3d50e16113dad69d7b33c14dda43976e83f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 22:22:57 -0400 Subject: [PATCH 0666/2835] add news item for git-annex 0.15 --- doc/news/version_0.10.mdwn | 11 ----------- doc/news/version_0.15.mdwn | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) delete mode 100644 doc/news/version_0.10.mdwn create mode 100644 doc/news/version_0.15.mdwn diff --git a/doc/news/version_0.10.mdwn b/doc/news/version_0.10.mdwn deleted file mode 100644 index 450b7d272f..0000000000 --- a/doc/news/version_0.10.mdwn +++ /dev/null @@ -1,11 +0,0 @@ -git-annex 0.10 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * In .gitattributes, the annex.numcopies attribute can be used - to control the number of copies to retain of different types of files. - * Bugfix: Always correctly handle gitattributes when in a subdirectory of - the repository. (Had worked ok for ones like "*.mp3", but failed for - ones like "dir/*".) - * fsck: Fix warning about not enough copies of a file, when locations - are known, but are not available in currently configured remotes. - * precommit: Optimise to avoid calling git-check-attr more than once. - * The git-annex-backend attribute has been renamed to annex.backend."""]] \ No newline at end of file diff --git a/doc/news/version_0.15.mdwn b/doc/news/version_0.15.mdwn new file mode 100644 index 0000000000..c3d4d2c0f7 --- /dev/null +++ b/doc/news/version_0.15.mdwn @@ -0,0 +1,15 @@ +git-annex 0.15 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Support scp-style urls for remotes (host:path). + * Support ssh urls containing "~". + * Add trust and untrust subcommands, to allow configuring repositories + that are trusted to retain files without explicit checking. + * Fix bug in numcopies handling when multiple remotes pointed to the + same repository. + * Introduce the git-annex-shell command. It's now possible to make + a user have it as a restricted login shell, similar to git-shell. + * Note that git-annex will always use git-annex-shell when accessing + a ssh remote, so all of your remotes need to be upgraded to this + version of git-annex at the same time. + * Now rsync is exclusively used for copying files to and from remotes. + scp is not longer supported."""]] \ No newline at end of file From 2eb621052599f3a112d57480a5e123bb47835644 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 13:13:27 -0400 Subject: [PATCH 0667/2835] fixed --- doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn index 6ee01d2300..6616758730 100644 --- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -2,3 +2,5 @@ git-annex does not seem to support all kinds of urls that git does. > Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls_]]. > --[[Joey]] + +>> And now all fixed! --[[Joey]] From 500ddd7f4bf36cdcaa7c0bdf5dae617c36535371 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:03:29 -0400 Subject: [PATCH 0668/2835] add a check for unknown UUID when logging --- LocationLog.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/LocationLog.hs b/LocationLog.hs index 6d52f4bdd1..e4dad03f5c 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -32,6 +32,7 @@ import System.Locale import qualified Data.Map as Map import System.Directory import System.Posix.Process +import Control.Monad (when) import qualified GitRepo as Git import Utility @@ -86,6 +87,8 @@ instance Read LogLine where - and returns the filename of the logfile. -} logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath logChange repo key u s = do + when (null u) $ + error $ "bug detected: unknown UUID for " ++ Git.repoDescribe repo line <- logNow s u ls <- readLog logfile writeLog logfile (compactLog $ line:ls) From ca60731e1c9617429b5c04892f165ae5451b0fab Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:15:39 -0400 Subject: [PATCH 0669/2835] refactor --- Remotes.hs | 103 +++++++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 297fa8d391..f1df2f6adf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -43,6 +43,39 @@ import RsyncFile list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes +{- Reads the configs of remotes. + - + - This has to be called before things that rely on eg, the UUID of + - remotes. Most such things will take care of running this themselves. + - + - As reading the config of remotes can be expensive, this + - function will only read configs once per git-annex run. It's + - assumed to be cheap to read the config of non-URL remotes, + - so this is done each time git-annex is run. Conversely, + - the config of an URL remote is only read when there is no + - cached UUID value. + - -} +readconfigs :: Annex () +readconfigs = do + g <- Annex.gitRepo + remotesread <- Annex.flagIsSet "remotesread" + unless remotesread $ do + let allremotes = Git.remotes g + let cheap = filter (not . Git.repoIsUrl) allremotes + let expensive = filter Git.repoIsUrl allremotes + doexpensive <- filterM cachedUUID expensive + unless (null doexpensive) $ + showNote $ "getting UUID for " ++ + list doexpensive ++ "..." + let todo = cheap ++ doexpensive + unless (null todo) $ do + _ <- mapM tryGitConfigRead todo + Annex.flagChange "remotesread" $ FlagBool True + where + cachedUUID r = do + u <- getUUID r + return $ null u + {- Cost ordered lists of remotes that the LocationLog indicate may have a key. - - The first list is of remotes that are trusted to have the key. @@ -54,54 +87,30 @@ list remotes = join ", " $ map Git.repoDescribe remotes -} keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID]) keyPossibilities key = do + readconfigs + allremotes <- remotesByCost - -- To determine if a remote has a key, its UUID needs to be known. - -- The locally cached UUIDs of remotes can fall out of date if - -- eg, a different drive is mounted at the same location. - -- But, reading the config of remotes can be expensive, so make - -- sure we only do it once per git-annex run. - remotesread <- Annex.flagIsSet "remotesread" - if remotesread - then partition allremotes - else do - -- We assume that it's cheap to read the config - -- of non-URL remotes, so that is done each time. - -- But reading the config of an URL remote is - -- only done when there is no cached UUID value. - let cheap = filter (not . Git.repoIsUrl) allremotes - let expensive = filter Git.repoIsUrl allremotes - doexpensive <- filterM cachedUUID expensive - unless (null doexpensive) $ - showNote $ "getting UUID for " ++ - list doexpensive ++ "..." - let todo = cheap ++ doexpensive - if not $ null todo - then do - _ <- mapM tryGitConfigRead todo - Annex.flagChange "remotesread" $ FlagBool True - keyPossibilities key - else partition allremotes - where - cachedUUID r = do - u <- getUUID r - return $ null u - partition remotes = do - g <- Annex.gitRepo - u <- getUUID g - trusted <- getTrusted - -- get uuids of other repositories that are - -- believed to have the key - uuids <- liftIO $ keyLocations g key - let validuuids = filter (/= u) uuids - -- get uuids trusted to have the key - -- note that validuuids is assumed to not have dups - let validtrusteduuids = intersect validuuids trusted - -- remotes that match uuids that have the key - validremotes <- reposByUUID remotes validuuids - -- partition out the trusted and untrusted remotes - trustedremotes <- reposByUUID validremotes validtrusteduuids - untrustedremotes <- reposWithoutUUID validremotes trusted - return (trustedremotes, untrustedremotes, validtrusteduuids) + g <- Annex.gitRepo + u <- getUUID g + trusted <- getTrusted + + -- get uuids of other repositories that are + -- believed to have the key + uuids <- liftIO $ keyLocations g key + let validuuids = filter (/= u) uuids + + -- get uuids trusted to have the key + -- note that validuuids is assumed to not have dups + let validtrusteduuids = intersect validuuids trusted + + -- remotes that match uuids that have the key + validremotes <- reposByUUID allremotes validuuids + + -- partition out the trusted and untrusted remotes + trustedremotes <- reposByUUID validremotes validtrusteduuids + untrustedremotes <- reposWithoutUUID validremotes trusted + + return (trustedremotes, untrustedremotes, validtrusteduuids) {- Checks if a given remote has the content for a key inAnnex. - If the remote cannot be accessed, returns a Left error. From 533419147c3578ae935150b31e1a8a01f8bcfe6f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:20:35 -0400 Subject: [PATCH 0670/2835] reorg --- Remotes.hs | 77 +++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index f1df2f6adf..17d1bd0b0a 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -43,7 +43,42 @@ import RsyncFile list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes -{- Reads the configs of remotes. +{- The git configs for the git repo's remotes is not read on startup + - because reading it may be expensive. This function tries to read the + - config for a specified remote, and updates state. If successful, it + - returns the updated git repo. -} +tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) +tryGitConfigRead r + | not $ Map.null $ Git.configMap r = return $ Right r -- already read + | Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] + | Git.repoIsUrl r = return $ Left r + | otherwise = store $ safely $ Git.configRead r + where + -- Reading config can fail due to IO error or + -- for other reasons; catch all possible exceptions. + safely a = do + result <- liftIO (try (a)::IO (Either SomeException Git.Repo)) + case result of + Left _ -> return r + Right r' -> return r' + pipedconfig cmd params = safely $ + pOpen ReadFromPipe cmd params $ + Git.hConfigRead r + store a = do + r' <- a + g <- Annex.gitRepo + let l = Git.remotes g + let g' = Git.remotesAdd g $ exchange l r' + Annex.gitRepoChange g' + return $ Right r' + exchange [] _ = [] + exchange (old:ls) new = + if Git.repoRemoteName old == Git.repoRemoteName new + then new : exchange ls new + else old : exchange ls new + + +{- Reads the configs of all remotes. - - This has to be called before things that rely on eg, the UUID of - remotes. Most such things will take care of running this themselves. @@ -55,8 +90,8 @@ list remotes = join ", " $ map Git.repoDescribe remotes - the config of an URL remote is only read when there is no - cached UUID value. - -} -readconfigs :: Annex () -readconfigs = do +readConfigs :: Annex () +readConfigs = do g <- Annex.gitRepo remotesread <- Annex.flagIsSet "remotesread" unless remotesread $ do @@ -87,7 +122,7 @@ readconfigs = do -} keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID]) keyPossibilities key = do - readconfigs + readConfigs allremotes <- remotesByCost g <- Annex.gitRepo @@ -201,40 +236,6 @@ byName name = do "there is no git remote named \"" ++ name ++ "\"" return $ head match -{- The git configs for the git repo's remotes is not read on startup - - because reading it may be expensive. This function tries to read the - - config for a specified remote, and updates state. If successful, it - - returns the updated git repo. -} -tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) -tryGitConfigRead r - | not $ Map.null $ Git.configMap r = return $ Right r -- already read - | Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] - | Git.repoIsUrl r = return $ Left r - | otherwise = store $ safely $ Git.configRead r - where - -- Reading config can fail due to IO error or - -- for other reasons; catch all possible exceptions. - safely a = do - result <- liftIO (try (a)::IO (Either SomeException Git.Repo)) - case result of - Left _ -> return r - Right r' -> return r' - pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd params $ - Git.hConfigRead r - store a = do - r' <- a - g <- Annex.gitRepo - let l = Git.remotes g - let g' = Git.remotesAdd g $ exchange l r' - Annex.gitRepoChange g' - return $ Right r' - exchange [] _ = [] - exchange (old:ls) new = - if Git.repoRemoteName old == Git.repoRemoteName new - then new : exchange ls new - else old : exchange ls new - {- Tries to copy a key's content from a remote's annex to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file From b001e6573c09235f9d22cb4daf034369f6bff11b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:33:24 -0400 Subject: [PATCH 0671/2835] expand --- doc/git-annex-shell.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index 9f51b6813d..1231ef668d 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -36,6 +36,10 @@ can be used as a login shell for SSH accounts. This runs rsync in server mode to transfer out the content of a key. +Note that the directory parameter should be an absolute path, otherwise +it is assumed to be relative to the user's home directory. Also the +first "/~/" or "/~user/" is expanded to the specified home directory. + Any other command is passed through to git-shell. # OPTIONS From a857e1f4ee247cae0cf0ce6696a9015460d117de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:34:14 -0400 Subject: [PATCH 0672/2835] git-annex-shell: Avoid exposing any git repo config except for the annex.uuid when doing configlist. --- Command/ConfigList.hs | 5 +++-- debian/changelog | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs index 0d9d789b54..b91c2a9160 100644 --- a/Command/ConfigList.hs +++ b/Command/ConfigList.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import Annex import Command -import qualified GitRepo as Git +import UUID command :: [Command] command = [Command "configlist" paramNothing seek @@ -23,5 +23,6 @@ seek = [withNothing start] start :: CommandStartNothing start = do g <- Annex.gitRepo - liftIO $ Git.run g ["config", "--list"] + u <- getUUID g + liftIO $ putStrLn $ "annex.uuid=" ++ u return Nothing diff --git a/debian/changelog b/debian/changelog index 12a2efaffa..1c265f1be4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (0.16) UNRELEASED; urgency=low + + * git-annex-shell: Avoid exposing any git repo config except for the + annex.uuid when doing configlist. + + -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 + git-annex (0.15) unstable; urgency=low * Support scp-style urls for remotes (host:path). From f1b747e6d9fae2b365f65fd43c6295da503218bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:45:27 -0400 Subject: [PATCH 0673/2835] bugfix: Running `move --to` with a remote whose UUID was not yet known * bugfix: Running `move --to` with a remote whose UUID was not yet known could result in git-annex not recording on the local side where the file was moved to. This could not result in data loss, or even a significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. --- Command/Move.hs | 1 + Remotes.hs | 2 +- debian/changelog | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Command/Move.hs b/Command/Move.hs index fa847e6bab..0077618f8b 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -75,6 +75,7 @@ toStart move file = isAnnexed file $ \(key, _) -> do return $ Just $ toPerform move key toPerform :: Bool -> Key -> CommandPerform toPerform move key = do + Remotes.readConfigs -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key diff --git a/Remotes.hs b/Remotes.hs index 17d1bd0b0a..da5b3e6229 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -7,8 +7,8 @@ module Remotes ( list, + readConfigs, keyPossibilities, - tryGitConfigRead, inAnnex, same, commandLineRemote, diff --git a/debian/changelog b/debian/changelog index 1c265f1be4..99bcc95657 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,12 @@ git-annex (0.16) UNRELEASED; urgency=low * git-annex-shell: Avoid exposing any git repo config except for the annex.uuid when doing configlist. + * bugfix: Running `move --to` with a remote whose UUID was not yet known + could result in git-annex not recording on the local side where the + file was moved to. This could not result in data loss, or even a + significant problem, since the remote *did* record that it had the file. + * Also, add a general guard to detect attempts to record information + about repositories with missing UUIDs. -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 From 759e860e4b6c514b74cafb2c0dd9c52c4d59316b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 21:05:31 -0400 Subject: [PATCH 0674/2835] add testcoverage target using hpc added a test for key read and show --- .gitignore | 2 ++ Makefile | 9 ++++++++- TypeInternals.hs | 18 ++++++++++++++++++ test.hs | 3 +++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d2f4c2b743..f68d1d0adf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ git-annex.1 git-annex-shell.1 doc/.ikiwiki html +*.tix +.hpc diff --git a/Makefile b/Makefile index 2f1fd05b91..8d124f1431 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,13 @@ test: $(GHCMAKE) test ./test +testcoverage: + rm -f test.tix test + ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test + ./test + hpc report test --exclude=Main --exclude=QC + hpc markup test --exclude=Main --exclude=QC --destdir=.hpc + # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. ifeq ($(shell which ikiwiki),) @@ -49,7 +56,7 @@ docs: $(mans) --exclude='news/.*' clean: - rm -rf build $(bins) $(mans) test configure SysConfig.hs + rm -rf build $(bins) $(mans) test configure SysConfig.hs *.tix .hpc rm -rf doc/.ikiwiki html .PHONY: $(bins) test install diff --git a/TypeInternals.hs b/TypeInternals.hs index 9acc06bb33..fe6e562f95 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -12,6 +12,7 @@ module TypeInternals where import Control.Monad.State (StateT) import Data.String.Utils import qualified Data.Map as M +import Test.QuickCheck import qualified GitRepo as Git import qualified GitQueue @@ -57,6 +58,23 @@ instance Read Key where b = head l k = join ":" $ drop 1 l +-- for quickcheck +instance Arbitrary Key where + arbitrary = do + backendname <- arbitrary + keyname <- arbitrary + return $ Key (backendname, keyname) + +prop_idempotent_key_read_show :: Key -> Bool +prop_idempotent_key_read_show k + -- filter out empty key or backend names + -- also backend names will not contain colons + | null kname || null bname || elem ':' bname = True + | otherwise = k == (read $ show k) + where + bname = backendName k + kname = keyName k + backendName :: Key -> BackendName backendName (Key (b,_)) = b keyName :: Key -> KeyName diff --git a/test.hs b/test.hs index 9d64e92607..28b54b78b8 100644 --- a/test.hs +++ b/test.hs @@ -4,14 +4,17 @@ import Test.HUnit.Tools import GitRepo import Locations import Utility +import TypeInternals alltests :: [Test] alltests = [ qctest "prop_idempotent_deencode" prop_idempotent_deencode, qctest "prop_idempotent_fileKey" prop_idempotent_fileKey, + qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show, qctest "prop_idempotent_shellescape" prop_idempotent_shellescape, qctest "prop_idempotent_shellescape_multiword" prop_idempotent_shellescape_multiword ] main :: IO (Counts, Int) main = runVerboseTests (TestList alltests) + From aedc46caca6f24755c914f105d1ac6d9787a7755 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 21:07:58 -0400 Subject: [PATCH 0675/2835] cleanup --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8d124f1431..338aa947e2 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,8 @@ testcoverage: rm -f test.tix test ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test ./test - hpc report test --exclude=Main --exclude=QC - hpc markup test --exclude=Main --exclude=QC --destdir=.hpc + @hpc report test --exclude=Main --exclude=QC + @hpc markup test --exclude=Main --exclude=QC --destdir=.hpc >/dev/null # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. From a323463726096bba15b6a353d35bc1eb0ae238b9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 21:27:08 -0400 Subject: [PATCH 0676/2835] add more tests --- Utility.hs | 31 ++++++++++++++++++++++++------- test.hs | 15 ++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/Utility.hs b/Utility.hs index 4ab5f09302..96bbc89ee2 100644 --- a/Utility.hs +++ b/Utility.hs @@ -19,8 +19,10 @@ module Utility ( readMaybe, safeWriteFile, - prop_idempotent_shellescape, - prop_idempotent_shellescape_multiword + prop_idempotent_shellEscape, + prop_idempotent_shellEscape_multiword, + prop_parentDir_basics, + prop_relPathDirToDir_basics ) where import System.IO @@ -45,7 +47,7 @@ readFileStrict :: FilePath -> IO String readFileStrict f = readFile f >>= \s -> length s `seq` return s {- Returns the parent directory of a path. Parent of / is "" -} -parentDir :: String -> String +parentDir :: FilePath -> FilePath parentDir dir = if not $ null dirs then slash ++ join s (take (length dirs - 1) dirs) @@ -55,6 +57,14 @@ parentDir dir = slash = if isAbsolute dir then s else "" s = [pathSeparator] +prop_parentDir_basics :: FilePath -> Bool +prop_parentDir_basics dir + | null dir = True + | dir == "/" = parentDir dir == "" + | otherwise = p /= dir + where + p = parentDir dir + {- Converts a filename into a normalized, absolute path. -} absPath :: FilePath -> IO FilePath absPath file = do @@ -97,6 +107,13 @@ relPathDirToDir from to = numcommon = length common path = join s $ dotdots ++ uncommon +prop_relPathDirToDir_basics :: FilePath -> FilePath -> Bool +prop_relPathDirToDir_basics from to + | from == to = null r + | otherwise = not (null r) && (last r == '/') + where + r = relPathDirToDir from to + {- Run a system command, and returns True or False - if it succeeded or failed. - @@ -150,10 +167,10 @@ shellUnEscape s = word:(shellUnEscape rest) | otherwise = inquote q (w++[c]) cs {- For quickcheck. -} -prop_idempotent_shellescape :: String -> Bool -prop_idempotent_shellescape s = [s] == (shellUnEscape $ shellEscape s) -prop_idempotent_shellescape_multiword :: [String] -> Bool -prop_idempotent_shellescape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) +prop_idempotent_shellEscape :: String -> Bool +prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s) +prop_idempotent_shellEscape_multiword :: [String] -> Bool +prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) {- Removes a FileMode from a file. - For example, call with otherWriteMode to chmod o-w -} diff --git a/test.hs b/test.hs index 28b54b78b8..77a8606b3a 100644 --- a/test.hs +++ b/test.hs @@ -7,14 +7,15 @@ import Utility import TypeInternals alltests :: [Test] -alltests = [ - qctest "prop_idempotent_deencode" prop_idempotent_deencode, - qctest "prop_idempotent_fileKey" prop_idempotent_fileKey, - qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show, - qctest "prop_idempotent_shellescape" prop_idempotent_shellescape, - qctest "prop_idempotent_shellescape_multiword" prop_idempotent_shellescape_multiword +alltests = + [ qctest "prop_idempotent_deencode" prop_idempotent_deencode + , qctest "prop_idempotent_fileKey" prop_idempotent_fileKey + , qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show + , qctest "prop_idempotent_shellEscape" prop_idempotent_shellEscape + , qctest "prop_idempotent_shellEscape_multiword" prop_idempotent_shellEscape_multiword + , qctest "prop_parentDir_basics" prop_parentDir_basics + , qctest "prop_relPathDirToDir_basics" prop_relPathDirToDir_basics ] main :: IO (Counts, Int) main = runVerboseTests (TestList alltests) - From 446978c1e6de0996144ba30adc178d358ed877b7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 22:14:24 -0400 Subject: [PATCH 0677/2835] fix reversion in annex-ignored --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index da5b3e6229..5bb2efa554 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -95,7 +95,7 @@ readConfigs = do g <- Annex.gitRepo remotesread <- Annex.flagIsSet "remotesread" unless remotesread $ do - let allremotes = Git.remotes g + allremotes <- filterM repoNotIgnored $ Git.remotes g let cheap = filter (not . Git.repoIsUrl) allremotes let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive From 27619497ec02685b740c838e512fc61e013c4ec3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 22:17:18 -0400 Subject: [PATCH 0678/2835] remove file before running cp This way, if a temp file was left behind, and permissions don't allow it to be written, cp won't fail. --- CopyFile.hs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CopyFile.hs b/CopyFile.hs index 8bd07dc35a..e913aa070d 100644 --- a/CopyFile.hs +++ b/CopyFile.hs @@ -7,13 +7,20 @@ module CopyFile (copyFile) where +import Control.Monad (when) +import System.Directory (doesFileExist, removeFile) + import Utility import qualified SysConfig {- The cp command is used, because I hate reinventing the wheel, - and because this allows easy access to features like cp --reflink. -} copyFile :: FilePath -> FilePath -> IO Bool -copyFile src dest = boolSystem "cp" opts +copyFile src dest = do + e <- doesFileExist dest + when e $ + removeFile dest + boolSystem "cp" opts where opts = if SysConfig.cp_reflink_auto then ["--reflink=auto", src, dest] From d134da6dab42d4c99a614fd4081d6d9b8b7b51ad Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 5 Jan 2011 20:28:50 -0400 Subject: [PATCH 0679/2835] tweak message --- Backend/File.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend/File.hs b/Backend/File.hs index 9bc5a2aa63..073a7c2267 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -137,7 +137,7 @@ showLocations key exclude = do showTriedRemotes :: [Git.Repo] -> Annex () showTriedRemotes [] = return () showTriedRemotes remotes = - showLongNote $ "I was unable to access these remotes: " ++ + showLongNote $ "Unable to access these remotes: " ++ Remotes.list remotes getNumCopies :: Maybe Int -> Annex Int From f936be30c36420072557a80a4baf1fabbeeb7b46 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 5 Jan 2011 20:29:11 -0400 Subject: [PATCH 0680/2835] message tweaked --- doc/walkthrough.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index c2ae583f02..2f868e3030 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -92,7 +92,7 @@ it: # git annex get video/hackity_hack_and_kaxxt.mov get video/_why_hackity_hack_and_kaxxt.mov (not available) - I was unable to access these remotes: usbdrive, server + Unable to access these remotes: usbdrive, server Try making some of these repositories available: 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive @@ -122,7 +122,7 @@ you'll see something like this. # git annex drop important_file other.iso drop important_file (unsafe) Could only verify the existence of 0 out of 1 necessary copies - I was unable to access these remotes: usbdrive + Unable to access these remotes: usbdrive Try making some of these repositories available: 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive From e15385c339ca8bf1d9896b542a866736b6dca924 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 18:12:52 -0400 Subject: [PATCH 0681/2835] git-annex setup --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..b98b07d7d2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git-annex/*.log merge=union From 2772faf921e3c3e68f10876303c158f76ce6fae6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 18:12:52 -0400 Subject: [PATCH 0682/2835] git annex init --- .git-annex/uuid.log | 1 + 1 file changed, 1 insertion(+) create mode 100644 .git-annex/uuid.log diff --git a/.git-annex/uuid.log b/.git-annex/uuid.log new file mode 100644 index 0000000000..8cd9452b64 --- /dev/null +++ b/.git-annex/uuid.log @@ -0,0 +1 @@ +1ac368a4-19e2-11e0-8c0f-8fcd42cf5a8d test repo From 901cdbde78c79a3cc6f5a53b10f925e21b8343b5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 20:09:49 -0400 Subject: [PATCH 0683/2835] added some toplevel git-annex subcommand tests note that test coverage doesn't work for those yet --- debian/changelog | 1 + test.hs | 108 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index 99bcc95657..d9aa1e4de3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. + * Test suite improvements. -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/test.hs b/test.hs index 77a8606b3a..b6b0c27405 100644 --- a/test.hs +++ b/test.hs @@ -1,14 +1,38 @@ +{- git-annex test suite + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + import Test.HUnit import Test.HUnit.Tools +import System.Directory +import System.Posix.Directory (changeWorkingDirectory) +import System.Posix.Files +import System.Posix.Env +import IO (bracket_, bracket) +import Control.Monad (unless, when) +import Data.List +import System.IO.Error -import GitRepo +import qualified GitRepo as Git import Locations import Utility import TypeInternals -alltests :: [Test] -alltests = - [ qctest "prop_idempotent_deencode" prop_idempotent_deencode +main :: IO (Counts, Int) +main = do + -- Add current directory to the from of PATH, so git-annex etc will + -- be used, no matter where it is run from. + cwd <- getCurrentDirectory + p <- getEnvDefault "PATH" "" + setEnv "PATH" (cwd++":"++p) True + runVerboseTests $ TestList [quickchecks, toplevels] + +quickchecks :: Test +quickchecks = TestLabel "quickchecks" $ TestList + [ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode , qctest "prop_idempotent_fileKey" prop_idempotent_fileKey , qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show , qctest "prop_idempotent_shellEscape" prop_idempotent_shellEscape @@ -17,5 +41,77 @@ alltests = , qctest "prop_relPathDirToDir_basics" prop_relPathDirToDir_basics ] -main :: IO (Counts, Int) -main = runVerboseTests (TestList alltests) +toplevels :: Test +toplevels = TestLabel "toplevel" $ TestList + [ test_init + , test_add + ] + +test_init :: Test +test_init = TestLabel "git-annex init" $ TestCase $ ingitrepo $ do + git_annex "init" ["-q", reponame] @? "init failed" + e <- doesFileExist annexlog + unless e $ + assertFailure $ annexlog ++ " not created" + c <- readFile annexlog + unless (isInfixOf reponame c) $ + assertFailure $ annexlog ++ " does not contain repo name" + where + annexlog = ".git-annex/uuid.log" + reponame = "test repo" + +test_add :: Test +test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do + writeFile file content + git_annex "add" ["-q", "foo"] @? "add failed" + s <- getSymbolicLinkStatus file + unless (isSymbolicLink s) $ + assertFailure "git-annex add did not create symlink" + c <- readFile file + unless (c == content) $ + assertFailure "file content changed during git-annex add" + r <- try (writeFile file $ content++"bar") + case r of + Left _ -> return () -- expected permission error + Right _ -> assertFailure "was able to modify annexed file content" + where + file = "foo" + content = "foo file content" + +git_annex :: String -> [String] -> IO Bool +git_annex command params = boolSystem "git-annex" (command:params) + +inannex :: Assertion -> Assertion +inannex a = ingitrepo $ do + git_annex "init" ["-q", reponame] @? "init failed" + a + where + reponame = "test repo" + +ingitrepo :: Assertion -> Assertion +ingitrepo a = withgitrepo $ \r -> do + cwd <- getCurrentDirectory + bracket_ (changeWorkingDirectory $ Git.workTree r) + (\_ -> changeWorkingDirectory cwd) + a + +withgitrepo :: (Git.Repo -> Assertion) -> Assertion +withgitrepo = bracket setup cleanup + where + tmpdir = ".t" + repodir = tmpdir ++ "/repo" + setup = do + cleanup True + createDirectory tmpdir + ok <- boolSystem "git" ["init", "-q", repodir] + unless ok $ + assertFailure "git init failed" + return $ Git.repoFromPath repodir + cleanup _ = do + e <- doesDirectoryExist tmpdir + when e $ do + -- git-annex prevents annexed file content + -- from being removed with permissions + -- bits; undo + _ <- boolSystem "chmod" ["+rw", "-R", tmpdir] + removeDirectoryRecursive tmpdir From 2533d826fc265b56556f8a6b9759d98771f79f53 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 20:26:57 -0400 Subject: [PATCH 0684/2835] make test suite link in git-annex's commands and run directly this way, test coverage works --- GitAnnex.hs | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ debian/changelog | 2 +- git-annex.hs | 66 +----------------------------------------- test.hs | 42 ++++++++++++++------------- 4 files changed, 99 insertions(+), 86 deletions(-) create mode 100644 GitAnnex.hs diff --git a/GitAnnex.hs b/GitAnnex.hs new file mode 100644 index 0000000000..05e98d3c3a --- /dev/null +++ b/GitAnnex.hs @@ -0,0 +1,75 @@ +{- git-annex main program + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module GitAnnex where + +import System.Console.GetOpt + +import Command +import Options + +import qualified Command.Add +import qualified Command.Unannex +import qualified Command.Drop +import qualified Command.Move +import qualified Command.Copy +import qualified Command.Get +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.SetKey +import qualified Command.Fix +import qualified Command.Init +import qualified Command.Fsck +import qualified Command.Unused +import qualified Command.DropUnused +import qualified Command.Unlock +import qualified Command.Lock +import qualified Command.PreCommit +import qualified Command.Find +import qualified Command.Uninit +import qualified Command.Trust +import qualified Command.Untrust + +cmds :: [Command] +cmds = concat + [ Command.Add.command + , Command.Get.command + , Command.Drop.command + , Command.Move.command + , Command.Copy.command + , Command.Unlock.command + , Command.Lock.command + , Command.Init.command + , Command.Unannex.command + , Command.Uninit.command + , Command.PreCommit.command + , Command.Trust.command + , Command.Untrust.command + , Command.FromKey.command + , Command.DropKey.command + , Command.SetKey.command + , Command.Fix.command + , Command.Fsck.command + , Command.Unused.command + , Command.DropUnused.command + , Command.Find.command + ] + +options :: [Option] +options = commonOptions ++ + [ Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + "skip files matching the glob pattern" + ] + +header :: String +header = "Usage: git-annex command [option ..]" diff --git a/debian/changelog b/debian/changelog index d9aa1e4de3..0aaaa75e60 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,7 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. - * Test suite improvements. + * Test suite improvements. Current top-level test coverage: 43% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/git-annex.hs b/git-annex.hs index dff67f9d84..f951817847 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,74 +6,10 @@ -} import System.Environment -import System.Console.GetOpt import qualified GitRepo as Git import CmdLine -import Command -import Options - -import qualified Command.Add -import qualified Command.Unannex -import qualified Command.Drop -import qualified Command.Move -import qualified Command.Copy -import qualified Command.Get -import qualified Command.FromKey -import qualified Command.DropKey -import qualified Command.SetKey -import qualified Command.Fix -import qualified Command.Init -import qualified Command.Fsck -import qualified Command.Unused -import qualified Command.DropUnused -import qualified Command.Unlock -import qualified Command.Lock -import qualified Command.PreCommit -import qualified Command.Find -import qualified Command.Uninit -import qualified Command.Trust -import qualified Command.Untrust - -cmds :: [Command] -cmds = concat - [ Command.Add.command - , Command.Get.command - , Command.Drop.command - , Command.Move.command - , Command.Copy.command - , Command.Unlock.command - , Command.Lock.command - , Command.Init.command - , Command.Unannex.command - , Command.Uninit.command - , Command.PreCommit.command - , Command.Trust.command - , Command.Untrust.command - , Command.FromKey.command - , Command.DropKey.command - , Command.SetKey.command - , Command.Fix.command - , Command.Fsck.command - , Command.Unused.command - , Command.DropUnused.command - , Command.Find.command - ] - -options :: [Option] -options = commonOptions ++ - [ Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) - "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) - "skip files matching the glob pattern" - ] - -header :: String -header = "Usage: git-annex command [option ..]" +import GitAnnex main :: IO () main = do diff --git a/test.hs b/test.hs index b6b0c27405..74cce41427 100644 --- a/test.hs +++ b/test.hs @@ -10,35 +10,30 @@ import Test.HUnit.Tools import System.Directory import System.Posix.Directory (changeWorkingDirectory) import System.Posix.Files -import System.Posix.Env import IO (bracket_, bracket) import Control.Monad (unless, when) import Data.List import System.IO.Error import qualified GitRepo as Git -import Locations -import Utility -import TypeInternals +import qualified Locations +import qualified Utility +import qualified TypeInternals +import qualified GitAnnex +import qualified CmdLine main :: IO (Counts, Int) -main = do - -- Add current directory to the from of PATH, so git-annex etc will - -- be used, no matter where it is run from. - cwd <- getCurrentDirectory - p <- getEnvDefault "PATH" "" - setEnv "PATH" (cwd++":"++p) True - runVerboseTests $ TestList [quickchecks, toplevels] +main = runVerboseTests $ TestList [quickchecks, toplevels] quickchecks :: Test quickchecks = TestLabel "quickchecks" $ TestList [ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode - , qctest "prop_idempotent_fileKey" prop_idempotent_fileKey - , qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show - , qctest "prop_idempotent_shellEscape" prop_idempotent_shellEscape - , qctest "prop_idempotent_shellEscape_multiword" prop_idempotent_shellEscape_multiword - , qctest "prop_parentDir_basics" prop_parentDir_basics - , qctest "prop_relPathDirToDir_basics" prop_relPathDirToDir_basics + , qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey + , qctest "prop_idempotent_key_read_show" TypeInternals.prop_idempotent_key_read_show + , qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape + , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword + , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics + , qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics ] toplevels :: Test @@ -79,7 +74,14 @@ test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do content = "foo file content" git_annex :: String -> [String] -> IO Bool -git_annex command params = boolSystem "git-annex" (command:params) +git_annex command params = do + gitrepo <- Git.repoFromCwd + r <- try $ + CmdLine.dispatch gitrepo (command:params) + GitAnnex.cmds GitAnnex.options GitAnnex.header + case r of + Right _ -> return True + Left _ -> return False inannex :: Assertion -> Assertion inannex a = ingitrepo $ do @@ -103,7 +105,7 @@ withgitrepo = bracket setup cleanup setup = do cleanup True createDirectory tmpdir - ok <- boolSystem "git" ["init", "-q", repodir] + ok <- Utility.boolSystem "git" ["init", "-q", repodir] unless ok $ assertFailure "git init failed" return $ Git.repoFromPath repodir @@ -113,5 +115,5 @@ withgitrepo = bracket setup cleanup -- git-annex prevents annexed file content -- from being removed with permissions -- bits; undo - _ <- boolSystem "chmod" ["+rw", "-R", tmpdir] + _ <- Utility.boolSystem "chmod" ["+rw", "-R", tmpdir] removeDirectoryRecursive tmpdir From 87f424eca7f8d9ce7a437fb3756755c042fe9002 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 21:39:26 -0400 Subject: [PATCH 0685/2835] more tests --- Makefile | 1 + debian/changelog | 2 +- test.hs | 139 ++++++++++++++++++++++++++++++++++++----------- 3 files changed, 108 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index 338aa947e2..e499d492c7 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ testcoverage: rm -f test.tix test ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test ./test + @echo "" @hpc report test --exclude=Main --exclude=QC @hpc markup test --exclude=Main --exclude=QC --destdir=.hpc >/dev/null diff --git a/debian/changelog b/debian/changelog index 0aaaa75e60..edcd349f55 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,7 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. - * Test suite improvements. Current top-level test coverage: 43% + * Test suite improvements. Current top-level test coverage: 53% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/test.hs b/test.hs index 74cce41427..3ad34971aa 100644 --- a/test.hs +++ b/test.hs @@ -14,6 +14,7 @@ import IO (bracket_, bracket) import Control.Monad (unless, when) import Data.List import System.IO.Error +import qualified Control.Exception.Extensible as E import qualified GitRepo as Git import qualified Locations @@ -23,7 +24,10 @@ import qualified GitAnnex import qualified CmdLine main :: IO (Counts, Int) -main = runVerboseTests $ TestList [quickchecks, toplevels] +main = do + r <- runVerboseTests $ TestList [quickchecks, toplevels] + cleanup tmpdir + return r quickchecks :: Test quickchecks = TestLabel "quickchecks" $ TestList @@ -38,12 +42,15 @@ quickchecks = TestLabel "quickchecks" $ TestList toplevels :: Test toplevels = TestLabel "toplevel" $ TestList + -- test order matters, later tests may rely on state from earlier [ test_init , test_add + , test_unannex + , test_drop ] test_init :: Test -test_init = TestLabel "git-annex init" $ TestCase $ ingitrepo $ do +test_init = TestLabel "git-annex init" $ TestCase $ innewrepo $ do git_annex "init" ["-q", reponame] @? "init failed" e <- doesFileExist annexlog unless e $ @@ -56,7 +63,7 @@ test_init = TestLabel "git-annex init" $ TestCase $ ingitrepo $ do reponame = "test repo" test_add :: Test -test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do +test_add = TestLabel "git-annex add" $ TestCase $ inoldrepo $ do writeFile file content git_annex "add" ["-q", "foo"] @? "add failed" s <- getSymbolicLinkStatus file @@ -65,7 +72,7 @@ test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do c <- readFile file unless (c == content) $ assertFailure "file content changed during git-annex add" - r <- try (writeFile file $ content++"bar") + r <- try $ writeFile file $ content++"bar" case r of Left _ -> return () -- expected permission error Right _ -> assertFailure "was able to modify annexed file content" @@ -73,47 +80,113 @@ test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do file = "foo" content = "foo file content" +test_unannex :: Test +test_unannex = TestLabel "git-annex unannex" $ TestCase $ intmpcopyrepo $ do + git_annex "unannex" ["-q", "foo"] @? "unannex failed" + s <- getSymbolicLinkStatus "foo" + when (isSymbolicLink s) $ + assertFailure "git-annex unannex left symlink" + +test_drop :: Test +test_drop = TestLabel "git-annex drop" $ TestCase $ intmpcopyrepo $ do + r <- git_annex "drop" ["-q", "foo"] + (not r) @? "drop wrongly succeeded with no known copy of file" + checklink + git_annex "drop" ["-q", "--force", "foo"] @? "drop --force failed" + checklink + r' <- try $ readFile "foo" + case r' of + Left _ -> return () -- expected; dangling link + Right _ -> assertFailure "drop did not remove file content" + where + checklink = do + s <- getSymbolicLinkStatus "foo" + unless (isSymbolicLink s) $ + assertFailure "git-annex drop killed symlink" + + + + git_annex :: String -> [String] -> IO Bool git_annex command params = do - gitrepo <- Git.repoFromCwd - r <- try $ - CmdLine.dispatch gitrepo (command:params) - GitAnnex.cmds GitAnnex.options GitAnnex.header + -- catch all errors, including normally fatal errors + r <- E.try (run)::IO (Either E.SomeException ()) case r of Right _ -> return True Left _ -> return False + where + run = do + gitrepo <- Git.repoFromCwd + CmdLine.dispatch gitrepo (command:params) + GitAnnex.cmds GitAnnex.options GitAnnex.header -inannex :: Assertion -> Assertion -inannex a = ingitrepo $ do +innewannex :: Assertion -> Assertion +innewannex a = innewrepo $ do git_annex "init" ["-q", reponame] @? "init failed" a where reponame = "test repo" -ingitrepo :: Assertion -> Assertion -ingitrepo a = withgitrepo $ \r -> do +innewrepo :: Assertion -> Assertion +innewrepo a = withgitrepo $ \r -> indir r a + +inoldrepo :: Assertion -> Assertion +inoldrepo = indir repodir + +intmpcopyrepo :: Assertion -> Assertion +intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a + +withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion +withtmpcopyrepo = bracket (copyrepo repodir tmprepodir) cleanup + +withgitrepo :: (FilePath -> Assertion) -> Assertion +withgitrepo = bracket (setuprepo repodir) return + +indir :: FilePath -> Assertion -> Assertion +indir dir a = do cwd <- getCurrentDirectory - bracket_ (changeWorkingDirectory $ Git.workTree r) + bracket_ (changeWorkingDirectory $ dir) (\_ -> changeWorkingDirectory cwd) a -withgitrepo :: (Git.Repo -> Assertion) -> Assertion -withgitrepo = bracket setup cleanup - where - tmpdir = ".t" - repodir = tmpdir ++ "/repo" - setup = do - cleanup True - createDirectory tmpdir - ok <- Utility.boolSystem "git" ["init", "-q", repodir] - unless ok $ - assertFailure "git init failed" - return $ Git.repoFromPath repodir - cleanup _ = do - e <- doesDirectoryExist tmpdir - when e $ do - -- git-annex prevents annexed file content - -- from being removed with permissions - -- bits; undo - _ <- Utility.boolSystem "chmod" ["+rw", "-R", tmpdir] - removeDirectoryRecursive tmpdir +setuprepo :: FilePath -> IO FilePath +setuprepo dir = do + cleanup dir + ensuretmpdir + ok <- Utility.boolSystem "git" ["init", "-q", dir] + unless ok $ + assertFailure "git init failed" + return dir + +copyrepo :: FilePath -> FilePath -> IO FilePath +copyrepo old new = do + cleanup new + ensuretmpdir + ok <- Utility.boolSystem "cp" ["-pr", old, new] + unless ok $ + assertFailure "cp -pr failed" + return new + +ensuretmpdir :: IO () +ensuretmpdir = do + e <- doesDirectoryExist tmpdir + unless e $ + createDirectory tmpdir + +cleanup :: FilePath -> IO () +cleanup dir = do + e <- doesDirectoryExist dir + when e $ do + -- git-annex prevents annexed file content from being + -- removed via permissions bits; undo + _ <- Utility.boolSystem "chmod" ["+rw", "-R", dir] + removeDirectoryRecursive dir + +tmpdir :: String +tmpdir = ".t" + +repodir :: String +repodir = tmpdir ++ "/repo" + +tmprepodir :: String +tmprepodir = tmpdir ++ "/tmprepo" From f4a26f01ea0ba956d968bba9b3b948298aa15568 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 22:22:09 -0400 Subject: [PATCH 0686/2835] more tests --- debian/changelog | 2 +- debian/control | 2 +- test.hs | 108 +++++++++++++++++++++++++++++++++-------------- 3 files changed, 78 insertions(+), 34 deletions(-) diff --git a/debian/changelog b/debian/changelog index edcd349f55..150268c9d9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,7 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. - * Test suite improvements. Current top-level test coverage: 53% + * Test suite improvements. Current top-level test coverage: 57% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/debian/control b/debian/control index d90041f77e..d0f8805d0f 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync, git | git-core Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex diff --git a/test.hs b/test.hs index 3ad34971aa..178842f8cd 100644 --- a/test.hs +++ b/test.hs @@ -47,6 +47,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_add , test_unannex , test_drop + , test_get ] test_init :: Test @@ -64,48 +65,43 @@ test_init = TestLabel "git-annex init" $ TestCase $ innewrepo $ do test_add :: Test test_add = TestLabel "git-annex add" $ TestCase $ inoldrepo $ do - writeFile file content - git_annex "add" ["-q", "foo"] @? "add failed" - s <- getSymbolicLinkStatus file - unless (isSymbolicLink s) $ - assertFailure "git-annex add did not create symlink" - c <- readFile file - unless (c == content) $ - assertFailure "file content changed during git-annex add" - r <- try $ writeFile file $ content++"bar" - case r of - Left _ -> return () -- expected permission error - Right _ -> assertFailure "was able to modify annexed file content" - where - file = "foo" - content = "foo file content" + writeFile foofile foocontent + git_annex "add" ["-q", foofile] @? "add failed" + checklink foofile + checkcontent foofile foocontent + checkunwritable foofile + ok <- Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "added foo"] + unless ok $ + assertFailure "git commit failed" test_unannex :: Test test_unannex = TestLabel "git-annex unannex" $ TestCase $ intmpcopyrepo $ do - git_annex "unannex" ["-q", "foo"] @? "unannex failed" - s <- getSymbolicLinkStatus "foo" + git_annex "unannex" ["-q", foofile] @? "unannex failed" + s <- getSymbolicLinkStatus foofile when (isSymbolicLink s) $ assertFailure "git-annex unannex left symlink" test_drop :: Test test_drop = TestLabel "git-annex drop" $ TestCase $ intmpcopyrepo $ do - r <- git_annex "drop" ["-q", "foo"] + r <- git_annex "drop" ["-q", foofile] (not r) @? "drop wrongly succeeded with no known copy of file" - checklink - git_annex "drop" ["-q", "--force", "foo"] @? "drop --force failed" - checklink - r' <- try $ readFile "foo" - case r' of - Left _ -> return () -- expected; dangling link - Right _ -> assertFailure "drop did not remove file content" - where - checklink = do - s <- getSymbolicLinkStatus "foo" - unless (isSymbolicLink s) $ - assertFailure "git-annex drop killed symlink" - - + checklink foofile + checkcontent foofile foocontent + git_annex "drop" ["-q", "--force", foofile] @? "drop --force failed" + checklink foofile + checkdangling foofile + git_annex "drop" ["-q", foofile] @? "drop of dropped file failed" +test_get :: Test +test_get = TestLabel "git-annex get" $ TestCase $ intmpclonerepo $ do + git_annex "get" ["-q", foofile] @? "get of file failed" + checklink foofile + checkcontent foofile foocontent + checkunwritable foofile + git_annex "get" ["-q", foofile] @? "get of file already here failed" + checklink foofile + checkcontent foofile foocontent + checkunwritable foofile git_annex :: String -> [String] -> IO Bool git_annex command params = do @@ -136,9 +132,15 @@ inoldrepo = indir repodir intmpcopyrepo :: Assertion -> Assertion intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a +intmpclonerepo :: Assertion -> Assertion +intmpclonerepo a = withtmpclonerepo $ \r -> indir r a + withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion withtmpcopyrepo = bracket (copyrepo repodir tmprepodir) cleanup +withtmpclonerepo :: (FilePath -> Assertion) -> Assertion +withtmpclonerepo = bracket (clonerepo repodir tmprepodir) cleanup + withgitrepo :: (FilePath -> Assertion) -> Assertion withgitrepo = bracket (setuprepo repodir) return @@ -166,6 +168,16 @@ copyrepo old new = do unless ok $ assertFailure "cp -pr failed" return new + +-- clones are always done as local clones; we cannot test ssh clones +clonerepo :: FilePath -> FilePath -> IO FilePath +clonerepo old new = do + cleanup new + ensuretmpdir + ok <- Utility.boolSystem "git" ["clone", "-q", old, new] + unless ok $ + assertFailure "git clone failed" + return new ensuretmpdir :: IO () ensuretmpdir = do @@ -181,6 +193,32 @@ cleanup dir = do -- removed via permissions bits; undo _ <- Utility.boolSystem "chmod" ["+rw", "-R", dir] removeDirectoryRecursive dir + +checklink :: FilePath -> Assertion +checklink f = do + s <- getSymbolicLinkStatus f + unless (isSymbolicLink s) $ + assertFailure $ f ++ " is not a symlink" + +checkcontent :: FilePath -> String -> Assertion +checkcontent f c = do + c' <- readFile f + unless (c' == c) $ + assertFailure $ f ++ " content unexpected" + +checkunwritable :: FilePath -> Assertion +checkunwritable f = do + r <- try $ writeFile f $ "dummy" + case r of + Left _ -> return () -- expected permission error + Right _ -> assertFailure $ "was able to modify annexed file's " ++ f ++ " content" + +checkdangling :: FilePath -> Assertion +checkdangling f = do + r <- try $ readFile f + case r of + Left _ -> return () -- expected; dangling link + Right _ -> assertFailure $ f ++ " was not a dangling link as expected" tmpdir :: String tmpdir = ".t" @@ -190,3 +228,9 @@ repodir = tmpdir ++ "/repo" tmprepodir :: String tmprepodir = tmpdir ++ "/tmprepo" + +foofile :: String +foofile = "foo" + +foocontent :: String +foocontent = "foo file content" From e29d2376937691a4170498b095191a6ffefc9875 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 01:02:06 -0400 Subject: [PATCH 0687/2835] more tests --- test.hs | 168 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 97 insertions(+), 71 deletions(-) diff --git a/test.hs b/test.hs index 178842f8cd..5a81f1a3e9 100644 --- a/test.hs +++ b/test.hs @@ -48,60 +48,89 @@ toplevels = TestLabel "toplevel" $ TestList , test_unannex , test_drop , test_get + , test_move ] test_init :: Test -test_init = TestLabel "git-annex init" $ TestCase $ innewrepo $ do +test_init = "git-annex init" ~: innewrepo $ do git_annex "init" ["-q", reponame] @? "init failed" e <- doesFileExist annexlog - unless e $ - assertFailure $ annexlog ++ " not created" + e @? (annexlog ++ " not created") c <- readFile annexlog - unless (isInfixOf reponame c) $ - assertFailure $ annexlog ++ " does not contain repo name" + isInfixOf reponame c @? annexlog ++ " does not contain repo name" where annexlog = ".git-annex/uuid.log" reponame = "test repo" test_add :: Test -test_add = TestLabel "git-annex add" $ TestCase $ inoldrepo $ do - writeFile foofile foocontent - git_annex "add" ["-q", foofile] @? "add failed" - checklink foofile - checkcontent foofile foocontent - checkunwritable foofile - ok <- Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "added foo"] - unless ok $ - assertFailure "git commit failed" +test_add = "git-annex add" ~: inoldrepo $ do + writeFile annexedfile $ content annexedfile + git_annex "add" ["-q", annexedfile] @? "add failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + writeFile ingitfile $ content ingitfile + Utility.boolSystem "git" ["add", ingitfile] @? "git add failed" + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" + git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op" + checkregularfile ingitfile test_unannex :: Test -test_unannex = TestLabel "git-annex unannex" $ TestCase $ intmpcopyrepo $ do - git_annex "unannex" ["-q", foofile] @? "unannex failed" - s <- getSymbolicLinkStatus foofile - when (isSymbolicLink s) $ - assertFailure "git-annex unannex left symlink" +test_unannex = "git-annex unannex" ~: intmpcopyrepo $ do + git_annex "unannex" ["-q", annexedfile] @? "unannex failed" + checkregularfile annexedfile + git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" + checkregularfile annexedfile + git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op" test_drop :: Test -test_drop = TestLabel "git-annex drop" $ TestCase $ intmpcopyrepo $ do - r <- git_annex "drop" ["-q", foofile] +test_drop = "git-annex drop" ~: intmpcopyrepo $ do + r <- git_annex "drop" ["-q", annexedfile] (not r) @? "drop wrongly succeeded with no known copy of file" - checklink foofile - checkcontent foofile foocontent - git_annex "drop" ["-q", "--force", foofile] @? "drop --force failed" - checklink foofile - checkdangling foofile - git_annex "drop" ["-q", foofile] @? "drop of dropped file failed" + checklink annexedfile + checkcontent annexedfile + git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed" + checklink annexedfile + checkdangling annexedfile + checkunwritable annexedfile + git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed" + git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile test_get :: Test -test_get = TestLabel "git-annex get" $ TestCase $ intmpclonerepo $ do - git_annex "get" ["-q", foofile] @? "get of file failed" - checklink foofile - checkcontent foofile foocontent - checkunwritable foofile - git_annex "get" ["-q", foofile] @? "get of file already here failed" - checklink foofile - checkcontent foofile foocontent - checkunwritable foofile +test_get = "git-annex get" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "get" ["-q", annexedfile] @? "get of file already here failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "get" ["-q", ingitfile] @? "get ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile + +test_move :: Test +test_move = "git-annex move" ~: intmpclonerepo $ do + git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file already here failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file failed" + checklink annexedfile + checkdangling annexedfile + checkunwritable annexedfile + git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already here failed" + checklink annexedfile + checkdangling annexedfile + checkunwritable annexedfile + git_annex :: String -> [String] -> IO Bool git_annex command params = do @@ -116,24 +145,17 @@ git_annex command params = do CmdLine.dispatch gitrepo (command:params) GitAnnex.cmds GitAnnex.options GitAnnex.header -innewannex :: Assertion -> Assertion -innewannex a = innewrepo $ do - git_annex "init" ["-q", reponame] @? "init failed" - a - where - reponame = "test repo" +innewrepo :: Assertion -> Test +innewrepo a = TestCase $ withgitrepo $ \r -> indir r a -innewrepo :: Assertion -> Assertion -innewrepo a = withgitrepo $ \r -> indir r a +inoldrepo :: Assertion -> Test +inoldrepo a = TestCase $ indir repodir a -inoldrepo :: Assertion -> Assertion -inoldrepo = indir repodir +intmpcopyrepo :: Assertion -> Test +intmpcopyrepo a = TestCase $ withtmpcopyrepo $ \r -> indir r a -intmpcopyrepo :: Assertion -> Assertion -intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a - -intmpclonerepo :: Assertion -> Assertion -intmpclonerepo a = withtmpclonerepo $ \r -> indir r a +intmpclonerepo :: Assertion -> Test +intmpclonerepo a = TestCase $ withtmpclonerepo $ \r -> indir r a withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion withtmpcopyrepo = bracket (copyrepo repodir tmprepodir) cleanup @@ -155,18 +177,14 @@ setuprepo :: FilePath -> IO FilePath setuprepo dir = do cleanup dir ensuretmpdir - ok <- Utility.boolSystem "git" ["init", "-q", dir] - unless ok $ - assertFailure "git init failed" + Utility.boolSystem "git" ["init", "-q", dir] @? "git init failed" return dir copyrepo :: FilePath -> FilePath -> IO FilePath copyrepo old new = do cleanup new ensuretmpdir - ok <- Utility.boolSystem "cp" ["-pr", old, new] - unless ok $ - assertFailure "cp -pr failed" + Utility.boolSystem "cp" ["-pr", old, new] @? "cp -pr failed" return new -- clones are always done as local clones; we cannot test ssh clones @@ -174,9 +192,7 @@ clonerepo :: FilePath -> FilePath -> IO FilePath clonerepo old new = do cleanup new ensuretmpdir - ok <- Utility.boolSystem "git" ["clone", "-q", old, new] - unless ok $ - assertFailure "git clone failed" + Utility.boolSystem "git" ["clone", "-q", old, new] @? "git clone failed" return new ensuretmpdir :: IO () @@ -197,14 +213,18 @@ cleanup dir = do checklink :: FilePath -> Assertion checklink f = do s <- getSymbolicLinkStatus f - unless (isSymbolicLink s) $ - assertFailure $ f ++ " is not a symlink" + isSymbolicLink s @? f ++ " is not a symlink" -checkcontent :: FilePath -> String -> Assertion -checkcontent f c = do - c' <- readFile f - unless (c' == c) $ - assertFailure $ f ++ " content unexpected" +checkregularfile :: FilePath -> Assertion +checkregularfile f = do + s <- getSymbolicLinkStatus f + isRegularFile s @? f ++ " is not a normal file" + return () + +checkcontent :: FilePath -> Assertion +checkcontent f = do + c <- readFile f + assertEqual ("checkcontent " ++ f) c (content f) checkunwritable :: FilePath -> Assertion checkunwritable f = do @@ -229,8 +249,14 @@ repodir = tmpdir ++ "/repo" tmprepodir :: String tmprepodir = tmpdir ++ "/tmprepo" -foofile :: String -foofile = "foo" +annexedfile :: String +annexedfile = "foo" -foocontent :: String -foocontent = "foo file content" +ingitfile :: String +ingitfile = "bar" + +content :: FilePath -> String +content f + | f == annexedfile = "annexed file content" + | f == ingitfile = "normal file content" + | otherwise = "unknown file " ++ f From 71a8278f9caea8d3e52a5d43cb60125018ec31fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 01:14:27 -0400 Subject: [PATCH 0688/2835] bugfix: Running `move --to` with a non-ssh remote failed. --- Remotes.hs | 7 +++++-- debian/changelog | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 5bb2efa554..2c8cdfdad1 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -249,8 +249,11 @@ copyToRemote r key | not $ Git.repoIsUrl r = do g <- Annex.gitRepo let keysrc = annexLocation g key - let keydest = annexLocation r key - liftIO $ copyFile keysrc keydest + -- run copy from perspective of remote + liftIO $ do + a <- Annex.new r [] + Annex.eval a $ Core.getViaTmp key $ \f -> + liftIO $ copyFile keysrc f | Git.repoIsSsh r = do g <- Annex.gitRepo let keysrc = annexLocation g key diff --git a/debian/changelog b/debian/changelog index 150268c9d9..969935a4e9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. + * bugfix: Running `move --to` with a non-ssh remote failed. * Test suite improvements. Current top-level test coverage: 57% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 From e43d4730c5157acd5ccc421472b33d0f12167a2c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:14:22 -0400 Subject: [PATCH 0689/2835] bugfix: Running `copy --to` when both local and remote had the key dropped it from local. --- Command/Move.hs | 2 +- debian/changelog | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index 0077618f8b..3e7fde3705 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -89,7 +89,7 @@ toPerform move key = do if ok then return $ Just $ toCleanup move remote key else return Nothing -- failed - Right True -> return $ Just $ Command.Drop.cleanup key + Right True -> return $ Just $ toCleanup move remote key toCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup toCleanup move remote key = do remoteHasKey remote key True diff --git a/debian/changelog b/debian/changelog index 969935a4e9..18aea04670 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,7 +9,9 @@ git-annex (0.16) UNRELEASED; urgency=low * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. * bugfix: Running `move --to` with a non-ssh remote failed. - * Test suite improvements. Current top-level test coverage: 57% + * bugfix: Running `copy --to` when both local and remote had the key + dropped it from local. + * Test suite improvements. Current top-level test coverage: 62% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 From 3fad3e527ea12aa5cab5fe646e60300444808f0b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:14:48 -0400 Subject: [PATCH 0690/2835] various test fixes --- test.hs | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/test.hs b/test.hs index 5a81f1a3e9..216da2033e 100644 --- a/test.hs +++ b/test.hs @@ -14,6 +14,7 @@ import IO (bracket_, bracket) import Control.Monad (unless, when) import Data.List import System.IO.Error +import System.Posix.Env import qualified Control.Exception.Extensible as E import qualified GitRepo as Git @@ -21,13 +22,18 @@ import qualified Locations import qualified Utility import qualified TypeInternals import qualified GitAnnex -import qualified CmdLine -main :: IO (Counts, Int) +main :: IO () main = do + tweakpath r <- runVerboseTests $ TestList [quickchecks, toplevels] cleanup tmpdir - return r + propigate r + +propigate :: (Counts, Int) -> IO () +propigate (Counts { errors = e }, _) + | e > 0 = error "failed" + | otherwise = return () quickchecks :: Test quickchecks = TestLabel "quickchecks" $ TestList @@ -49,6 +55,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_drop , test_get , test_move + , test_copy ] test_init :: Test @@ -126,7 +133,26 @@ test_move = "git-annex move" ~: intmpclonerepo $ do checklink annexedfile checkdangling annexedfile checkunwritable annexedfile - git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already here failed" + git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed" + checklink annexedfile + checkdangling annexedfile + checkunwritable annexedfile + +test_copy :: Test +test_copy = "git-annex copy" ~: intmpclonerepo $ do + git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file already here failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "copy" ["-q", "--to", "origin", annexedfile] @? "copy --to of file already there failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed" checklink annexedfile checkdangling annexedfile checkunwritable annexedfile @@ -140,10 +166,7 @@ git_annex command params = do Right _ -> return True Left _ -> return False where - run = do - gitrepo <- Git.repoFromCwd - CmdLine.dispatch gitrepo (command:params) - GitAnnex.cmds GitAnnex.options GitAnnex.header + run = GitAnnex.run (command:params) innewrepo :: Assertion -> Test innewrepo a = TestCase $ withgitrepo $ \r -> indir r a @@ -173,6 +196,14 @@ indir dir a = do (\_ -> changeWorkingDirectory cwd) a +-- While PATH is mostly avoided, the commit hook does run it. Make +-- sure that the just-built git annex is used. +tweakpath :: IO () +tweakpath = do + cwd <- getCurrentDirectory + p <- getEnvDefault "PATH" "" + setEnv "PATH" (cwd ++ ":" ++ p) True + setuprepo :: FilePath -> IO FilePath setuprepo dir = do cleanup dir From 55ca64d8510fc3acd34452844086fbcb4bca122c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:15:10 -0400 Subject: [PATCH 0691/2835] simplify --- GitAnnex.hs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/GitAnnex.hs b/GitAnnex.hs index 05e98d3c3a..24c9ace0ac 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -9,6 +9,8 @@ module GitAnnex where import System.Console.GetOpt +import qualified GitRepo as Git +import CmdLine import Command import Options @@ -73,3 +75,8 @@ options = commonOptions ++ header :: String header = "Usage: git-annex command [option ..]" + +run :: [String] -> IO () +run args = do + gitrepo <- Git.repoFromCwd + dispatch gitrepo args cmds options header From f189929b8b46016a254a6a938f9aba01184fb6c6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:15:23 -0400 Subject: [PATCH 0692/2835] workaround ghc weirdness with -odir The option cause it to always build to build/Main.o, no matter what binary it was building. This caused extra work, and in some cases, could cause the wrong code to be put into the final binary. --- .gitignore | 3 ++- Makefile | 7 ++++--- git-annex.hs | 8 ++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index f68d1d0adf..764a1af9d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -build/* +*.hi +*.o test configure SysConfig.hs diff --git a/Makefile b/Makefile index e499d492c7..44d4be02be 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr GHCFLAGS=-O2 -Wall -GHCMAKE=ghc -odir build -hidir build $(GHCFLAGS) --make +GHCMAKE=ghc $(GHCFLAGS) --make bins=git-annex git-annex-shell mans=git-annex.1 git-annex-shell.1 @@ -29,11 +29,11 @@ install: all rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \ fi -test: +test: $(bins) $(GHCMAKE) test ./test -testcoverage: +testcoverage: $(bins) rm -f test.tix test ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test ./test @@ -59,5 +59,6 @@ docs: $(mans) clean: rm -rf build $(bins) $(mans) test configure SysConfig.hs *.tix .hpc rm -rf doc/.ikiwiki html + find . \( -name \*.o -or -name \*.hi \) -exec rm {} \; .PHONY: $(bins) test install diff --git a/git-annex.hs b/git-annex.hs index f951817847..878d8bdbbc 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,4 +1,4 @@ -{- git-annex main program +{- git-annex main program stub - - Copyright 2010 Joey Hess - @@ -7,12 +7,8 @@ import System.Environment -import qualified GitRepo as Git -import CmdLine import GitAnnex - main :: IO () main = do args <- getArgs - gitrepo <- Git.repoFromCwd - dispatch gitrepo args cmds options header + run args From 7d15a5795aaad3b291406c1b52b381a1f246d6f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:18:39 -0400 Subject: [PATCH 0693/2835] more tests --- test.hs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test.hs b/test.hs index 216da2033e..18ccf331af 100644 --- a/test.hs +++ b/test.hs @@ -137,6 +137,12 @@ test_move = "git-annex move" ~: intmpclonerepo $ do checklink annexedfile checkdangling annexedfile checkunwritable annexedfile + git_annex "move" ["-q", "--to", "origin", ingitfile] @? "move of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile + git_annex "move" ["-q", "--from", "origin", ingitfile] @? "move of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile test_copy :: Test test_copy = "git-annex copy" ~: intmpclonerepo $ do @@ -156,6 +162,12 @@ test_copy = "git-annex copy" ~: intmpclonerepo $ do checklink annexedfile checkdangling annexedfile checkunwritable annexedfile + git_annex "copy" ["-q", "--to", "origin", ingitfile] @? "copy of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile + git_annex "copy" ["-q", "--from", "origin", ingitfile] @? "copy of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile git_annex :: String -> [String] -> IO Bool From 2684cbbd23feed32ffd50ccdd8a8303a4dfbfa28 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 12:34:44 -0400 Subject: [PATCH 0694/2835] clarify --- debian/changelog | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 18aea04670..b31a63b287 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,8 +9,7 @@ git-annex (0.16) UNRELEASED; urgency=low * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. * bugfix: Running `move --to` with a non-ssh remote failed. - * bugfix: Running `copy --to` when both local and remote had the key - dropped it from local. + * bugfix: Running `copy --to` with a non-ssh remote actually did a move. * Test suite improvements. Current top-level test coverage: 62% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 From 6cb1dff757ffad735e04d6d8134f5fbfdea71650 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 13:57:37 -0400 Subject: [PATCH 0695/2835] quiet git commits --- Command/Init.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Command/Init.hs b/Command/Init.hs index 8ad9f79d70..47ac8e4c08 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -50,7 +50,7 @@ cleanup = do g <- Annex.gitRepo logfile <- uuidLog liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] + liftIO $ Git.run g ["commit", "-q", "-m", "git annex init", logfile] return True {- configure git to use union merge driver on state files, if it is not @@ -71,7 +71,7 @@ gitAttributesWrite repo = do attributes = Git.attributes repo commit = do Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", + Git.run repo ["commit", "-q", "-m", "git-annex setup", attributes] attrLine :: String From f3472d3a5d71ffedc67ed212b87308052bb4c042 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 14:06:32 -0400 Subject: [PATCH 0696/2835] Test suite improvements. Current top-level test coverage: 65% --- debian/changelog | 2 +- test.hs | 273 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 185 insertions(+), 90 deletions(-) diff --git a/debian/changelog b/debian/changelog index b31a63b287..ead272dbe3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -10,7 +10,7 @@ git-annex (0.16) UNRELEASED; urgency=low about repositories with missing UUIDs. * bugfix: Running `move --to` with a non-ssh remote failed. * bugfix: Running `copy --to` with a non-ssh remote actually did a move. - * Test suite improvements. Current top-level test coverage: 62% + * Test suite improvements. Current top-level test coverage: 65% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/test.hs b/test.hs index 18ccf331af..e5563c5148 100644 --- a/test.hs +++ b/test.hs @@ -25,7 +25,7 @@ import qualified GitAnnex main :: IO () main = do - tweakpath + prepare r <- runVerboseTests $ TestList [quickchecks, toplevels] cleanup tmpdir propigate r @@ -56,10 +56,13 @@ toplevels = TestLabel "toplevel" $ TestList , test_get , test_move , test_copy + , test_lock + , test_edit + , test_fix ] test_init :: Test -test_init = "git-annex init" ~: innewrepo $ do +test_init = "git-annex init" ~: TestCase $ innewrepo $ do git_annex "init" ["-q", reponame] @? "init failed" e <- doesFileExist annexlog e @? (annexlog ++ " not created") @@ -70,105 +73,159 @@ test_init = "git-annex init" ~: innewrepo $ do reponame = "test repo" test_add :: Test -test_add = "git-annex add" ~: inoldrepo $ do +test_add = "git-annex add" ~: TestCase $ inmainrepo $ do writeFile annexedfile $ content annexedfile git_annex "add" ["-q", annexedfile] @? "add failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile writeFile ingitfile $ content ingitfile Utility.boolSystem "git" ["add", ingitfile] @? "git add failed" Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op" - checkregularfile ingitfile + unannexed ingitfile test_unannex :: Test -test_unannex = "git-annex unannex" ~: intmpcopyrepo $ do +test_unannex = "git-annex unannex" ~: TestCase $ intmpcopyrepo $ do git_annex "unannex" ["-q", annexedfile] @? "unannex failed" - checkregularfile annexedfile + unannexed annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" - checkregularfile annexedfile + unannexed annexedfile git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op" + unannexed ingitfile test_drop :: Test -test_drop = "git-annex drop" ~: intmpcopyrepo $ do - r <- git_annex "drop" ["-q", annexedfile] - (not r) @? "drop wrongly succeeded with no known copy of file" - checklink annexedfile - checkcontent annexedfile - git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed" - checklink annexedfile - checkdangling annexedfile - checkunwritable annexedfile - git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed" - git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile +test_drop = "git-annex drop" ~: TestList [nocopy, withcopy] + where + nocopy = "no remotes" ~: TestCase $ intmpcopyrepo $ do + r <- git_annex "drop" ["-q", annexedfile] + (not r) @? "drop wrongly succeeded with no known copy of file" + annexed_present annexedfile + git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed" + annexed_notpresent annexedfile + git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed" + git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op" + unannexed ingitfile + withcopy = "with remote" ~: TestCase $ intmpclonerepo $ do + git_annex "drop" ["-q", annexedfile] @? "drop failed though origin has copy" + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile test_get :: Test -test_get = "git-annex get" ~: intmpclonerepo $ do +test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile git_annex "get" ["-q", annexedfile] @? "get of file failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + inmainrepo $ annexed_present annexedfile + annexed_present annexedfile git_annex "get" ["-q", annexedfile] @? "get of file already here failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + inmainrepo $ annexed_present annexedfile + annexed_present annexedfile + inmainrepo $ unannexed ingitfile + unannexed ingitfile git_annex "get" ["-q", ingitfile] @? "get ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile + inmainrepo $ unannexed ingitfile + unannexed ingitfile test_move :: Test -test_move = "git-annex move" ~: intmpclonerepo $ do +test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_notpresent annexedfile git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file already here failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_notpresent annexedfile git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file failed" - checklink annexedfile - checkdangling annexedfile - checkunwritable annexedfile + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed" - checklink annexedfile - checkdangling annexedfile - checkunwritable annexedfile + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile git_annex "move" ["-q", "--to", "origin", ingitfile] @? "move of ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile git_annex "move" ["-q", "--from", "origin", ingitfile] @? "move of ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile test_copy :: Test -test_copy = "git-annex copy" ~: intmpclonerepo $ do +test_copy = "git-annex copy" ~: TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file already here failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile git_annex "copy" ["-q", "--to", "origin", annexedfile] @? "copy --to of file already there failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed" - checklink annexedfile - checkdangling annexedfile - checkunwritable annexedfile + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile git_annex "copy" ["-q", "--to", "origin", ingitfile] @? "copy of ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile git_annex "copy" ["-q", "--from", "origin", ingitfile] @? "copy of ingitfile should be no-op" checkregularfile ingitfile checkcontent ingitfile +test_lock :: Test +test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "unlock" ["-q", annexedfile] @? "unlock failed" + unannexed annexedfile + -- write different content, to verify that lock + -- throws it away + changecontent annexedfile + writeFile annexedfile $ (content annexedfile) ++ "foo" + git_annex "lock" ["-q", annexedfile] @? "lock failed" + annexed_present annexedfile + git_annex "unlock" ["-q", annexedfile] @? "unlock failed" + unannexed annexedfile + changecontent annexedfile + git_annex "add" ["-q", annexedfile] @? "add of modified file failed" + runchecks [checklink, checkunwritable] annexedfile + c <- readFile annexedfile + assertEqual ("content of modified file") c (changedcontent annexedfile) + r <- git_annex "drop" ["-q", annexedfile] + (not r) @? "drop wrongly succeeded with no known copy of modified file" + +test_edit :: Test +test_edit = "git-annex edit/commit" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "edit" ["-q", annexedfile] @? "edit failed" + unannexed annexedfile + changecontent annexedfile + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "content changed"] + @? "git commit of edited file failed" + runchecks [checklink, checkunwritable] annexedfile + c <- readFile annexedfile + assertEqual ("content of modified file") c (changedcontent annexedfile) + r <- git_annex "drop" ["-q", annexedfile] + (not r) @? "drop wrongly succeeded with no known copy of modified file" + +test_fix :: Test +test_fix = "git-annex fix" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + annexed_present annexedfile + createDirectory subdir + Utility.boolSystem "git" ["mv", annexedfile, subdir] + @? "git mv failed" + git_annex "fix" ["-q", newfile] @? "fix failed" + runchecks [checklink, checkunwritable] newfile + c <- readFile newfile + assertEqual ("content of moved file") c (content annexedfile) + where + subdir = "s" + newfile = subdir ++ "/" ++ annexedfile git_annex :: String -> [String] -> IO Bool git_annex command params = do @@ -180,42 +237,34 @@ git_annex command params = do where run = GitAnnex.run (command:params) -innewrepo :: Assertion -> Test -innewrepo a = TestCase $ withgitrepo $ \r -> indir r a +innewrepo :: Assertion -> Assertion +innewrepo a = withgitrepo $ \r -> indir r a -inoldrepo :: Assertion -> Test -inoldrepo a = TestCase $ indir repodir a +inmainrepo :: Assertion -> Assertion +inmainrepo a = indir mainrepodir a -intmpcopyrepo :: Assertion -> Test -intmpcopyrepo a = TestCase $ withtmpcopyrepo $ \r -> indir r a +intmpcopyrepo :: Assertion -> Assertion +intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a -intmpclonerepo :: Assertion -> Test -intmpclonerepo a = TestCase $ withtmpclonerepo $ \r -> indir r a +intmpclonerepo :: Assertion -> Assertion +intmpclonerepo a = withtmpclonerepo $ \r -> indir r a withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion -withtmpcopyrepo = bracket (copyrepo repodir tmprepodir) cleanup +withtmpcopyrepo = bracket (copyrepo mainrepodir tmprepodir) cleanup withtmpclonerepo :: (FilePath -> Assertion) -> Assertion -withtmpclonerepo = bracket (clonerepo repodir tmprepodir) cleanup +withtmpclonerepo = bracket (clonerepo mainrepodir tmprepodir) cleanup withgitrepo :: (FilePath -> Assertion) -> Assertion -withgitrepo = bracket (setuprepo repodir) return +withgitrepo = bracket (setuprepo mainrepodir) return indir :: FilePath -> Assertion -> Assertion indir dir a = do cwd <- getCurrentDirectory - bracket_ (changeWorkingDirectory $ dir) + bracket_ (changeToTmpDir $ dir) (\_ -> changeWorkingDirectory cwd) a --- While PATH is mostly avoided, the commit hook does run it. Make --- sure that the just-built git annex is used. -tweakpath :: IO () -tweakpath = do - cwd <- getCurrentDirectory - p <- getEnvDefault "PATH" "" - setEnv "PATH" (cwd ++ ":" ++ p) True - setuprepo :: FilePath -> IO FilePath setuprepo dir = do cleanup dir @@ -225,6 +274,8 @@ setuprepo dir = do copyrepo :: FilePath -> FilePath -> IO FilePath copyrepo old new = do + _ <- clonerepo old new + indir new $ Utility.boolSystem "git" ["remote", "rm", "origin"] @? "git remote failed" cleanup new ensuretmpdir Utility.boolSystem "cp" ["-pr", old, new] @? "cp -pr failed" @@ -236,6 +287,7 @@ clonerepo old new = do cleanup new ensuretmpdir Utility.boolSystem "git" ["clone", "-q", old, new] @? "git clone failed" + indir new $ git_annex "init" ["-q", new] @? "git annex init failed" return new ensuretmpdir :: IO () @@ -271,11 +323,18 @@ checkcontent f = do checkunwritable :: FilePath -> Assertion checkunwritable f = do - r <- try $ writeFile f $ "dummy" + r <- try $ writeFile f $ content f case r of Left _ -> return () -- expected permission error Right _ -> assertFailure $ "was able to modify annexed file's " ++ f ++ " content" +checkwritable :: FilePath -> Assertion +checkwritable f = do + r <- try $ writeFile f $ content f + case r of + Left _ -> assertFailure $ "unable to modify " ++ f + Right _ -> return () + checkdangling :: FilePath -> Assertion checkdangling f = do r <- try $ readFile f @@ -283,15 +342,45 @@ checkdangling f = do Left _ -> return () -- expected; dangling link Right _ -> assertFailure $ f ++ " was not a dangling link as expected" -tmpdir :: String +runchecks :: [FilePath -> Assertion] -> FilePath -> Assertion +runchecks [] _ = return () +runchecks (a:as) f = do + a f + runchecks as f + +annexed_notpresent :: FilePath -> Assertion +annexed_notpresent = runchecks [checklink, checkdangling, checkunwritable] + +annexed_present :: FilePath -> Assertion +annexed_present = runchecks [checklink, checkcontent, checkunwritable] + +unannexed :: FilePath -> Assertion +unannexed = runchecks [checkregularfile, checkcontent, checkwritable] + +prepare :: IO () +prepare = do + -- While PATH is mostly avoided, the commit hook does run it. Make + -- sure that the just-built git annex is used. + cwd <- getCurrentDirectory + p <- getEnvDefault "PATH" "" + setEnv "PATH" (cwd ++ ":" ++ p) True + setEnv "TOPDIR" cwd True + +changeToTmpDir :: FilePath -> IO () +changeToTmpDir t = do + -- Hack alert. Threading state to here was too much bother. + topdir <- getEnvDefault "TOPDIR" "" + changeWorkingDirectory $ topdir ++ "/" ++ t + +tmpdir :: String tmpdir = ".t" -repodir :: String -repodir = tmpdir ++ "/repo" +mainrepodir :: String +mainrepodir = tmpdir ++ "/repo" tmprepodir :: String tmprepodir = tmpdir ++ "/tmprepo" - + annexedfile :: String annexedfile = "foo" @@ -303,3 +392,9 @@ content f | f == annexedfile = "annexed file content" | f == ingitfile = "normal file content" | otherwise = "unknown file " ++ f + +changecontent :: FilePath -> IO () +changecontent f = writeFile f $ changedcontent f + +changedcontent :: FilePath -> String +changedcontent f = (content f) ++ " (modified)" From a14c881f023a55aa325db86fea3a5af1fce3d852 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 14:08:43 -0400 Subject: [PATCH 0697/2835] more --- test.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test.hs b/test.hs index e5563c5148..852d1b30f5 100644 --- a/test.hs +++ b/test.hs @@ -214,12 +214,17 @@ test_edit = "git-annex edit/commit" ~: intmpclonerepo $ do test_fix :: Test test_fix = "git-annex fix" ~: intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex "fix" ["-q", annexedfile] @? "fix of not present failed" + annexed_notpresent annexedfile git_annex "get" ["-q", annexedfile] @? "get of file failed" annexed_present annexedfile + git_annex "fix" ["-q", annexedfile] @? "fix of present file failed" + annexed_present annexedfile createDirectory subdir Utility.boolSystem "git" ["mv", annexedfile, subdir] @? "git mv failed" - git_annex "fix" ["-q", newfile] @? "fix failed" + git_annex "fix" ["-q", newfile] @? "fix of moved file failed" runchecks [checklink, checkunwritable] newfile c <- readFile newfile assertEqual ("content of moved file") c (content annexedfile) From ad9dfe322894763f7013b7a41e0bf9c905cc597c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 14:10:38 -0400 Subject: [PATCH 0698/2835] mention test suite --- doc/not.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/not.mdwn b/doc/not.mdwn index d965ab86f8..b138953f91 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -17,4 +17,6 @@ * git-annex is not some flaky script that was quickly thrown together. I wrote it in Haskell because I wanted it to be solid and to compile - down to a binary. + down to a binary. And it has a fairly extensive test suite. (Don't be + fooled by "make test" only showing a few dozen test cases; each test + involves checking dozens to hundreds of assertions.) From d31e61a90da100e188b45fe033337d405c6a2124 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 14:36:52 -0400 Subject: [PATCH 0699/2835] add news item for git-annex 0.16 --- debian/changelog | 6 +++--- doc/news/version_0.11.mdwn | 8 -------- doc/news/version_0.16.mdwn | 13 +++++++++++++ test.hs | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) delete mode 100644 doc/news/version_0.11.mdwn create mode 100644 doc/news/version_0.16.mdwn diff --git a/debian/changelog b/debian/changelog index ead272dbe3..bd986ad24c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.16) UNRELEASED; urgency=low +git-annex (0.16) unstable; urgency=low * git-annex-shell: Avoid exposing any git repo config except for the annex.uuid when doing configlist. @@ -10,9 +10,9 @@ git-annex (0.16) UNRELEASED; urgency=low about repositories with missing UUIDs. * bugfix: Running `move --to` with a non-ssh remote failed. * bugfix: Running `copy --to` with a non-ssh remote actually did a move. - * Test suite improvements. Current top-level test coverage: 65% + * Many test suite improvements. Current top-level test coverage: 65% - -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 + -- Joey Hess Fri, 07 Jan 2011 14:33:13 -0400 git-annex (0.15) unstable; urgency=low diff --git a/doc/news/version_0.11.mdwn b/doc/news/version_0.11.mdwn deleted file mode 100644 index 03c4d7d54d..0000000000 --- a/doc/news/version_0.11.mdwn +++ /dev/null @@ -1,8 +0,0 @@ -git-annex 0.11 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * If available, rsync will be used for file transfers from remote - repositories. This allows resuming interrupted transfers. - * Added remote.annex-rsync-options. - * Avoid deleting temp files when rsync fails. - * Improve detection of version 0 repos. - * Add uninit subcommand. Closes: #[605749](http://bugs.debian.org/605749)"""]] \ No newline at end of file diff --git a/doc/news/version_0.16.mdwn b/doc/news/version_0.16.mdwn new file mode 100644 index 0000000000..3247e302dd --- /dev/null +++ b/doc/news/version_0.16.mdwn @@ -0,0 +1,13 @@ +git-annex 0.16 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * git-annex-shell: Avoid exposing any git repo config except for the + annex.uuid when doing configlist. + * bugfix: Running `move --to` with a remote whose UUID was not yet known + could result in git-annex not recording on the local side where the + file was moved to. This could not result in data loss, or even a + significant problem, since the remote *did* record that it had the file. + * Also, add a general guard to detect attempts to record information + about repositories with missing UUIDs. + * bugfix: Running `move --to` with a non-ssh remote failed. + * bugfix: Running `copy --to` with a non-ssh remote actually did a move. + * Many test suite improvements. Current top-level test coverage: 65%"""]] \ No newline at end of file diff --git a/test.hs b/test.hs index 852d1b30f5..a8e0253099 100644 --- a/test.hs +++ b/test.hs @@ -266,7 +266,7 @@ withgitrepo = bracket (setuprepo mainrepodir) return indir :: FilePath -> Assertion -> Assertion indir dir a = do cwd <- getCurrentDirectory - bracket_ (changeToTmpDir $ dir) + bracket_ (changeToTmpDir dir) (\_ -> changeWorkingDirectory cwd) a From 82fe151f8796c5f0a37565134deb55142fdd1a99 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 8 Jan 2011 15:00:04 -0400 Subject: [PATCH 0700/2835] add another test and improve error handling --- test.hs | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/test.hs b/test.hs index a8e0253099..ea1e0728ea 100644 --- a/test.hs +++ b/test.hs @@ -84,18 +84,24 @@ test_add = "git-annex add" ~: TestCase $ inmainrepo $ do unannexed ingitfile test_unannex :: Test -test_unannex = "git-annex unannex" ~: TestCase $ intmpcopyrepo $ do - git_annex "unannex" ["-q", annexedfile] @? "unannex failed" - unannexed annexedfile - git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" - unannexed annexedfile - git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op" - unannexed ingitfile +test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] + where + nocopy = "no content" ~: intmpclonerepo $ do + r <- git_annex "unannex" ["-q", annexedfile] + not r @? "unannex incorrectly succeeded with no copy" + unannexed annexedfile + withcopy = "with content" ~: intmpcopyrepo $ do + git_annex "unannex" ["-q", annexedfile] @? "unannex failed" + unannexed annexedfile + git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" + unannexed annexedfile + git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op" + unannexed ingitfile test_drop :: Test -test_drop = "git-annex drop" ~: TestList [nocopy, withcopy] +test_drop = "git-annex drop" ~: TestList [noremote, withremote] where - nocopy = "no remotes" ~: TestCase $ intmpcopyrepo $ do + noremote = "no remotes" ~: TestCase $ intmpcopyrepo $ do r <- git_annex "drop" ["-q", annexedfile] (not r) @? "drop wrongly succeeded with no known copy of file" annexed_present annexedfile @@ -104,7 +110,7 @@ test_drop = "git-annex drop" ~: TestList [nocopy, withcopy] git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed" git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op" unannexed ingitfile - withcopy = "with remote" ~: TestCase $ intmpclonerepo $ do + withremote = "with remote" ~: TestCase $ intmpclonerepo $ do git_annex "drop" ["-q", annexedfile] @? "drop failed though origin has copy" annexed_notpresent annexedfile inmainrepo $ annexed_present annexedfile @@ -266,9 +272,15 @@ withgitrepo = bracket (setuprepo mainrepodir) return indir :: FilePath -> Assertion -> Assertion indir dir a = do cwd <- getCurrentDirectory - bracket_ (changeToTmpDir dir) + -- Assertion failures throw non-IO errors; catch + -- any type of error and change back to cwd before + -- rethrowing. + r <- bracket_ (changeToTmpDir dir) (\_ -> changeWorkingDirectory cwd) - a + (E.try (a)::IO (Either E.SomeException ())) + case r of + Right () -> return () + Left e -> error $ show e setuprepo :: FilePath -> IO FilePath setuprepo dir = do @@ -279,8 +291,6 @@ setuprepo dir = do copyrepo :: FilePath -> FilePath -> IO FilePath copyrepo old new = do - _ <- clonerepo old new - indir new $ Utility.boolSystem "git" ["remote", "rm", "origin"] @? "git remote failed" cleanup new ensuretmpdir Utility.boolSystem "cp" ["-pr", old, new] @? "cp -pr failed" From 32b0e103909035ad0f25427c57a1ff504aefcada Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 8 Jan 2011 15:14:41 -0400 Subject: [PATCH 0701/2835] unannex: Now skips files whose content is not present, rather than it being an error. This allows gradual conversion from one backend to another by running unannex followed by add in each repository. --- Command/Unannex.hs | 8 ++++++-- debian/changelog | 8 ++++++++ doc/walkthrough.mdwn | 13 +++++++++++++ test.hs | 10 ++++++---- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 288f9da44a..2c60a23bb9 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -29,8 +29,12 @@ seek = [withFilesInGit start] {- The unannex subcommand undoes an add. -} start :: CommandStartString start file = isAnnexed file $ \(key, backend) -> do - showStart "unannex" file - return $ Just $ perform file key backend + ishere <- inAnnex key + if ishere + then do + showStart "unannex" file + return $ Just $ perform file key backend + else return Nothing perform :: FilePath -> Key -> Backend -> CommandPerform perform file key backend = do diff --git a/debian/changelog b/debian/changelog index bd986ad24c..7ca74f9945 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +git-annex (0.17) UNRELEASED; urgency=low + + * unannex: Now skips files whose content is not present, rather than + it being an error. This allows gradual conversion from one backend + to another by running unannex followed by add in each repository. + + -- Joey Hess Sat, 08 Jan 2011 15:04:48 -0400 + git-annex (0.16) unstable; urgency=low * git-annex-shell: Avoid exposing any git repo config except for the diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 2f868e3030..47f05ebcf4 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -277,6 +277,19 @@ add something like this to `.gitattributes`: * annex.backend=SHA1 +## migrating between backends + +Perhaps you had been using the WORM backend, but now have configured +git-annex to use SHA1 for new files. Your old files are still in WORM. How +to migrate that content? A quick and dirty way is to use the unannex +subcommand, which removes a file from git-annex's control, followed by +a re-add of the file, to put it in the new backend. + + # git annex unannex my_cool_big_file + unannex my_cool_big_file ok + # git annex add my_cool_big_file + add my_cool_big_file (checksum ...) ok + ## unused data It's possible for data to accumulate in the annex that no files point to diff --git a/test.hs b/test.hs index ea1e0728ea..35085cc37e 100644 --- a/test.hs +++ b/test.hs @@ -16,6 +16,7 @@ import Data.List import System.IO.Error import System.Posix.Env import qualified Control.Exception.Extensible as E +import Control.Exception (throw) import qualified GitRepo as Git import qualified Locations @@ -87,10 +88,11 @@ test_unannex :: Test test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] where nocopy = "no content" ~: intmpclonerepo $ do - r <- git_annex "unannex" ["-q", annexedfile] - not r @? "unannex incorrectly succeeded with no copy" - unannexed annexedfile + annexed_notpresent annexedfile + git_annex "unannex" ["-q", annexedfile] @? "unannex failed with no copy" + annexed_notpresent annexedfile withcopy = "with content" ~: intmpcopyrepo $ do + annexed_present annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed" unannexed annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" @@ -280,7 +282,7 @@ indir dir a = do (E.try (a)::IO (Either E.SomeException ())) case r of Right () -> return () - Left e -> error $ show e + Left e -> throw e setuprepo :: FilePath -> IO FilePath setuprepo dir = do From a78b0555e1d46c4548cda3aaa1709040f6fa7f33 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 8 Jan 2011 15:54:14 -0400 Subject: [PATCH 0702/2835] New migrate subcommand can be used to switch files to using a different backend, safely and with no duplication of content. --- Command.hs | 6 +++++ Command/Add.hs | 5 ++-- Command/Migrate.hs | 63 ++++++++++++++++++++++++++++++++++++++++++++ GitAnnex.hs | 2 ++ TypeInternals.hs | 3 +++ debian/changelog | 7 ++--- doc/git-annex.mdwn | 8 ++++++ doc/walkthrough.mdwn | 35 +++++++++++++++--------- 8 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 Command/Migrate.hs diff --git a/Command.hs b/Command.hs index 690dd20ecf..b83e640b9b 100644 --- a/Command.hs +++ b/Command.hs @@ -117,6 +117,12 @@ withAttrFilesInGit attr a params = do files' <- filterFiles files pairs <- liftIO $ Git.checkAttr repo attr files' return $ map a pairs +withBackendFilesInGit :: CommandSeekBackendFiles +withBackendFilesInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ Git.inRepo repo params + files' <- filterFiles files + backendPairs a files' withFilesMissing :: CommandSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params diff --git a/Command/Add.hs b/Command/Add.hs index bc869a67de..c74b726e3f 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -42,11 +42,12 @@ perform (file, backend) = do stored <- Backend.storeFileKey file backend case stored of Nothing -> return Nothing - Just (key, _) -> return $ Just $ cleanup file key + Just (key, _) -> do + moveAnnex key file + return $ Just $ cleanup file key cleanup :: FilePath -> Key -> CommandCleanup cleanup file key = do - moveAnnex key file logStatus key ValuePresent link <- calcGitLink file key diff --git a/Command/Migrate.hs b/Command/Migrate.hs new file mode 100644 index 0000000000..0caded6d13 --- /dev/null +++ b/Command/Migrate.hs @@ -0,0 +1,63 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Migrate where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory + +import Command +import qualified Annex +import qualified Backend +import Locations +import Types +import Core +import Messages +import qualified Command.Add + +command :: [Command] +command = [Command "migrate" paramPath seek "switch data to different backend"] + +seek :: [CommandSeek] +seek = [withBackendFilesInGit start] + +start :: CommandStartBackendFile +start (_, Nothing) = return Nothing +start (file, Just newbackend) = isAnnexed file $ \(key, oldbackend) -> do + exists <- inAnnex key + if (newbackend /= oldbackend) && exists + then do + showStart "migrate" file + return $ Just $ perform file key newbackend + else + return Nothing + +perform :: FilePath -> Key -> Backend -> CommandPerform +perform file oldkey newbackend = do + g <- Annex.gitRepo + + -- Store the old backend's cached key in the new backend + -- (the file can't be stored as usual, because it's already a symlink). + -- The old backend's key is not dropped from it, because there may + -- be other files still pointing at that key. + let src = annexLocation g oldkey + stored <- Backend.storeFileKey src $ Just newbackend + case stored of + Nothing -> return Nothing + Just (newkey, _) -> do + ok <- getViaTmp newkey $ \t -> do + -- Make a hard link to the old backend's + -- cached key, to avoid wasting disk space. + liftIO $ createLink src t + return True + if ok + then do + -- Update symlink to use the new key. + liftIO $ removeFile file + return $ Just $ Command.Add.cleanup file newkey + else return Nothing diff --git a/GitAnnex.hs b/GitAnnex.hs index 24c9ace0ac..d9efdad2dd 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -32,6 +32,7 @@ import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit import qualified Command.Find +import qualified Command.Migrate import qualified Command.Uninit import qualified Command.Trust import qualified Command.Untrust @@ -59,6 +60,7 @@ cmds = concat , Command.Unused.command , Command.DropUnused.command , Command.Find.command + , Command.Migrate.command ] options :: [Option] diff --git a/TypeInternals.hs b/TypeInternals.hs index fe6e562f95..12a9080b33 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -103,3 +103,6 @@ data Backend = Backend { instance Show Backend where show backend = "Backend { name =\"" ++ name backend ++ "\" }" + +instance Eq Backend where + a == b = name a == name b diff --git a/debian/changelog b/debian/changelog index 7ca74f9945..85878113e9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,11 @@ git-annex (0.17) UNRELEASED; urgency=low * unannex: Now skips files whose content is not present, rather than - it being an error. This allows gradual conversion from one backend - to another by running unannex followed by add in each repository. + it being an error. + * New migrate subcommand can be used to switch files to using a different + backend, safely and with no duplication of content. - -- Joey Hess Sat, 08 Jan 2011 15:04:48 -0400 + -- Joey Hess Sat, 08 Jan 2011 13:45:06 -0400 git-annex (0.16) unstable; urgency=low diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e99be4e409..6d106fea48 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -144,6 +144,14 @@ Many git-annex commands will stage changes for later `git commit` by you. With no parameters, defaults to finding all files in the current directory and its subdirectories. +* migrate [path ...] + + Changes the specified annexed files to store their content in the + default backend (or the one specified with --backend). + + Note that the content is not removed from the backend it was previously in. + Use `git annex unused` to find and remove such content. + * unannex [path ...] Use this to undo an accidental add command. This is not the command you diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 47f05ebcf4..d2231c81e6 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -277,25 +277,32 @@ add something like this to `.gitattributes`: * annex.backend=SHA1 -## migrating between backends +## migrating data to a new backend -Perhaps you had been using the WORM backend, but now have configured -git-annex to use SHA1 for new files. Your old files are still in WORM. How -to migrate that content? A quick and dirty way is to use the unannex -subcommand, which removes a file from git-annex's control, followed by -a re-add of the file, to put it in the new backend. +Maybe you started out using the WORM backend, and have now configured +git-annex to use SHA1. But files you added to the annex before still +use the WORM backend. There is a simple command that can migrate that +data: - # git annex unannex my_cool_big_file - unannex my_cool_big_file ok - # git annex add my_cool_big_file - add my_cool_big_file (checksum ...) ok + # git annex migrate my_cool_big_file + migrate my_cool_big_file (checksum...) ok + +You can only migrate files whose content is currently available. Other +files will be skipped. + +After migrating a file to a new backend, the old content in the old backend +will still be present. That is necessary because multiple files +can point to the same content. The `git annex unused` sucommand can be +used to clear up that detritus later. Note that hard links are used, +to avoid wasting disk space. ## unused data It's possible for data to accumulate in the annex that no files point to -nymore. One way it can happen is if you `git rm` a file without +anymore. One way it can happen is if you `git rm` a file without first calling `git annex drop`. And, when you modify an annexed file, the old -content of the file remains in the annex. +content of the file remains in the annex. Another way is when migrating +between backends. This might be historical data you want to preserve, so git-annex defaults to preserving it. So from time to time, you may want to check for such data and @@ -318,6 +325,10 @@ data anymore, you can easily remove it: # git annex dropunused 1 dropunused 1 ok +Hint: To drop a lot of unused data, use a command like this: + + # git annex dropunused `seq 1 1000` + ## fsck: verifying your data You can use the fsck subcommand to check for problems in your data. From 9de982eab7091aed9f3e75122372bbf0ac141763 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 8 Jan 2011 16:09:17 -0400 Subject: [PATCH 0703/2835] copyright years --- Command/Migrate.hs | 2 +- debian/copyright | 2 +- test.hs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 0caded6d13..3d61214377 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2010 Joey Hess + - Copyright 2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} diff --git a/debian/copyright b/debian/copyright index 55638653c1..eceb2d9ffc 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,5 +1,5 @@ Files: * -Copyright: © 2010 Joey Hess +Copyright: © 2010,2011 Joey Hess License: GPL-3+ The full text of the GPL is distributed as doc/GPL in this package's source, or in /usr/share/common-licenses/GPL on Debian systems. diff --git a/test.hs b/test.hs index 35085cc37e..8c9298e679 100644 --- a/test.hs +++ b/test.hs @@ -1,6 +1,6 @@ {- git-annex test suite - - - Copyright 2010 Joey Hess + - Copyright 2010,2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} From 292c2796a42167378573028b97c4de582d107dc8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmlUJkSWTkiWuwzwilXh1Wd3jg8I33iTQg" Date: Sun, 9 Jan 2011 13:16:53 +0000 Subject: [PATCH 0704/2835] --- doc/bugs/problem_commit_normal_links.mdwn | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 doc/bugs/problem_commit_normal_links.mdwn diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn new file mode 100644 index 0000000000..a669d85fdd --- /dev/null +++ b/doc/bugs/problem_commit_normal_links.mdwn @@ -0,0 +1,35 @@ +Dear All, + +thank you for this wonderful tool! + +I am having an issue when I try to commit a normal link + +diokletian*194-> mkdir test +diokletian*195-> cd test +diokletian*196-> git init +Initialized empty Git repository in /home/henrus/test/.git/ +diokletian*197-> git annex init new +init new [master (root-commit) 49f5f91] git-annex setup + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 100644 .gitattributes +[master 76496ff] git annex init + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 100644 .git-annex/uuid.log +ok +diokletian*198-> mkdir subdir +diokletian*199-> ln -s subdir link +diokletian*200-> git add link +diokletian*201-> git commit -m "ok" +[master f12f62d] ok + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 link +diokletian*202-> ln -s subdir/ link2 +diokletian*203-> git add link2 +diokletian*204-> git commit -m "not ok" +git-annex: Prelude.head: empty list + +The trailing slash seems to make a difference! + +Best Regards, + +Henrik From 51327c242f7087587884903632505552917b844a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmlUJkSWTkiWuwzwilXh1Wd3jg8I33iTQg" Date: Sun, 9 Jan 2011 13:17:59 +0000 Subject: [PATCH 0705/2835] --- doc/bugs/problem_commit_normal_links.mdwn | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn index a669d85fdd..bc96ec9380 100644 --- a/doc/bugs/problem_commit_normal_links.mdwn +++ b/doc/bugs/problem_commit_normal_links.mdwn @@ -5,27 +5,49 @@ thank you for this wonderful tool! I am having an issue when I try to commit a normal link diokletian*194-> mkdir test + diokletian*195-> cd test + diokletian*196-> git init + Initialized empty Git repository in /home/henrus/test/.git/ + diokletian*197-> git annex init new + init new [master (root-commit) 49f5f91] git-annex setup + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 100644 .gitattributes + [master 76496ff] git annex init + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 100644 .git-annex/uuid.log + ok + diokletian*198-> mkdir subdir + diokletian*199-> ln -s subdir link + diokletian*200-> git add link + diokletian*201-> git commit -m "ok" + [master f12f62d] ok + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 link + diokletian*202-> ln -s subdir/ link2 + diokletian*203-> git add link2 + diokletian*204-> git commit -m "not ok" + git-annex: Prelude.head: empty list The trailing slash seems to make a difference! From f4ddb580c83cba1c0e66fc5deb20bfc49ee4cf1b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 10:04:16 -0400 Subject: [PATCH 0706/2835] bugfix: Fix crash caused by a symlink in the repo with link text ending in a "/". (Thanks Henrik for reporting.) --- Backend.hs | 3 ++- debian/changelog | 6 ++++-- doc/bugs/problem_commit_normal_links.mdwn | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Backend.hs b/Backend.hs index 8142e4707a..7e88ff3043 100644 --- a/Backend.hs +++ b/Backend.hs @@ -130,7 +130,8 @@ lookupFile file = do getsymlink = do l <- readSymbolicLink file return $ takeFileName l - makekey bs l = + makekey _ [] = return Nothing + makekey bs l = do case maybeLookupBackendName bs bname of Nothing -> do unless (null kname || null bname) $ diff --git a/debian/changelog b/debian/changelog index 85878113e9..2fd6d9d81e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,11 +1,13 @@ -git-annex (0.17) UNRELEASED; urgency=low +git-annex (0.17) unstable; urgency=low * unannex: Now skips files whose content is not present, rather than it being an error. * New migrate subcommand can be used to switch files to using a different backend, safely and with no duplication of content. + * bugfix: Fix crash caused by a symlink in the repo with link text ending in + a "/". (Thanks Henrik for reporting.) - -- Joey Hess Sat, 08 Jan 2011 13:45:06 -0400 + -- Joey Hess Sun, 09 Jan 2011 10:04:11 -0400 git-annex (0.16) unstable; urgency=low diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn index bc96ec9380..11c9f7d8fb 100644 --- a/doc/bugs/problem_commit_normal_links.mdwn +++ b/doc/bugs/problem_commit_normal_links.mdwn @@ -55,3 +55,5 @@ The trailing slash seems to make a difference! Best Regards, Henrik + +> Thanks for the bug report. This is fixed in 0.17. --[[Joey]] From 53170b1a1496084923531046386794c092cc64b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 10:48:04 -0400 Subject: [PATCH 0707/2835] fix real underlaying cause, and fix test suite to not skip it --- Backend.hs | 1 - TypeInternals.hs | 10 +++------- debian/changelog | 3 +-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Backend.hs b/Backend.hs index 7e88ff3043..a417c7247b 100644 --- a/Backend.hs +++ b/Backend.hs @@ -130,7 +130,6 @@ lookupFile file = do getsymlink = do l <- readSymbolicLink file return $ takeFileName l - makekey _ [] = return Nothing makekey bs l = do case maybeLookupBackendName bs bname of Nothing -> do diff --git a/TypeInternals.hs b/TypeInternals.hs index 12a9080b33..44db743faa 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -55,7 +55,7 @@ instance Read Key where readsPrec _ s = [(Key (b,k), "")] where l = split ":" s - b = head l + b = if null l then "" else head l k = join ":" $ drop 1 l -- for quickcheck @@ -67,13 +67,9 @@ instance Arbitrary Key where prop_idempotent_key_read_show :: Key -> Bool prop_idempotent_key_read_show k - -- filter out empty key or backend names - -- also backend names will not contain colons - | null kname || null bname || elem ':' bname = True + -- backend names will never contain colons + | elem ':' (backendName k) = True | otherwise = k == (read $ show k) - where - bname = backendName k - kname = keyName k backendName :: Key -> BackendName backendName (Key (b,_)) = b diff --git a/debian/changelog b/debian/changelog index 2fd6d9d81e..9ae31edc7b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,8 +4,7 @@ git-annex (0.17) unstable; urgency=low it being an error. * New migrate subcommand can be used to switch files to using a different backend, safely and with no duplication of content. - * bugfix: Fix crash caused by a symlink in the repo with link text ending in - a "/". (Thanks Henrik for reporting.) + * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.) -- Joey Hess Sun, 09 Jan 2011 10:04:11 -0400 From 5ddabe1bf70cb86be71c4561b88c5d8964fbf515 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 11:01:59 -0400 Subject: [PATCH 0708/2835] release --- doc/news/version_0.12.mdwn | 6 ------ doc/news/version_0.17.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 doc/news/version_0.12.mdwn create mode 100644 doc/news/version_0.17.mdwn diff --git a/doc/news/version_0.12.mdwn b/doc/news/version_0.12.mdwn deleted file mode 100644 index e7fccd1b36..0000000000 --- a/doc/news/version_0.12.mdwn +++ /dev/null @@ -1,6 +0,0 @@ -git-annex 0.12 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Add --exclude option to exclude files from processing. - * mwdn2man: Fix a bug in newline supression. Closes: #[606578](http://bugs.debian.org/606578) - * Bugfix to git annex add of an unlocked file in a subdir. Closes: #[606579](http://bugs.debian.org/606579) - * Makefile: Add PREFIX variable."""]] \ No newline at end of file diff --git a/doc/news/version_0.17.mdwn b/doc/news/version_0.17.mdwn new file mode 100644 index 0000000000..ddb59046eb --- /dev/null +++ b/doc/news/version_0.17.mdwn @@ -0,0 +1,7 @@ +git-annex 0.17 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * unannex: Now skips files whose content is not present, rather than + it being an error. + * New migrate subcommand can be used to switch files to using a different + backend, safely and with no duplication of content. + * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.)"""]] \ No newline at end of file From fd3efc086a98e298faff458aa64c2fe811081c5f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 11:02:04 -0400 Subject: [PATCH 0709/2835] close bug --- doc/bugs/problem_commit_normal_links.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn index 11c9f7d8fb..6dbd41fb4e 100644 --- a/doc/bugs/problem_commit_normal_links.mdwn +++ b/doc/bugs/problem_commit_normal_links.mdwn @@ -56,4 +56,4 @@ Best Regards, Henrik -> Thanks for the bug report. This is fixed in 0.17. --[[Joey]] +> Thanks for the bug report. This is fixed in 0.17. --[[Joey]] [[!tag done]] From 9de1192c574dc304f3e9dbfd01296f74c7ea5969 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 15:12:52 -0400 Subject: [PATCH 0710/2835] enable dpkg-mergechangelogs --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index b98b07d7d2..5d425843f2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -.git-annex/*.log merge=union +debian/changelog merge=dpkg-mergechangelogs From c2b13a88bf2e8bdeb126e4f13a5aeb0fbe09a0c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 18:23:58 -0400 Subject: [PATCH 0711/2835] use concat --- Command.hs | 2 +- GitRepo.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Command.hs b/Command.hs index b83e640b9b..31bb1f1f9b 100644 --- a/Command.hs +++ b/Command.hs @@ -64,7 +64,7 @@ data Command = Command { prepCmd :: Command -> [String] -> Annex [Annex Bool] prepCmd Command { cmdseek = seek } params = do lists <- mapM (\s -> s params) seek - return $ map doCommand $ foldl (++) [] lists + return $ map doCommand $ concat lists {- Runs a command through the start, perform and cleanup stages -} doCommand :: CommandStart -> CommandCleanup diff --git a/GitRepo.hs b/GitRepo.hs index 07f243b661..533038fc86 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -438,7 +438,7 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\"" e_num c = showoctal $ ord c -- unicode character is decomposed to -- Word8s and each is shown in octal - e_utf c = foldl (++) "" $ map showoctal $ + e_utf c = concat $ map showoctal $ (encode [c] :: [Word8]) From 3a844b1f3c5eb85d4571857ac10aa8183996513a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 15:43:29 -0400 Subject: [PATCH 0712/2835] test locationlog contents too --- test.hs | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/test.hs b/test.hs index 8c9298e679..20e8bc28ab 100644 --- a/test.hs +++ b/test.hs @@ -18,11 +18,15 @@ import System.Posix.Env import qualified Control.Exception.Extensible as E import Control.Exception (throw) +import qualified Annex +import qualified BackendList +import qualified Backend import qualified GitRepo as Git import qualified Locations import qualified Utility import qualified TypeInternals import qualified GitAnnex +import qualified LocationLog main :: IO () main = do @@ -359,6 +363,26 @@ checkdangling f = do Left _ -> return () -- expected; dangling link Right _ -> assertFailure $ f ++ " was not a dangling link as expected" +checklocationlog :: FilePath -> Bool -> Assertion +checklocationlog f expected = do + g <- Git.repoFromCwd + g' <- Git.configRead g + let thisuuid = Git.configGet g' "annex.uuid" "" + s <- Annex.new g BackendList.allBackends + r <- Annex.eval s $ Backend.lookupFile f + case r of + Just (k, _) -> do + uuids <- LocationLog.keyLocations g' k + assertEqual ("location log for " ++ f ++ " " ++ (show k) ++ " " ++ thisuuid) + expected (elem thisuuid uuids) + _ -> assertFailure $ f ++ " failed to look up key" + +inlocationlog :: FilePath -> Assertion +inlocationlog f = checklocationlog f True + +notinlocationlog :: FilePath -> Assertion +notinlocationlog f = checklocationlog f False + runchecks :: [FilePath -> Assertion] -> FilePath -> Assertion runchecks [] _ = return () runchecks (a:as) f = do @@ -366,10 +390,12 @@ runchecks (a:as) f = do runchecks as f annexed_notpresent :: FilePath -> Assertion -annexed_notpresent = runchecks [checklink, checkdangling, checkunwritable] +annexed_notpresent = runchecks + [checklink, checkdangling, checkunwritable, notinlocationlog] annexed_present :: FilePath -> Assertion -annexed_present = runchecks [checklink, checkcontent, checkunwritable] +annexed_present = runchecks + [checklink, checkcontent, checkunwritable, inlocationlog] unannexed :: FilePath -> Assertion unannexed = runchecks [checkregularfile, checkcontent, checkwritable] From a8ce30401dd69d1d203cfc33b791c1b4d175666e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 16:00:40 -0400 Subject: [PATCH 0713/2835] add checks that location log files are committed currently failing for move --to --- GitRepo.hs | 6 ++++++ LocationLog.hs | 1 + test.hs | 12 ++++++++++++ 3 files changed, 19 insertions(+) diff --git a/GitRepo.hs b/GitRepo.hs index 533038fc86..ec363fe73b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -38,6 +38,7 @@ module GitRepo ( inRepo, notInRepo, stagedFiles, + changedUnstagedFiles, checkAttr, decodeGitFile, encodeGitFile, @@ -249,6 +250,11 @@ stagedFiles repo l = pipeNullSplit repo $ ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", "--"] ++ l +{- Returns a list of files that have unstaged changes. -} +changedUnstagedFiles :: Repo -> [FilePath] -> IO [FilePath] +changedUnstagedFiles repo l = pipeNullSplit repo $ + ["diff", "--name-only", "-z", "--"] ++ l + {- Returns a list of the files in the specified locations that are staged - for commit, and whose type has changed. -} typeChangedStagedFiles :: Repo -> [FilePath] -> IO [FilePath] diff --git a/LocationLog.hs b/LocationLog.hs index e4dad03f5c..926939051c 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -23,6 +23,7 @@ module LocationLog ( LogStatus(..), logChange, + logFile, keyLocations ) where diff --git a/test.hs b/test.hs index 20e8bc28ab..0a5a365d97 100644 --- a/test.hs +++ b/test.hs @@ -375,6 +375,18 @@ checklocationlog f expected = do uuids <- LocationLog.keyLocations g' k assertEqual ("location log for " ++ f ++ " " ++ (show k) ++ " " ++ thisuuid) expected (elem thisuuid uuids) + + -- Location log files should always be checked + -- into git, and any modifications staged for + -- commit. This is a regression test, as some + -- commands forgot to. + let lf = LocationLog.logFile g' k + fs <- Git.inRepo g' [lf] + when (null fs) $ + assertFailure $ f ++ " logfile not added to git repo" + ufs <- Git.changedUnstagedFiles g' [lf] + when (not $ null ufs) $ + assertFailure $ f ++ " logfile changes not staged" _ -> assertFailure $ f ++ " failed to look up key" inlocationlog :: FilePath -> Assertion From 196c2fa78633aaaf9a43c8aa6a54b4144ffac3e2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 16:06:19 -0400 Subject: [PATCH 0714/2835] Bugfix: `copy --to` and `move --to` forgot to stage location log changes after transferring the file to the remote repository. (Did not affect ssh remotes.) --- Remotes.hs | 7 +++++-- debian/changelog | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 2c8cdfdad1..a7a1db4152 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -252,8 +252,11 @@ copyToRemote r key -- run copy from perspective of remote liftIO $ do a <- Annex.new r [] - Annex.eval a $ Core.getViaTmp key $ \f -> - liftIO $ copyFile keysrc f + Annex.eval a $ do + ok <- Core.getViaTmp key $ + \f -> liftIO $ copyFile keysrc f + Annex.queueRun + return ok | Git.repoIsSsh r = do g <- Annex.gitRepo let keysrc = annexLocation g key diff --git a/debian/changelog b/debian/changelog index 9ae31edc7b..96acc592c9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +git-annex (0.18) UNRELEASED; urgency=low + + * Bugfix: `copy --to` and `move --to` forgot to stage location log changes + after transferring the file to the remote repository. + (Did not affect ssh remotes.) + + -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 + git-annex (0.17) unstable; urgency=low * unannex: Now skips files whose content is not present, rather than From 8d6da87eec87de5317185f7bb8ebf50013e41c11 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 18:13:26 -0400 Subject: [PATCH 0715/2835] better types --- Annex.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Annex.hs b/Annex.hs index 55f9edb36e..765c9191fb 100644 --- a/Annex.hs +++ b/Annex.hs @@ -51,9 +51,9 @@ new gitrepo allbackends = do Annex.gitRepoChange gitrepo' {- performs an action in the Annex monad -} -run :: AnnexState -> StateT AnnexState IO a -> IO (a, AnnexState) +run :: AnnexState -> Annex a -> IO (a, AnnexState) run state action = runStateT action state -eval :: AnnexState -> StateT AnnexState IO a -> IO a +eval :: AnnexState -> Annex a -> IO a eval state action = evalStateT action state {- Returns the git repository being acted on -} From 730c273eae055eaed743593b6ab7274e210bf4e3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 18:49:32 -0400 Subject: [PATCH 0716/2835] robustness fixes for empty or missing trust log --- UUID.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/UUID.hs b/UUID.hs index 21f2b202e6..67695d3101 100644 --- a/UUID.hs +++ b/UUID.hs @@ -145,8 +145,12 @@ getTrusted :: Annex [UUID] getTrusted = do logfile <- trustLog s <- liftIO $ catch (readFile logfile) ignoreerror - return $ map (\l -> head $ words l) $ lines s + return $ parse s where + parse [] = [] + parse s = map firstword $ lines s + firstword [] = "" + firstword l = head $ words l ignoreerror _ = return "" {- Changes the list of trusted UUIDs. -} From caa0b6c0c275ed85567112c273b49e7905bb4b14 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 18:49:45 -0400 Subject: [PATCH 0717/2835] quiet git commit messages --- Command/Trust.hs | 2 +- Command/Untrust.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Command/Trust.hs b/Command/Trust.hs index 35ddefe842..c97d75ee47 100644 --- a/Command/Trust.hs +++ b/Command/Trust.hs @@ -40,5 +40,5 @@ perform repo = do g <- Annex.gitRepo logfile <- trustLog liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-m", "git annex untrust", logfile] + liftIO $ Git.run g ["commit", "-q", "-m", "git annex untrust", logfile] return $ Just $ return True diff --git a/Command/Untrust.hs b/Command/Untrust.hs index f49a2e989a..01b97b1c1f 100644 --- a/Command/Untrust.hs +++ b/Command/Untrust.hs @@ -40,5 +40,5 @@ perform repo = do g <- Annex.gitRepo logfile <- trustLog liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-m", "git annex untrust", logfile] + liftIO $ Git.run g ["commit", "-q", "-m", "git annex untrust", logfile] return $ Just $ return True From cebee3740142484f2061ebe323fec7b0f45fbd4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 18:50:18 -0400 Subject: [PATCH 0718/2835] add tests for trust/untrust --- test.hs | 59 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/test.hs b/test.hs index 0a5a365d97..7e56fd65c1 100644 --- a/test.hs +++ b/test.hs @@ -17,6 +17,7 @@ import System.IO.Error import System.Posix.Env import qualified Control.Exception.Extensible as E import Control.Exception (throw) +import Control.Monad.State (liftIO) import qualified Annex import qualified BackendList @@ -27,6 +28,8 @@ import qualified Utility import qualified TypeInternals import qualified GitAnnex import qualified LocationLog +import qualified UUID +import qualified Remotes main :: IO () main = do @@ -64,6 +67,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_lock , test_edit , test_fix + , test_trust ] test_init :: Test @@ -244,6 +248,28 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do subdir = "s" newfile = subdir ++ "/" ++ annexedfile +test_trust :: Test +test_trust = "git-annex trust/untrust" ~: intmpclonerepo $ do + trust False + git_annex "trust" ["-q", "origin"] @? "trust failed" + trust True + git_annex "trust" ["-q", "origin"] @? "trust of trusted failed" + trust True + git_annex "untrust" ["-q", "origin"] @? "untrust failed" + trust False + git_annex "untrust" ["-q", "origin"] @? "untrust of untrusted failed" + trust False + where + trust expected = do + istrusted <- annexeval $ do + uuids <- UUID.getTrusted + r <- Remotes.byName "origin" + u <- UUID.getUUID r + return $ elem u uuids + assertEqual "trust value" expected istrusted + +-- This is equivilant to running git-annex, but it's all run in-process +-- so test coverage collection works. git_annex :: String -> [String] -> IO Bool git_annex command params = do -- catch all errors, including normally fatal errors @@ -254,6 +280,15 @@ git_annex command params = do where run = GitAnnex.run (command:params) +-- Runs an action in the current annex. Note that shutdown actions +-- are not run; this should only be used for actions that query state. +annexeval :: TypeInternals.Annex a -> IO a +annexeval a = do + g <- Git.repoFromCwd + g' <- Git.configRead g + s <- Annex.new g' BackendList.allBackends + Annex.eval s a + innewrepo :: Assertion -> Assertion innewrepo a = withgitrepo $ \r -> indir r a @@ -365,26 +400,30 @@ checkdangling f = do checklocationlog :: FilePath -> Bool -> Assertion checklocationlog f expected = do - g <- Git.repoFromCwd - g' <- Git.configRead g - let thisuuid = Git.configGet g' "annex.uuid" "" - s <- Annex.new g BackendList.allBackends - r <- Annex.eval s $ Backend.lookupFile f + thisuuid <- annexeval $ do + g <- Annex.gitRepo + UUID.getUUID g + r <- annexeval $ Backend.lookupFile f case r of Just (k, _) -> do - uuids <- LocationLog.keyLocations g' k - assertEqual ("location log for " ++ f ++ " " ++ (show k) ++ " " ++ thisuuid) + uuids <- annexeval $ do + g <- Annex.gitRepo + liftIO $ LocationLog.keyLocations g k + assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid) expected (elem thisuuid uuids) -- Location log files should always be checked -- into git, and any modifications staged for -- commit. This is a regression test, as some -- commands forgot to. - let lf = LocationLog.logFile g' k - fs <- Git.inRepo g' [lf] + (fs, ufs) <- annexeval $ do + g <- Annex.gitRepo + let lf = LocationLog.logFile g k + fs <- liftIO $ Git.inRepo g [lf] + ufs <- liftIO $ Git.changedUnstagedFiles g [lf] + return (fs, ufs) when (null fs) $ assertFailure $ f ++ " logfile not added to git repo" - ufs <- Git.changedUnstagedFiles g' [lf] when (not $ null ufs) $ assertFailure $ f ++ " logfile changes not staged" _ -> assertFailure $ f ++ " failed to look up key" From cc7db6f058f69a1a0f08221f46d2d12d3d3764bf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 19:34:28 -0400 Subject: [PATCH 0719/2835] test fsck --- test.hs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test.hs b/test.hs index 7e56fd65c1..0c6e018842 100644 --- a/test.hs +++ b/test.hs @@ -30,6 +30,7 @@ import qualified GitAnnex import qualified LocationLog import qualified UUID import qualified Remotes +import qualified Core main :: IO () main = do @@ -68,6 +69,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_edit , test_fix , test_trust + , test_fsck ] test_init :: Test @@ -268,6 +270,20 @@ test_trust = "git-annex trust/untrust" ~: intmpclonerepo $ do return $ elem u uuids assertEqual "trust value" expected istrusted +test_fsck :: Test +test_fsck = "git-annex fsck" ~: intmpclonerepo $ do + git_annex "fsck" ["-q"] @? "fsck failed" + Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed" + r <- git_annex "fsck" ["-q"] + not r @? "fsck failed to fail with numcopies unsatisfied" + Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed" + + git_annex "get" ["-q", annexedfile] @? "get of file failed" + Core.allowWrite annexedfile + writeFile annexedfile (changedcontent annexedfile) + r <- git_annex "fsck" ["-q"] + not r @? "fsck failed to fail with corrupted file content" + -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. git_annex :: String -> [String] -> IO Bool From e2af0914faf487464046e0a60d20a638add1790d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 19:41:13 -0400 Subject: [PATCH 0720/2835] fsck: Fix bug in moving of corrupted files to .git/annex/bad/ --- Core.hs | 3 ++- debian/changelog | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 08e2265920..d59120d673 100644 --- a/Core.hs +++ b/Core.hs @@ -173,7 +173,8 @@ moveBad key = do g <- Annex.gitRepo let src = annexLocation g key let dest = annexBadLocation g ++ takeFileName src - liftIO $ createDirectoryIfMissing True dest + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ allowWrite (parentDir src) liftIO $ renameFile src dest liftIO $ removeDirectory (parentDir src) return dest diff --git a/debian/changelog b/debian/changelog index 96acc592c9..c826fb7d6e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.18) UNRELEASED; urgency=low * Bugfix: `copy --to` and `move --to` forgot to stage location log changes after transferring the file to the remote repository. (Did not affect ssh remotes.) + * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 From 485dbdd1a9b93274d04719b7a218a3a2728b5058 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 19:59:11 -0400 Subject: [PATCH 0721/2835] add tests of setkey/fromkey --- test.hs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/test.hs b/test.hs index 0c6e018842..b6edd8f138 100644 --- a/test.hs +++ b/test.hs @@ -18,6 +18,7 @@ import System.Posix.Env import qualified Control.Exception.Extensible as E import Control.Exception (throw) import Control.Monad.State (liftIO) +import Maybe import qualified Annex import qualified BackendList @@ -31,6 +32,7 @@ import qualified LocationLog import qualified UUID import qualified Remotes import qualified Core +import qualified Backend.SHA1 main :: IO () main = do @@ -60,6 +62,7 @@ toplevels = TestLabel "toplevel" $ TestList -- test order matters, later tests may rely on state from earlier [ test_init , test_add + , test_setkey , test_unannex , test_drop , test_get @@ -94,6 +97,17 @@ test_add = "git-annex add" ~: TestCase $ inmainrepo $ do git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op" unannexed ingitfile +test_setkey :: Test +test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do + writeFile tmp $ content sha1annexedfile + r <- annexeval $ TypeInternals.getKey Backend.SHA1.backend tmp + let sha1 = TypeInternals.keyName $ fromJust r + git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" + git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" + annexed_present sha1annexedfile + where + tmp = "tmpfile" + test_unannex :: Test test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] where @@ -281,8 +295,8 @@ test_fsck = "git-annex fsck" ~: intmpclonerepo $ do git_annex "get" ["-q", annexedfile] @? "get of file failed" Core.allowWrite annexedfile writeFile annexedfile (changedcontent annexedfile) - r <- git_annex "fsck" ["-q"] - not r @? "fsck failed to fail with corrupted file content" + r' <- git_annex "fsck" ["-q"] + not r' @? "fsck failed to fail with corrupted file content" -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. @@ -494,6 +508,9 @@ tmprepodir = tmpdir ++ "/tmprepo" annexedfile :: String annexedfile = "foo" +sha1annexedfile :: String +sha1annexedfile = "sha1foo" + ingitfile :: String ingitfile = "bar" @@ -501,6 +518,7 @@ content :: FilePath -> String content f | f == annexedfile = "annexed file content" | f == ingitfile = "normal file content" + | f == sha1annexedfile ="sha1 annexed file content" | otherwise = "unknown file " ++ f changecontent :: FilePath -> IO () From 098168559d13d0de4c881015ad251d339ac9b562 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 20:06:15 -0400 Subject: [PATCH 0722/2835] more fsck checks 72% coverage --- test.hs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test.hs b/test.hs index b6edd8f138..e36ab0955b 100644 --- a/test.hs +++ b/test.hs @@ -104,6 +104,7 @@ test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do let sha1 = TypeInternals.keyName $ fromJust r git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" annexed_present sha1annexedfile where tmp = "tmpfile" @@ -291,12 +292,16 @@ test_fsck = "git-annex fsck" ~: intmpclonerepo $ do r <- git_annex "fsck" ["-q"] not r @? "fsck failed to fail with numcopies unsatisfied" Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed" - - git_annex "get" ["-q", annexedfile] @? "get of file failed" - Core.allowWrite annexedfile - writeFile annexedfile (changedcontent annexedfile) - r' <- git_annex "fsck" ["-q"] - not r' @? "fsck failed to fail with corrupted file content" + corrupt annexedfile + corrupt sha1annexedfile + where + corrupt f = do + git_annex "get" ["-q", f] @? "get of file failed" + Core.allowWrite f + writeFile f (changedcontent f) + r <- git_annex "fsck" ["-q"] + not r @? "fsck failed to fail with corrupted file content" + git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem" -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. From 868486c6b6bad4614dcc6cad5a2e29f85833f967 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 20:08:15 -0400 Subject: [PATCH 0723/2835] update --- debian/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/changelog b/debian/changelog index c826fb7d6e..74fa4894a9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.18) UNRELEASED; urgency=low after transferring the file to the remote repository. (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ + * Test suite improvements. Current top-level test coverage: 72% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 From b13039d62eb3a99606562f2edca7405b96776787 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 20:48:58 -0400 Subject: [PATCH 0724/2835] test precommit; 74% --- test.hs | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/test.hs b/test.hs index e36ab0955b..28209d099e 100644 --- a/test.hs +++ b/test.hs @@ -231,19 +231,29 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do (not r) @? "drop wrongly succeeded with no known copy of modified file" test_edit :: Test -test_edit = "git-annex edit/commit" ~: intmpclonerepo $ do - git_annex "get" ["-q", annexedfile] @? "get of file failed" - annexed_present annexedfile - git_annex "edit" ["-q", annexedfile] @? "edit failed" - unannexed annexedfile - changecontent annexedfile - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "content changed"] - @? "git commit of edited file failed" - runchecks [checklink, checkunwritable] annexedfile - c <- readFile annexedfile - assertEqual ("content of modified file") c (changedcontent annexedfile) - r <- git_annex "drop" ["-q", annexedfile] - (not r) @? "drop wrongly succeeded with no known copy of modified file" +test_edit = "git-annex edit/commit" ~: TestList [t False, t True] + where t precommit = TestCase $ intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "edit" ["-q", annexedfile] @? "edit failed" + unannexed annexedfile + changecontent annexedfile + if precommit + then do + -- pre-commit depends on the file being + -- staged, normally git commit does this + Utility.boolSystem "git" ["add", annexedfile] + @? "git add of edited file failed" + git_annex "pre-commit" ["-q"] + @? "pre-commit failed" + else do + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "content changed"] + @? "git commit of edited file failed" + runchecks [checklink, checkunwritable] annexedfile + c <- readFile annexedfile + assertEqual ("content of modified file") c (changedcontent annexedfile) + r <- git_annex "drop" ["-q", annexedfile] + (not r) @? "drop wrongly succeeded with no known copy of modified file" test_fix :: Test test_fix = "git-annex fix" ~: intmpclonerepo $ do From b557a2ccf4cd634a2c276b9ae03b881814de58be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 21:11:32 -0400 Subject: [PATCH 0725/2835] test migrate; 75% --- debian/changelog | 2 +- test.hs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 74fa4894a9..6950bda20d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,7 @@ git-annex (0.18) UNRELEASED; urgency=low after transferring the file to the remote repository. (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ - * Test suite improvements. Current top-level test coverage: 72% + * Test suite improvements. Current top-level test coverage: 75% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 diff --git a/test.hs b/test.hs index 28209d099e..716d1a0dee 100644 --- a/test.hs +++ b/test.hs @@ -73,6 +73,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_fix , test_trust , test_fsck + , test_migrate ] test_init :: Test @@ -313,6 +314,35 @@ test_fsck = "git-annex fsck" ~: intmpclonerepo $ do not r @? "fsck failed to fail with corrupted file content" git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem" +test_migrate :: Test +test_migrate = "git-annex migrate" ~: TestList [t False, t True] + where t usegitattributes = TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + annexed_notpresent sha1annexedfile + git_annex "migrate" ["-q", annexedfile] @? "migrate of not present failed" + git_annex "migrate" ["-q", sha1annexedfile] @? "migrate of not present failed" + git_annex "get" ["-q", annexedfile] @? "get of file failed" + git_annex "get" ["-q", sha1annexedfile] @? "get of file failed" + annexed_present annexedfile + annexed_present sha1annexedfile + if usegitattributes + then do + writeFile ".gitattributes" "* annex.backend=SHA1" + git_annex "migrate" [sha1annexedfile] @? "migrate to same backend failed" + git_annex "migrate" [annexedfile] @? "migrate to different backend failed" + else do + git_annex "migrate" [sha1annexedfile, "--backend=SHA1"] @? "migrate to same backend failed" + git_annex "migrate" [annexedfile, "--backend=SHA1"] @? "migrate to different backend failed" + annexed_present annexedfile + annexed_present sha1annexedfile + backend annexedfile Backend.SHA1.backend + backend sha1annexedfile Backend.SHA1.backend + where + backend file expected = do + r <- annexeval $ Backend.lookupFile file + let b = snd $ fromJust r + assertEqual ("backend for " ++ file) expected b + -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. git_annex :: String -> [String] -> IO Bool From e18a4d566b4f205c8a60ddf79ce02ba023d34984 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 21:32:38 -0400 Subject: [PATCH 0726/2835] migrate: Fix support for --backend option. --- Command/Migrate.hs | 9 +++++++-- debian/changelog | 1 + test.hs | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 3d61214377..59ad36a2b4 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -27,15 +27,20 @@ seek :: [CommandSeek] seek = [withBackendFilesInGit start] start :: CommandStartBackendFile -start (_, Nothing) = return Nothing -start (file, Just newbackend) = isAnnexed file $ \(key, oldbackend) -> do +start (file, b) = isAnnexed file $ \(key, oldbackend) -> do exists <- inAnnex key + newbackend <- choosebackend b if (newbackend /= oldbackend) && exists then do showStart "migrate" file return $ Just $ perform file key newbackend else return Nothing + where + choosebackend Nothing = do + backends <- Backend.list + return $ head backends + choosebackend (Just backend) = return backend perform :: FilePath -> Key -> Backend -> CommandPerform perform file oldkey newbackend = do diff --git a/debian/changelog b/debian/changelog index 6950bda20d..510091b7bc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.18) UNRELEASED; urgency=low after transferring the file to the remote repository. (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ + * migrate: Fix support for --backend option. * Test suite improvements. Current top-level test coverage: 75% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 diff --git a/test.hs b/test.hs index 716d1a0dee..417d830e86 100644 --- a/test.hs +++ b/test.hs @@ -328,8 +328,8 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True] if usegitattributes then do writeFile ".gitattributes" "* annex.backend=SHA1" - git_annex "migrate" [sha1annexedfile] @? "migrate to same backend failed" - git_annex "migrate" [annexedfile] @? "migrate to different backend failed" + git_annex "migrate" ["-q", sha1annexedfile] @? "migrate to same backend failed" + git_annex "migrate" ["-q", annexedfile] @? "migrate to different backend failed" else do git_annex "migrate" [sha1annexedfile, "--backend=SHA1"] @? "migrate to same backend failed" git_annex "migrate" [annexedfile, "--backend=SHA1"] @? "migrate to different backend failed" From 611018e4bd1a79becdbb58846f8faa7464a57a01 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 23:02:21 -0400 Subject: [PATCH 0727/2835] I thought that reversion a migration might fail. It didn't. :) --- test.hs | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/test.hs b/test.hs index 417d830e86..6a8784d38c 100644 --- a/test.hs +++ b/test.hs @@ -33,6 +33,7 @@ import qualified UUID import qualified Remotes import qualified Core import qualified Backend.SHA1 +import qualified Backend.WORM main :: IO () main = do @@ -327,18 +328,34 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True] annexed_present sha1annexedfile if usegitattributes then do - writeFile ".gitattributes" "* annex.backend=SHA1" - git_annex "migrate" ["-q", sha1annexedfile] @? "migrate to same backend failed" - git_annex "migrate" ["-q", annexedfile] @? "migrate to different backend failed" + writeFile ".gitattributes" $ "* annex.backend=SHA1" + git_annex "migrate" ["-q", sha1annexedfile] + @? "migrate sha1annexedfile failed" + git_annex "migrate" ["-q", annexedfile] + @? "migrate annexedfile failed" else do - git_annex "migrate" [sha1annexedfile, "--backend=SHA1"] @? "migrate to same backend failed" - git_annex "migrate" [annexedfile, "--backend=SHA1"] @? "migrate to different backend failed" + git_annex "migrate" ["-q", sha1annexedfile, "--backend", "SHA1"] + @? "migrate sha1annexedfile failed" + git_annex "migrate" ["-q", annexedfile, "--backend", "SHA1"] + @? "migrate annexedfile failed" annexed_present annexedfile annexed_present sha1annexedfile - backend annexedfile Backend.SHA1.backend - backend sha1annexedfile Backend.SHA1.backend + checkbackend annexedfile Backend.SHA1.backend + checkbackend sha1annexedfile Backend.SHA1.backend + + -- check that reversing a migration works + writeFile ".gitattributes" $ "* annex.backend=WORM" + git_annex "migrate" ["-q", sha1annexedfile] + @? "migrate sha1annexedfile failed" + git_annex "migrate" ["-q", annexedfile] + @? "migrate annexedfile failed" + annexed_present annexedfile + annexed_present sha1annexedfile + checkbackend annexedfile Backend.WORM.backend + checkbackend sha1annexedfile Backend.WORM.backend + where - backend file expected = do + checkbackend file expected = do r <- annexeval $ Backend.lookupFile file let b = snd $ fromJust r assertEqual ("backend for " ++ file) expected b From ba6727f66342e59b7e420f102548db5c9a49c6b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Jan 2011 01:57:32 -0400 Subject: [PATCH 0728/2835] always write log, so it's empty if nothing is unused --- Command/Unused.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Command/Unused.hs b/Command/Unused.hs index d2dfc9aa3e..62bc5d023d 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -38,13 +38,13 @@ checkUnused :: Annex Bool checkUnused = do showNote "checking for unused data..." unused <- unusedKeys + let list = number 1 unused + g <- Annex.gitRepo + liftIO $ writeFile (annexUnusedLog g) $ unlines $ + map (\(n, k) -> show n ++ " " ++ show k) list if null unused then return True else do - let list = number 1 unused - g <- Annex.gitRepo - liftIO $ writeFile (annexUnusedLog g) $ unlines $ - map (\(n, k) -> show n ++ " " ++ show k) list showLongNote $ w list return False where From bb4a45f9ce9ea3b5b024bc6e46ab61b7b493de9b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Jan 2011 01:57:49 -0400 Subject: [PATCH 0729/2835] avoid crashing if run before unused log is present --- Command/DropUnused.hs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index ea2ff46eba..cfd6fc3d0e 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -9,6 +9,7 @@ module Command.DropUnused where import Control.Monad.State (liftIO) import qualified Data.Map as M +import System.Directory import Command import Types @@ -39,8 +40,13 @@ start s = do readUnusedLog :: Annex (M.Map String Key) readUnusedLog = do g <- Annex.gitRepo - l <- liftIO $ readFile (annexUnusedLog g) - return $ M.fromList $ map parse $ lines l + let f = annexUnusedLog g + e <- liftIO $ doesFileExist f + if e + then do + l <- liftIO $ readFile f + return $ M.fromList $ map parse $ lines l + else return $ M.empty where parse line = (head ws, tokey $ unwords $ tail ws) where From 5869e7ccd532735f991c4f386aecf86c74ec7fc4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Jan 2011 01:58:23 -0400 Subject: [PATCH 0730/2835] test unused et al --- debian/changelog | 2 +- test.hs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 510091b7bc..ed0b8923e9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,7 +5,7 @@ git-annex (0.18) UNRELEASED; urgency=low (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ * migrate: Fix support for --backend option. - * Test suite improvements. Current top-level test coverage: 75% + * Test suite improvements. Current top-level test coverage: 80% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 diff --git a/test.hs b/test.hs index 6a8784d38c..326fa549f8 100644 --- a/test.hs +++ b/test.hs @@ -19,6 +19,7 @@ import qualified Control.Exception.Extensible as E import Control.Exception (throw) import Control.Monad.State (liftIO) import Maybe +import qualified Data.Map as M import qualified Annex import qualified BackendList @@ -34,6 +35,7 @@ import qualified Remotes import qualified Core import qualified Backend.SHA1 import qualified Backend.WORM +import qualified Command.DropUnused main :: IO () main = do @@ -75,6 +77,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_trust , test_fsck , test_migrate + , test_unused ] test_init :: Test @@ -360,6 +363,39 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True] let b = snd $ fromJust r assertEqual ("backend for " ++ file) expected b +test_unused :: Test +test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do + -- keys have to be looked up before files are removed + annexedfilekey <- annexeval $ findkey annexedfile + sha1annexedfilekey <- annexeval $ findkey sha1annexedfile + git_annex "get" ["-q", annexedfile] @? "get of file failed" + git_annex "get" ["-q", sha1annexedfile] @? "get of file failed" + checkunused [] + Utility.boolSystem "git" ["rm", "-q", annexedfile] @? "git rm failed" + checkunused [annexedfilekey] + Utility.boolSystem "git" ["rm", "-q", sha1annexedfile] @? "git rm failed" + checkunused [annexedfilekey, sha1annexedfilekey] + + -- good opportunity to test dropkey also + git_annex "dropkey" ["-q", "--force", TypeInternals.keyName annexedfilekey] + @? "dropkey failed" + checkunused [sha1annexedfilekey] + + git_annex "dropunused" ["-q", "1", "2"] @? "dropunused failed" + checkunused [] + git_annex "dropunused" ["-q", "10", "501"] @? "dropunused failed on bogus numbers" + + where + checkunused expectedkeys = do + git_annex "unused" ["-q"] @? "unused failed" + unusedmap <- annexeval $ Command.DropUnused.readUnusedLog + let unusedkeys = M.elems unusedmap + assertEqual "unused keys differ" + (sort expectedkeys) (sort unusedkeys) + findkey f = do + r <- Backend.lookupFile f + return $ fst $ fromJust r + -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. git_annex :: String -> [String] -> IO Bool From 2a67721d51918339570f9db2b41d413205307e97 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 13 Jan 2011 18:53:15 -0400 Subject: [PATCH 0731/2835] DEP5 --- debian/copyright | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/copyright b/debian/copyright index eceb2d9ffc..692616a7c1 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,5 +1,8 @@ +Format: http://dep.debian.net/deps/dep5/ +Source: native package + Files: * -Copyright: © 2010,2011 Joey Hess +Copyright: © 2010-2011 Joey Hess License: GPL-3+ The full text of the GPL is distributed as doc/GPL in this package's source, or in /usr/share/common-licenses/GPL on Debian systems. From 23686cc63d38faf7fb4d66ece8cbeb5214b07e5f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 13 Jan 2011 23:36:58 -0400 Subject: [PATCH 0732/2835] clarify default values when no path is specified Much of the code to handle this was unnecessary, as git ls-files is used, and defaults to returning all files of the desired type. --- Command.hs | 13 ------------- Command/Find.hs | 2 +- Command/Fsck.hs | 2 +- Command/Uninit.hs | 2 +- doc/git-annex.mdwn | 4 +++- 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/Command.hs b/Command.hs index 31bb1f1f9b..9fafb18ef9 100644 --- a/Command.hs +++ b/Command.hs @@ -174,19 +174,6 @@ backendPairs a files = do pairs <- Backend.chooseBackends files return $ map a pairs -{- Default to acting on all files matching the seek action if - - none are specified. -} -withAll :: (a -> CommandSeek) -> a -> CommandSeek -withAll w a [] = do - g <- Annex.gitRepo - w a [Git.workTree g] -withAll w a p = w a p - -{- Provides a default parameter to act on if none is specified. -} -withDefault :: String-> (a -> CommandSeek) -> (a -> CommandSeek) -withDefault d w a [] = w a [d] -withDefault _ w a p = w a p - {- Filter out files from the state directory, and those matching the - exclude glob pattern, if it was specified. -} filterFiles :: [FilePath] -> Annex [FilePath] diff --git a/Command/Find.hs b/Command/Find.hs index 7cb781ce8c..6d94ea3f49 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -18,7 +18,7 @@ command = [Command "find" (paramOptional $ paramRepeating paramPath) seek "lists available files"] seek :: [CommandSeek] -seek = [withDefault "." withFilesInGit start] +seek = [withFilesInGit start] {- Output a list of files. -} start :: CommandStartString diff --git a/Command/Fsck.hs b/Command/Fsck.hs index d870bd4198..662c281c27 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -18,7 +18,7 @@ command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek "check for problems"] seek :: [CommandSeek] -seek = [withAll (withAttrFilesInGit "annex.numcopies") start] +seek = [withAttrFilesInGit "annex.numcopies" start] {- Checks a file's backend data for problems. -} start :: CommandStartAttrFile diff --git a/Command/Uninit.hs b/Command/Uninit.hs index 1a4e9b0d71..93465df373 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -25,7 +25,7 @@ command = [Command "uninit" paramPath seek "de-initialize git-annex and clean out repository"] seek :: [CommandSeek] -seek = [withAll withFilesInGit Command.Unannex.start, withNothing start] +seek = [withFilesInGit Command.Unannex.start, withNothing start] start :: CommandStartNothing start = do diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 6d106fea48..51a31ae159 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -59,7 +59,9 @@ content from the key-value store. Like many git commands, git-annex can be passed a path that is either a file or a directory. In the latter case it acts on all relevant -files in the directory. +files in the directory. If no path is specified, most git-annex commands +default to acting on all relevant files in the current directory (and +subdirectories). Many git-annex commands will stage changes for later `git commit` by you. From c1839fdccb286cb5e83f0cf2d1d2d8e15226b0eb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 13 Jan 2011 23:46:02 -0400 Subject: [PATCH 0733/2835] unlock: Fix behavior when file content is not present. --- Command/Unlock.hs | 9 +++++++-- debian/changelog | 1 + test.hs | 10 ++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 0e55585ae3..7c1625bf06 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -12,6 +12,7 @@ import System.Directory hiding (copyFile) import Command import qualified Annex +import qualified Backend import Types import Messages import Locations @@ -31,8 +32,12 @@ seek = [withFilesInGit start] - content. -} start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do - showStart "unlock" file - return $ Just $ perform file key + inbackend <- Backend.hasKey key + if not inbackend + then return Nothing + else do + showStart "unlock" file + return $ Just $ perform file key perform :: FilePath -> Key -> CommandPerform perform dest key = do diff --git a/debian/changelog b/debian/changelog index ed0b8923e9..b899fc4b41 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ git-annex (0.18) UNRELEASED; urgency=low (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ * migrate: Fix support for --backend option. + * unlock: Fix behavior when file content is not present. * Test suite improvements. Current top-level test coverage: 80% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 diff --git a/test.hs b/test.hs index 326fa549f8..ee29855b48 100644 --- a/test.hs +++ b/test.hs @@ -215,6 +215,12 @@ test_copy = "git-annex copy" ~: TestCase $ intmpclonerepo $ do test_lock :: Test test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do + -- regression test: unlock of not present file should skip it + annexed_notpresent annexedfile + r <- git_annex "unlock" ["-q", annexedfile] + r @? "unlock failed with not present file" + annexed_notpresent annexedfile + git_annex "get" ["-q", annexedfile] @? "get of file failed" annexed_present annexedfile git_annex "unlock" ["-q", annexedfile] @? "unlock failed" @@ -232,8 +238,8 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do runchecks [checklink, checkunwritable] annexedfile c <- readFile annexedfile assertEqual ("content of modified file") c (changedcontent annexedfile) - r <- git_annex "drop" ["-q", annexedfile] - (not r) @? "drop wrongly succeeded with no known copy of modified file" + r' <- git_annex "drop" ["-q", annexedfile] + not r' @? "drop wrongly succeeded with no known copy of modified file" test_edit :: Test test_edit = "git-annex edit/commit" ~: TestList [t False, t True] From 59c9eda9624a9d35f44400104e8f022afb0f2f7e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 14 Jan 2011 00:02:33 -0400 Subject: [PATCH 0734/2835] on second thought, unlock should fail if content is not present --- Command/Unlock.hs | 13 +++++++------ test.hs | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 7c1625bf06..4bd6e85998 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -7,6 +7,7 @@ module Command.Unlock where +import Control.Monad (when) import Control.Monad.State (liftIO) import System.Directory hiding (copyFile) @@ -32,15 +33,15 @@ seek = [withFilesInGit start] - content. -} start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do - inbackend <- Backend.hasKey key - if not inbackend - then return Nothing - else do - showStart "unlock" file - return $ Just $ perform file key + showStart "unlock" file + return $ Just $ perform file key perform :: FilePath -> Key -> CommandPerform perform dest key = do + inbackend <- Backend.hasKey key + when (not inbackend) $ + error "content not present" + g <- Annex.gitRepo let src = annexLocation g key liftIO $ removeFile dest diff --git a/test.hs b/test.hs index ee29855b48..2504bc7974 100644 --- a/test.hs +++ b/test.hs @@ -218,7 +218,7 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do -- regression test: unlock of not present file should skip it annexed_notpresent annexedfile r <- git_annex "unlock" ["-q", annexedfile] - r @? "unlock failed with not present file" + not r @? "unlock failed to fail with not present file" annexed_notpresent annexedfile git_annex "get" ["-q", annexedfile] @? "get of file failed" From 818111e0c2501a4e68419941655c10d657cc67d0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 14 Jan 2011 15:10:13 -0400 Subject: [PATCH 0735/2835] releasing version 0.18 --- debian/changelog | 4 ++-- debian/copyright | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index b899fc4b41..9ca3ea82b0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.18) UNRELEASED; urgency=low +git-annex (0.18) unstable; urgency=low * Bugfix: `copy --to` and `move --to` forgot to stage location log changes after transferring the file to the remote repository. @@ -8,7 +8,7 @@ git-annex (0.18) UNRELEASED; urgency=low * unlock: Fix behavior when file content is not present. * Test suite improvements. Current top-level test coverage: 80% - -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 + -- Joey Hess Fri, 14 Jan 2011 14:17:44 -0400 git-annex (0.17) unstable; urgency=low diff --git a/debian/copyright b/debian/copyright index 692616a7c1..90e26b7524 100644 --- a/debian/copyright +++ b/debian/copyright @@ -4,5 +4,6 @@ Source: native package Files: * Copyright: © 2010-2011 Joey Hess License: GPL-3+ - The full text of the GPL is distributed as doc/GPL in this package's - source, or in /usr/share/common-licenses/GPL on Debian systems. + The full text of version 3 of the GPL is distributed as doc/GPL in + this package's source, or in /usr/share/common-licenses/GPL-3 on + Debian systems. From 84836ed804633fa3d8ff50064331b8b90bb160dd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 14 Jan 2011 15:10:30 -0400 Subject: [PATCH 0736/2835] add news item for git-annex 0.18 --- doc/news/version_0.13.mdwn | 7 ------- doc/news/version_0.18.mdwn | 9 +++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) delete mode 100644 doc/news/version_0.13.mdwn create mode 100644 doc/news/version_0.18.mdwn diff --git a/doc/news/version_0.13.mdwn b/doc/news/version_0.13.mdwn deleted file mode 100644 index d4de64d3e2..0000000000 --- a/doc/news/version_0.13.mdwn +++ /dev/null @@ -1,7 +0,0 @@ -git-annex 0.13 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Makefile: Install man page and html (when built). - * Makefile: Add GHCFLAGS variable. - * Fix upgrade from 0.03. - * Support remotes using git+ssh and ssh+git as protocol. - Closes: #[607056](http://bugs.debian.org/607056)"""]] \ No newline at end of file diff --git a/doc/news/version_0.18.mdwn b/doc/news/version_0.18.mdwn new file mode 100644 index 0000000000..f2e0edb424 --- /dev/null +++ b/doc/news/version_0.18.mdwn @@ -0,0 +1,9 @@ +git-annex 0.18 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Bugfix: `copy --to` and `move --to` forgot to stage location log changes + after transferring the file to the remote repository. + (Did not affect ssh remotes.) + * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ + * migrate: Fix support for --backend option. + * unlock: Fix behavior when file content is not present. + * Test suite improvements. Current top-level test coverage: 80%"""]] \ No newline at end of file From e7b557ef5d347831142fd98eac901d79c7e1305d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 16 Jan 2011 16:05:05 -0400 Subject: [PATCH 0737/2835] got rid of Core module Most of it was to do with managing annexed Content, so put there --- Backend/File.hs | 2 +- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- CmdLine.hs | 56 +++++++++++++++++++++++++++++--- Command/Add.hs | 2 +- Command/Drop.hs | 2 +- Command/DropKey.hs | 2 +- Command/Find.hs | 2 +- Command/Fix.hs | 2 +- Command/FromKey.hs | 2 +- Command/Get.hs | 2 +- Command/InAnnex.hs | 2 +- Command/Migrate.hs | 2 +- Command/Move.hs | 2 +- Command/RecvKey.hs | 3 +- Command/SendKey.hs | 2 +- Command/SetKey.hs | 2 +- Command/Unannex.hs | 2 +- Command/Unlock.hs | 2 +- Command/Unused.hs | 13 +++++++- Core.hs => Content.hs | 75 +++++++++---------------------------------- Remotes.hs | 6 ++-- Upgrade.hs | 2 +- test.hs | 4 +-- 24 files changed, 104 insertions(+), 89 deletions(-) rename Core.hs => Content.hs (68%) diff --git a/Backend/File.hs b/Backend/File.hs index 073a7c2267..27b2a69015 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -22,7 +22,7 @@ import LocationLog import Locations import qualified Remotes import qualified GitRepo as Git -import Core +import Content import qualified Annex import UUID import Messages diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 68f7f683b5..2f3e2cf534 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -18,7 +18,7 @@ import TypeInternals import Messages import qualified Annex import Locations -import Core +import Content backend :: Backend backend = Backend.File.backend { diff --git a/Backend/WORM.hs b/Backend/WORM.hs index e9d8c42855..0c93012380 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -18,7 +18,7 @@ import qualified Backend.File import TypeInternals import Locations import qualified Annex -import Core +import Content import Messages backend :: Backend diff --git a/CmdLine.hs b/CmdLine.hs index fbcfb6405d..6772282c50 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -1,4 +1,4 @@ -{- git-annex command line parsing +{- git-annex command line parsing and dispatch - - Copyright 2010 Joey Hess - @@ -7,22 +7,27 @@ module CmdLine ( dispatch, - parseCmd, usage, + shutdown ) where +import System.IO.Error (try) import System.Console.GetOpt -import Control.Monad (when) import Control.Monad.State (liftIO) +import Control.Monad (when, unless) +import System.Directory import qualified Annex import qualified GitRepo as Git +import qualified GitQueue import Types import Command import BackendList -import Core import Upgrade import Options +import Messages +import UUID +import Locations {- Runs the passed command line. -} dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO () @@ -68,3 +73,46 @@ usage header cmds options = indent l = " " ++ l pad n s = replicate (n - length s) ' ' longest f = foldl max 0 $ map (length . f) cmds + +{- Runs a list of Annex actions. Catches IO errors and continues + - (but explicitly thrown errors terminate the whole command). + - Runs shutdown and propigates an overall error status at the end. + -} +tryRun :: AnnexState -> [Annex Bool] -> IO () +tryRun state actions = tryRun' state 0 actions +tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () +tryRun' state errnum (a:as) = do + result <- try $ Annex.run state a + case result of + Left err -> do + Annex.eval state $ showErr err + tryRun' state (errnum + 1) as + Right (True,state') -> tryRun' state' errnum as + Right (False,state') -> tryRun' state' (errnum + 1) as +tryRun' state errnum [] = do + _ <- try $ Annex.run state $ shutdown errnum + when (errnum > 0) $ error $ show errnum ++ " failed" + +{- Actions to perform each time ran. -} +startup :: Annex Bool +startup = do + prepUUID + return True + +{- Cleanup actions. -} +shutdown :: Integer -> Annex () +shutdown errnum = do + q <- Annex.queueGet + unless (q == GitQueue.empty) $ do + showSideAction "Recording state in git..." + Annex.queueRun + + -- If nothing failed, clean up any files left in the temp directory, + -- but leave the directory itself. If something failed, temp files + -- are left behind to allow resuming on re-run. + when (errnum == 0) $ do + g <- Annex.gitRepo + let tmp = annexTmpLocation g + exists <- liftIO $ doesDirectoryExist tmp + when exists $ liftIO $ removeDirectoryRecursive tmp + liftIO $ createDirectoryIfMissing True tmp diff --git a/Command/Add.hs b/Command/Add.hs index c74b726e3f..4b49297fc6 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -15,7 +15,7 @@ import qualified Annex import qualified Backend import LocationLog import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/Drop.hs b/Command/Drop.hs index a425c6138d..065e1743a1 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -13,7 +13,7 @@ import Command import qualified Backend import LocationLog import Types -import Core +import Content import Messages import Utility diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 29056139d3..6ba5c117c4 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -12,7 +12,7 @@ import qualified Annex import qualified Backend import LocationLog import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/Find.hs b/Command/Find.hs index 6d94ea3f49..3ed15c1537 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -11,7 +11,7 @@ import Control.Monad (when) import Control.Monad.State (liftIO) import Command -import Core +import Content command :: [Command] command = [Command "find" (paramOptional $ paramRepeating paramPath) seek diff --git a/Command/Fix.hs b/Command/Fix.hs index 8b08a26f6d..d67eca164c 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -14,7 +14,7 @@ import System.Directory import Command import qualified Annex import Utility -import Core +import Content import Messages command :: [Command] diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 0a13b8c734..9c4a3cfdcb 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -17,7 +17,7 @@ import qualified Annex import Utility import qualified Backend import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/Get.hs b/Command/Get.hs index e3668649ef..e0af6c4078 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -10,7 +10,7 @@ module Command.Get where import Command import qualified Backend import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs index d49539513b..68ac9a2c67 100644 --- a/Command/InAnnex.hs +++ b/Command/InAnnex.hs @@ -12,7 +12,7 @@ import System.Exit import Command import Types -import Core +import Content import qualified Backend command :: [Command] diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 59ad36a2b4..5bc54ceab5 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -16,7 +16,7 @@ import qualified Annex import qualified Backend import Locations import Types -import Core +import Content import Messages import qualified Command.Add diff --git a/Command/Move.hs b/Command/Move.hs index 3e7fde3705..2920c06616 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -14,7 +14,7 @@ import qualified Command.Drop import qualified Annex import LocationLog import Types -import Core +import Content import qualified GitRepo as Git import qualified Remotes import UUID diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 840b328613..0abea07f20 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -13,7 +13,8 @@ import System.Exit import Command import Types -import Core +import CmdLine +import Content import qualified Backend import RsyncFile diff --git a/Command/SendKey.hs b/Command/SendKey.hs index 0ddc0d23b9..aaa0b48369 100644 --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -15,7 +15,7 @@ import Locations import qualified Annex import Command import Types -import Core +import Content import qualified Backend import RsyncFile diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 5048d052f0..412504b2ee 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -16,7 +16,7 @@ import Utility import qualified Backend import LocationLog import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 2c60a23bb9..cdd577ba8b 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -16,7 +16,7 @@ import Utility import qualified Backend import LocationLog import Types -import Core +import Content import qualified GitRepo as Git import Messages diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 4bd6e85998..645fac8a25 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -17,7 +17,7 @@ import qualified Backend import Types import Messages import Locations -import Core +import Content import CopyFile command :: [Command] diff --git a/Command/Unused.hs b/Command/Unused.hs index 62bc5d023d..9fdf4cda65 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -9,13 +9,16 @@ module Command.Unused where import Control.Monad.State (liftIO) import qualified Data.Map as M +import Data.Maybe import Command import Types -import Core +import Content import Messages import Locations import qualified Annex +import qualified GitRepo as Git +import qualified Backend command :: [Command] command = [Command "unused" paramNothing seek "look for unused file content"] @@ -80,3 +83,11 @@ unusedKeys = do existsMap :: Ord k => [k] -> M.Map k Int existsMap l = M.fromList $ map (\k -> (k, 1)) l + +{- List of keys referenced by symlinks in the git repo. -} +getKeysReferenced :: Annex [Key] +getKeysReferenced = do + g <- Annex.gitRepo + files <- liftIO $ Git.inRepo g [Git.workTree g] + keypairs <- mapM Backend.lookupFile files + return $ map fst $ catMaybes keypairs diff --git a/Core.hs b/Content.hs similarity index 68% rename from Core.hs rename to Content.hs index d59120d673..0cbd6905cb 100644 --- a/Core.hs +++ b/Content.hs @@ -1,19 +1,30 @@ -{- git-annex core functions +{- git-annex file content managing - - Copyright 2010 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module Core where +module Content ( + inAnnex, + calcGitLink, + logStatus, + getViaTmp, + preventWrite, + allowWrite, + moveAnnex, + removeAnnex, + fromAnnex, + moveBad, + getKeysPresent +) where import System.IO.Error (try) import System.Directory import Control.Monad.State (liftIO) import System.Path -import Control.Monad (when, unless, filterM) +import Control.Monad (when, filterM) import System.Posix.Files -import Data.Maybe import System.FilePath import Types @@ -21,56 +32,8 @@ import Locations import LocationLog import UUID import qualified GitRepo as Git -import qualified GitQueue import qualified Annex -import qualified Backend import Utility -import Messages - -{- Runs a list of Annex actions. Catches IO errors and continues - - (but explicitly thrown errors terminate the whole command). - - Runs shutdown and propigates an overall error status at the end. - -} -tryRun :: AnnexState -> [Annex Bool] -> IO () -tryRun state actions = tryRun' state 0 actions -tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () -tryRun' state errnum (a:as) = do - result <- try $ Annex.run state a - case result of - Left err -> do - Annex.eval state $ showErr err - tryRun' state (errnum + 1) as - Right (True,state') -> tryRun' state' errnum as - Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' state errnum [] = do - _ <- try $ Annex.run state $ shutdown errnum - when (errnum > 0) $ error $ show errnum ++ " failed" - -{- Actions to perform each time ran. -} -startup :: Annex Bool -startup = do - prepUUID - return True - -{- When git-annex is done, it runs this. -} -shutdown :: Integer -> Annex Bool -shutdown errnum = do - q <- Annex.queueGet - unless (q == GitQueue.empty) $ do - showSideAction "Recording state in git..." - Annex.queueRun - - -- If nothing failed, clean up any files left in the temp directory, - -- but leave the directory itself. If something failed, temp files - -- are left behind to allow resuming on re-run. - when (errnum == 0) $ do - g <- Annex.gitRepo - let tmp = annexTmpLocation g - exists <- liftIO $ doesDirectoryExist tmp - when exists $ liftIO $ removeDirectoryRecursive tmp - liftIO $ createDirectoryIfMissing True tmp - - return True {- Checks if a given key is currently present in the annexLocation. -} inAnnex :: Key -> Annex Bool @@ -200,11 +163,3 @@ getKeysPresent' dir = do case result of Right s -> return $ isRegularFile s Left _ -> return False - -{- List of keys referenced by symlinks in the git repo. -} -getKeysReferenced :: Annex [Key] -getKeysReferenced = do - g <- Annex.gitRepo - files <- liftIO $ Git.inRepo g [Git.workTree g] - keypairs <- mapM Backend.lookupFile files - return $ map fst $ catMaybes keypairs diff --git a/Remotes.hs b/Remotes.hs index a7a1db4152..9004b33d00 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -34,7 +34,7 @@ import LocationLog import Locations import UUID import Utility -import qualified Core +import qualified Content import Messages import CopyFile import RsyncFile @@ -159,7 +159,7 @@ inAnnex r key = if Git.repoIsUrl r -- run a local check inexpensively, -- by making an Annex monad using the remote a <- Annex.new r [] - Annex.eval a (Core.inAnnex key) + Annex.eval a (Content.inAnnex key) checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- onRemote r (boolSystem, False) "inannex" @@ -253,7 +253,7 @@ copyToRemote r key liftIO $ do a <- Annex.new r [] Annex.eval a $ do - ok <- Core.getViaTmp key $ + ok <- Content.getViaTmp key $ \f -> liftIO $ copyFile keysrc f Annex.queueRun return ok diff --git a/Upgrade.hs b/Upgrade.hs index 2e1708439b..596d525db2 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -14,7 +14,7 @@ import Control.Monad (filterM) import System.Posix.Files import System.FilePath -import Core +import Content import Types import Locations import qualified GitRepo as Git diff --git a/test.hs b/test.hs index 2504bc7974..b8b264f0cf 100644 --- a/test.hs +++ b/test.hs @@ -32,7 +32,7 @@ import qualified GitAnnex import qualified LocationLog import qualified UUID import qualified Remotes -import qualified Core +import qualified Content import qualified Backend.SHA1 import qualified Backend.WORM import qualified Command.DropUnused @@ -318,7 +318,7 @@ test_fsck = "git-annex fsck" ~: intmpclonerepo $ do where corrupt f = do git_annex "get" ["-q", f] @? "get of file failed" - Core.allowWrite f + Content.allowWrite f writeFile f (changedcontent f) r <- git_annex "fsck" ["-q"] not r @? "fsck failed to fail with corrupted file content" From d5c18d71ef643f8ab55f1d0d4444621ba364b13d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Jan 2011 13:31:22 -0400 Subject: [PATCH 0738/2835] bug --- doc/bugs/tmp_file_handling.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/tmp_file_handling.mdwn diff --git a/doc/bugs/tmp_file_handling.mdwn b/doc/bugs/tmp_file_handling.mdwn new file mode 100644 index 0000000000..3c6c2a5976 --- /dev/null +++ b/doc/bugs/tmp_file_handling.mdwn @@ -0,0 +1,11 @@ +git-annex deletes all tmp files on shutdown, if everything succeeded. +This presents 2 problems: + +1. If git-annex is rsyncing something and another one is run, it will + delete the running instance's tmp files. +2. If a long-running rsync transfer is interrupted partway through, the + tmp file was expensive to obtain, and one needs to avoid running + git-annex to do anything else until that transfer can be resumed and + finished. + +--[[Joey]] From 1cc064d1a2b1686b0a45b38c53f8f9f922dd49e6 Mon Sep 17 00:00:00 2001 From: "http://jcftang.myopenid.com/" Date: Wed, 19 Jan 2011 11:00:17 +0000 Subject: [PATCH 0739/2835] --- ...figure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn diff --git a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn new file mode 100644 index 0000000000..b7ec0d3883 --- /dev/null +++ b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn @@ -0,0 +1 @@ +On RHEL5 (and clones) systems uuidgen is available as an alternative to uuid, the configure script fails, it should probably detect either uuid or uuidgen, or let the user decide? From a2428e7bdc993478cc197497f4735a11076de4e9 Mon Sep 17 00:00:00 2001 From: "http://jcftang.myopenid.com/" Date: Wed, 19 Jan 2011 12:12:22 +0000 Subject: [PATCH 0740/2835] --- ...igure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn index b7ec0d3883..83d1ae6641 100644 --- a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn +++ b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn @@ -1 +1 @@ -On RHEL5 (and clones) systems uuidgen is available as an alternative to uuid, the configure script fails, it should probably detect either uuid or uuidgen, or let the user decide? +On RHEL5 (and clones) systems uuidgen is available as an alternative to uuid, the configure script fails, it should probably detect either uuid or uuidgen, or let the user decide? - also uuidgen behaves differently from uuid on debian. From 1881b4d7fcb50d563d0e6890fc39dde0663d87a3 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 19 Jan 2011 17:43:16 +0000 Subject: [PATCH 0741/2835] small helper script for reflecting git-annex-add on rsync-style backups --- doc/forum/migration_to_git-annex_and_rsync.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/migration_to_git-annex_and_rsync.mdwn diff --git a/doc/forum/migration_to_git-annex_and_rsync.mdwn b/doc/forum/migration_to_git-annex_and_rsync.mdwn new file mode 100644 index 0000000000..51e03de0d9 --- /dev/null +++ b/doc/forum/migration_to_git-annex_and_rsync.mdwn @@ -0,0 +1,13 @@ +When migrating large file repositories to git-annex that are backuped in a way that uses an rsync-style mechanism (e.g. [dirvish](http://www.dirvish.org/)) and thus keeps incremental backups small by using hardlinks, space can be saved by manually reflecting the migration on the backup. So, instead of making a last pre-git-annex backup, migrating, and duplicating all backupped data with the next backup, I used the attached [[migrate.py]], and it saved me roughly a day of backuping. + +A note on terminology: "migrating" here means migrating from not using git-annex at all to using it, not to the ``git annex migrate`` command, for which a similar but different solution may be created. + +**WARNING**: This is a quickly hacked-together script. It worked for me, but is untested apart from that. It's just a dozen lines of code, so have a look at it and make sure you understand what it does, and what migrate.sh looks like. Take special care as this tampers with your backups, and if something goes wrong, well... + +First, have an up-to-date backup; then, git annex init / add etc as described in the [[walkthrough]]. In the directory in which you use git-annex, run: + + $ python migrate.py > migrate.sh + +Then copy the resulting migrate.sh to the equivalent location inside your backups and run it there. It will move all files that are now symlinked on the master to their new positions according to the symlinks (inside .git/annex/objects), but not create the symlinks (you will do a backup later anyway). + +After that, do a backup as usual. As rsync sees the moved files at their new locations, it will accept them and not duplicate the data. From 27325f212bfdf915d16eadfa9fc51b416d4177c0 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 19 Jan 2011 17:46:08 +0000 Subject: [PATCH 0742/2835] no attachments w/o admin, copy/pasting instead --- .../migration_to_git-annex_and_rsync.mdwn | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/forum/migration_to_git-annex_and_rsync.mdwn b/doc/forum/migration_to_git-annex_and_rsync.mdwn index 51e03de0d9..d99dab8728 100644 --- a/doc/forum/migration_to_git-annex_and_rsync.mdwn +++ b/doc/forum/migration_to_git-annex_and_rsync.mdwn @@ -1,4 +1,4 @@ -When migrating large file repositories to git-annex that are backuped in a way that uses an rsync-style mechanism (e.g. [dirvish](http://www.dirvish.org/)) and thus keeps incremental backups small by using hardlinks, space can be saved by manually reflecting the migration on the backup. So, instead of making a last pre-git-annex backup, migrating, and duplicating all backupped data with the next backup, I used the attached [[migrate.py]], and it saved me roughly a day of backuping. +When migrating large file repositories to git-annex that are backuped in a way that uses an rsync-style mechanism (e.g. [dirvish](http://www.dirvish.org/)) and thus keeps incremental backups small by using hardlinks, space can be saved by manually reflecting the migration on the backup. So, instead of making a last pre-git-annex backup, migrating, and duplicating all backupped data with the next backup, I used the attached migrate.py file below, and it saved me roughly a day of backuping. A note on terminology: "migrating" here means migrating from not using git-annex at all to using it, not to the ``git annex migrate`` command, for which a similar but different solution may be created. @@ -11,3 +11,23 @@ First, have an up-to-date backup; then, git annex init / add etc as described in Then copy the resulting migrate.sh to the equivalent location inside your backups and run it there. It will move all files that are now symlinked on the master to their new positions according to the symlinks (inside .git/annex/objects), but not create the symlinks (you will do a backup later anyway). After that, do a backup as usual. As rsync sees the moved files at their new locations, it will accept them and not duplicate the data. + +**migrate.py**: + + #!/usr/bin/env python + + import os + from pipes import quote + + print "#!/bin/sh" + print "set -e" + print "" + + for (dirpath, dirnames, filenames) in os.walk("."): + for f in filenames: + fn = os.path.join(dirpath, f) + if os.path.islink(fn): + link = os.path.normpath(os.path.join(dirpath, os.readlink(fn))) + assert link.startswith(".git/annex/objects/") + print "mkdir -p %s"%quote(os.path.dirname(link)) + print "mv %s %s"%(quote(fn), quote(link)) From dbb76c22d0f4f979fe90eeeff233dbbbfcf2346d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 19 Jan 2011 18:08:50 -0400 Subject: [PATCH 0743/2835] Support using the uuidgen command if the uuid command is not available. --- UUID.hs | 10 +- configure.hs | 96 ++++++++++++------- debian/changelog | 6 ++ ...d_detect_uuidgen_instead_of_just_uuid.mdwn | 7 +- doc/install.mdwn | 1 + 5 files changed, 82 insertions(+), 38 deletions(-) diff --git a/UUID.hs b/UUID.hs index 67695d3101..26a64523fc 100644 --- a/UUID.hs +++ b/UUID.hs @@ -33,6 +33,7 @@ import Types import Locations import qualified Annex import Utility +import qualified SysConfig type UUID = String @@ -42,7 +43,14 @@ configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} genUUID :: IO UUID -genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h +genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h + where + command = SysConfig.uuid + params = if (command == "uuid") + -- request a random uuid be generated + then ["-m"] + -- uuidgen generates random uuid by default + else [] {- Looks up a repo's UUID. May return "" if none is known. - diff --git a/configure.hs b/configure.hs index 1abdc8914d..9f50328d33 100644 --- a/configure.hs +++ b/configure.hs @@ -5,24 +5,33 @@ import System.Cmd import System.Exit import System.Directory -type Test = IO Bool -data TestCase = TestCase String String Test -data Config = Config String Bool +type ConfigKey = String +data ConfigValue = BoolConfig Bool | StringConfig String +data Config = Config ConfigKey ConfigValue + +type Test = IO Config +type TestName = String +data TestCase = TestCase TestName Test instance Show Config where show (Config key value) = unlines [ - key ++ " :: Bool" - , key ++ " = " ++ show value + key ++ " :: " ++ valuetype value + , key ++ " = " ++ showvalue value ] + where + valuetype (BoolConfig _) = "Bool" + valuetype (StringConfig _) = "String" + showvalue (BoolConfig b) = show b + showvalue (StringConfig s) = show s tests :: [TestCase] tests = [ - TestCase "cp -a" "cp_a" $ testCp "-a" - , TestCase "cp -p" "cp_p" $ testCp "-p" - , TestCase "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" - , TestCase "uuid" "uuid" $ requireCmd "uuid" "uuid" - , TestCase "xargs -0" "xargs_0" $ requireCmd "xargs -0" "xargs -0 /dev/null" + TestCase "cp -a" $ testCp "cp_a" "-a" + , TestCase "cp -p" $ testCp "cp_p" "-p" + , TestCase "cp --reflink=auto" $ testCp "cp_reflink_auto" "--reflink=auto" + , TestCase "uuid" $ selectCmd "uuid" ["uuid", "uuidgen"] + , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0" "xargs -0 /dev/null" ] tmpDir :: String @@ -31,34 +40,49 @@ tmpDir = "tmp" testFile :: String testFile = tmpDir ++ "/testfile" +requireCmd :: ConfigKey -> String -> String -> Test +requireCmd k c cmdline = do + ret <- testCmd k cmdline + handle ret + where + handle r@(Config _ (BoolConfig True)) = return r + handle r = do + testEnd r + error $ "** the " ++ c ++ " command is required to use git-annex" + +testCp :: ConfigKey -> String -> Test +testCp k option = testCmd k $ + "cp " ++ option ++ " " ++ testFile ++ " " ++ testFile ++ ".new" + +testCmd :: ConfigKey -> String -> Test +testCmd k c = do + ret <- system $ quiet c + return $ Config k (BoolConfig $ ret == ExitSuccess) + +selectCmd :: ConfigKey -> [String] -> Test +selectCmd k cmds = search cmds + where + search [] = do + testEnd $ Config k (BoolConfig False) + error $ "* need one of these commands, but none are available: " ++ show cmds + search (c:cs) = do + ret <- system $ quiet c + if (ret == ExitSuccess) + then return $ Config k (StringConfig c) + else search cs + quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" -requireCmd :: String -> String -> Test -requireCmd c cmdline = do - ret <- testCmd $ quiet cmdline - if ret - then return True - else do - testEnd False - error $ "** the " ++ c ++ " command is required to use git-annex" - -testCp :: String -> Test -testCp option = testCmd $ quiet $ "cp " ++ option ++ " " ++ testFile ++ - " " ++ testFile ++ ".new" - -testCmd :: String -> Test -testCmd c = do - ret <- system c - return $ ret == ExitSuccess - -testStart :: String -> IO () +testStart :: TestName -> IO () testStart s = do putStr $ " checking " ++ s ++ "..." hFlush stdout -testEnd :: Bool -> IO () -testEnd r = putStrLn $ " " ++ show r +testEnd :: Config -> IO () +testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes" +testEnd (Config _ (BoolConfig False)) = putStrLn $ " no" +testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s writeSysConfig :: [Config] -> IO () writeSysConfig config = writeFile "SysConfig.hs" body @@ -73,12 +97,12 @@ writeSysConfig config = writeFile "SysConfig.hs" body runTests :: [TestCase] -> IO [Config] runTests [] = return [] -runTests ((TestCase tname key t):ts) = do +runTests ((TestCase tname t):ts) = do testStart tname - val <- t - testEnd val + c <- t + testEnd c rest <- runTests ts - return $ (Config key val):rest + return $ c:rest setup :: IO () setup = do diff --git a/debian/changelog b/debian/changelog index 9ca3ea82b0..23358486d1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.19) UNRELEASED; urgency=low + + * Support using the uuidgen command if the uuid command is not available. + + -- Joey Hess Wed, 19 Jan 2011 18:07:51 -0400 + git-annex (0.18) unstable; urgency=low * Bugfix: `copy --to` and `move --to` forgot to stage location log changes diff --git a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn index 83d1ae6641..2b9c773678 100644 --- a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn +++ b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn @@ -1 +1,6 @@ -On RHEL5 (and clones) systems uuidgen is available as an alternative to uuid, the configure script fails, it should probably detect either uuid or uuidgen, or let the user decide? - also uuidgen behaves differently from uuid on debian. +On RHEL5 (and clones) systems uuidgen is available as an alternative to +uuid, the configure script fails, it should probably detect either uuid or +uuidgen, or let the user decide? - also uuidgen behaves differently from +uuid on debian. + +> uuidgen is now supported. --[[Joey]] [[done]] diff --git a/doc/install.mdwn b/doc/install.mdwn index bad1d9f258..732660c507 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,6 +5,7 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * `uuid`: + (or uuidgen from util-linux) * `xargs`: * `rsync`: * Then just [[download]] git-annex and run: `make; make install` From 4465689cc22564e650be4bad759006d587b41307 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 19 Jan 2011 20:02:48 -0400 Subject: [PATCH 0744/2835] refactor --- Makefile | 2 +- TestConfig.hs | 93 ++++++++++++++++++++++++++++++++++++++++++++++ configure.hs | 100 ++++++-------------------------------------------- 3 files changed, 105 insertions(+), 90 deletions(-) create mode 100644 TestConfig.hs diff --git a/Makefile b/Makefile index 44d4be02be..831c004dc5 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ mans=git-annex.1 git-annex-shell.1 all: $(bins) $(mans) docs -SysConfig.hs: configure.hs +SysConfig.hs: configure.hs TestConfig.hs $(GHCMAKE) configure ./configure diff --git a/TestConfig.hs b/TestConfig.hs new file mode 100644 index 0000000000..5e59681ddf --- /dev/null +++ b/TestConfig.hs @@ -0,0 +1,93 @@ +{- Tests the system and generates SysConfig.hs. -} + +module TestConfig where + +import System.IO +import System.Cmd +import System.Exit + +type ConfigKey = String +data ConfigValue = BoolConfig Bool | StringConfig String +data Config = Config ConfigKey ConfigValue + +type Test = IO Config +type TestName = String +data TestCase = TestCase TestName Test + +instance Show ConfigValue where + show (BoolConfig b) = show b + show (StringConfig s) = show s + +instance Show Config where + show (Config key value) = unlines + [ key ++ " :: " ++ valuetype value + , key ++ " = " ++ show value + ] + where + valuetype (BoolConfig _) = "Bool" + valuetype (StringConfig _) = "String" + +writeSysConfig :: [Config] -> IO () +writeSysConfig config = writeFile "SysConfig.hs" body + where + body = unlines $ header ++ map show config ++ footer + header = [ + "{- Automatically generated. -}" + , "module SysConfig where" + , "" + ] + footer = [] + +runTests :: [TestCase] -> IO [Config] +runTests [] = return [] +runTests ((TestCase tname t):ts) = do + testStart tname + c <- t + testEnd c + rest <- runTests ts + return $ c:rest + +{- Tests that a command is available, aborting if not. -} +requireCmd :: ConfigKey -> String -> Test +requireCmd k cmdline = do + ret <- testCmd k cmdline + handle ret + where + handle r@(Config _ (BoolConfig True)) = return r + handle r = do + testEnd r + error $ "** the " ++ c ++ " command is required" + c = (words cmdline) !! 0 + +{- Checks if a command is available by running a command line. -} +testCmd :: ConfigKey -> String -> Test +testCmd k cmdline = do + ret <- system $ quiet cmdline + return $ Config k (BoolConfig $ ret == ExitSuccess) + +{- Ensures that one of a set of commands is available by running each in + - turn. The Config is set to the first one found. -} +selectCmd :: ConfigKey -> [String] -> Test +selectCmd k cmds = search cmds + where + search [] = do + testEnd $ Config k (BoolConfig False) + error $ "* need one of these commands, but none are available: " ++ show cmds + search (c:cs) = do + ret <- system $ quiet c + if (ret == ExitSuccess) + then return $ Config k (StringConfig c) + else search cs + +quiet :: String -> String +quiet s = s ++ " >/dev/null 2>&1" + +testStart :: TestName -> IO () +testStart s = do + putStr $ " checking " ++ s ++ "..." + hFlush stdout + +testEnd :: Config -> IO () +testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes" +testEnd (Config _ (BoolConfig False)) = putStrLn $ " no" +testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s diff --git a/configure.hs b/configure.hs index 9f50328d33..8d1c117a71 100644 --- a/configure.hs +++ b/configure.hs @@ -1,37 +1,17 @@ {- Checks system configuration and generates SysConfig.hs. -} -import System.IO -import System.Cmd -import System.Exit import System.Directory -type ConfigKey = String -data ConfigValue = BoolConfig Bool | StringConfig String -data Config = Config ConfigKey ConfigValue - -type Test = IO Config -type TestName = String -data TestCase = TestCase TestName Test - -instance Show Config where - show (Config key value) = unlines [ - key ++ " :: " ++ valuetype value - , key ++ " = " ++ showvalue value - ] - where - valuetype (BoolConfig _) = "Bool" - valuetype (StringConfig _) = "String" - showvalue (BoolConfig b) = show b - showvalue (StringConfig s) = show s +import TestConfig tests :: [TestCase] tests = [ - TestCase "cp -a" $ testCp "cp_a" "-a" - , TestCase "cp -p" $ testCp "cp_p" "-p" - , TestCase "cp --reflink=auto" $ testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid" $ selectCmd "uuid" ["uuid", "uuidgen"] - , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0" "xargs -0 /dev/null" + testCp "cp_a" "-a" + , testCp "cp_p" "-p" + , testCp "cp_reflink_auto" "--reflink=auto" + , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] + , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" ] tmpDir :: String @@ -40,69 +20,11 @@ tmpDir = "tmp" testFile :: String testFile = tmpDir ++ "/testfile" -requireCmd :: ConfigKey -> String -> String -> Test -requireCmd k c cmdline = do - ret <- testCmd k cmdline - handle ret +testCp :: ConfigKey -> String -> TestCase +testCp k option = TestCase cmd $ testCmd k run where - handle r@(Config _ (BoolConfig True)) = return r - handle r = do - testEnd r - error $ "** the " ++ c ++ " command is required to use git-annex" - -testCp :: ConfigKey -> String -> Test -testCp k option = testCmd k $ - "cp " ++ option ++ " " ++ testFile ++ " " ++ testFile ++ ".new" - -testCmd :: ConfigKey -> String -> Test -testCmd k c = do - ret <- system $ quiet c - return $ Config k (BoolConfig $ ret == ExitSuccess) - -selectCmd :: ConfigKey -> [String] -> Test -selectCmd k cmds = search cmds - where - search [] = do - testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show cmds - search (c:cs) = do - ret <- system $ quiet c - if (ret == ExitSuccess) - then return $ Config k (StringConfig c) - else search cs - -quiet :: String -> String -quiet s = s ++ " >/dev/null 2>&1" - -testStart :: TestName -> IO () -testStart s = do - putStr $ " checking " ++ s ++ "..." - hFlush stdout - -testEnd :: Config -> IO () -testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes" -testEnd (Config _ (BoolConfig False)) = putStrLn $ " no" -testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s - -writeSysConfig :: [Config] -> IO () -writeSysConfig config = writeFile "SysConfig.hs" body - where - body = unlines $ header ++ map show config ++ footer - header = [ - "{- Automatically generated by configure. -}" - , "module SysConfig where" - , "" - ] - footer = [] - -runTests :: [TestCase] -> IO [Config] -runTests [] = return [] -runTests ((TestCase tname t):ts) = do - testStart tname - c <- t - testEnd c - rest <- runTests ts - return $ c:rest + cmd = "cp " ++ option + run = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new" setup :: IO () setup = do From 778966b4f41407078372185c57ba18ae0c578ef7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 25 Jan 2011 18:54:27 -0400 Subject: [PATCH 0745/2835] improve man building --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 831c004dc5..565edfb0e0 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ SysConfig.hs: configure.hs TestConfig.hs $(bins): SysConfig.hs $(GHCMAKE) $@ -git-annex.1: +git-annex.1: doc/git-annex.mdwn ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 -git-annex-shell.1: - ./mdwn2man git-annex 1 doc/git-annex-shell.mdwn > git-annex-shell.1 +git-annex-shell.1: doc/git-annex-shell.mdwn + ./mdwn2man git-annex-shell 1 doc/git-annex-shell.mdwn > git-annex-shell.1 install: all install -d $(DESTDIR)$(PREFIX)/bin From f8e303e1c9a9d976a4dc339295aa0f09eb997b7f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 25 Jan 2011 18:54:34 -0400 Subject: [PATCH 0746/2835] document interaction of annex-ignore with --from/--to --- doc/git-annex.mdwn | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 51a31ae159..9d89413b96 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -295,8 +295,13 @@ Here are all the supported configuration settings. * `remote..annex-ignore` If set to `true`, prevents git-annex - from ever using this remote. This is, for example, useful if the - remote is a bare repository, which git-annex does not currently support. + from using this remote by default. (You can still request it be used + by the --from and --to options.) + + This is, for example, useful if the remote is a bare repository, + which git-annex does not currently support. Or, it could be used + if the network connection between two repositories is too slow + to be used normally. * `remote..annex-uuid` From 109a719b03dbeb70eb317be17f7e18567efa9dac Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 25 Jan 2011 21:02:34 -0400 Subject: [PATCH 0747/2835] parameterize Backend type This allows the Backend type to not depend on the Annex type, and so the Annex type can later be moved out of TypeInternals. --- Annex.hs | 11 ++++++----- Backend.hs | 22 +++++++++++----------- Backend/File.hs | 2 +- Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Backend/WORM.hs | 2 +- BackendList.hs | 2 +- Command.hs | 4 ++-- Command/Drop.hs | 2 +- Command/Fsck.hs | 2 +- Command/Get.hs | 2 +- Command/Migrate.hs | 2 +- Command/Unannex.hs | 2 +- TypeInternals.hs | 27 ++++++++++++++------------- 14 files changed, 43 insertions(+), 41 deletions(-) diff --git a/Annex.hs b/Annex.hs index 765c9191fb..a0de630874 100644 --- a/Annex.hs +++ b/Annex.hs @@ -33,14 +33,15 @@ import Types import qualified TypeInternals as Internals {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> [Backend] -> IO AnnexState +new :: Git.Repo -> [Backend Annex] -> IO AnnexState new gitrepo allbackends = do let s = Internals.AnnexState { Internals.repo = gitrepo, Internals.backends = [], Internals.supportedBackends = allbackends, Internals.flags = M.empty, - Internals.repoqueue = GitQueue.empty + Internals.repoqueue = GitQueue.empty, + Internals.quiet = False } (_,s') <- Annex.run s prep return s' @@ -69,19 +70,19 @@ gitRepoChange r = do put state { Internals.repo = r } {- Returns the backends being used. -} -backends :: Annex [Backend] +backends :: Annex [Backend Annex] backends = do state <- get return (Internals.backends state) {- Sets the backends to use. -} -backendsChange :: [Backend] -> Annex () +backendsChange :: [Backend Annex] -> Annex () backendsChange b = do state <- get put state { Internals.backends = b } {- Returns the full list of supported backends. -} -supportedBackends :: Annex [Backend] +supportedBackends :: Annex [Backend Annex] supportedBackends = do state <- get return (Internals.supportedBackends state) diff --git a/Backend.hs b/Backend.hs index a417c7247b..caf50005ad 100644 --- a/Backend.hs +++ b/Backend.hs @@ -42,7 +42,7 @@ import qualified TypeInternals as Internals import Messages {- List of backends in the order to try them when storing a new key. -} -list :: Annex [Backend] +list :: Annex [Backend Annex] list = do l <- Annex.backends -- list is cached here if not $ null l @@ -64,12 +64,12 @@ list = do else map (lookupBackendName bs) $ words s {- Looks up a backend in a list. May fail if unknown. -} -lookupBackendName :: [Backend] -> String -> Backend +lookupBackendName :: [Backend Annex] -> String -> Backend Annex lookupBackendName bs s = case maybeLookupBackendName bs s of Just b -> b Nothing -> error $ "unknown backend " ++ s -maybeLookupBackendName :: [Backend] -> String -> Maybe Backend +maybeLookupBackendName :: [Backend Annex] -> String -> Maybe (Backend Annex) maybeLookupBackendName bs s = if 1 /= length matches then Nothing @@ -77,14 +77,14 @@ maybeLookupBackendName bs s = where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} -storeFileKey :: FilePath -> Maybe Backend -> Annex (Maybe (Key, Backend)) +storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex)) storeFileKey file trybackend = do bs <- list let bs' = case trybackend of Nothing -> bs Just backend -> backend:bs storeFileKey' bs' file -storeFileKey' :: [Backend] -> FilePath -> Annex (Maybe (Key, Backend)) +storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex)) storeFileKey' [] _ = return Nothing storeFileKey' (b:bs) file = do result <- (Internals.getKey b) file @@ -100,11 +100,11 @@ storeFileKey' (b:bs) file = do {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool +retrieveKeyFile :: Backend Annex -> Key -> FilePath -> Annex Bool retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} -removeKey :: Backend -> Key -> Maybe Int -> Annex Bool +removeKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool removeKey backend key numcopies = (Internals.removeKey backend) key numcopies {- Checks if a key is present in its backend. -} @@ -114,12 +114,12 @@ hasKey key = do (Internals.hasKey backend) key {- Checks a key's backend for problems. -} -fsckKey :: Backend -> Key -> Maybe Int -> Annex Bool +fsckKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} -lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) +lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex)) lookupFile file = do bs <- Annex.supportedBackends tl <- liftIO $ try getsymlink @@ -147,7 +147,7 @@ lookupFile file = do {- Looks up the backends that should be used for each file in a list. - That can be configured on a per-file basis in the gitattributes file. -} -chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe Backend)] +chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))] chooseBackends fs = do g <- Annex.gitRepo bs <- Annex.supportedBackends @@ -155,7 +155,7 @@ chooseBackends fs = do return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs {- Returns the backend to use for a key. -} -keyBackend :: Key -> Annex Backend +keyBackend :: Key -> Annex (Backend Annex) keyBackend key = do bs <- Annex.supportedBackends return $ lookupBackendName bs $ backendName key diff --git a/Backend/File.hs b/Backend/File.hs index 27b2a69015..c8ddd59381 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -27,7 +27,7 @@ import qualified Annex import UUID import Messages -backend :: Backend +backend :: Backend Annex backend = Backend { name = mustProvide, getKey = mustProvide, diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 2f3e2cf534..e665e5da75 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -20,7 +20,7 @@ import qualified Annex import Locations import Content -backend :: Backend +backend :: Backend Annex backend = Backend.File.backend { name = "SHA1", getKey = keyValue, diff --git a/Backend/URL.hs b/Backend/URL.hs index 3eb7376e01..8ed354aed8 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -14,7 +14,7 @@ import TypeInternals import Utility import Messages -backend :: Backend +backend :: Backend Annex backend = Backend { name = "URL", getKey = keyValue, diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 0c93012380..cd4254e2bf 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -21,7 +21,7 @@ import qualified Annex import Content import Messages -backend :: Backend +backend :: Backend Annex backend = Backend.File.backend { name = "WORM", getKey = keyValue, diff --git a/BackendList.hs b/BackendList.hs index d1180d22f9..5ae78bcc7f 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -13,7 +13,7 @@ import qualified Backend.SHA1 import qualified Backend.URL import Types -allBackends :: [Backend] +allBackends :: [Backend Annex] allBackends = [ Backend.WORM.backend , Backend.SHA1.backend diff --git a/Command.hs b/Command.hs index 9fafb18ef9..06fc704bd9 100644 --- a/Command.hs +++ b/Command.hs @@ -43,7 +43,7 @@ type CommandCleanup = Annex Bool - functions. -} type CommandSeekStrings = CommandStartString -> CommandSeek type CommandStartString = String -> CommandStart -type BackendFile = (FilePath, Maybe Backend) +type BackendFile = (FilePath, Maybe (Backend Annex)) type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek type CommandStartBackendFile = BackendFile -> CommandStart type AttrFile = (FilePath, String) @@ -95,7 +95,7 @@ notAnnexed file a = do Just _ -> return Nothing Nothing -> a -isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) +isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a) isAnnexed file a = do r <- Backend.lookupFile file case r of diff --git a/Command/Drop.hs b/Command/Drop.hs index 065e1743a1..fdc55969f0 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -37,7 +37,7 @@ start (file, attr) = isAnnexed file $ \(key, backend) -> do where numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> Maybe Int -> CommandPerform +perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform perform key backend numcopies = do success <- Backend.removeKey backend key numcopies if success diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 662c281c27..fc9bd7f527 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -28,7 +28,7 @@ start (file, attr) = isAnnexed file $ \(key, backend) -> do where numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> Maybe Int -> CommandPerform +perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform perform key backend numcopies = do success <- Backend.fsckKey backend key numcopies if success diff --git a/Command/Get.hs b/Command/Get.hs index e0af6c4078..2aa3c0c150 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -30,7 +30,7 @@ start file = isAnnexed file $ \(key, backend) -> do showStart "get" file return $ Just $ perform key backend -perform :: Key -> Backend -> CommandPerform +perform :: Key -> Backend Annex -> CommandPerform perform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if ok diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 5bc54ceab5..566b508c0c 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -42,7 +42,7 @@ start (file, b) = isAnnexed file $ \(key, oldbackend) -> do return $ head backends choosebackend (Just backend) = return backend -perform :: FilePath -> Key -> Backend -> CommandPerform +perform :: FilePath -> Key -> Backend Annex -> CommandPerform perform file oldkey newbackend = do g <- Annex.gitRepo diff --git a/Command/Unannex.hs b/Command/Unannex.hs index cdd577ba8b..4134439697 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -36,7 +36,7 @@ start file = isAnnexed file $ \(key, backend) -> do return $ Just $ perform file key backend else return Nothing -perform :: FilePath -> Key -> Backend -> CommandPerform +perform :: FilePath -> Key -> Backend Annex -> CommandPerform perform file key backend = do -- force backend to always remove ok <- Backend.removeKey backend key (Just 0) diff --git a/TypeInternals.hs b/TypeInternals.hs index 44db743faa..99f3049730 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -28,10 +28,11 @@ data Flag = -- but it uses Backend, so has to be here to avoid a depends loop. data AnnexState = AnnexState { repo :: Git.Repo, - backends :: [Backend], - supportedBackends :: [Backend], + backends :: [Backend Annex], + supportedBackends :: [Backend Annex], flags :: M.Map FlagName Flag, - repoqueue :: GitQueue.Queue + repoqueue :: GitQueue.Queue, + quiet :: Bool } deriving (Show) -- git-annex's monad @@ -43,7 +44,7 @@ type BackendName = String data Key = Key (BackendName, KeyName) deriving (Eq, Ord) -- constructs a key in a backend -genKey :: Backend -> KeyName -> Key +genKey :: Backend a -> KeyName -> Key genKey b f = Key (name b,f) -- show a key to convert it to a string; the string includes the @@ -77,28 +78,28 @@ keyName :: Key -> KeyName keyName (Key (_,k)) = k -- this structure represents a key-value backend -data Backend = Backend { +data Backend a = Backend { -- name of this backend name :: String, -- converts a filename to a key - getKey :: FilePath -> Annex (Maybe Key), + getKey :: FilePath -> a (Maybe Key), -- stores a file's contents to a key - storeFileKey :: FilePath -> Key -> Annex Bool, + storeFileKey :: FilePath -> Key -> a Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> Annex Bool, + retrieveKeyFile :: Key -> FilePath -> a Bool, -- removes a key, optionally checking that enough copies are stored -- elsewhere - removeKey :: Key -> Maybe Int -> Annex Bool, + removeKey :: Key -> Maybe Int -> a Bool, -- checks if a backend is storing the content of a key - hasKey :: Key -> Annex Bool, + hasKey :: Key -> a Bool, -- called during fsck to check a key -- (second parameter may be the number of copies that there should -- be of the key) - fsckKey :: Key -> Maybe Int -> Annex Bool + fsckKey :: Key -> Maybe Int -> a Bool } -instance Show Backend where +instance Show (Backend a) where show backend = "Backend { name =\"" ++ name backend ++ "\" }" -instance Eq Backend where +instance Eq (Backend a) where a == b = name a == name b From 082b022f9ae56b1446b6607cf7851cd4f1d4f904 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 25 Jan 2011 21:49:04 -0400 Subject: [PATCH 0748/2835] successfully split Annex and AnnexState out of TypeInternals --- Annex.hs | 91 +++++++++++++++++++++++++++--------------------- Backend.hs | 10 +++--- Backend/File.hs | 1 + Backend/SHA1.hs | 1 + Backend/URL.hs | 1 + Backend/WORM.hs | 1 + CmdLine.hs | 4 +-- Options.hs | 8 ++--- Remotes.hs | 2 +- TypeInternals.hs | 21 ----------- Types.hs | 6 ++-- test.hs | 3 +- 12 files changed, 72 insertions(+), 77 deletions(-) diff --git a/Annex.hs b/Annex.hs index a0de630874..a67ea48631 100644 --- a/Annex.hs +++ b/Annex.hs @@ -6,18 +6,20 @@ -} module Annex ( + Annex, + AnnexState(..), + getState, new, run, eval, gitRepo, gitRepoChange, - backends, backendsChange, - supportedBackends, + FlagName, + Flag(..), flagIsSet, flagChange, flagGet, - Flag(..), queue, queueGet, queueRun, @@ -29,19 +31,38 @@ import qualified Data.Map as M import qualified GitRepo as Git import qualified GitQueue -import Types -import qualified TypeInternals as Internals +import qualified TypeInternals + +-- git-annex's monad +type Annex = StateT AnnexState IO + +-- internal state storage +data AnnexState = AnnexState { + repo :: Git.Repo, + backends :: [TypeInternals.Backend Annex], + supportedBackends :: [TypeInternals.Backend Annex], + flags :: M.Map FlagName Flag, + repoqueue :: GitQueue.Queue, + quiet :: Bool +} deriving (Show) + +-- command-line flags +type FlagName = String +data Flag = + FlagBool Bool | + FlagString String + deriving (Eq, Read, Show) {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> [Backend Annex] -> IO AnnexState +new :: Git.Repo -> [TypeInternals.Backend Annex] -> IO AnnexState new gitrepo allbackends = do - let s = Internals.AnnexState { - Internals.repo = gitrepo, - Internals.backends = [], - Internals.supportedBackends = allbackends, - Internals.flags = M.empty, - Internals.repoqueue = GitQueue.empty, - Internals.quiet = False + let s = AnnexState { + repo = gitrepo, + backends = [], + supportedBackends = allbackends, + flags = M.empty, + repoqueue = GitQueue.empty, + quiet = False } (_,s') <- Annex.run s prep return s' @@ -57,41 +78,33 @@ run state action = runStateT action state eval :: AnnexState -> Annex a -> IO a eval state action = evalStateT action state +{- gets a value from the internal Annex state -} +getState :: (AnnexState -> a) -> Annex a +getState a = do + state <- get + return (a state) + {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo -gitRepo = do - state <- get - return (Internals.repo state) +gitRepo = getState repo {- Changes the git repository being acted on. -} gitRepoChange :: Git.Repo -> Annex () gitRepoChange r = do state <- get - put state { Internals.repo = r } - -{- Returns the backends being used. -} -backends :: Annex [Backend Annex] -backends = do - state <- get - return (Internals.backends state) + put state { repo = r } {- Sets the backends to use. -} -backendsChange :: [Backend Annex] -> Annex () +backendsChange :: [TypeInternals.Backend Annex] -> Annex () backendsChange b = do state <- get - put state { Internals.backends = b } - -{- Returns the full list of supported backends. -} -supportedBackends :: Annex [Backend Annex] -supportedBackends = do - state <- get - return (Internals.supportedBackends state) + put state { backends = b } {- Return True if a Bool flag is set. -} flagIsSet :: FlagName -> Annex Bool flagIsSet name = do state <- get - case (M.lookup name $ Internals.flags state) of + case (M.lookup name $ flags state) of Just (FlagBool True) -> return True _ -> return False @@ -99,13 +112,13 @@ flagIsSet name = do flagChange :: FlagName -> Flag -> Annex () flagChange name val = do state <- get - put state { Internals.flags = M.insert name val $ Internals.flags state } + put state { flags = M.insert name val $ flags state } {- Gets the value of a String flag (or "" if there is no such String flag) -} flagGet :: FlagName -> Annex String flagGet name = do state <- get - case (M.lookup name $ Internals.flags state) of + case (M.lookup name $ flags state) of Just (FlagString s) -> return s _ -> return "" @@ -113,23 +126,23 @@ flagGet name = do queue :: String -> [String] -> FilePath -> Annex () queue command params file = do state <- get - let q = Internals.repoqueue state - put state { Internals.repoqueue = GitQueue.add q command params file } + let q = repoqueue state + put state { repoqueue = GitQueue.add q command params file } {- Returns the queue. -} queueGet :: Annex GitQueue.Queue queueGet = do state <- get - return (Internals.repoqueue state) + return (repoqueue state) {- Runs (and empties) the queue. -} queueRun :: Annex () queueRun = do state <- get - let q = Internals.repoqueue state + let q = repoqueue state g <- gitRepo liftIO $ GitQueue.run g q - put state { Internals.repoqueue = GitQueue.empty } + put state { repoqueue = GitQueue.empty } {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () diff --git a/Backend.hs b/Backend.hs index caf50005ad..551c041a80 100644 --- a/Backend.hs +++ b/Backend.hs @@ -44,11 +44,11 @@ import Messages {- List of backends in the order to try them when storing a new key. -} list :: Annex [Backend Annex] list = do - l <- Annex.backends -- list is cached here + l <- Annex.getState Annex.backends -- list is cached here if not $ null l then return l else do - bs <- Annex.supportedBackends + bs <- Annex.getState Annex.supportedBackends g <- Annex.gitRepo let defaults = parseBackendList bs $ Git.configGet g "annex.backends" "" backendflag <- Annex.flagGet "backend" @@ -121,7 +121,7 @@ fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies - by examining what the file symlinks to. -} lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex)) lookupFile file = do - bs <- Annex.supportedBackends + bs <- Annex.getState Annex.supportedBackends tl <- liftIO $ try getsymlink case tl of Left _ -> return Nothing @@ -150,12 +150,12 @@ lookupFile file = do chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))] chooseBackends fs = do g <- Annex.gitRepo - bs <- Annex.supportedBackends + bs <- Annex.getState Annex.supportedBackends pairs <- liftIO $ Git.checkAttr g "annex.backend" fs return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs {- Returns the backend to use for a key. -} keyBackend :: Key -> Annex (Backend Annex) keyBackend key = do - bs <- Annex.supportedBackends + bs <- Annex.getState Annex.supportedBackends return $ lookupBackendName bs $ backendName key diff --git a/Backend/File.hs b/Backend/File.hs index c8ddd59381..962d09909b 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -24,6 +24,7 @@ import qualified Remotes import qualified GitRepo as Git import Content import qualified Annex +import Types import UUID import Messages diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index e665e5da75..be41264b0e 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -19,6 +19,7 @@ import Messages import qualified Annex import Locations import Content +import Types backend :: Backend Annex backend = Backend.File.backend { diff --git a/Backend/URL.hs b/Backend/URL.hs index 8ed354aed8..d67b7db847 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -10,6 +10,7 @@ module Backend.URL (backend) where import Control.Monad.State (liftIO) import Data.String.Utils +import Types import TypeInternals import Utility import Messages diff --git a/Backend/WORM.hs b/Backend/WORM.hs index cd4254e2bf..0110183938 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -20,6 +20,7 @@ import Locations import qualified Annex import Content import Messages +import Types backend :: Backend Annex backend = Backend.File.backend { diff --git a/CmdLine.hs b/CmdLine.hs index 6772282c50..39dd61e99a 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -78,9 +78,9 @@ usage header cmds options = - (but explicitly thrown errors terminate the whole command). - Runs shutdown and propigates an overall error status at the end. -} -tryRun :: AnnexState -> [Annex Bool] -> IO () +tryRun :: Annex.AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions -tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () +tryRun' :: Annex.AnnexState -> Integer -> [Annex Bool] -> IO () tryRun' state errnum (a:as) = do result <- try $ Annex.run state a case result of diff --git a/Options.hs b/Options.hs index 5f367c9dd4..2d4bb85fb2 100644 --- a/Options.hs +++ b/Options.hs @@ -18,10 +18,10 @@ import Command -} type Option = OptDescr (Annex ()) -storeOptBool :: FlagName -> Bool -> Annex () -storeOptBool name val = Annex.flagChange name $ FlagBool val -storeOptString :: FlagName -> String -> Annex () -storeOptString name val = Annex.flagChange name $ FlagString val +storeOptBool :: Annex.FlagName -> Bool -> Annex () +storeOptBool name val = Annex.flagChange name $ Annex.FlagBool val +storeOptString :: Annex.FlagName -> String -> Annex () +storeOptString name val = Annex.flagChange name $ Annex.FlagString val commonOptions :: [Option] commonOptions = diff --git a/Remotes.hs b/Remotes.hs index 9004b33d00..e5aa80e1c2 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -105,7 +105,7 @@ readConfigs = do let todo = cheap ++ doexpensive unless (null todo) $ do _ <- mapM tryGitConfigRead todo - Annex.flagChange "remotesread" $ FlagBool True + Annex.flagChange "remotesread" $ Annex.FlagBool True where cachedUUID r = do u <- getUUID r diff --git a/TypeInternals.hs b/TypeInternals.hs index 99f3049730..abafe8711c 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -17,27 +17,6 @@ import Test.QuickCheck import qualified GitRepo as Git import qualified GitQueue --- command-line flags -type FlagName = String -data Flag = - FlagBool Bool | - FlagString String - deriving (Eq, Read, Show) - --- git-annex's runtime state type doesn't really belong here, --- but it uses Backend, so has to be here to avoid a depends loop. -data AnnexState = AnnexState { - repo :: Git.Repo, - backends :: [Backend Annex], - supportedBackends :: [Backend Annex], - flags :: M.Map FlagName Flag, - repoqueue :: GitQueue.Queue, - quiet :: Bool -} deriving (Show) - --- git-annex's monad -type Annex = StateT AnnexState IO - -- annexed filenames are mapped through a backend into keys type KeyName = String type BackendName = String diff --git a/Types.hs b/Types.hs index b94a4170af..8c19bbbb39 100644 --- a/Types.hs +++ b/Types.hs @@ -7,14 +7,12 @@ module Types ( Annex, - AnnexState, Backend, Key, genKey, backendName, - keyName, - FlagName, - Flag(..) + keyName ) where import TypeInternals +import Annex diff --git a/test.hs b/test.hs index b8b264f0cf..2528e6398e 100644 --- a/test.hs +++ b/test.hs @@ -28,6 +28,7 @@ import qualified GitRepo as Git import qualified Locations import qualified Utility import qualified TypeInternals +import qualified Types import qualified GitAnnex import qualified LocationLog import qualified UUID @@ -416,7 +417,7 @@ git_annex command params = do -- Runs an action in the current annex. Note that shutdown actions -- are not run; this should only be used for actions that query state. -annexeval :: TypeInternals.Annex a -> IO a +annexeval :: Types.Annex a -> IO a annexeval a = do g <- Git.repoFromCwd g' <- Git.configRead g From 6a97b10fcb3e1fa6a230d92a25b42ded587ff743 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 00:17:38 -0400 Subject: [PATCH 0749/2835] rework config storage Moved away from a map of flags to storing config directly in the AnnexState structure. Got rid of most accessor functions in Annex. This allowed supporting multiple --exclude flags. --- Annex.hs | 134 +++++++++++++++++---------------------------- Backend.hs | 28 ++++++---- Backend/File.hs | 2 +- CmdLine.hs | 2 +- Command.hs | 21 ++++++- Command/DropKey.hs | 2 +- Command/FromKey.hs | 19 +++---- Command/Move.hs | 77 +++++++++++++------------- Command/SetKey.hs | 26 ++++----- GitAnnex.hs | 14 +++-- Messages.hs | 2 +- Options.hs | 17 +++--- Remotes.hs | 30 ++++------ debian/changelog | 1 + doc/git-annex.mdwn | 2 + 15 files changed, 179 insertions(+), 198 deletions(-) diff --git a/Annex.hs b/Annex.hs index a67ea48631..d47d449677 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,26 +8,18 @@ module Annex ( Annex, AnnexState(..), - getState, new, run, eval, + getState, + changeState, gitRepo, - gitRepoChange, - backendsChange, - FlagName, - Flag(..), - flagIsSet, - flagChange, - flagGet, queue, - queueGet, queueRun, setConfig ) where import Control.Monad.State -import qualified Data.Map as M import qualified GitRepo as Git import qualified GitQueue @@ -37,40 +29,42 @@ import qualified TypeInternals type Annex = StateT AnnexState IO -- internal state storage -data AnnexState = AnnexState { - repo :: Git.Repo, - backends :: [TypeInternals.Backend Annex], - supportedBackends :: [TypeInternals.Backend Annex], - flags :: M.Map FlagName Flag, - repoqueue :: GitQueue.Queue, - quiet :: Bool -} deriving (Show) +data AnnexState = AnnexState + { repo :: Git.Repo + , backends :: [TypeInternals.Backend Annex] + , supportedBackends :: [TypeInternals.Backend Annex] + , repoqueue :: GitQueue.Queue + , quiet :: Bool + , force :: Bool + , defaultbackend :: Maybe String + , defaultkey :: Maybe String + , toremote :: Maybe String + , fromremote :: Maybe String + , exclude :: [String] + , remotesread :: Bool + } deriving (Show) --- command-line flags -type FlagName = String -data Flag = - FlagBool Bool | - FlagString String - deriving (Eq, Read, Show) +newState :: Git.Repo -> [TypeInternals.Backend Annex] -> AnnexState +newState gitrepo allbackends = AnnexState + { repo = gitrepo + , backends = [] + , supportedBackends = allbackends + , repoqueue = GitQueue.empty + , quiet = False + , force = False + , defaultbackend = Nothing + , defaultkey = Nothing + , toremote = Nothing + , fromremote = Nothing + , exclude = [] + , remotesread = False + } {- Create and returns an Annex state object for the specified git repo. -} new :: Git.Repo -> [TypeInternals.Backend Annex] -> IO AnnexState new gitrepo allbackends = do - let s = AnnexState { - repo = gitrepo, - backends = [], - supportedBackends = allbackends, - flags = M.empty, - repoqueue = GitQueue.empty, - quiet = False - } - (_,s') <- Annex.run s prep - return s' - where - prep = do - -- read git config and update state - gitrepo' <- liftIO $ Git.configRead gitrepo - Annex.gitRepoChange gitrepo' + gitrepo' <- liftIO $ Git.configRead gitrepo + return $ newState gitrepo' allbackends {- performs an action in the Annex monad -} run :: AnnexState -> Annex a -> IO (a, AnnexState) @@ -78,50 +72,26 @@ run state action = runStateT action state eval :: AnnexState -> Annex a -> IO a eval state action = evalStateT action state -{- gets a value from the internal Annex state -} +{- Gets a value from the internal state, selected by the passed value + - constructor. -} getState :: (AnnexState -> a) -> Annex a -getState a = do +getState c = do state <- get - return (a state) + return (c state) + +{- Applies a state mutation function to change the internal state. + - + - Example: changeState (\s -> s { quiet = True }) + -} +changeState :: (AnnexState -> AnnexState) -> Annex () +changeState a = do + state <- get + put (a state) {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo gitRepo = getState repo -{- Changes the git repository being acted on. -} -gitRepoChange :: Git.Repo -> Annex () -gitRepoChange r = do - state <- get - put state { repo = r } - -{- Sets the backends to use. -} -backendsChange :: [TypeInternals.Backend Annex] -> Annex () -backendsChange b = do - state <- get - put state { backends = b } - -{- Return True if a Bool flag is set. -} -flagIsSet :: FlagName -> Annex Bool -flagIsSet name = do - state <- get - case (M.lookup name $ flags state) of - Just (FlagBool True) -> return True - _ -> return False - -{- Sets the value of a flag. -} -flagChange :: FlagName -> Flag -> Annex () -flagChange name val = do - state <- get - put state { flags = M.insert name val $ flags state } - -{- Gets the value of a String flag (or "" if there is no such String flag) -} -flagGet :: FlagName -> Annex String -flagGet name = do - state <- get - case (M.lookup name $ flags state) of - Just (FlagString s) -> return s - _ -> return "" - {- Adds a git command to the queue. -} queue :: String -> [String] -> FilePath -> Annex () queue command params file = do @@ -129,12 +99,6 @@ queue command params file = do let q = repoqueue state put state { repoqueue = GitQueue.add q command params file } -{- Returns the queue. -} -queueGet :: Annex GitQueue.Queue -queueGet = do - state <- get - return (repoqueue state) - {- Runs (and empties) the queue. -} queueRun :: Annex () queueRun = do @@ -146,9 +110,9 @@ queueRun = do {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () -setConfig key value = do +setConfig k value = do g <- Annex.gitRepo - liftIO $ Git.run g ["config", key, value] + liftIO $ Git.run g ["config", k, value] -- re-read git config and update the repo's state g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' + Annex.changeState $ \s -> s { Annex.repo = g' } diff --git a/Backend.hs b/Backend.hs index 551c041a80..055c5b8ab3 100644 --- a/Backend.hs +++ b/Backend.hs @@ -48,20 +48,24 @@ list = do if not $ null l then return l else do + s <- getstandard + d <- Annex.getState Annex.defaultbackend + handle d s + where + parseBackendList l [] = l + parseBackendList bs s = map (lookupBackendName bs) $ words s + handle Nothing s = return s + handle (Just "") s = return s + handle (Just name) s = do + bs <- Annex.getState Annex.supportedBackends + let l' = (lookupBackendName bs name):s + Annex.changeState $ \state -> state { Annex.backends = l' } + return l' + getstandard = do bs <- Annex.getState Annex.supportedBackends g <- Annex.gitRepo - let defaults = parseBackendList bs $ Git.configGet g "annex.backends" "" - backendflag <- Annex.flagGet "backend" - let l' = if not $ null backendflag - then (lookupBackendName bs backendflag):defaults - else defaults - Annex.backendsChange l' - return l' - where - parseBackendList bs s = - if null s - then bs - else map (lookupBackendName bs) $ words s + return $ parseBackendList bs $ + Git.configGet g "annex.backends" "" {- Looks up a backend in a list. May fail if unknown. -} lookupBackendName :: [Backend Annex] -> String -> Backend Annex diff --git a/Backend/File.hs b/Backend/File.hs index 962d09909b..d0c1e0e22a 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -90,7 +90,7 @@ copyKeyFile key file = do - error if not. -} checkRemoveKey :: Key -> Maybe Int -> Annex Bool checkRemoveKey key numcopiesM = do - force <- Annex.flagIsSet "force" + force <- Annex.getState Annex.force if force || numcopiesM == Just 0 then return True else do diff --git a/CmdLine.hs b/CmdLine.hs index 39dd61e99a..1b5daadeb0 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -102,7 +102,7 @@ startup = do {- Cleanup actions. -} shutdown :: Integer -> Annex () shutdown errnum = do - q <- Annex.queueGet + q <- Annex.getState Annex.repoqueue unless (q == GitQueue.empty) $ do showSideAction "Recording state in git..." Annex.queueRun diff --git a/Command.hs b/Command.hs index 06fc704bd9..cbfb265002 100644 --- a/Command.hs +++ b/Command.hs @@ -179,11 +179,11 @@ backendPairs a files = do filterFiles :: [FilePath] -> Annex [FilePath] filterFiles l = do let l' = filter notState l - exclude <- Annex.flagGet "exclude" + exclude <- Annex.getState Annex.exclude if null exclude then return l' else do - let regexp = compile ("^" ++ wildToRegex exclude) [] + let regexp = compile (toregex exclude) [] return $ filter (notExcluded regexp) l' where notState f = stateLoc /= take stateLocLen f @@ -191,6 +191,10 @@ filterFiles l = do notExcluded r f = case match r f [] of Nothing -> True Just _ -> False + toregex exclude = "^(" ++ toregex' exclude "" ++ ")" + toregex' [] c = c + toregex' (w:ws) "" = toregex' ws (wildToRegex w) + toregex' (w:ws) c = toregex' ws (c ++ "|" ++ wildToRegex w) {- filter out symlinks -} notSymlink :: FilePath -> IO Bool @@ -219,3 +223,16 @@ paramName :: String paramName = "NAME" paramNothing :: String paramNothing = "" + +{- The Key specified by the --key and --backend parameters. -} +cmdlineKey :: Annex Key +cmdlineKey = do + k <- Annex.getState Annex.defaultkey + backends <- Backend.list + return $ genKey (head backends) (keyname' k) + where + keyname' Nothing = badkey + keyname' (Just "") = badkey + keyname' (Just n) = n + badkey = error "please specify the key with --key" + diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 6ba5c117c4..8c7566df84 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -28,7 +28,7 @@ start keyname = do backends <- Backend.list let key = genKey (head backends) keyname present <- inAnnex key - force <- Annex.flagIsSet "force" + force <- Annex.getState Annex.force if not present then return Nothing else if not force diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 9c4a3cfdcb..8817942580 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -10,7 +10,7 @@ module Command.FromKey where import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory -import Control.Monad (when, unless) +import Control.Monad (unless) import Command import qualified Annex @@ -30,22 +30,21 @@ seek = [withFilesMissing start] {- Adds a file pointing at a manually-specified key -} start :: CommandStartString start file = do - keyname <- Annex.flagGet "key" - when (null keyname) $ error "please specify the key with --key" - backends <- Backend.list - let key = genKey (head backends) keyname - + key <- cmdlineKey inbackend <- Backend.hasKey key unless inbackend $ error $ - "key ("++keyname++") is not present in backend" + "key ("++keyName key++") is not present in backend" showStart "fromkey" file - return $ Just $ perform file key -perform :: FilePath -> Key -> CommandPerform -perform file key = do + return $ Just $ perform file + +perform :: FilePath -> CommandPerform +perform file = do + key <- cmdlineKey link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ cleanup file + cleanup :: FilePath -> CommandCleanup cleanup file = do Annex.queue "add" ["--"] file diff --git a/Command/Move.hs b/Command/Move.hs index 2920c06616..4416134c04 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -34,12 +34,16 @@ seek = [withFilesInGit $ start True] - moving data in the key-value backend. -} start :: Bool -> CommandStartString start move file = do - fromName <- Annex.flagGet "fromrepository" - toName <- Annex.flagGet "torepository" - case (fromName, toName) of - ("", "") -> error "specify either --from or --to" - ("", _) -> toStart move file - (_ , "") -> fromStart move file + to <- Annex.getState Annex.toremote + from <- Annex.getState Annex.fromremote + case (from, to) of + (Nothing, Nothing) -> error "specify either --from or --to" + (Nothing, Just name) -> do + dest <- Remotes.byName name + toStart dest move file + (Just name, Nothing) -> do + src <- Remotes.byName name + fromStart src move file (_ , _) -> error "only one of --from or --to can be specified" showAction :: Bool -> FilePath -> Annex () @@ -65,34 +69,33 @@ remoteHasKey remote key present = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -toStart :: Bool -> CommandStartString -toStart move file = isAnnexed file $ \(key, _) -> do +toStart :: Git.Repo -> Bool -> CommandStartString +toStart dest move file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if not ishere then return Nothing -- not here, so nothing to do else do showAction move file - return $ Just $ toPerform move key -toPerform :: Bool -> Key -> CommandPerform -toPerform move key = do + return $ Just $ toPerform dest move key +toPerform :: Git.Repo -> Bool -> Key -> CommandPerform +toPerform dest move key = do Remotes.readConfigs -- checking the remote is expensive, so not done in the start step - remote <- Remotes.commandLineRemote - isthere <- Remotes.inAnnex remote key + isthere <- Remotes.inAnnex dest key case isthere of Left err -> do showNote $ show err return Nothing Right False -> do - showNote $ "to " ++ Git.repoDescribe remote ++ "..." - ok <- Remotes.copyToRemote remote key + showNote $ "to " ++ Git.repoDescribe dest ++ "..." + ok <- Remotes.copyToRemote dest key if ok - then return $ Just $ toCleanup move remote key + then return $ Just $ toCleanup dest move key else return Nothing -- failed - Right True -> return $ Just $ toCleanup move remote key -toCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup -toCleanup move remote key = do - remoteHasKey remote key True + Right True -> return $ Just $ toCleanup dest move key +toCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup +toCleanup dest move key = do + remoteHasKey dest key True if move then Command.Drop.cleanup key else return True @@ -103,36 +106,34 @@ toCleanup move remote key = do - If the current repository already has the content, it is still removed - from the other repository when moving. -} -fromStart :: Bool -> CommandStartString -fromStart move file = isAnnexed file $ \(key, _) -> do - remote <- Remotes.commandLineRemote +fromStart :: Git.Repo -> Bool -> CommandStartString +fromStart src move file = isAnnexed file $ \(key, _) -> do (trusted, untrusted, _) <- Remotes.keyPossibilities key - if null $ filter (\r -> Remotes.same r remote) (trusted ++ untrusted) + if null $ filter (\r -> Remotes.same r src) (trusted ++ untrusted) then return Nothing else do showAction move file - return $ Just $ fromPerform move key -fromPerform :: Bool -> Key -> CommandPerform -fromPerform move key = do - remote <- Remotes.commandLineRemote + return $ Just $ fromPerform src move key +fromPerform :: Git.Repo -> Bool -> Key -> CommandPerform +fromPerform src move key = do ishere <- inAnnex key if ishere - then return $ Just $ fromCleanup move remote key + then return $ Just $ fromCleanup src move key else do - showNote $ "from " ++ Git.repoDescribe remote ++ "..." - ok <- getViaTmp key $ Remotes.copyFromRemote remote key + showNote $ "from " ++ Git.repoDescribe src ++ "..." + ok <- getViaTmp key $ Remotes.copyFromRemote src key if ok - then return $ Just $ fromCleanup move remote key + then return $ Just $ fromCleanup src move key else return Nothing -- fail -fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup -fromCleanup True remote key = do - ok <- Remotes.onRemote remote (boolSystem, False) "dropkey" +fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup +fromCleanup src True key = do + ok <- Remotes.onRemote src (boolSystem, False) "dropkey" ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] - -- better safe than sorry: assume the remote dropped the key + -- better safe than sorry: assume the src dropped the key -- even if it seemed to fail; the failure could have occurred -- after it really dropped it - remoteHasKey remote key False + remoteHasKey src key False return ok -fromCleanup False _ _ = return True +fromCleanup _ False _ = return True diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 412504b2ee..388392cd60 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -8,14 +8,10 @@ module Command.SetKey where import Control.Monad.State (liftIO) -import Control.Monad (when) import Command -import qualified Annex import Utility -import qualified Backend import LocationLog -import Types import Content import Messages @@ -29,26 +25,24 @@ seek = [withTempFile start] {- Sets cached content for a key. -} start :: CommandStartString start file = do - keyname <- Annex.flagGet "key" - when (null keyname) $ error "please specify the key with --key" - backends <- Backend.list - let key = genKey (head backends) keyname showStart "setkey" file - return $ Just $ perform file key -perform :: FilePath -> Key -> CommandPerform -perform file key = do + return $ Just $ perform file + +perform :: FilePath -> CommandPerform +perform file = do + key <- cmdlineKey -- the file might be on a different filesystem, so mv is used - -- rather than simply calling moveToObjectDir key file + -- rather than simply calling moveToObjectDir ok <- getViaTmp key $ \dest -> do if dest /= file then liftIO $ boolSystem "mv" [file, dest] else return True if ok - then return $ Just $ cleanup key + then return $ Just $ cleanup else error "mv failed!" -cleanup :: Key -> CommandCleanup -cleanup key = do +cleanup :: CommandCleanup +cleanup = do + key <- cmdlineKey logStatus key ValuePresent return True - diff --git a/GitAnnex.hs b/GitAnnex.hs index d9efdad2dd..378b6e538d 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -13,6 +13,7 @@ import qualified GitRepo as Git import CmdLine import Command import Options +import qualified Annex import qualified Command.Add import qualified Command.Unannex @@ -65,15 +66,20 @@ cmds = concat options :: [Option] options = commonOptions ++ - [ Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + [ Option ['k'] ["key"] (ReqArg setkey paramKey) "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + , Option ['t'] ["to"] (ReqArg setto paramRemote) "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + , Option ['f'] ["from"] (ReqArg setfrom paramRemote) "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + , Option ['x'] ["exclude"] (ReqArg addexclude paramGlob) "skip files matching the glob pattern" ] + where + setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v } + setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v } + setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v } + addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:(Annex.exclude s) } header :: String header = "Usage: git-annex command [option ..]" diff --git a/Messages.hs b/Messages.hs index 2934de4287..2b98622309 100644 --- a/Messages.hs +++ b/Messages.hs @@ -17,7 +17,7 @@ import qualified Annex verbose :: Annex () -> Annex () verbose a = do - q <- Annex.flagIsSet "quiet" + q <- Annex.getState Annex.quiet unless q a showSideAction :: String -> Annex () diff --git a/Options.hs b/Options.hs index 2d4bb85fb2..4cd62c2222 100644 --- a/Options.hs +++ b/Options.hs @@ -18,19 +18,18 @@ import Command -} type Option = OptDescr (Annex ()) -storeOptBool :: Annex.FlagName -> Bool -> Annex () -storeOptBool name val = Annex.flagChange name $ Annex.FlagBool val -storeOptString :: Annex.FlagName -> String -> Annex () -storeOptString name val = Annex.flagChange name $ Annex.FlagString val - commonOptions :: [Option] commonOptions = - [ Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) + [ Option ['f'] ["force"] (NoArg (setforce True)) "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) + , Option ['q'] ["quiet"] (NoArg (setquiet True)) "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) + , Option ['v'] ["verbose"] (NoArg (setquiet False)) "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) + , Option ['b'] ["backend"] (ReqArg setdefaultbackend paramName) "specify default key-value backend to use" ] + where + setforce v = Annex.changeState $ \s -> s { Annex.force = v } + setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v } + setdefaultbackend v = Annex.changeState $ \s -> s { Annex.defaultbackend = Just v } diff --git a/Remotes.hs b/Remotes.hs index e5aa80e1c2..e04874f7dc 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -11,7 +11,6 @@ module Remotes ( keyPossibilities, inAnnex, same, - commandLineRemote, byName, copyFromRemote, copyToRemote, @@ -69,7 +68,7 @@ tryGitConfigRead r g <- Annex.gitRepo let l = Git.remotes g let g' = Git.remotesAdd g $ exchange l r' - Annex.gitRepoChange g' + Annex.changeState $ \s -> s { Annex.repo = g' } return $ Right r' exchange [] _ = [] exchange (old:ls) new = @@ -93,7 +92,7 @@ tryGitConfigRead r readConfigs :: Annex () readConfigs = do g <- Annex.gitRepo - remotesread <- Annex.flagIsSet "remotesread" + remotesread <- Annex.getState Annex.remotesread unless remotesread $ do allremotes <- filterM repoNotIgnored $ Git.remotes g let cheap = filter (not . Git.repoIsUrl) allremotes @@ -105,7 +104,7 @@ readConfigs = do let todo = cheap ++ doexpensive unless (null todo) $ do _ <- mapM tryGitConfigRead todo - Annex.flagChange "remotesread" $ Annex.FlagBool True + Annex.changeState $ \s -> s { Annex.remotesread = True } where cachedUUID r = do u <- getUUID r @@ -204,27 +203,22 @@ repoCost r = do repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do ignored <- repoConfig r "ignore" "false" - fromName <- Annex.flagGet "fromrepository" - toName <- Annex.flagGet "torepository" - let name = if null fromName then toName else fromName - if not $ null name - then return $ match name + to <- match Annex.toremote + from <- match Annex.fromremote + if to || from + then return True else return $ not $ Git.configTrue ignored where - match name = name == Git.repoRemoteName r + match a = do + name <- Annex.getState a + case name of + Nothing -> return False + Just n -> return $ n == Git.repoRemoteName r {- Checks if two repos are the same, by comparing their remote names. -} same :: Git.Repo -> Git.Repo -> Bool same a b = Git.repoRemoteName a == Git.repoRemoteName b -{- Returns the remote specified by --from or --to, may fail with error. -} -commandLineRemote :: Annex Git.Repo -commandLineRemote = do - fromName <- Annex.flagGet "fromrepository" - toName <- Annex.flagGet "torepository" - let name = if null fromName then toName else fromName - byName name - {- Looks up a remote by name. -} byName :: String -> Annex Git.Repo byName name = do diff --git a/debian/changelog b/debian/changelog index 23358486d1..53b6e20a6c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.19) UNRELEASED; urgency=low * Support using the uuidgen command if the uuid command is not available. + * Allow --exclude to be specified more than once. -- Joey Hess Wed, 19 Jan 2011 18:07:51 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 9d89413b96..8e9418fd13 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -258,6 +258,8 @@ Many git-annex commands will stage changes for later `git commit` by you. Skips files matching the glob pattern. The glob is matched relative to the current directory. + This option can be specified multiple times. + * --backend=name Specifies which key-value backend to use. From aa2ca533bcef1848a9dc94bfea8d33a99e8f2df0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 00:29:32 -0400 Subject: [PATCH 0750/2835] trim cruft --- TypeInternals.hs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/TypeInternals.hs b/TypeInternals.hs index abafe8711c..d3592f4822 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -9,14 +9,9 @@ module TypeInternals where -import Control.Monad.State (StateT) import Data.String.Utils -import qualified Data.Map as M import Test.QuickCheck -import qualified GitRepo as Git -import qualified GitQueue - -- annexed filenames are mapped through a backend into keys type KeyName = String type BackendName = String From 616d1d4a208693c46f41781d9099c1f04ae067e6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 00:37:50 -0400 Subject: [PATCH 0751/2835] rename TypeInternals to BackendTypes Now that it only contains types used by the backends --- Annex.hs | 10 ++-- Backend.hs | 16 +++---- Backend/File.hs | 2 +- Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Backend/WORM.hs | 2 +- TypeInternals.hs => BackendTypes.hs | 71 ++++++++++++++--------------- Types.hs | 2 +- test.hs | 10 ++-- 9 files changed, 58 insertions(+), 59 deletions(-) rename TypeInternals.hs => BackendTypes.hs (89%) diff --git a/Annex.hs b/Annex.hs index d47d449677..4a1b89dcf2 100644 --- a/Annex.hs +++ b/Annex.hs @@ -23,7 +23,7 @@ import Control.Monad.State import qualified GitRepo as Git import qualified GitQueue -import qualified TypeInternals +import qualified BackendTypes -- git-annex's monad type Annex = StateT AnnexState IO @@ -31,8 +31,8 @@ type Annex = StateT AnnexState IO -- internal state storage data AnnexState = AnnexState { repo :: Git.Repo - , backends :: [TypeInternals.Backend Annex] - , supportedBackends :: [TypeInternals.Backend Annex] + , backends :: [BackendTypes.Backend Annex] + , supportedBackends :: [BackendTypes.Backend Annex] , repoqueue :: GitQueue.Queue , quiet :: Bool , force :: Bool @@ -44,7 +44,7 @@ data AnnexState = AnnexState , remotesread :: Bool } deriving (Show) -newState :: Git.Repo -> [TypeInternals.Backend Annex] -> AnnexState +newState :: Git.Repo -> [BackendTypes.Backend Annex] -> AnnexState newState gitrepo allbackends = AnnexState { repo = gitrepo , backends = [] @@ -61,7 +61,7 @@ newState gitrepo allbackends = AnnexState } {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> [TypeInternals.Backend Annex] -> IO AnnexState +new :: Git.Repo -> [BackendTypes.Backend Annex] -> IO AnnexState new gitrepo allbackends = do gitrepo' <- liftIO $ Git.configRead gitrepo return $ newState gitrepo' allbackends diff --git a/Backend.hs b/Backend.hs index 055c5b8ab3..d9bf35f0dd 100644 --- a/Backend.hs +++ b/Backend.hs @@ -38,7 +38,7 @@ import Locations import qualified GitRepo as Git import qualified Annex import Types -import qualified TypeInternals as Internals +import qualified BackendTypes as B import Messages {- List of backends in the order to try them when storing a new key. -} @@ -78,7 +78,7 @@ maybeLookupBackendName bs s = if 1 /= length matches then Nothing else Just $ head matches - where matches = filter (\b -> s == Internals.name b) bs + where matches = filter (\b -> s == B.name b) bs {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex)) @@ -91,11 +91,11 @@ storeFileKey file trybackend = do storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex)) storeFileKey' [] _ = return Nothing storeFileKey' (b:bs) file = do - result <- (Internals.getKey b) file + result <- (B.getKey b) file case result of Nothing -> nextbackend Just key -> do - stored <- (Internals.storeFileKey b) file key + stored <- (B.storeFileKey b) file key if (not stored) then nextbackend else return $ Just (key, b) @@ -105,21 +105,21 @@ storeFileKey' (b:bs) file = do {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} retrieveKeyFile :: Backend Annex -> Key -> FilePath -> Annex Bool -retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest +retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} removeKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool -removeKey backend key numcopies = (Internals.removeKey backend) key numcopies +removeKey backend key numcopies = (B.removeKey backend) key numcopies {- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool hasKey key = do backend <- keyBackend key - (Internals.hasKey backend) key + (B.hasKey backend) key {- Checks a key's backend for problems. -} fsckKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool -fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies +fsckKey backend key numcopies = (B.fsckKey backend) key numcopies {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/Backend/File.hs b/Backend/File.hs index d0c1e0e22a..ac6e4a910f 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -17,7 +17,7 @@ module Backend.File (backend, checkKey) where import Control.Monad.State import System.Directory -import TypeInternals +import BackendTypes import LocationLog import Locations import qualified Remotes diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index be41264b0e..f8dbea4b03 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -14,7 +14,7 @@ import System.IO import System.Directory import qualified Backend.File -import TypeInternals +import BackendTypes import Messages import qualified Annex import Locations diff --git a/Backend/URL.hs b/Backend/URL.hs index d67b7db847..45a204b07d 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import Data.String.Utils import Types -import TypeInternals +import BackendTypes import Utility import Messages diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 0110183938..56f243396e 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -15,7 +15,7 @@ import System.Directory import Data.String.Utils import qualified Backend.File -import TypeInternals +import BackendTypes import Locations import qualified Annex import Content diff --git a/TypeInternals.hs b/BackendTypes.hs similarity index 89% rename from TypeInternals.hs rename to BackendTypes.hs index d3592f4822..e4b155f98f 100644 --- a/TypeInternals.hs +++ b/BackendTypes.hs @@ -1,22 +1,53 @@ -{- git-annex internal data types +{- git-annex key/value backend data types - - - Most things should not need this, using Types and/or Annex instead. + - Most things should not need this, using Types instead - - Copyright 2010 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module TypeInternals where +module BackendTypes where import Data.String.Utils import Test.QuickCheck --- annexed filenames are mapped through a backend into keys type KeyName = String type BackendName = String data Key = Key (BackendName, KeyName) deriving (Eq, Ord) +data Backend a = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: FilePath -> a (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: FilePath -> Key -> a Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> FilePath -> a Bool, + -- removes a key, optionally checking that enough copies are stored + -- elsewhere + removeKey :: Key -> Maybe Int -> a Bool, + -- checks if a backend is storing the content of a key + hasKey :: Key -> a Bool, + -- called during fsck to check a key + -- (second parameter may be the number of copies that there should + -- be of the key) + fsckKey :: Key -> Maybe Int -> a Bool +} + +instance Show (Backend a) where + show backend = "Backend { name =\"" ++ name backend ++ "\" }" + +instance Eq (Backend a) where + a == b = name a == name b + +-- accessors for the parts of a key +keyName :: Key -> KeyName +keyName (Key (_,k)) = k +backendName :: Key -> BackendName +backendName (Key (b,_)) = b + -- constructs a key in a backend genKey :: Backend a -> KeyName -> Key genKey b f = Key (name b,f) @@ -45,35 +76,3 @@ prop_idempotent_key_read_show k -- backend names will never contain colons | elem ':' (backendName k) = True | otherwise = k == (read $ show k) - -backendName :: Key -> BackendName -backendName (Key (b,_)) = b -keyName :: Key -> KeyName -keyName (Key (_,k)) = k - --- this structure represents a key-value backend -data Backend a = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: FilePath -> a (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: FilePath -> Key -> a Bool, - -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> a Bool, - -- removes a key, optionally checking that enough copies are stored - -- elsewhere - removeKey :: Key -> Maybe Int -> a Bool, - -- checks if a backend is storing the content of a key - hasKey :: Key -> a Bool, - -- called during fsck to check a key - -- (second parameter may be the number of copies that there should - -- be of the key) - fsckKey :: Key -> Maybe Int -> a Bool -} - -instance Show (Backend a) where - show backend = "Backend { name =\"" ++ name backend ++ "\" }" - -instance Eq (Backend a) where - a == b = name a == name b diff --git a/Types.hs b/Types.hs index 8c19bbbb39..0890efd5e3 100644 --- a/Types.hs +++ b/Types.hs @@ -14,5 +14,5 @@ module Types ( keyName ) where -import TypeInternals +import BackendTypes import Annex diff --git a/test.hs b/test.hs index 2528e6398e..0c47da310c 100644 --- a/test.hs +++ b/test.hs @@ -27,7 +27,7 @@ import qualified Backend import qualified GitRepo as Git import qualified Locations import qualified Utility -import qualified TypeInternals +import qualified BackendTypes import qualified Types import qualified GitAnnex import qualified LocationLog @@ -54,7 +54,7 @@ quickchecks :: Test quickchecks = TestLabel "quickchecks" $ TestList [ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode , qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey - , qctest "prop_idempotent_key_read_show" TypeInternals.prop_idempotent_key_read_show + , qctest "prop_idempotent_key_read_show" BackendTypes.prop_idempotent_key_read_show , qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics @@ -106,8 +106,8 @@ test_add = "git-annex add" ~: TestCase $ inmainrepo $ do test_setkey :: Test test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do writeFile tmp $ content sha1annexedfile - r <- annexeval $ TypeInternals.getKey Backend.SHA1.backend tmp - let sha1 = TypeInternals.keyName $ fromJust r + r <- annexeval $ BackendTypes.getKey Backend.SHA1.backend tmp + let sha1 = BackendTypes.keyName $ fromJust r git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" @@ -384,7 +384,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do checkunused [annexedfilekey, sha1annexedfilekey] -- good opportunity to test dropkey also - git_annex "dropkey" ["-q", "--force", TypeInternals.keyName annexedfilekey] + git_annex "dropkey" ["-q", "--force", BackendTypes.keyName annexedfilekey] @? "dropkey failed" checkunused [sha1annexedfilekey] From ff11803c8983173b00ca8a74e3d323679a9e128c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 12:34:15 -0400 Subject: [PATCH 0752/2835] add --- doc/todo/S3.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/todo/S3.mdwn diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn new file mode 100644 index 0000000000..ec2d403ced --- /dev/null +++ b/doc/todo/S3.mdwn @@ -0,0 +1 @@ +Support Amazon S3 as a file storage backend. From b4179f5081414f3ac903a878538b976931442913 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 12:43:21 -0400 Subject: [PATCH 0753/2835] mention git-media --- doc/not.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index b138953f91..c9c5754d7e 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -20,3 +20,13 @@ down to a binary. And it has a fairly extensive test suite. (Don't be fooled by "make test" only showing a few dozen test cases; each test involves checking dozens to hundreds of assertions.) + +* git-annex is not [git-media](https://github.com/schacon/git-media), + although they both approach the same problem from a similar direction. + I only learned of git-media after writing git-annex, but I probably + would have still written git-annex instead of using it. Currently, + git-media has the advantage of using git smudge filters rather than + git-annex's pile of symlinks, and it may be a tighter fit for certian + situations. It lacks git-annex's support for widely distributed storage, + using only a single backend data store. It also does not support + partial checkouts of file contents, like git-annex does. From 06ca13b103eb22afe5c514a9f26f97862cb37384 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 13:21:51 -0400 Subject: [PATCH 0754/2835] add --- doc/todo/smudge.mdwn | 65 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 doc/todo/smudge.mdwn diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn new file mode 100644 index 0000000000..65cfb0fdab --- /dev/null +++ b/doc/todo/smudge.mdwn @@ -0,0 +1,65 @@ +git-annex should use smudge/clean filters. + +The trick is doing it efficiently. Since git a2b665d, 2011-01-05, +something like this works to provide a filename to the clean script: + + git config --global filter.huge.clean huge-clean %f + +This avoids it needing to read all the current file content from stdin +when doing eg, a git status or git commit. Instead it is passed the +filename that git is operating on, I think that's from the working +directory. + +So, WORM could just look at that file and easily tell if it is one +it already knows (same mtime and size). If so, it can short-circuit and +do nothing, file content is already cached. + +SHA1 has a harder job. Would not want to re-sha1 the file every time, +probably. So it'd need a cache of file stat info, mapped to known objects. + +On the smudge side, I have not heard of a way to have the smudge filter +point to an existing file, it probably still needs to cat it out. Luckily +that is only done at checkout anyway. + +---- + +The other trick may be doing it with partial content availability. +When a smudge filter fails, git leaves the tree and index in a very weird +state. More investigation needed. + +### test files + +huge-smudge: + +
+#!/bin/sh
+read sha1
+echo "smudging $sha1" >&2
+cat ~/$sha1
+
+ +huge-clean: + +
+#!/bin/sh
+cat >temp
+sha1=`sha1sum temp | cut -d' ' -f1`
+echo "cleaning $sha1" >&2
+ls -l temp >&2
+mv temp ~/$sha1
+echo $sha1
+
+ +.gitattributes: + +
+*.huge filter=huge
+
+ +in .git/config: + +
+[filter "huge"]
+        clean = huge-clean
+        smudge = huge-smudge
+

From 758019cc18e03c203b023efb3c0d76c053265362 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 13:34:39 -0400
Subject: [PATCH 0755/2835] update

---
 doc/todo/smudge.mdwn | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 65cfb0fdab..855d9c7f89 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -21,11 +21,19 @@ On the smudge side, I have not heard of a way to have the smudge filter
 point to an existing file, it probably still needs to cat it out. Luckily
 that is only done at checkout anyway.
 
-----
+### dealing with partial content availability
+
+The smudge filter cannot be allowed to fail, that leaves the tree and
+index in a weird state. So if a file's content is requested by calling
+the smudge filter, the trick is to instead provide dummy content,
+indicating it is not available (and perhaps saying to run "git-annex get").
+
+Then, in the clean filter, it has to detect that it's cleaning a file
+with that dummy content, and make sure to provide the same identifier as
+it would if the file content was there. 
+
+I've a demo implementation of this technique in the scripts below.
 
-The other trick may be doing it with partial content availability.
-When a smudge filter fails, git leaves the tree and index in a very weird
-state. More investigation needed.
 
 ### test files
 
@@ -35,7 +43,11 @@ huge-smudge:
 #!/bin/sh
 read sha1
 echo "smudging $sha1" >&2
-cat ~/$sha1
+if [ -e ~/$sha1 ]; then
+	cat ~/$sha1
+else
+	echo "$sha1 not available"
+fi
 

 
 huge-clean:
@@ -43,6 +55,11 @@ huge-clean:
 
 #!/bin/sh
 cat >temp
+if grep -q 'not available' temp; then
+	awk '{print $1}' temp # provide what we would if the content were avail!
+	rm temp
+	exit 0
+fi
 sha1=`sha1sum temp | cut -d' ' -f1`
 echo "cleaning $sha1" >&2
 ls -l temp >&2

From 07769fc94968ab65a828c0c001c2fa443a328d99 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 13:40:11 -0400
Subject: [PATCH 0756/2835] more

---
 doc/todo/smudge.mdwn | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 855d9c7f89..f51f45a391 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -17,10 +17,6 @@ do nothing, file content is already cached.
 SHA1 has a harder job. Would not want to re-sha1 the file every time,
 probably. So it'd need a cache of file stat info, mapped to known objects.
 
-On the smudge side, I have not heard of a way to have the smudge filter
-point to an existing file, it probably still needs to cat it out. Luckily
-that is only done at checkout anyway.
-
 ### dealing with partial content availability
 
 The smudge filter cannot be allowed to fail, that leaves the tree and
@@ -34,6 +30,13 @@ it would if the file content was there.
 
 I've a demo implementation of this technique in the scripts below.
 
+----
+
+It may further be possible to use the %f with the smudge filter
+(docs say it's supported), and instead of outputting the dummy content, 
+it could create a dangling symlink, which would be more like git-annex's
+behavior now, and makes it easy to tell what content is not available
+with `ls`.
 
 ### test files
 

From f7e3d6eea2f71efe14c3ccb29ef4e88840384d02 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 14:09:06 -0400
Subject: [PATCH 0757/2835] document 3-level trust

---
 debian/changelog   |  3 +++
 doc/copies.mdwn    |  1 +
 doc/git-annex.mdwn |  7 ++++---
 doc/trust.mdwn     | 39 ++++++++++++++++++++++++++++++++++-----
 4 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 53b6e20a6c..529a0896ba 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,9 @@ git-annex (0.19) UNRELEASED; urgency=low
 
   * Support using the uuidgen command if the uuid command is not available.
   * Allow --exclude to be specified more than once.
+  * There are now three levels of repository trust.
+  * untrust: Now marks the current repository as untrusted.
+  * semitrust: Now restores the default trust level. (What untrust used to do.)
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
 
diff --git a/doc/copies.mdwn b/doc/copies.mdwn
index 98f315081f..39a714d3bb 100644
--- a/doc/copies.mdwn
+++ b/doc/copies.mdwn
@@ -4,6 +4,7 @@ your git repository's `.git` directory, not in some external data store.
 It's important that data not get lost by an ill-considered `git annex drop`
 command.  So, then using those backends, git-annex can be configured to try
 to keep N copies of a file's content available across all repositories. 
+(Although [[untrusted_repositories|trust]] don't count toward this total.)
 
 By default, N is 1; it is configured by annex.numcopies. This default
 can be overridden on a per-file-type basis by the annex.numcopies
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 8e9418fd13..49e67a83ea 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -183,12 +183,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * trust [repository ...]
 
-  Records that a repository is [[trusted]] to not unexpectedly lose content.
-  Use with care.
+  Records that a repository is [[trusted|trust]] to not unexpectedly lose
+  content. Use with care.
 
 * untrust [repository ...]
 
-  Undoes a trust command.
+  Records that a repository is [[not trusted|trusted]] and could lose content
+  at any time.
 
 * fromkey file
 
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index 6ce43f30eb..edec6568e7 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -1,3 +1,11 @@
+Git-annex supports three levels of trust of a repository:
+
+* semitrusted (default)
+* untrusted
+* trusted
+
+## semi-trusted
+
 Normally, git-annex does not fully trust its stored [[location_tracking]]
 information. When removing content, it will directly check
 that other repositories have enough [[copies]].
@@ -7,14 +15,35 @@ Generally that explicit checking is a good idea. Consider that the current
 out. Or, a remote may have suffered a catastrophic loss of data, or itself
 been lost.
 
-Sometimes though, you may have reasons to trust the location tracking
-information for a remote repository. For example, it may be an offline
+There is still some trust involved here. A semi-trusted repository is
+dependended on to retain a copy of the file content; possibly the only
+[[copy|copies]].
+
+(Being semitrusted is the default. The `git annex semitrust` command
+restores a repository to this default, when it has been overridden.)
+
+## untrusted
+
+An untrusted repository is not trusted to retain data at all. Git-annex
+will not count data in such a repository as a of the data, and will
+retain sufficient [[copies]] elsewhere.
+
+This is a good choice for eg, portable drives that could get lost. Or,
+if a disk is known to be dying, you can set it to untrusted and let
+`git annex fsck` warn about data that needs to be copied off it.
+
+To configure a repository as untrusted, use the `git annex untrust`
+command.
+
+## trusted
+
+Sometimes, you may have reasons to fully trust the location tracking
+information for a repository. For example, it may be an offline
 archival drive, from which you rarely or never remove content. Deciding
 when it makes sense to trust the tracking info is up to you.
 
 One way to handle this is just to use `--force` when a command cannot
 access a remote you trust.
 
-Another option is to configure which remotes you trust with the 
-`git annex trust` command, or by manually adding the UUIDs of trusted remotes
-to `.git-annex/trust.log`.
+To configure a repository as fully trusted, use the `git annex trust`
+command.

From 268cb35e644754093db003aee08d050a1f3f9466 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 15:37:16 -0400
Subject: [PATCH 0758/2835] implement 3 level trust storage in trust.log

---
 Command/Semitrust.hs | 35 ++++++++++++++++++
 Command/Trust.hs     | 13 ++-----
 Command/Untrust.hs   | 13 ++-----
 GitAnnex.hs          |  2 ++
 Remotes.hs           |  3 +-
 Trust.hs             | 86 ++++++++++++++++++++++++++++++++++++++++++++
 UUID.hs              | 30 +---------------
 doc/git-annex.mdwn   |  9 +++--
 test.hs              | 27 ++++++++------
 9 files changed, 153 insertions(+), 65 deletions(-)
 create mode 100644 Command/Semitrust.hs
 create mode 100644 Trust.hs

diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
new file mode 100644
index 0000000000..8ed95f5a3c
--- /dev/null
+++ b/Command/Semitrust.hs
@@ -0,0 +1,35 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Semitrust where
+
+import Command
+import qualified GitRepo as Git
+import qualified Remotes
+import UUID
+import Trust
+import Messages
+
+command :: [Command]
+command = [Command "semitrust" (paramRepeating paramRemote) seek
+	"return repository to default trust level"]
+
+seek :: [CommandSeek]
+seek = [withString start]
+
+{- Marks a remote as not trusted. -}
+start :: CommandStartString
+start name = do
+	r <- Remotes.byName name
+	showStart "untrust" name
+	return $ Just $ perform r
+
+perform :: Git.Repo -> CommandPerform
+perform repo = do
+	uuid <- getUUID repo
+	trustSet uuid SemiTrusted
+	return $ Just $ return True
diff --git a/Command/Trust.hs b/Command/Trust.hs
index c97d75ee47..a40c8dccce 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -7,13 +7,10 @@
 
 module Command.Trust where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (unless)
-
 import Command
-import qualified Annex
 import qualified GitRepo as Git
 import qualified Remotes
+import Trust
 import UUID
 import Messages
 
@@ -34,11 +31,5 @@ start name = do
 perform :: Git.Repo -> CommandPerform
 perform repo = do
 	uuid <- getUUID repo
-	trusted <- getTrusted
-	unless (elem uuid trusted) $ do
-		setTrusted $ uuid:trusted
-		g <- Annex.gitRepo
-		logfile <- trustLog
-		liftIO $ Git.run g ["add", logfile]
-		liftIO $ Git.run g ["commit", "-q", "-m", "git annex untrust", logfile]
+	trustSet uuid Trusted
 	return $ Just $ return True
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 01b97b1c1f..9e884e812f 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -7,14 +7,11 @@
 
 module Command.Untrust where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when)
-
 import Command
-import qualified Annex
 import qualified GitRepo as Git
 import qualified Remotes
 import UUID
+import Trust
 import Messages
 
 command :: [Command]
@@ -34,11 +31,5 @@ start name = do
 perform :: Git.Repo -> CommandPerform
 perform repo = do
 	uuid <- getUUID repo
-	trusted <- getTrusted
-	when (elem uuid trusted) $ do
-		setTrusted $ filter (\u -> u /= uuid) trusted
-		g <- Annex.gitRepo
-		logfile <- trustLog
-		liftIO $ Git.run g ["add", logfile]
-		liftIO $ Git.run g ["commit", "-q", "-m", "git annex untrust", logfile]
+	trustSet uuid UnTrusted
 	return $ Just $ return True
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 378b6e538d..b09ec82ffc 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -37,6 +37,7 @@ import qualified Command.Migrate
 import qualified Command.Uninit
 import qualified Command.Trust
 import qualified Command.Untrust
+import qualified Command.Semitrust
 
 cmds :: [Command]
 cmds = concat
@@ -53,6 +54,7 @@ cmds = concat
 	, Command.PreCommit.command
 	, Command.Trust.command
 	, Command.Untrust.command
+	, Command.Semitrust.command
 	, Command.FromKey.command
 	, Command.DropKey.command
 	, Command.SetKey.command
diff --git a/Remotes.hs b/Remotes.hs
index e04874f7dc..e8a78a59ed 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -32,6 +32,7 @@ import qualified Annex
 import LocationLog
 import Locations
 import UUID
+import Trust
 import Utility
 import qualified Content
 import Messages
@@ -126,7 +127,7 @@ keyPossibilities key = do
 	allremotes <- remotesByCost
 	g <- Annex.gitRepo
 	u <- getUUID g
-	trusted <- getTrusted
+	trusted <- trustGet Trusted
 
 	-- get uuids of other repositories that are
 	-- believed to have the key
diff --git a/Trust.hs b/Trust.hs
new file mode 100644
index 0000000000..9474d47d7a
--- /dev/null
+++ b/Trust.hs
@@ -0,0 +1,86 @@
+{- git-annex trust levels
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Trust (
+	TrustLevel(..),
+	trustLog,
+	trustGet,
+	trustMap,
+	trustMapParse,
+	trustSet
+) where
+
+import Control.Monad.State
+import qualified Data.Map as M
+
+import qualified GitRepo as Git
+import Types
+import UUID
+import Locations
+import qualified Annex
+import Utility
+
+data TrustLevel = SemiTrusted | UnTrusted | Trusted
+	deriving Eq
+
+instance Show TrustLevel where
+        show SemiTrusted = "?"
+        show UnTrusted = "0"
+        show Trusted = "1"
+
+instance Read TrustLevel where
+        readsPrec _ "1" = [(Trusted, "")]
+        readsPrec _ "0" = [(UnTrusted, "")]
+	readsPrec _ _ = [(SemiTrusted, "")]
+
+{- Filename of trust.log. -}
+trustLog :: Annex FilePath
+trustLog = do
+	g <- Annex.gitRepo
+	return $ gitStateDir g ++ "trust.log"
+
+{- Returns a list of UUIDs at the specified trust level. -}
+trustGet :: TrustLevel -> Annex [UUID]
+trustGet level = do
+	m <- trustMap
+	return $ M.keys $ M.filter (== level) m
+
+{- Read the trustLog into a map. -}
+trustMap :: Annex (M.Map UUID TrustLevel)
+trustMap = do
+	logfile <- trustLog
+	s <- liftIO $ catch (readFile logfile) ignoreerror
+	return $ trustMapParse s
+	where
+                ignoreerror _ = return ""
+
+{- Trust map parser. -}
+trustMapParse :: String -> M.Map UUID TrustLevel
+trustMapParse s = M.fromList $ map pair $ filter (not . null) $ lines s
+	where
+		pair l
+			| length w > 1 = (w !! 0, read (w !! 1) :: TrustLevel)
+			-- for back-compat; the trust log used to only
+			-- list trusted uuids
+			| otherwise = (w !! 0, Trusted)
+			where
+				w = words l
+
+{- Changes the trust level for a uuid in the trustLog, and commits it. -}
+trustSet :: UUID -> TrustLevel -> Annex ()
+trustSet uuid level = do
+        m <- trustMap
+	when (M.lookup uuid m /= Just level) $ do
+		let m' = M.insert uuid level m
+	        logfile <- trustLog
+	        liftIO $ safeWriteFile logfile (serialize m')
+		g <- Annex.gitRepo
+		liftIO $ Git.run g ["add", logfile]
+		liftIO $ Git.run g ["commit", "-q", "-m", "git annex trust change", logfile]
+        where
+                serialize m = unlines $ map showpair $ M.toList m
+		showpair (u, t) = u ++ " " ++ show t
diff --git a/UUID.hs b/UUID.hs
index 26a64523fc..ec67026894 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -17,10 +17,7 @@ module UUID (
 	reposWithoutUUID,
 	prettyPrintUUIDs,
 	describeUUID,
-	uuidLog,
-	trustLog,
-	getTrusted,
-	setTrusted
+	uuidLog
 ) where
 
 import Control.Monad.State
@@ -141,28 +138,3 @@ uuidLog :: Annex FilePath
 uuidLog = do
 	g <- Annex.gitRepo
 	return $ gitStateDir g ++ "uuid.log"
-
-{- Filename of trust.log. -}
-trustLog :: Annex FilePath
-trustLog = do
-	g <- Annex.gitRepo
-	return $ gitStateDir g ++ "trust.log"
-
-{- List of trusted UUIDs. -}
-getTrusted :: Annex [UUID]
-getTrusted = do
-	logfile <- trustLog
-	s <- liftIO $ catch (readFile logfile) ignoreerror
-	return $ parse s
-	where
-		parse [] = []
-		parse s = map firstword $ lines s
-		firstword [] = ""
-		firstword l = head $ words l
-		ignoreerror _ = return ""
-
-{- Changes the list of trusted UUIDs. -}
-setTrusted :: [UUID] -> Annex ()
-setTrusted u = do
-	logfile <- trustLog
-	liftIO $ safeWriteFile logfile $ unlines u
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 49e67a83ea..9c2be8dc90 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -188,9 +188,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * untrust [repository ...]
 
-  Records that a repository is [[not trusted|trusted]] and could lose content
+  Records that a repository is [[not trusted|trust]] and could lose content
   at any time.
 
+* semitrust [repository ...]
+
+  Returns a repository to the default [[semi trusted|trust]] state.
+
 * fromkey file
 
   This can be used to maually set up a file to link to a specified key
@@ -356,7 +360,8 @@ available. Annexed files in your git repository symlink to that content.
 `.git-annex/uuid.log` is used to map between repository UUID and
 decscriptions.
 
-`.git-annex/trust.log` is used to list the UUIDs of trusted repositories.
+`.git-annex/trust.log` is used to indicate which repositories are trusted
+and untrusted.
 
 `.git-annex/*.log` is where git-annex records its content tracking
 information. These files should be committed to git.
diff --git a/test.hs b/test.hs
index 0c47da310c..c5e755966c 100644
--- a/test.hs
+++ b/test.hs
@@ -32,6 +32,7 @@ import qualified Types
 import qualified GitAnnex
 import qualified LocationLog
 import qualified UUID
+import qualified Trust
 import qualified Remotes
 import qualified Content
 import qualified Backend.SHA1
@@ -288,24 +289,28 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 		newfile = subdir ++ "/" ++ annexedfile
 
 test_trust :: Test
-test_trust = "git-annex trust/untrust" ~: intmpclonerepo $ do
-	trust False
+test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
+	trustcheck Trust.SemiTrusted
 	git_annex "trust" ["-q", "origin"] @? "trust failed"
-	trust True
+	trustcheck Trust.Trusted
 	git_annex "trust" ["-q", "origin"] @? "trust of trusted failed"
-	trust True
+	trustcheck Trust.Trusted
 	git_annex "untrust" ["-q", "origin"] @? "untrust failed"
-	trust False
+	trustcheck Trust.UnTrusted
 	git_annex "untrust" ["-q", "origin"] @? "untrust of untrusted failed"
-	trust False
+	trustcheck Trust.UnTrusted
+	git_annex "semitrust" ["-q", "origin"] @? "semitrust failed"
+	trustcheck Trust.SemiTrusted
+	git_annex "semitrust" ["-q", "origin"] @? "semitrust of semitrusted failed"
+	trustcheck Trust.SemiTrusted
 	where
-		trust expected = do
-			istrusted <- annexeval $ do
-				uuids <- UUID.getTrusted
+		trustcheck expected = do
+			present <- annexeval $ do
+				l <- Trust.trustGet expected
 				r <- Remotes.byName "origin"
 				u <- UUID.getUUID r
-				return $ elem u uuids
-			assertEqual "trust value" expected istrusted
+				return $ elem u l
+			assertEqual ("trust value " ++ show expected) True present
 
 test_fsck :: Test
 test_fsck = "git-annex fsck" ~: intmpclonerepo $ do

From 7b2da21ab7bc51785203a69cc05ab811a8629ecb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 15:59:10 -0400
Subject: [PATCH 0759/2835] avoid moving if src and dest are the same

---
 Command/Move.hs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 4416134c04..d52ca07df8 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -71,8 +71,9 @@ remoteHasKey remote key present	= do
  -}
 toStart :: Git.Repo -> Bool -> CommandStartString
 toStart dest move file = isAnnexed file $ \(key, _) -> do
+	g <- Annex.gitRepo
 	ishere <- inAnnex key
-	if not ishere
+	if not ishere || g == dest
 		then return Nothing -- not here, so nothing to do
 		else do
 			showAction move file
@@ -108,8 +109,9 @@ toCleanup dest move key = do
  -}
 fromStart :: Git.Repo -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
+	g <- Annex.gitRepo
 	(trusted, untrusted, _) <- Remotes.keyPossibilities key
-	if null $ filter (\r -> Remotes.same r src) (trusted ++ untrusted)
+	if (g == src) || (null $ filter (\r -> Remotes.same r src) (trusted ++ untrusted))
 		then return Nothing
 		else do
 			showAction move file

From 7f6af79232eee685daf58e72737c5284f80cf482 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 16:20:28 -0400
Subject: [PATCH 0760/2835] trust setting improvements

---
 Command/Semitrust.hs |  4 +--
 Command/Trust.hs     |  4 +--
 Command/Untrust.hs   |  4 +--
 Remotes.hs           |  1 +
 Trust.hs             |  2 ++
 debian/changelog     |  1 +
 doc/git-annex.mdwn   |  2 ++
 test.hs              | 64 ++++++++++++++++++++++++--------------------
 8 files changed, 47 insertions(+), 35 deletions(-)

diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 8ed95f5a3c..a91d25359c 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -21,11 +21,11 @@ command = [Command "semitrust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withString start]
 
-{- Marks a remote as not trusted. -}
 start :: CommandStartString
 start name = do
+	showStart "semitrust" name
+	Remotes.readConfigs
 	r <- Remotes.byName name
-	showStart "untrust" name
 	return $ Just $ perform r
 
 perform :: Git.Repo -> CommandPerform
diff --git a/Command/Trust.hs b/Command/Trust.hs
index a40c8dccce..3fbff68b89 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -21,11 +21,11 @@ command = [Command "trust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withString start]
 
-{- Marks a remote as trusted. -}
 start :: CommandStartString
 start name = do
-	r <- Remotes.byName name
 	showStart "trust" name
+	Remotes.readConfigs
+	r <- Remotes.byName name
 	return $ Just $ perform r
 
 perform :: Git.Repo -> CommandPerform
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 9e884e812f..69d0ab391a 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -21,11 +21,11 @@ command = [Command "untrust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withString start]
 
-{- Marks a remote as not trusted. -}
 start :: CommandStartString
 start name = do
-	r <- Remotes.byName name
 	showStart "untrust" name
+	Remotes.readConfigs
+	r <- Remotes.byName name
 	return $ Just $ perform r
 
 perform :: Git.Repo -> CommandPerform
diff --git a/Remotes.hs b/Remotes.hs
index e8a78a59ed..fe5b7f767e 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -222,6 +222,7 @@ same a b = Git.repoRemoteName a == Git.repoRemoteName b
 
 {- Looks up a remote by name. -}
 byName :: String -> Annex Git.Repo
+byName "." = Annex.gitRepo -- special case to refer to current repository
 byName name = do
 	when (null name) $ error "no remote specified"
 	g <- Annex.gitRepo
diff --git a/Trust.hs b/Trust.hs
index 9474d47d7a..695059a932 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -73,6 +73,8 @@ trustMapParse s = M.fromList $ map pair $ filter (not . null) $ lines s
 {- Changes the trust level for a uuid in the trustLog, and commits it. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
+	when (null uuid) $
+		error "unknown UUID; cannot modify trust level"
         m <- trustMap
 	when (M.lookup uuid m /= Just level) $ do
 		let m' = M.insert uuid level m
diff --git a/debian/changelog b/debian/changelog
index 529a0896ba..8b863b947b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (0.19) UNRELEASED; urgency=low
   * There are now three levels of repository trust.
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
+  * fsck: Warn if content is only in untrusted repositories.
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9c2be8dc90..d2cde35a0d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -186,6 +186,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Records that a repository is [[trusted|trust]] to not unexpectedly lose
   content. Use with care.
 
+  To trust the current repository, use "."
+
 * untrust [repository ...]
 
   Records that a repository is [[not trusted|trust]] and could lose content
diff --git a/test.hs b/test.hs
index c5e755966c..817b1bafbe 100644
--- a/test.hs
+++ b/test.hs
@@ -42,7 +42,7 @@ import qualified Command.DropUnused
 main :: IO ()
 main = do
 	prepare
-	r <- runVerboseTests $ TestList [quickchecks, toplevels]
+	r <- runVerboseTests $ TestList [quickcheck, blackbox]
 	cleanup tmpdir
 	propigate r
 
@@ -51,8 +51,8 @@ propigate (Counts { errors = e }, _)
 	| e > 0 = error "failed"
 	| otherwise = return ()
 
-quickchecks :: Test
-quickchecks = TestLabel "quickchecks" $ TestList
+quickcheck :: Test
+quickcheck = TestLabel "quickcheck" $ TestList
 	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" BackendTypes.prop_idempotent_key_read_show
@@ -62,8 +62,8 @@ quickchecks = TestLabel "quickchecks" $ TestList
 	, qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics
 	]
 
-toplevels :: Test
-toplevels = TestLabel "toplevel" $ TestList
+blackbox :: Test
+blackbox = TestLabel "blackbox" $ TestList
 	-- test order matters, later tests may rely on state from earlier
 	[ test_init
 	, test_add
@@ -290,38 +290,44 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 
 test_trust :: Test
 test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
-	trustcheck Trust.SemiTrusted
-	git_annex "trust" ["-q", "origin"] @? "trust failed"
-	trustcheck Trust.Trusted
-	git_annex "trust" ["-q", "origin"] @? "trust of trusted failed"
-	trustcheck Trust.Trusted
-	git_annex "untrust" ["-q", "origin"] @? "untrust failed"
-	trustcheck Trust.UnTrusted
-	git_annex "untrust" ["-q", "origin"] @? "untrust of untrusted failed"
-	trustcheck Trust.UnTrusted
-	git_annex "semitrust" ["-q", "origin"] @? "semitrust failed"
-	trustcheck Trust.SemiTrusted
-	git_annex "semitrust" ["-q", "origin"] @? "semitrust of semitrusted failed"
-	trustcheck Trust.SemiTrusted
+	git_annex "trust" ["-q", repo] @? "trust failed"
+	trustcheck Trust.Trusted "trusted 1"
+	git_annex "trust" ["-q", repo] @? "trust of trusted failed"
+	trustcheck Trust.Trusted "trusted 2"
+	git_annex "untrust" ["-q", repo] @? "untrust failed"
+	trustcheck Trust.UnTrusted "untrusted 1"
+	git_annex "untrust" ["-q", repo] @? "untrust of untrusted failed"
+	trustcheck Trust.UnTrusted "untrusted 2"
+	git_annex "semitrust" ["-q", repo] @? "semitrust failed"
+	trustcheck Trust.SemiTrusted "semitrusted 1"
+	git_annex "semitrust" ["-q", repo] @? "semitrust of semitrusted failed"
+	trustcheck Trust.SemiTrusted "semitrusted 2"
 	where
-		trustcheck expected = do
+		trustcheck expected msg = do
 			present <- annexeval $ do
+				Remotes.readConfigs
 				l <- Trust.trustGet expected
-				r <- Remotes.byName "origin"
+				r <- Remotes.byName repo
 				u <- UUID.getUUID r
 				return $ elem u l
-			assertEqual ("trust value " ++ show expected) True present
+			assertBool msg present
+		repo = "origin"
 
 test_fsck :: Test
-test_fsck = "git-annex fsck" ~: intmpclonerepo $ do
-	git_annex "fsck" ["-q"] @? "fsck failed"
-	Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed"
-	r <- git_annex "fsck" ["-q"]
-	not r @? "fsck failed to fail with numcopies unsatisfied"
-	Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed"
-	corrupt annexedfile
-	corrupt sha1annexedfile
+test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted]
 	where
+		basicfsck = TestCase $ intmpclonerepo $ do
+			git_annex "fsck" ["-q"] @? "fsck failed"
+			Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed"
+			r <- git_annex "fsck" ["-q"]
+			not r @? "fsck failed to fail with numcopies unsatisfied"
+			Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed"
+			corrupt annexedfile
+			corrupt sha1annexedfile
+		withlocaluntrusted = TestCase $ intmpcopyrepo $ do
+			git_annex "untrust" ["-q", "."] @? "untrust of current repo failed"
+			r <- git_annex "fsck" ["-q"]
+			not r @? "fsck failed to fail with content only available in untrusted (current) repository"
 		corrupt f = do
 			git_annex "get" ["-q", f] @? "get of file failed"
 			Content.allowWrite f

From b7903eb2d149ceb164d7422fb56573735d83ebde Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 16:44:14 -0400
Subject: [PATCH 0761/2835] move partitioning out of keyPossibilities

And a bug fix in passing.
---
 Backend/File.hs  |  8 ++++----
 Command/Move.hs  |  4 ++--
 Remotes.hs       | 16 +++-------------
 debian/changelog |  2 ++
 4 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index ac6e4a910f..358bc4b7cc 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -50,8 +50,7 @@ dummyStore _ _ = return True
  - and copy it over to this one. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(trusted, untrusted, _) <- Remotes.keyPossibilities key
-	let remotes = trusted ++ untrusted
+	(remotes, _) <- Remotes.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -94,9 +93,10 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(_, untrusted, have) <- Remotes.keyPossibilities key
+			(remotes, trusteduuids) <- Remotes.keyPossibilities key
+			untrusted <- reposWithoutUUID remotes trusteduuids
 			numcopies <- getNumCopies numcopiesM
-			findcopies numcopies have untrusted []
+			findcopies numcopies trusteduuids untrusted []
 	where
 		findcopies need have [] bad
 			| length have >= need = return True
diff --git a/Command/Move.hs b/Command/Move.hs
index d52ca07df8..3eadf30c6f 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -110,8 +110,8 @@ toCleanup dest move key = do
 fromStart :: Git.Repo -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
-	(trusted, untrusted, _) <- Remotes.keyPossibilities key
-	if (g == src) || (null $ filter (\r -> Remotes.same r src) (trusted ++ untrusted))
+	(remotes, _) <- Remotes.keyPossibilities key
+	if (g == src) || (null $ filter (\r -> Remotes.same r src) remotes)
 		then return Nothing
 		else do
 			showAction move file
diff --git a/Remotes.hs b/Remotes.hs
index fe5b7f767e..be4c1383af 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -112,15 +112,11 @@ readConfigs = do
 			return $ null u
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
- -
- - The first list is of remotes that are trusted to have the key.
- -
- - The second is of untrusted remotes that may have the key.
  -
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
  -}
-keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID])
+keyPossibilities :: Key -> Annex ([Git.Repo], [UUID])
 keyPossibilities key = do
 	readConfigs
 
@@ -129,23 +125,17 @@ keyPossibilities key = do
 	u <- getUUID g
 	trusted <- trustGet Trusted
 
-	-- get uuids of other repositories that are
-	-- believed to have the key
+	-- get uuids of all repositories that are recorded to have the key
 	uuids <- liftIO $ keyLocations g key
 	let validuuids = filter (/= u) uuids
 
-	-- get uuids trusted to have the key
 	-- note that validuuids is assumed to not have dups
 	let validtrusteduuids = intersect validuuids trusted
 
 	-- remotes that match uuids that have the key
 	validremotes <- reposByUUID allremotes validuuids
 
-	-- partition out the trusted and untrusted remotes
-	trustedremotes <- reposByUUID validremotes validtrusteduuids
-	untrustedremotes <- reposWithoutUUID validremotes trusted
-
-	return (trustedremotes, untrustedremotes, validtrusteduuids)
+	return (validremotes, validtrusteduuids)
 
 {- Checks if a given remote has the content for a key inAnnex.
  - If the remote cannot be accessed, returns a Left error.
diff --git a/debian/changelog b/debian/changelog
index 8b863b947b..01e99ae2b3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (0.19) UNRELEASED; urgency=low
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
   * fsck: Warn if content is only in untrusted repositories.
+  * bugfix: Files were copied from trusted remotes first even if their
+    annex.cost was higher than other remotes.
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
 

From ba748a11989f6ba7ea1f306be99b8b3f259ede68 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 17:44:40 -0400
Subject: [PATCH 0762/2835] fsck: handle untrusted repos

---
 Backend/File.hs      | 36 +++++++++++++++++++++++++++---------
 debian/changelog     |  2 +-
 doc/trust.mdwn       |  3 +--
 doc/walkthrough.mdwn | 27 +++++++++++++++++++++++++++
 4 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 358bc4b7cc..dea69ef9b6 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -16,6 +16,7 @@ module Backend.File (backend, checkKey) where
 
 import Control.Monad.State
 import System.Directory
+import Data.List
 
 import BackendTypes
 import LocationLog
@@ -27,6 +28,7 @@ import qualified Annex
 import Types
 import UUID
 import Messages
+import Trust
 
 backend :: Backend Annex
 backend = Backend {
@@ -150,8 +152,8 @@ getNumCopies Nothing = do
 		config = "annex.numcopies"
 
 {- This is used to check that numcopies is satisfied for the key on fsck.
- - This trusts the location log, and so checks all keys, even those with
- - data not present in the current annex.
+ - This trusts data in the the location log, and so can check all keys, even
+ - those with data not present in the current annex.
  -
  - The passed action is first run to allow backends deriving this one
  - to do their own checks.
@@ -167,15 +169,31 @@ checkKeyNumCopies key numcopies = do
 	needed <- getNumCopies numcopies
 	g <- Annex.gitRepo
 	locations <- liftIO $ keyLocations g key
-	let present = length locations
+	untrusted <- trustGet UnTrusted
+	let untrustedlocations = intersect untrusted locations
+	let safelocations = filter (\l -> not $ l `elem` untrusted) locations
+	let present = length safelocations
 	if present < needed
 		then do
-			warning $ note present needed
+			ppuuids <- prettyPrintUUIDs untrustedlocations
+			missingNote present needed ppuuids
 			return False
 		else return True
 	where
-		note 0 _ = "** No known copies of "++show key++" exist!"
-		note present needed = 
-			"Only " ++ show present ++ " of " ++ show needed ++ 
-			" copies of "++show key++" exist. " ++
-			"Back it up with git-annex copy."
+
+missingNote :: Int -> Int -> String -> Annex ()
+missingNote 0 _ [] = showLongNote $ "** No known copies of this file exist!"
+missingNote 0 _ untrusted = do
+	showLongNote $
+		"Only these untrusted locations may have copies of this file!" ++
+		"\n" ++ untrusted ++
+		"Back it up to trusted locations with git-annex copy."
+missingNote present needed untrusted = do
+	showLongNote $
+		"Only " ++ show present ++ " of " ++ show needed ++ 
+		" trustworthy copies of this file exist." ++
+		"\nBack it up with git-annex copy."
+	when (not $ null untrusted) $ do
+		showLongNote $
+			"\nThe following untrusted copies may also exist: " ++
+			"\n" ++ untrusted
diff --git a/debian/changelog b/debian/changelog
index 01e99ae2b3..fcd986e1b6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (0.19) UNRELEASED; urgency=low
   * There are now three levels of repository trust.
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
-  * fsck: Warn if content is only in untrusted repositories.
+  * fsck: Take untrusted repositories into account.
   * bugfix: Files were copied from trusted remotes first even if their
     annex.cost was higher than other remotes.
 
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index edec6568e7..dbddfe4269 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -25,8 +25,7 @@ restores a repository to this default, when it has been overridden.)
 ## untrusted
 
 An untrusted repository is not trusted to retain data at all. Git-annex
-will not count data in such a repository as a of the data, and will
-retain sufficient [[copies]] elsewhere.
+will retain sufficient [[copies]] of data elsewhere.
 
 This is a good choice for eg, portable drives that could get lost. Or,
 if a disk is known to be dying, you can set it to untrusted and let
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index d2231c81e6..f375d4e444 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -391,3 +391,30 @@ it so anything put in there is backed up more thoroughly:
 	# echo "* annex.numcopies=3" > important_stuff/.gitattributes
 
 For more details about the numcopies setting, see [[copies]].
+
+## untrusted repositories
+
+Suppose you have a portable USB drive and are using it as a git annex
+repository. You don't trust the drive, because you could lose it, or
+just because portable USB drives don't tend to last very long. You can
+let git-annex know about this, and it will adjust its behavior to avoid
+relying on that drive's continued availability.
+	
+	# cd /media/usb
+	# git annex untrust .
+	untrust . ok
+
+Now when you do a fsck, you'll be warned appropriately:
+
+	# git annex fsck .
+	fsck my_big_file
+	  Only these untrusted locations may have copies of this file!
+	  	05e296c4-2989-11e0-bf40-bad1535567fe  -- portable USB drive
+	  Back it up to trusted locations with git-annex copy.
+	failed
+
+Also, git-annex will refuse to drop a file from elsewhere just because
+it can see a copy on the untrusted drive.
+
+It's also possible to tell git-annex that you have an unusually high
+level of trust for a repository. See [[trust]] for details.

From 6b48f740f1e55b8461757bed670f29bc87e186a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 17:47:02 -0400
Subject: [PATCH 0763/2835] rework note

---
 Backend/File.hs | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index dea69ef9b6..8c39c3f803 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -176,24 +176,22 @@ checkKeyNumCopies key numcopies = do
 	if present < needed
 		then do
 			ppuuids <- prettyPrintUUIDs untrustedlocations
-			missingNote present needed ppuuids
+			showLongNote $ missingNote present needed ppuuids
 			return False
 		else return True
 	where
 
-missingNote :: Int -> Int -> String -> Annex ()
-missingNote 0 _ [] = showLongNote $ "** No known copies of this file exist!"
-missingNote 0 _ untrusted = do
-	showLongNote $
+missingNote :: Int -> Int -> String -> String
+missingNote 0 _ [] = 
+		"** No known copies of this file exist!"
+missingNote 0 _ untrusted =
 		"Only these untrusted locations may have copies of this file!" ++
 		"\n" ++ untrusted ++
 		"Back it up to trusted locations with git-annex copy."
-missingNote present needed untrusted = do
-	showLongNote $
+missingNote present needed [] =
 		"Only " ++ show present ++ " of " ++ show needed ++ 
 		" trustworthy copies of this file exist." ++
 		"\nBack it up with git-annex copy."
-	when (not $ null untrusted) $ do
-		showLongNote $
-			"\nThe following untrusted copies may also exist: " ++
-			"\n" ++ untrusted
+missingNote present needed untrusted = missingNote present needed [] ++
+		"\nThe following untrusted copies may also exist: " ++
+		"\n" ++ untrusted

From ff3c12725502451c8ab7da72797b2d69c4a66900 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 19:35:29 -0400
Subject: [PATCH 0764/2835] wording

---
 doc/walkthrough.mdwn | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index f375d4e444..231c3e543c 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -394,15 +394,19 @@ For more details about the numcopies setting, see [[copies]].
 
 ## untrusted repositories
 
-Suppose you have a portable USB drive and are using it as a git annex
+Suppose you have a USB thunb drive and are using it as a git annex
 repository. You don't trust the drive, because you could lose it, or
-just because portable USB drives don't tend to last very long. You can
-let git-annex know about this, and it will adjust its behavior to avoid
-relying on that drive's continued availability.
+accidentially run it through the laundry. Or, maybe you have a drive that
+you know is dying, and you'd like to be warned if there are any files
+on it not backed up somewhere else. Maybe the drive has already died
+or been lost.
+
+You can let git-annex know that you don't trust a repository, and it will
+adjust its behavior to avoid relying on that repositories's continued
+availability.
 	
-	# cd /media/usb
-	# git annex untrust .
-	untrust . ok
+	# git annex untrust usbdrive
+	untrust usbdrive ok
 
 Now when you do a fsck, you'll be warned appropriately:
 
@@ -414,7 +418,7 @@ Now when you do a fsck, you'll be warned appropriately:
 	failed
 
 Also, git-annex will refuse to drop a file from elsewhere just because
-it can see a copy on the untrusted drive.
+it can see a copy on the untrusted repository.
 
 It's also possible to tell git-annex that you have an unusually high
 level of trust for a repository. See [[trust]] for details.

From 1a11085a50e79fbcce829d4ea89539107f5b306c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 19:35:35 -0400
Subject: [PATCH 0765/2835] drop: suppprt untrusted repos

---
 Backend/File.hs  | 22 +++++++++++++++-------
 debian/changelog |  2 +-
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 8c39c3f803..ab14b82339 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -96,9 +96,10 @@ checkRemoveKey key numcopiesM = do
 		then return True
 		else do
 			(remotes, trusteduuids) <- Remotes.keyPossibilities key
-			untrusted <- reposWithoutUUID remotes trusteduuids
+			untrusteduuids <- trustGet UnTrusted
+			tocheck <- reposWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
-			findcopies numcopies trusteduuids untrusted []
+			findcopies numcopies trusteduuids tocheck []
 	where
 		findcopies need have [] bad
 			| length have >= need = return True
@@ -131,11 +132,18 @@ showLocations key exclude = do
 	g <- Annex.gitRepo
 	u <- getUUID g
 	uuids <- liftIO $ keyLocations g key
-	let uuidsf = filter (\l -> l /= u && (not $ elem l exclude)) uuids
-	ppuuids <- prettyPrintUUIDs uuidsf
-	if null uuidsf
-		then showLongNote $ "No other repository is known to contain the file."
-		else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids
+	untrusteduuids <- trustGet UnTrusted
+	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
+	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
+	ppuuidswanted <- prettyPrintUUIDs uuidswanted
+	ppuuidsskipped <- prettyPrintUUIDs uuidsskipped
+	showLongNote $ message ppuuidswanted ppuuidsskipped
+	where
+		filteruuids list x = filter (\l -> not $ elem l x) list
+		message [] [] = "No other repository is known to contain the file."
+		message rs [] = "Try making some of these repositories available:\n" ++ rs
+		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
+		message rs us = message rs [] ++ message [] us
 
 showTriedRemotes :: [Git.Repo] -> Annex ()
 showTriedRemotes [] = return ()	
diff --git a/debian/changelog b/debian/changelog
index fcd986e1b6..c675d1456d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (0.19) UNRELEASED; urgency=low
   * There are now three levels of repository trust.
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
-  * fsck: Take untrusted repositories into account.
+  * fsck, drop: Take untrusted repositories into account.
   * bugfix: Files were copied from trusted remotes first even if their
     annex.cost was higher than other remotes.
 

From 15d27232bdaea0cf4f70cbbe6fa925d593d18e73 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:03:03 -0400
Subject: [PATCH 0766/2835] test untrusted repo handling

---
 test.hs | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/test.hs b/test.hs
index 817b1bafbe..4e02c2a6f2 100644
--- a/test.hs
+++ b/test.hs
@@ -133,11 +133,11 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy]
 			unannexed ingitfile
 
 test_drop :: Test
-test_drop = "git-annex drop" ~: TestList [noremote, withremote]
+test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpcopyrepo $ do
 			r <- git_annex "drop" ["-q", annexedfile]
-			(not r) @? "drop wrongly succeeded with no known copy of file"
+			not r @? "drop wrongly succeeded with no known copy of file"
 			annexed_present annexedfile
 			git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed"
 			annexed_notpresent annexedfile
@@ -145,9 +145,19 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote]
 			git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op"
 			unannexed ingitfile
 		withremote = "with remote" ~: TestCase $ intmpclonerepo $ do
+			git_annex "get" ["-q", annexedfile] @? "get failed"
+			annexed_present annexedfile
 			git_annex "drop" ["-q", annexedfile] @? "drop failed though origin has copy"
 			annexed_notpresent annexedfile
 			inmainrepo $ annexed_present annexedfile
+		untrustedremote = "untrusted remote" ~: TestCase $ intmpclonerepo $ do
+			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
+			git_annex "get" ["-q", annexedfile] @? "get failed"
+			annexed_present annexedfile
+			r <- git_annex "drop" ["-q", annexedfile]
+			not r @? "drop wrongly suceeded with only an untrusted copy of the file"
+			annexed_present annexedfile
+			inmainrepo $ annexed_present annexedfile
 
 test_get :: Test
 test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do
@@ -314,20 +324,27 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 		repo = "origin"
 
 test_fsck :: Test
-test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted]
+test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withremoteuntrusted]
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
 			git_annex "fsck" ["-q"] @? "fsck failed"
 			Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed"
-			r <- git_annex "fsck" ["-q"]
-			not r @? "fsck failed to fail with numcopies unsatisfied"
+			fsck_should_fail "numcopies unsatisfied"
 			Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
 		withlocaluntrusted = TestCase $ intmpcopyrepo $ do
 			git_annex "untrust" ["-q", "."] @? "untrust of current repo failed"
-			r <- git_annex "fsck" ["-q"]
-			not r @? "fsck failed to fail with content only available in untrusted (current) repository"
+			fsck_should_fail "content only available in untrusted (current) repository"
+			git_annex "trust" ["-q", "."] @? "trust of current repo failed"
+			git_annex "fsck" ["-q"] @? "fsck failed on trusted repo"
+		withremoteuntrusted = TestCase $ intmpclonerepo $ do
+			Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed"
+			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "get" ["-q", sha1annexedfile] @? "get failed"
+			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
+			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
+			fsck_should_fail "content not replicated to enough non-untrusted repositories"
 		corrupt f = do
 			git_annex "get" ["-q", f] @? "get of file failed"
 			Content.allowWrite f
@@ -335,6 +352,9 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted]
 			r <- git_annex "fsck" ["-q"]
 			not r @? "fsck failed to fail with corrupted file content"
 			git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem"
+		fsck_should_fail m = do
+			r <- git_annex "fsck" ["-q"]
+			not r @? "fsck failed to fail with " ++ m
 
 test_migrate :: Test
 test_migrate = "git-annex migrate" ~: TestList [t False, t True]

From ee2e94f08703ee1b5f51bf5c2d26de141b58298c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:03:12 -0400
Subject: [PATCH 0767/2835] this should be a warning

---
 Backend/File.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index ab14b82339..f31752d90d 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -184,7 +184,7 @@ checkKeyNumCopies key numcopies = do
 	if present < needed
 		then do
 			ppuuids <- prettyPrintUUIDs untrustedlocations
-			showLongNote $ missingNote present needed ppuuids
+			warning $ missingNote present needed ppuuids
 			return False
 		else return True
 	where

From 3cb5cb6bf6f03ad3ef574a14ed35275916ac44b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:08:37 -0400
Subject: [PATCH 0768/2835] bring back display of keys

in fsck -q, that's the only way to know what file it means
---
 Backend/File.hs | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index f31752d90d..10304dd91c 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -184,22 +184,24 @@ checkKeyNumCopies key numcopies = do
 	if present < needed
 		then do
 			ppuuids <- prettyPrintUUIDs untrustedlocations
-			warning $ missingNote present needed ppuuids
+			warning $ missingNote (show key) present needed ppuuids
 			return False
 		else return True
 	where
 
-missingNote :: Int -> Int -> String -> String
-missingNote 0 _ [] = 
-		"** No known copies of this file exist!"
-missingNote 0 _ untrusted =
-		"Only these untrusted locations may have copies of this file!" ++
+missingNote :: String -> Int -> Int -> String -> String
+missingNote file 0 _ [] = 
+		"** No known copies of " ++ file ++ " exist!"
+missingNote file 0 _ untrusted =
+		"Only these untrusted locations may have copies of " ++ file ++
+		"\n-- they could lose it at any time!" ++
 		"\n" ++ untrusted ++
 		"Back it up to trusted locations with git-annex copy."
-missingNote present needed [] =
+missingNote file present needed [] =
 		"Only " ++ show present ++ " of " ++ show needed ++ 
-		" trustworthy copies of this file exist." ++
+		" trustworthy copies of " ++ file ++ " exist." ++
 		"\nBack it up with git-annex copy."
-missingNote present needed untrusted = missingNote present needed [] ++
+missingNote file present needed untrusted = 
+		missingNote file present needed [] ++
 		"\nThe following untrusted copies may also exist: " ++
 		"\n" ++ untrusted

From c30d38e108ade7eb4fa57552631b64dd3b9582c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:30:07 -0400
Subject: [PATCH 0769/2835] better warnings display

---
 CmdLine.hs       |  1 -
 Messages.hs      | 15 +++++++++------
 debian/changelog |  2 +-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 1b5daadeb0..76402a8218 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -70,7 +70,6 @@ usage header cmds options =
 			cmdparams c ++
 			pad (longest cmdparams + 2) (cmdparams c) ++
 			cmddesc c
-		indent l = "  " ++ l
 		pad n s = replicate (n - length s) ' '
 		longest f = foldl max 0 $ map (length . f) cmds
 
diff --git a/Messages.hs b/Messages.hs
index 2b98622309..6f4ec1e626 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -37,9 +37,8 @@ showProgress :: Annex ()
 showProgress = verbose $ liftIO $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indented
-	where
-		indented = join "\n" $ map (\l -> "  " ++ l) $ lines s 
+showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indent s
+
 showEndOk :: Annex ()
 showEndOk = verbose $ liftIO $ putStrLn "ok"
 
@@ -48,9 +47,13 @@ showEndFail = verbose $ liftIO $ putStrLn "\nfailed"
 
 {- Exception pretty-printing. -}
 showErr :: (Show a) => a -> Annex ()
-showErr e = warning $ show e
+showErr e = warning $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
-warning s = do
+warning w = do
 	verbose $ liftIO $ putStr "\n"
-	liftIO $ hPutStrLn stderr $ "git-annex: " ++ s ++ " "
+	liftIO $ hFlush stdout
+	liftIO $ hPutStrLn stderr $ indent w
+
+indent :: String -> String
+indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
diff --git a/debian/changelog b/debian/changelog
index c675d1456d..92b21173c8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,7 +6,7 @@ git-annex (0.19) UNRELEASED; urgency=low
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
   * fsck, drop: Take untrusted repositories into account.
-  * bugfix: Files were copied from trusted remotes first even if their
+  * Bugfix: Files were copied from trusted remotes first even if their
     annex.cost was higher than other remotes.
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400

From e1d213d6e3c9fe0cda6e2b80c4abeb17db5d3a16 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:37:46 -0400
Subject: [PATCH 0770/2835] make filename available to fsck messages

---
 Backend.hs      |  4 ++--
 Backend/File.hs | 16 +++++++++-------
 Backend/URL.hs  |  4 ++--
 BackendTypes.hs |  5 +++--
 Command/Fsck.hs |  8 ++++----
 5 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index d9bf35f0dd..bdceea2f5b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -118,8 +118,8 @@ hasKey key = do
 	(B.hasKey backend) key
 
 {- Checks a key's backend for problems. -}
-fsckKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool
-fsckKey backend key numcopies = (B.fsckKey backend) key numcopies
+fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+fsckKey backend key file numcopies = (B.fsckKey backend) key file numcopies
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
diff --git a/Backend/File.hs b/Backend/File.hs
index 10304dd91c..26a6add1a4 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -166,14 +166,14 @@ getNumCopies Nothing = do
  - The passed action is first run to allow backends deriving this one
  - to do their own checks.
  -}
-checkKey :: (Key -> Annex Bool) -> Key -> Maybe Int -> Annex Bool
-checkKey a key numcopies = do
+checkKey :: (Key -> Annex Bool) -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKey a key file numcopies = do
 	a_ok <- a key
-	copies_ok <- checkKeyNumCopies key numcopies
+	copies_ok <- checkKeyNumCopies key file numcopies
 	return $ a_ok && copies_ok
 
-checkKeyNumCopies :: Key -> Maybe Int -> Annex Bool
-checkKeyNumCopies key numcopies = do
+checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
 	g <- Annex.gitRepo
 	locations <- liftIO $ keyLocations g key
@@ -184,10 +184,12 @@ checkKeyNumCopies key numcopies = do
 	if present < needed
 		then do
 			ppuuids <- prettyPrintUUIDs untrustedlocations
-			warning $ missingNote (show key) present needed ppuuids
+			warning $ missingNote (filename file key) present needed ppuuids
 			return False
 		else return True
 	where
+		filename Nothing k = show k
+		filename (Just f) _ = f
 
 missingNote :: String -> Int -> Int -> String -> String
 missingNote file 0 _ [] = 
@@ -203,5 +205,5 @@ missingNote file present needed [] =
 		"\nBack it up with git-annex copy."
 missingNote file present needed untrusted = 
 		missingNote file present needed [] ++
-		"\nThe following untrusted copies may also exist: " ++
+		"\nThe following untrusted locations may also have copies: " ++
 		"\n" ++ untrusted
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 45a204b07d..38954e5a31 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -41,8 +41,8 @@ dummyStore _ _ = return False
 dummyRemove :: Key -> Maybe a -> Annex Bool
 dummyRemove _ _ = return False
 
-dummyFsck :: Key -> Maybe a -> Annex Bool
-dummyFsck _ _ = return True
+dummyFsck :: Key -> Maybe FilePath -> Maybe a -> Annex Bool
+dummyFsck _ _ _ = return True
 
 dummyOk :: Key -> Annex Bool
 dummyOk _ = return True
diff --git a/BackendTypes.hs b/BackendTypes.hs
index e4b155f98f..fd4a61b989 100644
--- a/BackendTypes.hs
+++ b/BackendTypes.hs
@@ -31,9 +31,10 @@ data Backend a = Backend {
 	-- checks if a backend is storing the content of a key
 	hasKey :: Key -> a Bool,
 	-- called during fsck to check a key
-	-- (second parameter may be the number of copies that there should
+	-- (second parameter may be the filename associated with it)
+	-- (third parameter may be the number of copies that there should
 	-- be of the key)
-	fsckKey :: Key -> Maybe Int -> a Bool
+	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool
 }
 
 instance Show (Backend a) where
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index fc9bd7f527..b6f330d4c2 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -24,13 +24,13 @@ seek = [withAttrFilesInGit "annex.numcopies" start]
 start :: CommandStartAttrFile
 start (file, attr) = isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
-	return $ Just $ perform key backend numcopies
+	return $ Just $ perform key file backend numcopies
 	where
 		numcopies = readMaybe attr :: Maybe Int
 
-perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform
-perform key backend numcopies = do
-	success <- Backend.fsckKey backend key numcopies
+perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
+perform key file backend numcopies = do
+	success <- Backend.fsckKey backend key (Just file) numcopies
 	if success
 		then return $ Just $ return True
 		else return Nothing

From 2e0cbcdf5db31ec6f092ba81fbb7fc6685c051a1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Thu, 27 Jan 2011 01:29:08 +0000
Subject: [PATCH 0771/2835] Feature request for a sort of "dedup" for filenames

---
 ...4___command_that_will_skip_duplicates.mdwn | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
new file mode 100644
index 0000000000..31734e01b9
--- /dev/null
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -0,0 +1,22 @@
+(Hi, this is paulproteus/Asheesh Laroia).
+
+I've been enjoying using git-annex to archive my data.
+
+It's great that, by using git-annex and the SHA1 backend, I get a space-saving kind of deduplication through the symbolic links.
+
+My question is, is there a frontend by which I can look at some files and only store the ones that are not already in the repository? That would help me in terms of personal file organization.
+
+It seems there is not, so this is a wishlist bug filed so that maybe such a thing might exist. What I would really like to do is:
+
+ $ git annex add --no-add-if-already-present .
+ $ git commit -m "Slurping in some photos I found on my old laptop hard drive"
+
+And then I'd do something like:
+
+ $ git clean -f
+
+to remove the files that didn't get annexed in this run. That way, only one filename would ever point to a particular SHA1.
+
+I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks.
+
+(I would be even happier for "git annex add --unlink-duplicates .")

From 8dc7af780b8432d95dd6421bf7d54f11a2d312ba Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Thu, 27 Jan 2011 01:29:35 +0000
Subject: [PATCH 0772/2835] Fix some formatting.

---
 ...git_annex__34___command_that_will_skip_duplicates.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index 31734e01b9..f4f0b7bced 100644
--- a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -1,4 +1,4 @@
-(Hi, this is paulproteus/Asheesh Laroia).
+(Hi, this is paulproteus@debian, AKA Asheesh).
 
 I've been enjoying using git-annex to archive my data.
 
@@ -8,12 +8,12 @@ My question is, is there a frontend by which I can look at some files and only s
 
 It seems there is not, so this is a wishlist bug filed so that maybe such a thing might exist. What I would really like to do is:
 
- $ git annex add --no-add-if-already-present .
- $ git commit -m "Slurping in some photos I found on my old laptop hard drive"
+* $ git annex add --no-add-if-already-present .
+* $ git commit -m "Slurping in some photos I found on my old laptop hard drive"
 
 And then I'd do something like:
 
- $ git clean -f
+* $ git clean -f
 
 to remove the files that didn't get annexed in this run. That way, only one filename would ever point to a particular SHA1.
 

From e51b4d172b3ad2d6053bcf99ac72f8f1d72707d6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Thu, 27 Jan 2011 01:30:34 +0000
Subject: [PATCH 0773/2835]

---
 ...__34__git_annex__34___command_that_will_skip_duplicates.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index f4f0b7bced..68efc0d1ac 100644
--- a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -17,6 +17,6 @@ And then I'd do something like:
 
 to remove the files that didn't get annexed in this run. That way, only one filename would ever point to a particular SHA1.
 
-I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks.
+I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks. If this feature existed, I could comfortably toss them all into one git annex that grew, bit by bit, to store all of these files exactly once.
 
 (I would be even happier for "git annex add --unlink-duplicates .")

From 7ab481448c4cdc39bae746423f535ae742b8784a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 21:52:51 -0400
Subject: [PATCH 0774/2835] wording

---
 doc/trust.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index dbddfe4269..f843d3858c 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -4,7 +4,7 @@ Git-annex supports three levels of trust of a repository:
 * untrusted
 * trusted
 
-## semi-trusted
+## semitrusted
 
 Normally, git-annex does not fully trust its stored [[location_tracking]]
 information. When removing content, it will directly check
@@ -15,7 +15,7 @@ Generally that explicit checking is a good idea. Consider that the current
 out. Or, a remote may have suffered a catastrophic loss of data, or itself
 been lost.
 
-There is still some trust involved here. A semi-trusted repository is
+There is still some trust involved here. A semitrusted repository is
 dependended on to retain a copy of the file content; possibly the only
 [[copy|copies]].
 

From 346ade8de55a527b5d8576ca11cc5cc03d8147cf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Thu, 27 Jan 2011 05:59:26 +0000
Subject: [PATCH 0775/2835]

---
 ...__git_annex__34___command_that_will_skip_duplicates.mdwn | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index 68efc0d1ac..eda17aea66 100644
--- a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -4,7 +4,7 @@ I've been enjoying using git-annex to archive my data.
 
 It's great that, by using git-annex and the SHA1 backend, I get a space-saving kind of deduplication through the symbolic links.
 
-My question is, is there a frontend by which I can look at some files and only store the ones that are not already in the repository? That would help me in terms of personal file organization.
+I'm looking for the ability to filter files, before they get added to the annex, so that I don't add new files whose content is already in the annex.look  That would help me in terms of personal file organization.
 
 It seems there is not, so this is a wishlist bug filed so that maybe such a thing might exist. What I would really like to do is:
 
@@ -20,3 +20,7 @@ to remove the files that didn't get annexed in this run. That way, only one file
 I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks. If this feature existed, I could comfortably toss them all into one git annex that grew, bit by bit, to store all of these files exactly once.
 
 (I would be even happier for "git annex add --unlink-duplicates .")
+
+(Another way to do this would be to "git annex add" them all, and then use a "git annex remove-duplicates" that could prompt me about which files are duplicates of each other, and then I could pipe that command's output into xargs git rm.)
+
+(As I write this, I realize it's possible to parse the destination of the symlink in a way that does this..)

From 9d3ba00a15ecc67eb98a9cf99561f2a54ade15c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 14:01:30 -0400
Subject: [PATCH 0776/2835] add test of adding a second file with the same sha1

---
 test.hs | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/test.hs b/test.hs
index 4e02c2a6f2..d7a6bd152f 100644
--- a/test.hs
+++ b/test.hs
@@ -94,15 +94,27 @@ test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 		reponame = "test repo"
 
 test_add :: Test
-test_add = "git-annex add" ~: TestCase $ inmainrepo $ do
-	writeFile annexedfile $ content annexedfile
-	git_annex "add" ["-q", annexedfile] @? "add failed"
-	annexed_present annexedfile
-	writeFile ingitfile $ content ingitfile
-	Utility.boolSystem "git" ["add", ingitfile] @? "git add failed"
-	Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed"
-	git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
-	unannexed ingitfile
+test_add = "git-annex add" ~: TestList [basic, sha1dup]
+	where
+		-- this test case runs in the main repo, to set up a basic
+		-- annexed file that later tests will use
+		basic = TestCase $ inmainrepo $ do
+			writeFile annexedfile $ content annexedfile
+			git_annex "add" ["-q", annexedfile] @? "add failed"
+			annexed_present annexedfile
+			writeFile ingitfile $ content ingitfile
+			Utility.boolSystem "git" ["add", ingitfile] @? "git add failed"
+			Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed"
+			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
+			unannexed ingitfile
+		sha1dup = TestCase $ intmpclonerepo $ do
+			writeFile sha1annexedfile $ content sha1annexedfile
+			git_annex "add" ["-q", sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
+			annexed_present sha1annexedfile
+			writeFile sha1annexedfiledup $ content sha1annexedfiledup
+			git_annex "add" ["-q", sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
+			annexed_present sha1annexedfiledup
+			annexed_present sha1annexedfile
 
 test_setkey :: Test
 test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
@@ -647,6 +659,9 @@ annexedfile = "foo"
 sha1annexedfile :: String
 sha1annexedfile = "sha1foo"
 
+sha1annexedfiledup :: String
+sha1annexedfiledup = "sha1foodup"
+
 ingitfile :: String
 ingitfile = "bar"
 
@@ -655,6 +670,7 @@ content f
 	| f == annexedfile = "annexed file content"
 	| f == ingitfile = "normal file content"
 	| f == sha1annexedfile ="sha1 annexed file content"
+	| f == sha1annexedfiledup = content sha1annexedfile
 	| otherwise = "unknown file " ++ f
 
 changecontent :: FilePath -> IO ()

From 2087c8ef5e21d3a6c9427b08b59c6814be767baf Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 27 Jan 2011 18:29:44 +0000
Subject: [PATCH 0777/2835] Added a comment

---
 ..._fd213310ee548d8726791d2b02237fde._comment | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment
new file mode 100644
index 0000000000..094e4526eb
--- /dev/null
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-01-27T18:29:44Z"
+ content="""
+Hey Asheesh, I'm happy you're finding git-annex useful.
+
+So, there are two forms of duplication going on here. There's duplication of the content, and duplication of the filenames 
+pointing at that content.
+
+Duplication of the filenames is probably not a concern, although it's what I thought you were talking about at first. It's probably info worth recording that backup-2010/some_dir/foo and backup-2009/other_dir/foo are two names you've used for the same content in the past. If you really wanted to remove backup-2009/foo, you could do it by writing a script that looks at the basenames of the symlink targets and removes files that point to the same content as other files.
+
+Using SHA1 ensures that the same key is used for identical files, so generally avoids duplication of content. But if you have 2 disks with an identical file on each, and make them both into annexes, then git-annex will happily retain both
+copies of the content, one per disk. It generally considers keeping copies of content a good thing. :)
+
+So, what if you want to remove the unnecessary copies? Well, there's a really simple way:
+
+
+cd /media/usb-1
+git remote add other-disk /media/usb-0
+git annex add
+git annex drop
+

+
+This asks git-annex to add everything to the annex, but then remove any file contents that it can safely remove. What can it safely remove? Well, anything that it can verify is on another repository such as \"other-disk\"! So, this will happily drop any duplicated file contents, while leaving all the rest alone.
+
+In practice, you might not want to have all your old backup disks mounted at the same time and configured as remotes. Look into configuring [[trust]] to avoid needing do to that. If usb-0 is already a trusted disk, all you need is a simple \"git annex drop\" on usb-1.
+"""]]

From 5e54eb79b8dbb58f9e772fc4c7c9fe900d045217 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 15:12:38 -0400
Subject: [PATCH 0778/2835] less verbose

---
 Backend/File.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 26a6add1a4..35cbc01911 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -196,7 +196,6 @@ missingNote file 0 _ [] =
 		"** No known copies of " ++ file ++ " exist!"
 missingNote file 0 _ untrusted =
 		"Only these untrusted locations may have copies of " ++ file ++
-		"\n-- they could lose it at any time!" ++
 		"\n" ++ untrusted ++
 		"Back it up to trusted locations with git-annex copy."
 missingNote file present needed [] =

From dee9655237d753cfeffc5cb7ffccaa86b464dcc2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 15:45:22 -0400
Subject: [PATCH 0779/2835] bugfix to move --to

Due to recent changes, the remotes config was not read before the remote
to act on was picked.
---
 Command/Move.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 3eadf30c6f..6dc2e48746 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -34,6 +34,7 @@ seek = [withFilesInGit $ start True]
  - moving data in the key-value backend. -}
 start :: Bool -> CommandStartString
 start move file = do
+	Remotes.readConfigs
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
 	case (from, to) of
@@ -80,7 +81,6 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 			return $ Just $ toPerform dest move key
 toPerform :: Git.Repo -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
-	Remotes.readConfigs
 	-- checking the remote is expensive, so not done in the start step
 	isthere <- Remotes.inAnnex dest key
 	case isthere of

From 96404170673e4ff42b68de1ba967a09a88dd550e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 16:10:45 -0400
Subject: [PATCH 0780/2835] avoid warning when symlink in the repo contains a
 colon but is not a pointer to annexed content

---
 Backend.hs   | 3 ++-
 Locations.hs | 6 ++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/Backend.hs b/Backend.hs
index bdceea2f5b..74f71f8bea 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -137,7 +137,8 @@ lookupFile file = do
 		makekey bs l = do
 			case maybeLookupBackendName bs bname of
 				Nothing -> do
-					unless (null kname || null bname) $
+					unless (null kname || null bname ||
+					        not (isLinkToAnnex l)) $
 						warning skip
 					return Nothing
 				Just backend -> return $ Just (k, backend)
diff --git a/Locations.hs b/Locations.hs
index 6c541e937c..327c099e38 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -15,6 +15,7 @@ module Locations (
 	annexTmpLocation,
 	annexBadLocation,
 	annexUnusedLog,
+	isLinkToAnnex,
 	annexDir,
 	annexObjectDir,
 
@@ -22,6 +23,7 @@ module Locations (
 ) where
 
 import Data.String.Utils
+import Data.List
 
 import Types
 import qualified GitRepo as Git
@@ -69,6 +71,10 @@ annexBadLocation r = annexDir r ++ "/bad/"
 annexUnusedLog :: Git.Repo -> FilePath
 annexUnusedLog r = annexDir r ++ "/unused"
 
+{- Checks a symlink target to see if it appears to point to annexed content. -}
+isLinkToAnnex :: FilePath -> Bool
+isLinkToAnnex s = isInfixOf "/.git/annex/objects/" s
+
 {- Converts a key into a filename fragment.
  -
  - Escape "/" in the key name, to keep a flat tree of files and avoid

From 6be516ae3bddb8f05ea62661019836e03be12a2c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 16:31:29 -0400
Subject: [PATCH 0781/2835] use isPrefixOf

---
 Command.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command.hs b/Command.hs
index cbfb265002..bedb18cc97 100644
--- a/Command.hs
+++ b/Command.hs
@@ -13,6 +13,7 @@ import System.Posix.Files
 import Control.Monad (filterM)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
+import Data.List
 
 import Types
 import qualified Backend
@@ -186,8 +187,7 @@ filterFiles l = do
 			let regexp = compile (toregex exclude) []
 			return $ filter (notExcluded regexp) l'
 	where
-		notState f = stateLoc /= take stateLocLen f
-		stateLocLen = length stateLoc
+		notState f = not $ isPrefixOf stateLoc f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False

From 167523f09d48777f3a5931fdcbc21b9d363e0e6c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 17:00:32 -0400
Subject: [PATCH 0782/2835] better directory handling

Rename Locations functions for better consitency, and make their values
more consistent too.

Used  rather than manually building paths. There are still more places
that manually do so, but are tricky, due to the behavior of  when
the second FilePath is absolute. So I only changed places where
it obviously was relative.
---
 Backend/File.hs       |  2 +-
 Backend/SHA1.hs       |  2 +-
 Backend/WORM.hs       |  2 +-
 CmdLine.hs            |  2 +-
 Command.hs            |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Init.hs       |  3 +-
 Command/Migrate.hs    |  2 +-
 Command/SendKey.hs    |  2 +-
 Command/Unlock.hs     |  2 +-
 Command/Unused.hs     |  2 +-
 Content.hs            | 20 ++++++------
 Locations.hs          | 74 +++++++++++++++++++++++++++----------------
 Remotes.hs            |  6 ++--
 Upgrade.hs            |  4 +--
 Version.hs            |  8 ++---
 16 files changed, 78 insertions(+), 57 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 35cbc01911..68dd4db271 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -77,7 +77,7 @@ copyKeyFile key file = do
 		-- before going on to the next remote.)
 		probablyPresent r =
 			if not $ Git.repoIsUrl r
-				then liftIO $ doesFileExist $ annexLocation r key
+				then liftIO $ doesFileExist $ gitAnnexLocation r key
 				else return True
 		docopy r continue = do
 			showNote $ "copying from " ++ Git.repoDescribe r ++ "..."
diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs
index f8dbea4b03..3d868dbd19 100644
--- a/Backend/SHA1.hs
+++ b/Backend/SHA1.hs
@@ -48,7 +48,7 @@ keyValue file = do
 checkKeySHA1 :: Key -> Annex Bool
 checkKeySHA1 key = do
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
 	if not present
 		then return True
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 56f243396e..20a81d8411 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -57,7 +57,7 @@ keySize key = read $ section !! 1
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
 	if not present
 		then return True
diff --git a/CmdLine.hs b/CmdLine.hs
index 76402a8218..68d1e0dd31 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -111,7 +111,7 @@ shutdown errnum = do
 	-- are left behind to allow resuming on re-run.
 	when (errnum == 0) $ do
 		g <- Annex.gitRepo
-		let tmp = annexTmpLocation g
+		let tmp = gitAnnexTmpDir g
 		exists <- liftIO $ doesDirectoryExist tmp
 		when exists $ liftIO $ removeDirectoryRecursive tmp
 		liftIO $ createDirectoryIfMissing True tmp
diff --git a/Command.hs b/Command.hs
index bedb18cc97..bebbf3f1be 100644
--- a/Command.hs
+++ b/Command.hs
@@ -187,7 +187,7 @@ filterFiles l = do
 			let regexp = compile (toregex exclude) []
 			return $ filter (notExcluded regexp) l'
 	where
-		notState f = not $ isPrefixOf stateLoc f
+		notState f = not $ isPrefixOf stateDir f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index cfd6fc3d0e..9427f81035 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -40,7 +40,7 @@ start s = do
 readUnusedLog :: Annex (M.Map String Key)
 readUnusedLog = do
 	g <- Annex.gitRepo
-	let f = annexUnusedLog g
+	let f = gitAnnexUnusedLog g
 	e <- liftIO $ doesFileExist f
 	if e
 		then do
diff --git a/Command/Init.hs b/Command/Init.hs
index 47ac8e4c08..e780c88634 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -10,6 +10,7 @@ module Command.Init where
 import Control.Monad.State (liftIO)
 import Control.Monad (when)
 import System.Directory
+import System.FilePath
 
 import Command
 import qualified Annex
@@ -75,7 +76,7 @@ gitAttributesWrite repo = do
 					attributes]
 
 attrLine :: String		
-attrLine = stateLoc ++ "*.log merge=union"
+attrLine = stateDir  "*.log merge=union"
 
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 566b508c0c..c0e80c5b47 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -50,7 +50,7 @@ perform file oldkey newbackend = do
 	-- (the file can't be stored as usual, because it's already a symlink).
 	-- The old backend's key is not dropped from it, because there may
 	-- be other files still pointing at that key.
-	let src = annexLocation g oldkey
+	let src = gitAnnexLocation g oldkey
 	stored <- Backend.storeFileKey src $ Just newbackend
 	case stored of
 		Nothing -> return Nothing
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index aaa0b48369..cb883b53aa 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -32,7 +32,7 @@ start keyname = do
 	let key = genKey (head backends) keyname
 	present <- inAnnex key
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	when present $
 		liftIO $ rsyncServerSend file
 	liftIO exitFailure
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 645fac8a25..bd1021cc3c 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -43,7 +43,7 @@ perform dest key = do
 		error "content not present"
 
 	g <- Annex.gitRepo
-	let src = annexLocation g key
+	let src = gitAnnexLocation g key
 	liftIO $ removeFile dest
 	showNote "copying..."
 	ok <- liftIO $ copyFile src dest
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 9fdf4cda65..5e5698e38d 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -43,7 +43,7 @@ checkUnused = do
 	unused <- unusedKeys
 	let list = number 1 unused
 	g <- Annex.gitRepo
-	liftIO $ writeFile (annexUnusedLog g) $ unlines $ 
+	liftIO $ writeFile (gitAnnexUnusedLog g) $ unlines $ 
 		map (\(n, k) -> show n ++ " " ++ show k) list
 	if null unused
 		then return True
diff --git a/Content.hs b/Content.hs
index 0cbd6905cb..d0ed8d861c 100644
--- a/Content.hs
+++ b/Content.hs
@@ -35,12 +35,12 @@ import qualified GitRepo as Git
 import qualified Annex
 import Utility
 
-{- Checks if a given key is currently present in the annexLocation. -}
+{- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
 inAnnex key = do
 	g <- Annex.gitRepo
 	when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo"
-	liftIO $ doesFileExist $ annexLocation g key
+	liftIO $ doesFileExist $ gitAnnexLocation g key
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
@@ -51,7 +51,7 @@ calcGitLink file key = do
 		Just f -> f
 		Nothing -> error $ "unable to normalize " ++ file
 	return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++
-		annexLocationRelative key
+		annexLocation key
 
 {- Updates the LocationLog when a key's presence changes. -}
 logStatus :: Key -> LogStatus -> Annex ()
@@ -67,7 +67,7 @@ logStatus key status = do
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
 	g <- Annex.gitRepo
-	let tmp = annexTmpLocation g ++ keyFile key
+	let tmp = gitAnnexTmpDir g  keyFile key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	success <- action tmp
 	if success
@@ -97,7 +97,7 @@ allowWrite f = do
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
 	g <- Annex.gitRepo
-	let dest = annexLocation g key
+	let dest = gitAnnexLocation g key
 	let dir = parentDir dest
 	liftIO $ do
 		createDirectoryIfMissing True dir
@@ -110,7 +110,7 @@ moveAnnex key src = do
 removeAnnex :: Key -> Annex ()
 removeAnnex key = do
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	let dir = parentDir file
 	liftIO $ do
 		allowWrite dir
@@ -121,7 +121,7 @@ removeAnnex key = do
 fromAnnex :: Key -> FilePath -> Annex ()
 fromAnnex key dest = do
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	let dir = parentDir file
 	liftIO $ do
 		allowWrite dir
@@ -134,8 +134,8 @@ fromAnnex key dest = do
 moveBad :: Key -> Annex FilePath
 moveBad key = do
 	g <- Annex.gitRepo
-	let src = annexLocation g key
-	let dest = annexBadLocation g ++ takeFileName src
+	let src = gitAnnexLocation g key
+	let dest = gitAnnexBadDir g  takeFileName src
 	liftIO $ createDirectoryIfMissing True (parentDir dest)
 	liftIO $ allowWrite (parentDir src)
 	liftIO $ renameFile src dest
@@ -146,7 +146,7 @@ moveBad key = do
 getKeysPresent :: Annex [Key]
 getKeysPresent = do
 	g <- Annex.gitRepo
-	getKeysPresent' $ annexObjectDir g
+	getKeysPresent' $ gitAnnexObjectDir g
 getKeysPresent' :: FilePath -> Annex [Key]
 getKeysPresent' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
diff --git a/Locations.hs b/Locations.hs
index 327c099e38..b2624754ec 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -7,73 +7,93 @@
 
 module Locations (
 	gitStateDir,
-	stateLoc,
+	stateDir,
 	keyFile,
 	fileKey,
+	gitAnnexLocation,
 	annexLocation,
-	annexLocationRelative,
-	annexTmpLocation,
-	annexBadLocation,
-	annexUnusedLog,
+	gitAnnexDir,
+	gitAnnexObjectDir,
+	gitAnnexTmpDir,
+	gitAnnexBadDir,
+	gitAnnexUnusedLog,
 	isLinkToAnnex,
-	annexDir,
-	annexObjectDir,
 
 	prop_idempotent_fileKey
 ) where
 
+import System.FilePath
 import Data.String.Utils
 import Data.List
 
 import Types
 import qualified GitRepo as Git
 
+{- Conventions:
+ -
+ - Functions ending in "Dir" should always return values ending with a
+ - trailing path separator. Most code does not rely on that, but a few
+ - things do. 
+ -
+ - Everything else should not end in a trailing path sepatator. 
+ -
+ - Only functions (with names starting with "git") that build a path
+ - based on a git repository should return an absolute path.
+ - Everything else should use relative paths.
+ -}
+
 {- Long-term, cross-repo state is stored in files inside the .git-annex
  - directory, in the git repository's working tree. -}
-stateLoc :: String
-stateLoc = ".git-annex/"
+stateDir :: FilePath
+stateDir = addTrailingPathSeparator $ ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
-gitStateDir repo = Git.workTree repo ++ "/" ++ stateLoc
+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
 
-{- Annexed file's absolute location. -}
-annexLocation :: Git.Repo -> Key -> FilePath
-annexLocation r key = 
-	Git.workTree r ++ "/" ++ annexLocationRelative key
+{- Annexed content is stored in .git/annex/objects; .git/annex is used
+ - for other temporary storage also. -}
+annexDir :: FilePath
+annexDir = addTrailingPathSeparator $ ".git/annex"
+objectDir :: FilePath
+objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
 {- Annexed file's location relative to git's working tree. 
  -
  - Note: Assumes repo is NOT bare.-}
-annexLocationRelative :: Key -> FilePath
-annexLocationRelative key = ".git/annex/objects/" ++ f ++ "/" ++ f
+annexLocation :: Key -> FilePath
+annexLocation key = ".git/annex/objects"  f  f
 	where
 		f = keyFile key
 
+{- Annexed file's absolute location in a repository. -}
+gitAnnexLocation :: Git.Repo -> Key -> FilePath
+gitAnnexLocation r key = Git.workTree r  annexLocation key
+
 {- The annex directory of a repository.
  -
  - Note: Assumes repo is NOT bare. -}
-annexDir :: Git.Repo -> FilePath
-annexDir r = Git.workTree r ++ "/.git/annex"
+gitAnnexDir :: Git.Repo -> FilePath
+gitAnnexDir r = addTrailingPathSeparator $ Git.workTree r  annexDir
 
 {- The part of the annex directory where file contents are stored.
  -}
-annexObjectDir :: Git.Repo -> FilePath
-annexObjectDir r = annexDir r ++ "/objects"
+gitAnnexObjectDir :: Git.Repo -> FilePath
+gitAnnexObjectDir r = addTrailingPathSeparator $ Git.workTree r  objectDir
 
 {- .git-annex/tmp/ is used for temp files -}
-annexTmpLocation :: Git.Repo -> FilePath
-annexTmpLocation r = annexDir r ++ "/tmp/"
+gitAnnexTmpDir :: Git.Repo -> FilePath
+gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 
 {- .git-annex/bad/ is used for bad files found during fsck -}
-annexBadLocation :: Git.Repo -> FilePath
-annexBadLocation r = annexDir r ++ "/bad/"
+gitAnnexBadDir :: Git.Repo -> FilePath
+gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
 
 {- .git/annex/unused is used to number possibly unused keys -}
-annexUnusedLog :: Git.Repo -> FilePath
-annexUnusedLog r = annexDir r ++ "/unused"
+gitAnnexUnusedLog :: Git.Repo -> FilePath
+gitAnnexUnusedLog r = gitAnnexDir r  "unused"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = isInfixOf "/.git/annex/objects/" s
+isLinkToAnnex s = isInfixOf ("/" ++ objectDir) s
 
 {- Converts a key into a filename fragment.
  -
diff --git a/Remotes.hs b/Remotes.hs
index be4c1383af..616db225ef 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -225,7 +225,7 @@ byName name = do
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
-	| not $ Git.repoIsUrl r = liftIO $ copyFile (annexLocation r key) file
+	| not $ Git.repoIsUrl r = liftIO $ copyFile (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsynchelper r True key file
 	| otherwise = error "copying from non-ssh repo not supported"
 
@@ -234,7 +234,7 @@ copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
 	| not $ Git.repoIsUrl r = do
 		g <- Annex.gitRepo
-		let keysrc = annexLocation g key
+		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
 		liftIO $ do
 			a <- Annex.new r []
@@ -245,7 +245,7 @@ copyToRemote r key
 				return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo
-		let keysrc = annexLocation g key
+		let keysrc = gitAnnexLocation g key
 		rsynchelper r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
diff --git a/Upgrade.hs b/Upgrade.hs
index 596d525db2..1e70e68d56 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -39,9 +39,9 @@ upgradeFrom0 = do
 	g <- Annex.gitRepo
 
 	-- do the reorganisation of the files
-	let olddir = annexDir g
+	let olddir = gitAnnexDir g
 	keys <- getKeysPresent0' olddir
-	_ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys
+	_ <- mapM (\k -> moveAnnex k $ olddir  keyFile k) keys
 
 	-- update the symlinks to the files
 	files <- liftIO $ Git.inRepo g [Git.workTree g]
diff --git a/Version.hs b/Version.hs
index fc1ce3d7ec..9e31d3c9eb 100644
--- a/Version.hs
+++ b/Version.hs
@@ -29,10 +29,10 @@ getVersion = do
 		then return $ Just v
 		else do
 			-- version 0 was not recorded in .git/config;
-			-- such a repo should have an annexDir but no
-			-- annexObjectDir
-			d <- liftIO $ doesDirectoryExist $ annexDir g
-			o <- liftIO $ doesDirectoryExist $ annexObjectDir g
+			-- such a repo should have an gitAnnexDir but no
+			-- gitAnnexObjectDir
+			d <- liftIO $ doesDirectoryExist $ gitAnnexDir g
+			o <- liftIO $ doesDirectoryExist $ gitAnnexObjectDir g
 			if d && not o
 				then return $ Just "0"
 				else return Nothing -- no version yet

From 4f9336bb60ca4dc08ca271605de71415c98ec3ff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 17:58:30 -0400
Subject: [PATCH 0783/2835] idiom

---
 Command.hs         | 2 +-
 GitRepo.hs         | 4 ++--
 Locations.hs       | 2 +-
 git-annex-shell.hs | 2 +-
 test.hs            | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command.hs b/Command.hs
index bebbf3f1be..298234fa05 100644
--- a/Command.hs
+++ b/Command.hs
@@ -187,7 +187,7 @@ filterFiles l = do
 			let regexp = compile (toregex exclude) []
 			return $ filter (notExcluded regexp) l'
 	where
-		notState f = not $ isPrefixOf stateDir f
+		notState f = not $ stateDir `isPrefixOf` f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False
diff --git a/GitRepo.hs b/GitRepo.hs
index ec363fe73b..4e69544d43 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -323,7 +323,7 @@ configRemotes repo = map construct remotepairs
 			| otherwise = repoFromPath v
 		-- git remotes can be written scp style -- [user@]host:dir
 		-- where dir is relative to the user's home directory.
-		scpstyle v = isInfixOf ":" v && (not $ isInfixOf "//" v)
+		scpstyle v = ":" `isInfixOf` v && (not $ "//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
 			where
 				bits = split ":" v
@@ -458,7 +458,7 @@ prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)
  -}
 absDir :: String -> IO String
 absDir d
-	| isPrefixOf "/" d = expandt d
+	| "/" `isPrefixOf` d = expandt d
 	| otherwise = do
 		h <- myhomedir
 		return $ h ++ d
diff --git a/Locations.hs b/Locations.hs
index b2624754ec..75843c29eb 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -93,7 +93,7 @@ gitAnnexUnusedLog r = gitAnnexDir r  "unused"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = isInfixOf ("/" ++ objectDir) s
+isLinkToAnnex s = ("/" ++ objectDir) `isInfixOf` s
 
 {- Converts a key into a filename fragment.
  -
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 78dd777904..fa2a7f606a 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -51,7 +51,7 @@ main' c@(cmd:dir:params)
 main' c@(cmd:_)
 	-- Handle the case of being the user's login shell. It will be passed
 	-- a single string containing all the real parameters.
-	| isPrefixOf "git-annex-shell " cmd = main' $ drop 1 $ shellUnEscape cmd
+	| "git-annex-shell " `isPrefixOf` cmd = main' $ drop 1 $ shellUnEscape cmd
 	| elem cmd builtins = failure
 	| otherwise = external c
 
diff --git a/test.hs b/test.hs
index d7a6bd152f..5dda3b835a 100644
--- a/test.hs
+++ b/test.hs
@@ -88,7 +88,7 @@ test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 	e <- doesFileExist annexlog
 	e @? (annexlog ++ " not created")
 	c <- readFile annexlog
-	isInfixOf reponame c @? annexlog ++ " does not contain repo name"
+	reponame `isInfixOf` c @? annexlog ++ " does not contain repo name"
 	where
 		annexlog = ".git-annex/uuid.log"
 		reponame = "test repo"

From 5ab922ce0e47e200f943e8a8c9aeb77d2a863b3e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Fri, 28 Jan 2011 07:30:06 +0000
Subject: [PATCH 0784/2835] Added a comment: I actually *do* want to avoid
 duplication of filenames

---
 ...2_4394bde1c6fd44acae649baffe802775._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment
new file mode 100644
index 0000000000..04d58a4598
--- /dev/null
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
+ nickname="Asheesh"
+ subject="I actually *do* want to avoid duplication of filenames"
+ date="2011-01-28T07:30:05Z"
+ content="""
+I really do want just one filename per file, at least for some cases.
+
+For my photos, there's no benefit to having a few filenames point to the same file. As I'm putting them all into the git-annex, that is a good time to remove the pure duplicates so that I don't e.g. see them twice when browsing the directory as a gallery. Also, I am uploading my photos to the web, and I want to avoid uploading the same photo (by content) twice.
+
+I hope that makes things clearer!
+
+For now I'm just doing this:
+
+* paulproteus@renaissance:/mnt/backups-terabyte/paulproteus/sd-card-from-2011-01-06/sd-cards/DCIM/100CANON $ for file in *; do hash=$(sha1sum \"$file\"); if ls /home/paulproteus/Photos/in-flickr/.git-annex | grep -q \"$hash\"; then echo already annexed ; else flickr_upload \"$file\" && mv \"$file\" \"/home/paulproteus/Photos/in-flickr/2011-01-28/from-some-nested-sd-card-bk\" && (cd /home/paulproteus/Photos/in-flickr/2011-01-28/from-some-nested-sd-card-bk && git annex add . && git commit -m ...) ; fi; done
+
+(Yeah, Flickr for my photos for now. I feel sad about betraying the principle of autonomo.us-ness.)
+"""]]

From 04fe906ac6e611fd59ef44244a01e8fe61abec6f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Jan 2011 12:35:51 -0400
Subject: [PATCH 0785/2835] use safewritefile

---
 Command/Init.hs   | 5 +++--
 Command/Uninit.hs | 2 +-
 Command/Unused.hs | 3 ++-
 LocationLog.hs    | 8 +-------
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index e780c88634..2976b988d1 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -20,6 +20,7 @@ import Version
 import Messages
 import Locations
 import Types
+import Utility
 	
 command :: [Command]
 command = [Command "init" paramDesc seek
@@ -61,7 +62,7 @@ gitAttributesWrite repo = do
 	exists <- doesFileExist attributes
 	if not exists
 		then do
-			writeFile attributes $ attrLine ++ "\n"
+			safeWriteFile attributes $ attrLine ++ "\n"
 			commit
 		else do
 			content <- readFile attributes
@@ -85,7 +86,7 @@ gitPreCommitHookWrite repo = do
 	if exists
 		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
 		else liftIO $ do
-			writeFile hook preCommitScript
+			safeWriteFile hook preCommitScript
 			p <- getPermissions hook
 			setPermissions hook $ p {executable = True}
 	where
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 93465df373..e9406ce3af 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -59,5 +59,5 @@ gitAttributesUnWrite repo = do
 	attrexists <- doesFileExist attributes
 	when attrexists $ do
 		c <- readFileStrict attributes
-		writeFile attributes $ unlines $
+		safeWriteFile attributes $ unlines $
 			filter (/= Command.Init.attrLine) $ lines c
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5e5698e38d..b9dc62a324 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,6 +16,7 @@ import Types
 import Content
 import Messages
 import Locations
+import Utility
 import qualified Annex
 import qualified GitRepo as Git
 import qualified Backend
@@ -43,7 +44,7 @@ checkUnused = do
 	unused <- unusedKeys
 	let list = number 1 unused
 	g <- Annex.gitRepo
-	liftIO $ writeFile (gitAnnexUnusedLog g) $ unlines $ 
+	liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ unlines $ 
 		map (\(n, k) -> show n ++ " " ++ show k) list
 	if null unused
 		then return True
diff --git a/LocationLog.hs b/LocationLog.hs
index 926939051c..56953bc029 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -32,7 +32,6 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as Map
 import System.Directory
-import System.Posix.Process
 import Control.Monad (when)
 
 import qualified GitRepo as Git
@@ -112,12 +111,7 @@ readLog file = do
 
 {- Writes a set of lines to a log file -}
 writeLog :: FilePath -> [LogLine] -> IO ()
-writeLog file ls = do
-	pid <- getProcessID
-	let tmpfile = file ++ ".tmp" ++ show pid
-	createDirectoryIfMissing True (parentDir file)
-	writeFile tmpfile $ unlines $ map show ls
-	renameFile tmpfile file
+writeLog file ls = safeWriteFile file (unlines $ map show ls)
 
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> UUID -> IO LogLine

From e6da7eb1770e506661d8c6755736ed285a17554a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Jan 2011 14:10:50 -0400
Subject: [PATCH 0786/2835] Improved temp file handling

* Improved temp file handling. Transfers of content can now be resumed
  from temp files later; the resume does not have to be the immediate
  next git-annex run.
* unused: Include partially transferred content in the list.
---
 CmdLine.hs                      | 10 ----
 Command/DropUnused.hs           |  7 +++
 Command/Unused.hs               | 95 +++++++++++++++++++++++----------
 Content.hs                      |  2 +-
 Locations.hs                    |  5 ++
 debian/changelog                |  4 ++
 doc/bugs/tmp_file_handling.mdwn |  2 +
 doc/git-annex.mdwn              |  2 +-
 8 files changed, 87 insertions(+), 40 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 68d1e0dd31..82a21d0fc7 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -105,13 +105,3 @@ shutdown errnum = do
 	unless (q == GitQueue.empty) $ do
 		showSideAction "Recording state in git..."
 		Annex.queueRun
-
-	-- If nothing failed, clean up any files left in the temp directory,
-	-- but leave the directory itself. If something failed, temp files
-	-- are left behind to allow resuming on re-run.
-	when (errnum == 0) $ do
-		g <- Annex.gitRepo
-		let tmp = gitAnnexTmpDir g
-		exists <- liftIO $ doesDirectoryExist tmp
-		when exists $ liftIO $ removeDirectoryRecursive tmp
-		liftIO $ createDirectoryIfMissing True tmp
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 9427f81035..63216ce4f1 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -7,6 +7,7 @@
 
 module Command.DropUnused where
 
+import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Directory
@@ -33,8 +34,14 @@ start s = do
 	case M.lookup s m of
 		Nothing -> return Nothing
 		Just key -> do
+			g <- Annex.gitRepo
 			showStart "dropunused" s
 			backend <- keyBackend key
+			-- drop both content in the backend and any tmp
+			-- file for the key
+			let tmp = gitAnnexTmpLocation g key
+			tmp_exists <- liftIO $ doesFileExist tmp
+			when tmp_exists $ liftIO $ removeFile tmp
 			return $ Just $ Command.Drop.perform key backend (Just 0)
 
 readUnusedLog :: Annex (M.Map String Key)
diff --git a/Command/Unused.hs b/Command/Unused.hs
index b9dc62a324..4ffa6a17f8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -7,9 +7,12 @@
 
 module Command.Unused where
 
+import Control.Monad (filterM, unless)
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import Data.Maybe
+import System.FilePath
+import System.Directory
 
 import Command
 import Types
@@ -41,49 +44,71 @@ perform = do
 checkUnused :: Annex Bool
 checkUnused = do
 	showNote "checking for unused data..."
-	unused <- unusedKeys
-	let list = number 1 unused
+	(unused, staletmp) <- unusedKeys
+	let unusedlist = number 0 unused
+	let staletmplist = number (length unused) staletmp
+	let list = unusedlist ++ staletmplist
 	g <- Annex.gitRepo
 	liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ unlines $ 
 		map (\(n, k) -> show n ++ " " ++ show k) list
-	if null unused
-		then return True
-		else do
-			showLongNote $ w list
-			return False
+	unless (null unused) $
+		showLongNote $ unusedmsg unusedlist
+	unless (null staletmp) $
+		showLongNote $ staletmpmsg staletmplist
+	unless (null list) $
+		showLongNote $ "\n"
+	return $ null list
+
 	where
-		w u = unlines $
-			["Some annexed data is no longer pointed to by any files in the repository:",
-			 "  NUMBER  KEY"]
-			++ map cols u ++
-			["(To see where data was previously used, try: git log --stat -S'KEY')",
-			 "(To remove unwanted data: git-annex dropunused NUMBER)",
-			 ""]
+		unusedmsg u = unlines $
+			["Some annexed data is no longer pointed to by any files in the repository:"]
+			++ table u ++
+			["(To see where data was previously used, try: git log --stat -S'KEY')"] ++
+			dropmsg
+		staletmpmsg t = unlines $ 
+			["Some partially transferred data exists in temporary files:"]
+			++ table t ++ dropmsg
+		dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"]
+			
+		table l = ["  NUMBER  KEY"] ++ map cols l
 		cols (n,k) = "  " ++ pad 6 (show n) ++ "  " ++ show k
 		pad n s = s ++ replicate (n - length s) ' '
 
-number :: Integer -> [a] -> [(Integer, a)]
+number :: Int -> [a] -> [(Int, a)]
 number _ [] = []
-number n (x:xs) = (n, x):(number (n+1) xs)
+number n (x:xs) = (n+1, x):(number (n+1) xs)
 
 {- Finds keys whose content is present, but that do not seem to be used
- - by any files in the git repo. -}
-unusedKeys :: Annex [Key]
+ - by any files in the git repo, or that are only present as tmp files. -}
+unusedKeys :: Annex ([Key], [Key])
 unusedKeys = do
+	g <- Annex.gitRepo
 	present <- getKeysPresent
 	referenced <- getKeysReferenced
 	
-	-- Constructing a single map, of the set that tends to be smaller,
-	-- appears more efficient in both memory and CPU than constructing
-	-- and taking the M.difference of two maps.
-	let present_m = existsMap present
-	let unused_m = remove referenced present_m
-	return $ M.keys unused_m
-	where
-		remove a b = foldl (flip M.delete) b a
+	let unused = present `exclude` referenced
 
-existsMap :: Ord k => [k] -> M.Map k Int
-existsMap l = M.fromList $ map (\k -> (k, 1)) l
+	-- Some tmp files may be dups copies of content that is fully present.
+	-- Simply delete those, while including the keys for the rest of
+	-- the temp files in the returned list for the user to deal with.
+	tmps <- tmpKeys
+	let staletmp = tmps `exclude` present
+	let duptmp = tmps `exclude` staletmp
+	_ <- liftIO $ mapM (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
+
+	return (unused, staletmp)
+
+	where
+		-- Constructing a single map, of the set that tends to be
+		-- smaller, appears more efficient in both memory and CPU
+		-- than constructing and taking the M.difference of two maps.
+		exclude [] _ = [] -- optimisation
+		exclude smaller larger = M.keys $ remove larger $ existsMap smaller
+
+		existsMap :: Ord k => [k] -> M.Map k Int
+		existsMap l = M.fromList $ map (\k -> (k, 1)) l
+
+		remove a b = foldl (flip M.delete) b a
 
 {- List of keys referenced by symlinks in the git repo. -}
 getKeysReferenced :: Annex [Key]
@@ -92,3 +117,17 @@ getKeysReferenced = do
 	files <- liftIO $ Git.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
+
+{- List of keys that have temp files in the git repo. -}
+tmpKeys :: Annex [Key]
+tmpKeys = do
+	g <- Annex.gitRepo
+	let tmp = gitAnnexTmpDir g
+	exists <- liftIO $ doesDirectoryExist tmp
+	if (not exists)
+		then return []
+		else do
+			contents <- liftIO $ getDirectoryContents tmp
+			files <- liftIO $ filterM doesFileExist $
+				map (tmp ) contents
+			return $ map (fileKey . takeFileName) files
diff --git a/Content.hs b/Content.hs
index d0ed8d861c..e16ad883c2 100644
--- a/Content.hs
+++ b/Content.hs
@@ -67,7 +67,7 @@ logStatus key status = do
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
 	g <- Annex.gitRepo
-	let tmp = gitAnnexTmpDir g  keyFile key
+	let tmp = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	success <- action tmp
 	if success
diff --git a/Locations.hs b/Locations.hs
index 75843c29eb..d30ceb1367 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -15,6 +15,7 @@ module Locations (
 	gitAnnexDir,
 	gitAnnexObjectDir,
 	gitAnnexTmpDir,
+	gitAnnexTmpLocation,
 	gitAnnexBadDir,
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
@@ -83,6 +84,10 @@ gitAnnexObjectDir r = addTrailingPathSeparator $ Git.workTree r  objectDir
 gitAnnexTmpDir :: Git.Repo -> FilePath
 gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 
+{- The temp file to use for a given key. -}
+gitAnnexTmpLocation :: Git.Repo -> Key -> FilePath
+gitAnnexTmpLocation r key = gitAnnexTmpDir r  keyFile key
+
 {- .git-annex/bad/ is used for bad files found during fsck -}
 gitAnnexBadDir :: Git.Repo -> FilePath
 gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
diff --git a/debian/changelog b/debian/changelog
index 92b21173c8..cf17e8a00f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,10 @@ git-annex (0.19) UNRELEASED; urgency=low
   * fsck, drop: Take untrusted repositories into account.
   * Bugfix: Files were copied from trusted remotes first even if their
     annex.cost was higher than other remotes.
+  * Improved temp file handling. Transfers of content can now be resumed
+    from temp files later; the resume does not have to be the immediate
+    next git-annex run.
+  * unused: Include partially transferred content in the list.
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
 
diff --git a/doc/bugs/tmp_file_handling.mdwn b/doc/bugs/tmp_file_handling.mdwn
index 3c6c2a5976..9db932e571 100644
--- a/doc/bugs/tmp_file_handling.mdwn
+++ b/doc/bugs/tmp_file_handling.mdwn
@@ -9,3 +9,5 @@ This presents 2 problems:
    finished.
 
 --[[Joey]] 
+
+[[done]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d2cde35a0d..83a286b0e1 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,7 +132,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * unused
 
   Checks the annex for data that is not used by any files currently
-  in the annex, and prints a numbered list of the data. 
+  in the annex, and prints a numbered list of the data.
 
 * dropunused [number ...]
 

From 3c491130239dba5a4c1d882a6e91f1e138938f94 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Jan 2011 14:42:06 -0400
Subject: [PATCH 0787/2835] releasing version 0.19

---
 debian/changelog | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index cf17e8a00f..7abdee85cb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
-git-annex (0.19) UNRELEASED; urgency=low
+git-annex (0.19) unstable; urgency=low
 
-  * Support using the uuidgen command if the uuid command is not available.
+  * configure: Support using the uuidgen command if the uuid command is
+    not available.
   * Allow --exclude to be specified more than once.
   * There are now three levels of repository trust.
   * untrust: Now marks the current repository as untrusted.
@@ -12,8 +13,10 @@ git-annex (0.19) UNRELEASED; urgency=low
     from temp files later; the resume does not have to be the immediate
     next git-annex run.
   * unused: Include partially transferred content in the list.
+  * Bugfix: Running a second git-annex while a first has a transfer in
+    progress no longer deletes the first processes's temp file.
 
- -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
+ -- Joey Hess   Fri, 28 Jan 2011 14:31:37 -0400
 
 git-annex (0.18) unstable; urgency=low
 

From f88fd8d9fcc70365bc562f62b569b89109bb7e9a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Jan 2011 14:42:24 -0400
Subject: [PATCH 0788/2835] add news item for git-annex 0.19

---
 doc/news/version_0.14.mdwn |  5 -----
 doc/news/version_0.19.mdwn | 17 +++++++++++++++++
 2 files changed, 17 insertions(+), 5 deletions(-)
 delete mode 100644 doc/news/version_0.14.mdwn
 create mode 100644 doc/news/version_0.19.mdwn

diff --git a/doc/news/version_0.14.mdwn b/doc/news/version_0.14.mdwn
deleted file mode 100644
index fc6c42bc19..0000000000
--- a/doc/news/version_0.14.mdwn
+++ /dev/null
@@ -1,5 +0,0 @@
-git-annex 0.14 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Bugfix to git annex unused in a repository with nothing yet annexed.
-   * Support upgrading from a v0 annex with nothing in it.
-   * Avoid multiple calls to git ls-files when passed eg, "*"."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.19.mdwn b/doc/news/version_0.19.mdwn
new file mode 100644
index 0000000000..5d6ab47be0
--- /dev/null
+++ b/doc/news/version_0.19.mdwn
@@ -0,0 +1,17 @@
+git-annex 0.19 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * configure: Support using the uuidgen command if the uuid command is
+     not available.
+   * Allow --exclude to be specified more than once.
+   * There are now three levels of repository trust.
+   * untrust: Now marks the current repository as untrusted.
+   * semitrust: Now restores the default trust level. (What untrust used to do.)
+   * fsck, drop: Take untrusted repositories into account.
+   * Bugfix: Files were copied from trusted remotes first even if their
+     annex.cost was higher than other remotes.
+   * Improved temp file handling. Transfers of content can now be resumed
+     from temp files later; the resume does not have to be the immediate
+     next git-annex run.
+   * unused: Include partially transferred content in the list.
+   * Bugfix: Running a second git-annex while a first has a transfer in
+     progress no longer deletes the first processes's temp file."""]]
\ No newline at end of file

From b8318384858c59d73f231d225323a8a0a4a42868 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Jan 2011 23:27:17 -0400
Subject: [PATCH 0789/2835] cleanup

---
 CmdLine.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 82a21d0fc7..308b04d1a9 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -15,7 +15,6 @@ import System.IO.Error (try)
 import System.Console.GetOpt
 import Control.Monad.State (liftIO)
 import Control.Monad (when, unless)
-import System.Directory
 
 import qualified Annex
 import qualified GitRepo as Git
@@ -27,7 +26,6 @@ import Upgrade
 import Options
 import Messages
 import UUID
-import Locations
 
 {- Runs the passed command line. -}
 dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO ()

From 5584ccc8adbbcd629b3809fd39b5c00479fd0a25 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Jan 2011 23:30:08 -0400
Subject: [PATCH 0790/2835] factor out pure code

---
 Command/Unused.hs | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 4ffa6a17f8..32d70aa19d 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -51,12 +51,9 @@ checkUnused = do
 	g <- Annex.gitRepo
 	liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ unlines $ 
 		map (\(n, k) -> show n ++ " " ++ show k) list
-	unless (null unused) $
-		showLongNote $ unusedmsg unusedlist
-	unless (null staletmp) $
-		showLongNote $ staletmpmsg staletmplist
-	unless (null list) $
-		showLongNote $ "\n"
+	unless (null unused) $ showLongNote $ unusedmsg unusedlist
+	unless (null staletmp) $ showLongNote $ staletmpmsg staletmplist
+	unless (null list) $ showLongNote $ "\n"
 	return $ null list
 
 	where
@@ -85,20 +82,23 @@ unusedKeys = do
 	g <- Annex.gitRepo
 	present <- getKeysPresent
 	referenced <- getKeysReferenced
-	
-	let unused = present `exclude` referenced
-
-	-- Some tmp files may be dups copies of content that is fully present.
-	-- Simply delete those, while including the keys for the rest of
-	-- the temp files in the returned list for the user to deal with.
 	tmps <- tmpKeys
-	let staletmp = tmps `exclude` present
-	let duptmp = tmps `exclude` staletmp
+	
+	let (unused, staletmp, duptmp) = calcUnusedKeys present referenced tmps
+
+	-- Tmp files that are dups of content already present can simply
+	-- be removed.
 	_ <- liftIO $ mapM (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
 
 	return (unused, staletmp)
 
+calcUnusedKeys :: [Key] -> [Key] -> [Key] -> ([Key], [Key], [Key])
+calcUnusedKeys present referenced tmps = (unused, staletmp, duptmp)
 	where
+		unused = present `exclude` referenced
+		staletmp = tmps `exclude` present
+		duptmp = tmps `exclude` staletmp
+
 		-- Constructing a single map, of the set that tends to be
 		-- smaller, appears more efficient in both memory and CPU
 		-- than constructing and taking the M.difference of two maps.

From c64b50a0ce725df2d5317b7ae4918b61aafa25ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Jan 2011 23:32:32 -0400
Subject: [PATCH 0791/2835] shutdown no longer a special case

---
 CmdLine.hs         | 12 ++++++------
 Command/RecvKey.hs |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 308b04d1a9..030637a738 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -32,7 +32,7 @@ dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO ()
 dispatch gitrepo args cmds options header = do
 	state <- Annex.new gitrepo allBackends
 	(actions, state') <- Annex.run state $ parseCmd args header cmds options
-	tryRun state' $ [startup, upgrade] ++ actions
+	tryRun state' $ [startup, upgrade] ++ actions ++ [shutdown]
 
 {- Parses command line, stores configure flags, and returns a 
  - list of actions to be run in the Annex monad. -}
@@ -73,7 +73,6 @@ usage header cmds options =
 
 {- Runs a list of Annex actions. Catches IO errors and continues
  - (but explicitly thrown errors terminate the whole command).
- - Runs shutdown and propigates an overall error status at the end.
  -}
 tryRun :: Annex.AnnexState -> [Annex Bool] -> IO ()
 tryRun state actions = tryRun' state 0 actions
@@ -86,8 +85,7 @@ tryRun' state errnum (a:as) = do
 			tryRun' state (errnum + 1) as
 		Right (True,state') -> tryRun' state' errnum as
 		Right (False,state') -> tryRun' state' (errnum + 1) as
-tryRun' state errnum [] = do
-	_ <- try $ Annex.run state $ shutdown errnum
+tryRun' _ errnum [] = do
 	when (errnum > 0) $ error $ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
@@ -97,9 +95,11 @@ startup = do
 	return True
 
 {- Cleanup actions. -}
-shutdown :: Integer -> Annex ()
-shutdown errnum = do
+shutdown :: Annex Bool
+shutdown = do
 	q <- Annex.getState Annex.repoqueue
 	unless (q == GitQueue.empty) $ do
 		showSideAction "Recording state in git..."
 		Annex.queueRun
+
+	return True
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 0abea07f20..8a96730503 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -38,6 +38,6 @@ start keyname = do
 		then do
 			-- forcibly quit after receiving one key,
 			-- and shutdown cleanly so queued git commands run
-			_ <- shutdown 0
+			_ <- shutdown
 			liftIO exitSuccess
 		else liftIO exitFailure

From 6d3df8a0833fdef290b76681e8ba85ec94b76e36 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Jan 2011 23:47:10 -0400
Subject: [PATCH 0792/2835] more pure code refactoring

---
 CmdLine.hs     |  1 -
 Command.hs     | 13 ++++++++-----
 LocationLog.hs | 10 +++++++---
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 030637a738..d65739791f 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -101,5 +101,4 @@ shutdown = do
 	unless (q == GitQueue.empty) $ do
 		showSideAction "Recording state in git..."
 		Annex.queueRun
-
 	return True
diff --git a/Command.hs b/Command.hs
index 298234fa05..e54add707b 100644
--- a/Command.hs
+++ b/Command.hs
@@ -184,17 +184,20 @@ filterFiles l = do
 	if null exclude
 		then return l'
 		else do
-			let regexp = compile (toregex exclude) []
+			let regexp = compile (wildsRegex exclude) []
 			return $ filter (notExcluded regexp) l'
 	where
 		notState f = not $ stateDir `isPrefixOf` f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False
-		toregex exclude = "^(" ++ toregex' exclude "" ++ ")"
-		toregex' [] c = c
-		toregex' (w:ws) "" = toregex' ws (wildToRegex w)
-		toregex' (w:ws) c = toregex' ws (c ++ "|" ++ wildToRegex w)
+
+wildsRegex :: [String] -> String
+wildsRegex ws = "^(" ++ wildsRegex' ws "" ++ ")"
+wildsRegex' :: [String] -> String -> String
+wildsRegex' [] c = c
+wildsRegex' (w:ws) "" = wildsRegex' ws (wildToRegex w)
+wildsRegex' (w:ws) c = wildsRegex' ws (c ++ "|" ++ wildToRegex w)
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
diff --git a/LocationLog.hs b/LocationLog.hs
index 56953bc029..f778df3864 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -104,11 +104,15 @@ readLog file = do
 	if exists
 		then do
 			s <- readFile file
-			-- filter out any unparsable lines
-			return $ filter (\l -> status l /= Undefined )
-				$ map read $ lines s
+			return $ parseLog s
 		else return []
 
+parseLog :: String -> [LogLine]
+parseLog s = filter parsable $ map read $ lines s
+	where
+		-- some lines may be unparseable, avoid them
+		parsable l = status l /= Undefined
+
 {- Writes a set of lines to a log file -}
 writeLog :: FilePath -> [LogLine] -> IO ()
 writeLog file ls = safeWriteFile file (unlines $ map show ls)

From 53677d764710d522394f59e494fb6b632aca2326 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Jan 2011 00:08:17 -0400
Subject: [PATCH 0793/2835] tweak

---
 Command.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index e54add707b..1c1ff6bbcf 100644
--- a/Command.hs
+++ b/Command.hs
@@ -183,17 +183,16 @@ filterFiles l = do
 	exclude <- Annex.getState Annex.exclude
 	if null exclude
 		then return l'
-		else do
-			let regexp = compile (wildsRegex exclude) []
-			return $ filter (notExcluded regexp) l'
+		else return $ filter (notExcluded $ wildsRegex exclude) l'
 	where
 		notState f = not $ stateDir `isPrefixOf` f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False
 
-wildsRegex :: [String] -> String
-wildsRegex ws = "^(" ++ wildsRegex' ws "" ++ ")"
+wildsRegex :: [String] -> Regex
+wildsRegex ws = compile regex []
+	where regex = "^(" ++ wildsRegex' ws "" ++ ")"
 wildsRegex' :: [String] -> String -> String
 wildsRegex' [] c = c
 wildsRegex' (w:ws) "" = wildsRegex' ws (wildToRegex w)

From 96e561bc477bd0a4bbffb769369fabe1e1e1982f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Jan 2011 01:41:15 -0400
Subject: [PATCH 0794/2835] use Set instead of existence Map

more efficient and idiomatic

I did try using Set.difference, it's still slower than my method.
---
 Command/Unused.hs | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 32d70aa19d..28a26f82c6 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -9,7 +9,7 @@ module Command.Unused where
 
 import Control.Monad (filterM, unless)
 import Control.Monad.State (liftIO)
-import qualified Data.Map as M
+import qualified Data.Set as S
 import Data.Maybe
 import System.FilePath
 import System.Directory
@@ -99,16 +99,12 @@ calcUnusedKeys present referenced tmps = (unused, staletmp, duptmp)
 		staletmp = tmps `exclude` present
 		duptmp = tmps `exclude` staletmp
 
-		-- Constructing a single map, of the set that tends to be
+		-- Constructing a single set, of the list that tends to be
 		-- smaller, appears more efficient in both memory and CPU
-		-- than constructing and taking the M.difference of two maps.
+		-- than constructing and taking the S.difference of two sets.
 		exclude [] _ = [] -- optimisation
-		exclude smaller larger = M.keys $ remove larger $ existsMap smaller
-
-		existsMap :: Ord k => [k] -> M.Map k Int
-		existsMap l = M.fromList $ map (\k -> (k, 1)) l
-
-		remove a b = foldl (flip M.delete) b a
+		exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
+		remove a b = foldl (flip S.delete) b a
 
 {- List of keys referenced by symlinks in the git repo. -}
 getKeysReferenced :: Annex [Key]

From 1b0edc1ab2f3516dc532b0cf4ea39a0af2f6392f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Jan 2011 12:01:56 -0400
Subject: [PATCH 0795/2835] idiomatic elem

---
 Backend/File.hs    | 6 +++---
 BackendTypes.hs    | 2 +-
 UUID.hs            | 4 ++--
 git-annex-shell.hs | 4 ++--
 test.hs            | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 68dd4db271..d76cd29391 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -108,7 +108,7 @@ checkRemoveKey key numcopiesM = do
 			| length have >= need = return True
 			| otherwise = do
 				u <- getUUID r
-				let dup = elem u have
+				let dup = u `elem` have
 				haskey <- Remotes.inAnnex r key
 				case (dup, haskey) of
 					(False, Right True)	-> findcopies need (u:have) rs bad
@@ -139,7 +139,7 @@ showLocations key exclude = do
 	ppuuidsskipped <- prettyPrintUUIDs uuidsskipped
 	showLongNote $ message ppuuidswanted ppuuidsskipped
 	where
-		filteruuids list x = filter (\l -> not $ elem l x) list
+		filteruuids list x = filter (`notElem` x) list
 		message [] [] = "No other repository is known to contain the file."
 		message rs [] = "Try making some of these repositories available:\n" ++ rs
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
@@ -179,7 +179,7 @@ checkKeyNumCopies key file numcopies = do
 	locations <- liftIO $ keyLocations g key
 	untrusted <- trustGet UnTrusted
 	let untrustedlocations = intersect untrusted locations
-	let safelocations = filter (\l -> not $ l `elem` untrusted) locations
+	let safelocations = filter (`notElem` untrusted) locations
 	let present = length safelocations
 	if present < needed
 		then do
diff --git a/BackendTypes.hs b/BackendTypes.hs
index fd4a61b989..c4927ab20c 100644
--- a/BackendTypes.hs
+++ b/BackendTypes.hs
@@ -75,5 +75,5 @@ instance Arbitrary Key where
 prop_idempotent_key_read_show :: Key -> Bool
 prop_idempotent_key_read_show k
 	-- backend names will never contain colons
-	| elem ':' (backendName k) = True
+	| ':' `elem` (backendName k) = True
 	| otherwise = k == (read $ show k)
diff --git a/UUID.hs b/UUID.hs
index ec67026894..a654424b4f 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -88,7 +88,7 @@ reposByUUID repos uuids = filterM match repos
 	where
 		match r = do
 			u <- getUUID r
-			return $ elem u uuids
+			return $ u `elem` uuids
 
 {- Filters a list of repos to ones that do not have the listed UUIDs. -}
 reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
@@ -96,7 +96,7 @@ reposWithoutUUID repos uuids = filterM unmatch repos
 	where
 		unmatch r = do
 			u <- getUUID r
-			return $ not $ elem u uuids
+			return $ u `notElem` uuids
 
 {- Pretty-prints a list of UUIDs -}
 prettyPrintUUIDs :: [UUID] -> Annex String
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index fa2a7f606a..fee4091ef8 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -46,13 +46,13 @@ main' [] = failure
 main' ("-c":p) = main' p
 -- a command can be either a builtin or something to pass to git-shell
 main' c@(cmd:dir:params)
-	| elem cmd builtins = builtin cmd dir params
+	| cmd `elem` builtins = builtin cmd dir params
 	| otherwise = external c
 main' c@(cmd:_)
 	-- Handle the case of being the user's login shell. It will be passed
 	-- a single string containing all the real parameters.
 	| "git-annex-shell " `isPrefixOf` cmd = main' $ drop 1 $ shellUnEscape cmd
-	| elem cmd builtins = failure
+	| cmd `elem` builtins = failure
 	| otherwise = external c
 
 builtins :: [String]
diff --git a/test.hs b/test.hs
index 5dda3b835a..f7d3f7c53d 100644
--- a/test.hs
+++ b/test.hs
@@ -331,7 +331,7 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 				l <- Trust.trustGet expected
 				r <- Remotes.byName repo
 				u <- UUID.getUUID r
-				return $ elem u l
+				return $ u `elem` l
 			assertBool msg present
 		repo = "origin"
 
@@ -588,7 +588,7 @@ checklocationlog f expected = do
 				g <- Annex.gitRepo
 				liftIO $ LocationLog.keyLocations g k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
-				expected (elem thisuuid uuids)
+				expected (thisuuid `elem` uuids)
 
 			-- Location log files should always be checked
 			-- into git, and any modifications staged for

From 4a0fe24f394e9c5489bacaa5630b3cabefdf95e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 11:42:22 -0400
Subject: [PATCH 0796/2835] use a newtype for Key

should be more efficient
---
 BackendTypes.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BackendTypes.hs b/BackendTypes.hs
index c4927ab20c..c0705a550f 100644
--- a/BackendTypes.hs
+++ b/BackendTypes.hs
@@ -14,7 +14,7 @@ import Test.QuickCheck
 
 type KeyName = String
 type BackendName = String
-data Key = Key (BackendName, KeyName) deriving (Eq, Ord)
+newtype Key = Key (BackendName, KeyName) deriving (Eq, Ord)
 
 data Backend a = Backend {
 	-- name of this backend

From d007e58a5437daa72062453fc6f44c5beb7d271d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 13:52:11 -0400
Subject: [PATCH 0797/2835] use mapM_

---
 Command/Unused.hs | 2 +-
 GitQueue.hs       | 2 +-
 Remotes.hs        | 2 +-
 Upgrade.hs        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 28a26f82c6..fecfec7422 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -88,7 +88,7 @@ unusedKeys = do
 
 	-- Tmp files that are dups of content already present can simply
 	-- be removed.
-	_ <- liftIO $ mapM (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
+	liftIO $ mapM_ (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
 
 	return (unused, staletmp)
 
diff --git a/GitQueue.hs b/GitQueue.hs
index d8ba861366..0cb64caefb 100644
--- a/GitQueue.hs
+++ b/GitQueue.hs
@@ -45,7 +45,7 @@ add queue subcommand params file = M.insertWith (++) action [file] queue
 {- Runs a queue on a git repository. -}
 run :: Git.Repo -> Queue -> IO ()
 run repo queue = do
-	_ <- mapM (uncurry $ runAction repo) $ M.toList queue
+	mapM_ (uncurry $ runAction repo) $ M.toList queue
 	return ()
 
 {- Runs an Action on a list of files in a git repository.
diff --git a/Remotes.hs b/Remotes.hs
index 616db225ef..9f1e2ee507 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -104,7 +104,7 @@ readConfigs = do
 				list doexpensive ++ "..."
 		let todo = cheap ++ doexpensive
 		unless (null todo) $ do
-			_ <- mapM tryGitConfigRead todo
+			mapM_ tryGitConfigRead todo
 			Annex.changeState $ \s -> s { Annex.remotesread = True }
 	where
 		cachedUUID r = do
diff --git a/Upgrade.hs b/Upgrade.hs
index 1e70e68d56..9c5a57a0c2 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -41,7 +41,7 @@ upgradeFrom0 = do
 	-- do the reorganisation of the files
 	let olddir = gitAnnexDir g
 	keys <- getKeysPresent0' olddir
-	_ <- mapM (\k -> moveAnnex k $ olddir  keyFile k) keys
+	mapM_ (\k -> moveAnnex k $ olddir  keyFile k) keys
 
 	-- update the symlinks to the files
 	files <- liftIO $ Git.inRepo g [Git.workTree g]

From 9fe5865a07ac5a66c7ecaad4a5682b205939735d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 16:06:27 -0400
Subject: [PATCH 0798/2835] annoyance

---
 doc/bugs/ordering.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/ordering.mdwn

diff --git a/doc/bugs/ordering.mdwn b/doc/bugs/ordering.mdwn
new file mode 100644
index 0000000000..902cf7676c
--- /dev/null
+++ b/doc/bugs/ordering.mdwn
@@ -0,0 +1,10 @@
+One would expect "git annex get foo bar" to first retrieve foo, and then
+bar. Actually though, it will operate on them in alphabetical order
+(probably). This is annoying when you wanted to 1st list the most important
+files to get. Maybe you'll run out of time before all can be gotten. The
+workaround of course is to run "git annex get" twice.
+
+This ordering comes from "git ls-files". git-annex passes it all the files
+the user specified. This is a useful optimisation -- earlier it would
+run "git ls-files" once per parameter, and so "git annex get *" could be
+rather slow. But, it produces this ordering problem.

From 37c62eebb72fa0c216336435669cf8a05c2dbc88 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 20:06:34 -0400
Subject: [PATCH 0799/2835] Preserve specified file ordering when instructed to
 act on multiple files or directories.

---
 Command.hs             | 47 ++++++++++++++++++++++++++++++++++++------
 debian/changelog       |  7 +++++++
 doc/bugs/ordering.mdwn |  2 ++
 3 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/Command.hs b/Command.hs
index 1c1ff6bbcf..859f713a0c 100644
--- a/Command.hs
+++ b/Command.hs
@@ -14,6 +14,7 @@ import Control.Monad (filterM)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
+import System.Path
 
 import Types
 import qualified Backend
@@ -108,20 +109,20 @@ isAnnexed file a = do
 withFilesInGit :: CommandSeekStrings
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo repo params
+	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	return $ map a files'
 withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo repo params
+	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	pairs <- liftIO $ Git.checkAttr repo attr files'
 	return $ map a pairs
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo repo params
+	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	backendPairs a files'
 withFilesMissing :: CommandSeekStrings
@@ -136,7 +137,7 @@ withFilesMissing a params = do
 withFilesNotInGit :: CommandSeekBackendFiles
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
-	newfiles <- liftIO $ Git.notInRepo repo params
+	newfiles <- liftIO $ runPreserverOrder (Git.notInRepo repo) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
 withString :: CommandSeekStrings
@@ -146,7 +147,7 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ Git.stagedFiles repo params
+	tocommit <- liftIO $ runPreserverOrder (Git.stagedFiles repo) params
 	tocommit' <- filterFiles tocommit
 	return $ map a tocommit'
 withFilesUnlocked :: CommandSeekBackendFiles
@@ -157,7 +158,7 @@ withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBa
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
 	repo <- Annex.gitRepo
-	typechangedfiles <- liftIO $ typechanged repo params
+	typechangedfiles <- liftIO $ runPreserverOrder (typechanged repo) params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
 	unlockedfiles' <- filterFiles unlockedfiles
@@ -238,3 +239,37 @@ cmdlineKey  = do
 		keyname' (Just n) = n
 		badkey = error "please specify the key with --key"
 
+{- Given an original list of files, and an expanded list derived from it,
+ - ensures that the original list's ordering is preserved. 
+ -
+ - The input list may contain a directory, like "dir" or "dir/". Any
+ - items in the expanded list that are contained in that directory will
+ - appear at the same position as it did in the input list.
+ -}
+preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]
+-- optimisation, only one item in original list, so no reordering needed
+preserveOrder [_] new = new
+preserveOrder orig new = collect orig new
+	where
+		collect [] n = n
+		collect [_] n = n -- optimisation
+		collect (l:ls) n = found ++ collect ls rest
+			where (found, rest)=partition (l `dirContains`) n
+
+runPreserverOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
+runPreserverOrder a files = do
+	r <- a files
+	return $ preserveOrder files r
+
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
+	where
+		norm p = case (absNormPath p ".") of
+			Just r -> r
+			Nothing -> ""
+		a' = norm a
+		b' = norm b
diff --git a/debian/changelog b/debian/changelog
index 7abdee85cb..eee71a5e94 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (0.20) UNRELEASED; urgency=low
+
+  * Preserve specified file ordering when instructed to act on multiple
+    files or directories.
+
+ -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
+
 git-annex (0.19) unstable; urgency=low
 
   * configure: Support using the uuidgen command if the uuid command is
diff --git a/doc/bugs/ordering.mdwn b/doc/bugs/ordering.mdwn
index 902cf7676c..536bfce36a 100644
--- a/doc/bugs/ordering.mdwn
+++ b/doc/bugs/ordering.mdwn
@@ -8,3 +8,5 @@ This ordering comes from "git ls-files". git-annex passes it all the files
 the user specified. This is a useful optimisation -- earlier it would
 run "git ls-files" once per parameter, and so "git annex get *" could be
 rather slow. But, it produces this ordering problem.
+
+[[done]]

From 27056daccd1a2f541cd104a835a32523a532d4da Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 20:14:08 -0400
Subject: [PATCH 0800/2835] cleanup last change

---
 Command.hs       | 38 ++++++++++++++++----------------------
 Utility.hs       | 15 +++++++++++++++
 debian/changelog |  3 ++-
 3 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/Command.hs b/Command.hs
index 859f713a0c..0bbc6088c5 100644
--- a/Command.hs
+++ b/Command.hs
@@ -14,7 +14,6 @@ import Control.Monad (filterM)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
-import System.Path
 
 import Types
 import qualified Backend
@@ -22,6 +21,7 @@ import Messages
 import qualified Annex
 import qualified GitRepo as Git
 import Locations
+import Utility
 
 {- A command runs in four stages.
  -
@@ -109,20 +109,20 @@ isAnnexed file a = do
 withFilesInGit :: CommandSeekStrings
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	return $ map a files'
 withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	pairs <- liftIO $ Git.checkAttr repo attr files'
 	return $ map a pairs
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	backendPairs a files'
 withFilesMissing :: CommandSeekStrings
@@ -137,7 +137,7 @@ withFilesMissing a params = do
 withFilesNotInGit :: CommandSeekBackendFiles
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
-	newfiles <- liftIO $ runPreserverOrder (Git.notInRepo repo) params
+	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
 withString :: CommandSeekStrings
@@ -147,7 +147,7 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ runPreserverOrder (Git.stagedFiles repo) params
+	tocommit <- liftIO $ runPreserveOrder (Git.stagedFiles repo) params
 	tocommit' <- filterFiles tocommit
 	return $ map a tocommit'
 withFilesUnlocked :: CommandSeekBackendFiles
@@ -158,7 +158,7 @@ withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBa
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
 	repo <- Annex.gitRepo
-	typechangedfiles <- liftIO $ runPreserverOrder (typechanged repo) params
+	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
 	unlockedfiles' <- filterFiles unlockedfiles
@@ -256,20 +256,14 @@ preserveOrder orig new = collect orig new
 		collect (l:ls) n = found ++ collect ls rest
 			where (found, rest)=partition (l `dirContains`) n
 
-runPreserverOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
-runPreserverOrder a files = do
+{- Runs an action that takes a list of FilePaths, and ensures that 
+ - its return list preserves order.
+ -
+ - This assumes that it's cheaper to call preserveOrder on the result,
+ - than it would be to run the action once with each param. In the case
+ - of git file list commands, that assumption tends to hold.
+ -}
+runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
+runPreserveOrder a files = do
 	r <- a files
 	return $ preserveOrder files r
-
-{- Checks if the first FilePath is, or could be said to contain the second.
- - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
- - are all equivilant.
- -}
-dirContains :: FilePath -> FilePath -> Bool
-dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
-	where
-		norm p = case (absNormPath p ".") of
-			Just r -> r
-			Nothing -> ""
-		a' = norm a
-		b' = norm b
diff --git a/Utility.hs b/Utility.hs
index 96bbc89ee2..2bb623532d 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -18,6 +18,7 @@ module Utility (
 	unsetFileMode,
 	readMaybe,
 	safeWriteFile,
+	dirContains,
 	
 	prop_idempotent_shellEscape,
 	prop_idempotent_shellEscape_multiword,
@@ -36,6 +37,7 @@ import System.Path
 import System.FilePath
 import System.Directory
 import Foreign (complement)
+import Data.List
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -65,6 +67,19 @@ prop_parentDir_basics dir
 	where
 		p = parentDir dir
 
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
+	where
+		norm p = case (absNormPath p ".") of
+			Just r -> r
+			Nothing -> ""
+		a' = norm a
+		b' = norm b
+
 {- Converts a filename into a normalized, absolute path. -}
 absPath :: FilePath -> IO FilePath
 absPath file = do
diff --git a/debian/changelog b/debian/changelog
index eee71a5e94..d7edc1733f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,8 @@
 git-annex (0.20) UNRELEASED; urgency=low
 
   * Preserve specified file ordering when instructed to act on multiple
-    files or directories.
+    files or directories. For example, "git annex get a b" will now always
+    get "a" before "b". Previously it could operate in either order.
 
  -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
 

From 755029ae0e46e9d4ca2b7416d5a67b2be34eec0a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 23:27:53 -0400
Subject: [PATCH 0801/2835] use forM_ in a few places

---
 Command/Unused.hs | 4 ++--
 GitQueue.hs       | 4 ++--
 Upgrade.hs        | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index fecfec7422..d9f4e39783 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -7,7 +7,7 @@
 
 module Command.Unused where
 
-import Control.Monad (filterM, unless)
+import Control.Monad (filterM, unless, forM_)
 import Control.Monad.State (liftIO)
 import qualified Data.Set as S
 import Data.Maybe
@@ -88,7 +88,7 @@ unusedKeys = do
 
 	-- Tmp files that are dups of content already present can simply
 	-- be removed.
-	liftIO $ mapM_ (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
+	liftIO $ forM_ duptmp $ \t -> removeFile $ gitAnnexTmpLocation g t
 
 	return (unused, staletmp)
 
diff --git a/GitQueue.hs b/GitQueue.hs
index 0cb64caefb..4a777af4db 100644
--- a/GitQueue.hs
+++ b/GitQueue.hs
@@ -16,7 +16,7 @@ import qualified Data.Map as M
 import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
-import Control.Monad (unless)
+import Control.Monad (unless, forM_)
 
 import qualified GitRepo as Git
 
@@ -45,7 +45,7 @@ add queue subcommand params file = M.insertWith (++) action [file] queue
 {- Runs a queue on a git repository. -}
 run :: Git.Repo -> Queue -> IO ()
 run repo queue = do
-	mapM_ (uncurry $ runAction repo) $ M.toList queue
+	forM_ (M.toList queue) $ uncurry $ runAction repo
 	return ()
 
 {- Runs an Action on a list of files in a git repository.
diff --git a/Upgrade.hs b/Upgrade.hs
index 9c5a57a0c2..b584b2666f 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -10,7 +10,7 @@ module Upgrade where
 import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
-import Control.Monad (filterM)
+import Control.Monad (filterM, forM_)
 import System.Posix.Files
 import System.FilePath
 
@@ -41,7 +41,7 @@ upgradeFrom0 = do
 	-- do the reorganisation of the files
 	let olddir = gitAnnexDir g
 	keys <- getKeysPresent0' olddir
-	mapM_ (\k -> moveAnnex k $ olddir  keyFile k) keys
+	forM_ keys $ \k -> moveAnnex k $ olddir  keyFile k
 
 	-- update the symlinks to the files
 	files <- liftIO $ Git.inRepo g [Git.workTree g]

From 3c13f62f11d1aca82ea48cfa9cf66c4bd88a7b7b Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Tue, 1 Feb 2011 23:39:12 +0000
Subject: [PATCH 0802/2835] getting git-annex files back to git

---
 doc/forum/unannex_alternatives.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/unannex_alternatives.mdwn

diff --git a/doc/forum/unannex_alternatives.mdwn b/doc/forum/unannex_alternatives.mdwn
new file mode 100644
index 0000000000..efd05838e6
--- /dev/null
+++ b/doc/forum/unannex_alternatives.mdwn
@@ -0,0 +1,9 @@
+what is the work flow to get a file that is in git-annex out of there and into git? (current situation: `git-annex add`ed a bunch of pictures, later found make files in there which i'd rather have in git for proper source code control)
+
+the most intuitive thing to do is `git unannex`, which at first seemed to do the right thing, but when committing there came the hook and everything was back to where it was before.
+
+i could disable the hook as a workaround, but that doesn't smell like a good work flow.
+
+the [[man page|git-annex]] does warn that `unannex` is only supposed to be used against unintentional `git annex add`s (probably meaning that it should be used before something is committed), but the alternatives it suggests (`git rm` and `git annex drop`) don't to what i want to do.
+
+am i missing something or is there really no work flow for this? --[[chrysn]]

From 97d1cba498c12c0caac919d9763884646de04aec Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 2 Feb 2011 00:39:10 +0000
Subject: [PATCH 0803/2835] Added a comment

---
 ..._dcd4cd41280b41512bbdffafaf307993._comment | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment

diff --git a/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment b/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment
new file mode 100644
index 0000000000..7f278d2bc9
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment
@@ -0,0 +1,46 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-02-02T00:39:10Z"
+ content="""
+Git-annex's commit hook does not prevent unannex being used. The file you unannex will not be checked into git anymore and will be a regular file again, not a git-annex symlink. 
+
+For example, here's a transcript:
+
+
+joey@gnu:~/tmp>mkdir demo
+joey@gnu:~/tmp>cd demo
+joey@gnu:~/tmp/demo>git init
+Initialized empty Git repository in /home/joey/tmp/demo/.git/
+joey@gnu:~/tmp/demo>git annex init demo
+init demo ok
+joey@gnu:~/tmp/demo>echo hi > file
+joey@gnu:~/tmp/demo>git annex add file 
+add file ok
+(Recording state in git...)
+joey@gnu:~/tmp/demo>git commit -m add
+[master 64cf267] add
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 100644 .git-annex/WORM:1296607093:3:file.log
+ create mode 120000 file
+joey@gnu:~/tmp/demo>git annex unannex file
+unannex file ok
+(Recording state in git...)
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git commit
+[master 78a09cc] unannex
+ 2 files changed, 1 insertions(+), 2 deletions(-)
+ delete mode 120000 file
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git status
+# On branch master
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	file
+nothing added to commit but untracked files present (use \"git add\" to track)
+

+"""]]

From b9f40099866201a23b998fcc9f15fce859907228 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 2 Feb 2011 00:41:24 +0000
Subject: [PATCH 0804/2835] Added a comment

---
 ..._58a72a9fe0f58c7af0b4d7927a2dd21d._comment | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment

diff --git a/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment b/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment
new file mode 100644
index 0000000000..91ddadf8c6
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-02-02T00:41:24Z"
+ content="""
+And following on to my transcript, you can then add the file to git in the regular git way, and it works fine:
+
+
+joey@gnu:~/tmp/demo>git add file
+joey@gnu:~/tmp/demo>git commit
+[master 225ffc0] added as regular git file, not in annex
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 100644 file
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git log file
+commit 225ffc048f5af7c0466b3b1fe549a6d5e9a9e9fe
+Author: Joey Hess 
+Date:   Tue Feb 1 20:43:13 2011 -0400
+
+    added as regular git file, not in annex
+
+commit 78a09cc791b875c3b859ca9401e5b6472bf19d08
+Author: Joey Hess 
+Date:   Tue Feb 1 20:38:30 2011 -0400
+
+    unannex
+
+commit 64cf267734adae05c020d9fd4d5a7ff7c64390db
+Author: Joey Hess 
+Date:   Tue Feb 1 20:38:18 2011 -0400
+
+    add
+

+"""]]

From 513da0f1ef4a1e05f4b4f9461b2eb1fad31fcb43 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 2 Feb 2011 00:46:00 +0000
Subject: [PATCH 0805/2835] Added a comment

---
 ...mment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment

diff --git a/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
new file mode 100644
index 0000000000..548f7aacbb
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-02-02T00:46:00Z"
+ content="""
+Sorry for all the followups, but I see now that if you unannex, then add the file to git normally, and commit, the hook *does* misbehave.
+
+This seems to be a bug. git-annex's hook thinks that you have used git annex unlock (or \"git annex edit\") on the file and are now committing a changed version, and the right thing to do there is to add the new content to the annex and update the symlink accordingly. I'll track this bug over at [[bugs/unannex_vs_unlock_hook_confusion]].
+
+So, committing after unannex, and before checking the file into git in the usual way, is a workaround.
+"""]]

From 3ceef4410c49100b603bc41e2c8f9487ea093274 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 2 Feb 2011 00:47:43 +0000
Subject: [PATCH 0806/2835]

---
 doc/bugs/unannex_vs_unlock_hook_confusion._comment | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/unannex_vs_unlock_hook_confusion._comment

diff --git a/doc/bugs/unannex_vs_unlock_hook_confusion._comment b/doc/bugs/unannex_vs_unlock_hook_confusion._comment
new file mode 100644
index 0000000000..8a63303775
--- /dev/null
+++ b/doc/bugs/unannex_vs_unlock_hook_confusion._comment
@@ -0,0 +1,9 @@
+See [[forum/unannex_alternatives]] for problem description.
+
+If an unannex is followed by a "git add; git commit", git-annex's hook thinks
+that you have used git annex unlock on the file and are
+now committing a changed version, and the right thing to do there is to add the
+new content to the annex and update the symlink accordingly.
+
+Can we tell the difference between an unannexed file that has yet to be committed
+and an unlocked file? --[[Joey||

From ed089c22554ca5b3cd8c8f532db4a9fcc65385d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 Feb 2011 21:01:43 -0400
Subject: [PATCH 0807/2835] rename (caused by ikiwiki bug, now fixed)

---
 ...k_confusion._comment => unannex_vs_unlock_hook_confusion.mdwn} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/bugs/{unannex_vs_unlock_hook_confusion._comment => unannex_vs_unlock_hook_confusion.mdwn} (100%)

diff --git a/doc/bugs/unannex_vs_unlock_hook_confusion._comment b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
similarity index 100%
rename from doc/bugs/unannex_vs_unlock_hook_confusion._comment
rename to doc/bugs/unannex_vs_unlock_hook_confusion.mdwn

From 9aecf4110afb2fb1f6b10a9a9234ad0afafda56e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 Feb 2011 21:04:43 -0400
Subject: [PATCH 0808/2835] followup

---
 doc/bugs/unannex_vs_unlock_hook_confusion.mdwn | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
index 8a63303775..7e4a9f2914 100644
--- a/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
+++ b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
@@ -6,4 +6,8 @@ now committing a changed version, and the right thing to do there is to add the
 new content to the annex and update the symlink accordingly.
 
 Can we tell the difference between an unannexed file that has yet to be committed
-and an unlocked file? --[[Joey||
+and has been re-added as a normal file, vs an unlocked file? --[[Joey||
+
+> Hmm, not really. An unannexed file's content will have been dropped from
+> the backend, but that's about the only difference. Perhaps unannex should
+> just commit the removal of the file itself? --[[Joey]]

From c77ac11acc10efc23acfa3d81e1deaac11cb724f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 Feb 2011 21:26:19 -0400
Subject: [PATCH 0809/2835] unannex: Commit staged changes at end

to avoid some confusing behavior with the pre-commit hook, which would see
some types of commits after an unannex as checking in of an unlocked file.
---
 Command/Unannex.hs                                    |  7 ++++++-
 debian/changelog                                      |  3 +++
 doc/bugs/unannex_vs_unlock_hook_confusion.mdwn        |  2 ++
 ...omment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment |  6 +++++-
 doc/git-annex.mdwn                                    | 11 +++++++----
 5 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 4134439697..3810cca1c2 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -55,5 +55,10 @@ cleanup file key = do
 
 	fromAnnex key file
 	logStatus key ValueMissing
-
+	
+	-- Commit staged changes at end to avoid confusing the
+	-- pre-commit hook if this file is later added back to
+	-- git as a normal, non-annexed file.
+	Annex.queue "commit" ["-m", "content removed from git annex"] "--"
+	
 	return True
diff --git a/debian/changelog b/debian/changelog
index d7edc1733f..6bfc946442 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (0.20) UNRELEASED; urgency=low
   * Preserve specified file ordering when instructed to act on multiple
     files or directories. For example, "git annex get a b" will now always
     get "a" before "b". Previously it could operate in either order.
+  * unannex: Commit staged changes at end, to avoid some confusing behavior
+    with the pre-commit hook, which would see some types of commits after
+    an unannex as checking in of an unlocked file.
 
  -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
 
diff --git a/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
index 7e4a9f2914..c03990c203 100644
--- a/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
+++ b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
@@ -11,3 +11,5 @@ and has been re-added as a normal file, vs an unlocked file? --[[Joey||
 > Hmm, not really. An unannexed file's content will have been dropped from
 > the backend, but that's about the only difference. Perhaps unannex should
 > just commit the removal of the file itself? --[[Joey]]
+
+> [[done]], staged changes committed at end.
diff --git a/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
index 548f7aacbb..9f3223578b 100644
--- a/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
+++ b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
@@ -8,5 +8,9 @@ Sorry for all the followups, but I see now that if you unannex, then add the fil
 
 This seems to be a bug. git-annex's hook thinks that you have used git annex unlock (or \"git annex edit\") on the file and are now committing a changed version, and the right thing to do there is to add the new content to the annex and update the symlink accordingly. I'll track this bug over at [[bugs/unannex_vs_unlock_hook_confusion]].
 
-So, committing after unannex, and before checking the file into git in the usual way, is a workaround.
+So, committing after unannex, and before checking the file into git in the
+usual way, is a workaround. But only if you do a "git commit" to commit
+staged changes.
+
+Anyway, this confusing point is fixed in git now!
 """]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 83a286b0e1..68a1672df0 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -156,10 +156,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * unannex [path ...]
 
-  Use this to undo an accidental add command. This is not the command you
-  should use if you intentionally annexed a file and don't want its contents
-  any more. In that case you should use `git annex drop` instead, and you
-  can also `git rm` the file.
+  Use this to undo an accidental `git annex add` command. You can use
+  `git annex unannex` to move content out of the annex at any point,
+  even if you've already committed it.
+
+  This is not the command you should use if you intentionally annexed a
+  file and don't want its contents any more. In that case you should use
+  `git annex drop` instead, and you can also `git rm` the file.
 
 * uninit
 

From 0e7984a79354135f265d2342608953104d15db2e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 Feb 2011 21:58:47 -0400
Subject: [PATCH 0810/2835] add check for unclean tree

---
 .git-annex/uuid.log |  1 -
 Command.hs          |  2 +-
 Command/Unannex.hs  |  6 ++++++
 GitRepo.hs          | 17 +++++++++++++----
 4 files changed, 20 insertions(+), 6 deletions(-)
 delete mode 100644 .git-annex/uuid.log

diff --git a/.git-annex/uuid.log b/.git-annex/uuid.log
deleted file mode 100644
index 8cd9452b64..0000000000
--- a/.git-annex/uuid.log
+++ /dev/null
@@ -1 +0,0 @@
-1ac368a4-19e2-11e0-8c0f-8fcd42cf5a8d test repo
diff --git a/Command.hs b/Command.hs
index 0bbc6088c5..601b584642 100644
--- a/Command.hs
+++ b/Command.hs
@@ -147,7 +147,7 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ runPreserveOrder (Git.stagedFiles repo) params
+	tocommit <- liftIO $ runPreserveOrder (Git.stagedFilesNotDeleted repo) params
 	tocommit' <- filterFiles tocommit
 	return $ map a tocommit'
 withFilesUnlocked :: CommandSeekBackendFiles
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 3810cca1c2..c663b29ab0 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -8,6 +8,7 @@
 module Command.Unannex where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (unless)
 import System.Directory
 
 import Command
@@ -32,6 +33,11 @@ start file = isAnnexed file $ \(key, backend) -> do
 	ishere <- inAnnex key
 	if ishere
 		then do
+			g <- Annex.gitRepo
+			staged <- liftIO $ Git.stagedFiles g [Git.workTree g]
+			unless (null staged) $
+				error "This command cannot be run when there are already files staged for commit."
+
 			showStart "unannex" file
 			return $ Just $ perform file key backend
 		else return Nothing
diff --git a/GitRepo.hs b/GitRepo.hs
index 4e69544d43..031a9cbe21 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -38,6 +38,7 @@ module GitRepo (
 	inRepo,
 	notInRepo,
 	stagedFiles,
+	stagedFilesNotDeleted,
 	changedUnstagedFiles,
 	checkAttr,
 	decodeGitFile,
@@ -243,12 +244,20 @@ notInRepo :: Repo -> [FilePath] -> IO [FilePath]
 notInRepo repo l = pipeNullSplit repo $
 	["ls-files", "--others", "--exclude-standard", "-z", "--"] ++ l
 
+{- Returns a list of all files that are staged for commit. -}
+stagedFiles :: Repo -> [FilePath] -> IO [FilePath]
+stagedFiles repo l = stagedFiles' repo l []
+
 {- Returns a list of the files, staged for commit, that are being added,
  - moved, or changed (but not deleted), from the specified locations. -}
-stagedFiles :: Repo -> [FilePath] -> IO [FilePath]
-stagedFiles repo l = pipeNullSplit repo $
-	["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", 
-		"--"] ++ l
+stagedFilesNotDeleted :: Repo -> [FilePath] -> IO [FilePath]
+stagedFilesNotDeleted repo l = stagedFiles' repo l ["--diff-filter=ACMRT"]
+
+stagedFiles' :: Repo -> [FilePath] -> [String] -> IO [FilePath]
+stagedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+	where
+		start = ["diff", "--cached", "--name-only", "-z"]
+		end = ["--"] ++ l
 
 {- Returns a list of files that have unstaged changes. -}
 changedUnstagedFiles :: Repo -> [FilePath] -> IO [FilePath]

From 8f0b86dab5db2882aefd2b3653de186b94fec4f4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
 
Date: Thu, 3 Feb 2011 14:07:52 +0000
Subject: [PATCH 0811/2835]

---
 doc/bugs/Problems_running_make_on_osx.mdwn | 29 ++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx.mdwn

diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn
new file mode 100644
index 0000000000..6708981b90
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx.mdwn
@@ -0,0 +1,29 @@
+Followed the instructions over here: http://git-annex.branchable.com/forum/git-annex_on_OSX/
+
+and had to install the following extra packages to be able to get make to start:
+
+[realizes pcre-light is needed but pcre not installed on my mac]
+sudo port install pcre
+sudo cabal install pcre-light
+
+But then I got the following error:
+
+ghc -O2 -Wall --make git-annex
+[ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o 
+
+BackendTypes.hs:71:17:
+    No instance for (Arbitrary Char)
+      arising from a use of `arbitrary' at BackendTypes.hs:71:17-25
+    Possible fix: add an instance declaration for (Arbitrary Char)
+    In a stmt of a 'do' expression: backendname <- arbitrary
+    In the expression:
+        do backendname <- arbitrary
+           keyname <- arbitrary
+             return $ Key (backendname, keyname)
+    In the definition of `arbitrary':
+        arbitrary = do backendname <- arbitrary
+                       keyname <- arbitrary
+                         return $ Key (backendname, keyname)
+make: *** [git-annex] Error 1
+
+My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary so any help would be appreciated.

From 41656301ef734a9dd810ec77adb1fd1ff350fd39 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
 
Date: Thu, 3 Feb 2011 14:08:38 +0000
Subject: [PATCH 0812/2835]

---
 doc/bugs/Problems_running_make_on_osx.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn
index 6708981b90..d14c058e41 100644
--- a/doc/bugs/Problems_running_make_on_osx.mdwn
+++ b/doc/bugs/Problems_running_make_on_osx.mdwn
@@ -5,7 +5,6 @@ and had to install the following extra packages to be able to get make to start:
 [realizes pcre-light is needed but pcre not installed on my mac]
 sudo port install pcre
 sudo cabal install pcre-light
-
 But then I got the following error:
 
 ghc -O2 -Wall --make git-annex

From 3595abe1ca77c7942bcee420e9058b99b198708c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
 
Date: Thu, 3 Feb 2011 14:10:28 +0000
Subject: [PATCH 0813/2835]

---
 doc/bugs/Problems_running_make_on_osx.mdwn | 40 +++++++++++-----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn
index d14c058e41..96626cc376 100644
--- a/doc/bugs/Problems_running_make_on_osx.mdwn
+++ b/doc/bugs/Problems_running_make_on_osx.mdwn
@@ -2,27 +2,27 @@ Followed the instructions over here: http://git-annex.branchable.com/forum/git-a
 
 and had to install the following extra packages to be able to get make to start:
 
-[realizes pcre-light is needed but pcre not installed on my mac]
-sudo port install pcre
-sudo cabal install pcre-light
-But then I got the following error:
+[realizes pcre-light is needed but pcre not installed on my mac]  
+sudo port install pcre  
+sudo cabal install pcre-light  
+But then I got the following error:  
 
-ghc -O2 -Wall --make git-annex
-[ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o 
+ghc -O2 -Wall --make git-annex  
+[ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o   
 
-BackendTypes.hs:71:17:
-    No instance for (Arbitrary Char)
-      arising from a use of `arbitrary' at BackendTypes.hs:71:17-25
-    Possible fix: add an instance declaration for (Arbitrary Char)
-    In a stmt of a 'do' expression: backendname <- arbitrary
-    In the expression:
-        do backendname <- arbitrary
-           keyname <- arbitrary
-             return $ Key (backendname, keyname)
-    In the definition of `arbitrary':
-        arbitrary = do backendname <- arbitrary
-                       keyname <- arbitrary
-                         return $ Key (backendname, keyname)
-make: *** [git-annex] Error 1
+BackendTypes.hs:71:17:  
+    No instance for (Arbitrary Char)  
+      arising from a use of `arbitrary' at BackendTypes.hs:71:17-25  
+    Possible fix: add an instance declaration for (Arbitrary Char)  
+    In a stmt of a 'do' expression: backendname <- arbitrary  
+    In the expression:  
+        do backendname <- arbitrary  
+           keyname <- arbitrary  
+             return $ Key (backendname, keyname)  
+    In the definition of `arbitrary':  
+        arbitrary = do backendname <- arbitrary  
+                       keyname <- arbitrary  
+                         return $ Key (backendname, keyname)  
+make: *** [git-annex] Error 1  
 
 My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary so any help would be appreciated.

From d29187b1518e5821a8367b74315442f8557d5617 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 14:03:24 -0400
Subject: [PATCH 0814/2835] update instructions for pcre

---
 doc/forum/git-annex_on_OSX.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn
index 44c275ff1c..302ad10994 100644
--- a/doc/forum/git-annex_on_OSX.mdwn
+++ b/doc/forum/git-annex_on_OSX.mdwn
@@ -8,6 +8,8 @@ sudo port install haskell-platform git-core ossp-uuid md5sha1sum
 sudo cabal  update
 sudo cabal install missingh
 sudo cabal install utf8-string
+sudo port install pcre
+sudo cabal install pcre-light
 
 git clone  git://git.kitenet.net/git-annex
 

From b1caa49248a10a54b3c5c38acda11dd81ce60d11 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 14:11:28 -0400
Subject: [PATCH 0815/2835] workaround

---
 doc/bugs/Problems_running_make_on_osx.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn
index 96626cc376..bd0f2e64a8 100644
--- a/doc/bugs/Problems_running_make_on_osx.mdwn
+++ b/doc/bugs/Problems_running_make_on_osx.mdwn
@@ -5,8 +5,14 @@ and had to install the following extra packages to be able to get make to start:
 [realizes pcre-light is needed but pcre not installed on my mac]  
 sudo port install pcre  
 sudo cabal install pcre-light  
+
+> Ah right, that is a new dependency. I've updated the forum page
+> with this info.
+> --[[Joey]] 
+
 But then I got the following error:  
 
+
 ghc -O2 -Wall --make git-annex  
 [ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o   
 
@@ -24,5 +30,14 @@ BackendTypes.hs:71:17:
                        keyname <- arbitrary  
                          return $ Key (backendname, keyname)  
 make: *** [git-annex] Error 1  
+

 
 My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary so any help would be appreciated.
+
+> Hmm, it seems you may be missing part of the quickcheck haskell
+> library, or have a different version than me.
+> 
+> The easy fix is probably to just edit BackendTypes.hs and delete the
+> entire end of the file from line 68, "for quickcheck" down. This code
+> is only used by the test suite (so "make test" will fail), 
+> but it should get it to build. --[[Joey]]

From 14bc885de96dd3ec52ab33ec6bbb02974d0a381c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 18:47:14 -0400
Subject: [PATCH 0816/2835] more accessor functions and better bad url handling

---
 GitRepo.hs | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 031a9cbe21..b5a94d4263 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -16,11 +16,13 @@ module GitRepo (
 	repoIsUrl,
 	repoIsSsh,
 	repoDescribe,
+	repoLocation,
 	workTree,
 	gitDir,
 	relative,
 	urlPath,
 	urlHost,
+	urlScheme,
 	configGet,
 	configMap,
 	configRead,
@@ -101,7 +103,10 @@ repoFromUrl :: String -> Repo
 repoFromUrl url
 	| startswith "file://" url = repoFromPath $ uriPath u
 	| otherwise = newFrom $ Url u
-		where u = fromJust $ parseURI url
+		where
+			u = case (parseURI url) of
+				Just v -> v
+				Nothing -> error $ "bad url " ++ url
 
 {- User-visible description of a git repo. -}
 repoDescribe :: Repo -> String
@@ -109,6 +114,11 @@ repoDescribe Repo { remoteName = Just name } = name
 repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
 
+{- Location of the repo, either as a path or url. -}
+repoLocation :: Repo -> String
+repoLocation Repo { location = Url url } = show url
+repoLocation Repo { location = Dir dir } = dir
+
 {- Constructs and returns an updated version of a repo with
  - different remotes list. -}
 remotesAdd :: Repo -> [Repo] -> Repo
@@ -192,10 +202,16 @@ relative repo@(Repo { location = Dir d }) file = do
 			Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo
 relative repo _ = assertLocal repo $ error "internal"
 
+{- Scheme of an URL repo. -}
+urlScheme :: Repo -> String
+urlScheme Repo { location = Url u } = uriScheme u
+urlScheme repo = assertUrl repo $ error "internal"
+
 {- Hostname of an URL repo. (May include a username and/or port too.) -}
 urlHost :: Repo -> String
 urlHost Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a
-	where a = fromJust $ uriAuthority u
+	where
+		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
 urlHost repo = assertUrl repo $ error "internal"
 
 {- Path of an URL repo. -}

From 0c7d17ae062c136e549cc9800dae85f3e3793237 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 18:55:12 -0400
Subject: [PATCH 0817/2835] new map subcommand, basically working

Still todo:

- add repos from uuid.log that were not directly found
- group repos into their respective hosts
- display inaccessible repos and broken remote connections in red
- anonymize the url display somewhat, so the maps can be shared
- use uuid info to tell when two apparently different repos are actually
  the same repo accessed in different ways
---
 Command/Map.hs     | 153 +++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |   2 +
 Remotes.hs         |   7 ++-
 UUID.hs            |  10 ++-
 debian/changelog   |   2 +
 doc/git-annex.mdwn |  17 +++++
 6 files changed, 185 insertions(+), 6 deletions(-)
 create mode 100644 Command/Map.hs

diff --git a/Command/Map.hs b/Command/Map.hs
new file mode 100644
index 0000000000..753d6ebdcb
--- /dev/null
+++ b/Command/Map.hs
@@ -0,0 +1,153 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Map where
+
+import Control.Monad.State (liftIO)
+import Control.Exception.Extensible
+import System.Cmd.Utils
+
+import Command
+import qualified Annex
+import qualified GitRepo as Git
+import qualified Remotes
+import Messages
+import Types
+import Utility
+
+-- a link from the first repository to the second (its remote)
+data Link = Link Git.Repo Git.Repo
+
+command :: [Command]
+command = [Command "map" paramNothing seek "generate map of repositories"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStartNothing
+start = do
+	g <- Annex.gitRepo
+	rs <- spider g
+
+	liftIO $ writeFile file (dotGraph rs)
+	showLongNote $ "running: dot -Tx11 " ++ file ++ "\n"
+	r <- liftIO $ boolSystem "dot" ["-Tx11", file]
+	return $ Just $ return $ Just $ return r
+	where
+		file = "map.dot"
+
+{- Generates a graph for dot(1). Each repository is displayed
+ - as a node, and each of its remotes is represented as an edge
+ - pointing at the node for the remote. -}
+dotGraph :: [Git.Repo] -> String
+dotGraph rs = unlines $ [header] ++ map dotGraphRepo rs ++ [footer]
+	where
+		header = "digraph map {"
+		footer= "}"
+
+dotGraphRepo :: Git.Repo -> String
+dotGraphRepo r = unlines $ map dotline (node:edges)
+	where
+		node = nodename r ++
+			" [ label=" ++ dotquote (Git.repoDescribe r) ++ " ]"
+		edges = map edge (Git.remotes r)
+		edge e = nodename r ++ " -> " ++ nodename (makeabs r e)
+		nodename n = dotquote (Git.repoLocation n)
+		dotquote s = "\"" ++ s ++ "\""
+		dotline s = "\t" ++ s ++ ";"
+
+{- Recursively searches out remotes starting with the specified repo. -}
+spider :: Git.Repo -> Annex [Git.Repo]
+spider r = spider' [r] []
+spider' :: [Git.Repo] -> [Git.Repo] -> Annex [Git.Repo]
+spider' [] known = return known
+spider' (r:rs) known
+	| any (same r) known = spider' rs known
+	| otherwise = do
+		r' <- scan r
+		let remotes = map (makeabs r') (Git.remotes r')
+		spider' (rs ++ remotes) (r':known)
+
+{- Makes a remote have an absolute url, rather than a host-local path. -}
+makeabs :: Git.Repo -> Git.Repo -> Git.Repo
+makeabs repo remote
+	| Git.repoIsUrl remote = remote
+	| not $ Git.repoIsUrl repo = remote
+	| otherwise = Git.repoFromUrl combinedurl
+		where
+			combinedurl =
+				Git.urlScheme repo ++ "//" ++
+				Git.urlHost repo ++
+				Git.workTree remote
+
+{- Checks if two repos are the same. -}
+same :: Git.Repo -> Git.Repo -> Bool
+same a b
+	| both Git.repoIsSsh = matching Git.urlHost && matching Git.workTree
+	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
+	| otherwise = False
+		
+	where
+		matching t = t a == t b
+		both t = t a && t b
+		neither t = not (t a) && not (t b)
+
+{- reads the config of a remote, with progress display -}
+scan :: Git.Repo -> Annex Git.Repo
+scan r = do
+	showStart "map" (Git.repoDescribe r)
+	v <- tryScan r
+	case v of
+		Just r' -> do
+			showEndOk
+			return r'
+		Nothing -> do
+			showEndFail
+			return r
+
+{- tries to read the config of a remote, returning it only if it can
+ - be accessed -}
+tryScan :: Git.Repo -> Annex (Maybe Git.Repo)
+tryScan r
+	| Git.repoIsSsh r = sshscan
+	| Git.repoIsUrl r = return Nothing
+	| otherwise = safely $ Git.configRead r
+	where
+		safely a = do
+			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
+			case result of
+				Left _ -> return Nothing
+				Right r' -> return $ Just r'
+		pipedconfig cmd params = safely $
+			pOpen ReadFromPipe cmd params $
+				Git.hConfigRead r
+
+		configlist =
+			Remotes.onRemote r (pipedconfig, Nothing) "configlist" []
+		manualconfiglist = do
+			sshoptions <- Remotes.repoConfig r "ssh-options" ""
+			let sshcmd =
+				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
+				"git config --list"
+			liftIO $ pipedconfig "ssh" $
+				words sshoptions ++ [Git.urlHost r, sshcmd]
+
+		-- First, try sshing and running git config manually,
+		-- only fall back to git-annex-shell configlist if that
+		-- fails.
+		-- 
+		-- This is done for two reasons, first I'd like this
+		-- subcommand to be usable on non-git-annex repos.
+		-- Secondly, configlist doesn't include information about
+		-- the remote's remotes.
+		sshscan = do
+			showNote "sshing..."
+			showProgress
+			v <- manualconfiglist
+			case v of
+				Nothing -> configlist
+				ok -> return ok
diff --git a/GitAnnex.hs b/GitAnnex.hs
index b09ec82ffc..3be222874f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -38,6 +38,7 @@ import qualified Command.Uninit
 import qualified Command.Trust
 import qualified Command.Untrust
 import qualified Command.Semitrust
+import qualified Command.Map
 
 cmds :: [Command]
 cmds = concat
@@ -64,6 +65,7 @@ cmds = concat
 	, Command.DropUnused.command
 	, Command.Find.command
 	, Command.Migrate.command
+	, Command.Map.command
 	]
 
 options :: [Option]
diff --git a/Remotes.hs b/Remotes.hs
index 9f1e2ee507..89b4032475 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -7,6 +7,7 @@
 
 module Remotes (
 	list,
+	tryGitConfigRead,
 	readConfigs,
 	keyPossibilities,
 	inAnnex,
@@ -14,7 +15,8 @@ module Remotes (
 	byName,
 	copyFromRemote,
 	copyToRemote,
-	onRemote
+	onRemote,
+	repoConfig
 ) where
 
 import Control.Exception.Extensible
@@ -77,7 +79,6 @@ tryGitConfigRead r
 				then new : exchange ls new
 				else old : exchange ls new
 
-
 {- Reads the configs of all remotes.
  -
  - This has to be called before things that rely on eg, the UUID of
@@ -92,9 +93,9 @@ tryGitConfigRead r
  - -}
 readConfigs :: Annex ()
 readConfigs = do
-	g <- Annex.gitRepo
 	remotesread <- Annex.getState Annex.remotesread
 	unless remotesread $ do
+		g <- Annex.gitRepo
 		allremotes <- filterM repoNotIgnored $ Git.remotes g
 		let cheap = filter (not . Git.repoIsUrl) allremotes
 		let expensive = filter Git.repoIsUrl allremotes
diff --git a/UUID.hs b/UUID.hs
index a654424b4f..6c719b41ef 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -11,13 +11,15 @@
 module UUID (
 	UUID,
 	getUUID,
+	getUncachedUUID,
 	prepUUID,
 	genUUID,
 	reposByUUID,
 	reposWithoutUUID,
 	prettyPrintUUIDs,
 	describeUUID,
-	uuidLog
+	uuidLog,
+	uuidMap
 ) where
 
 import Control.Monad.State
@@ -60,7 +62,7 @@ getUUID r = do
 	g <- Annex.gitRepo
 
 	let c = cached g
-	let u = uncached
+	let u = getUncachedUUID r
 	
 	if c /= u && u /= ""
 		then do
@@ -68,11 +70,13 @@ getUUID r = do
 			return u
 		else return c
 	where
-		uncached = Git.configGet r "annex.uuid" ""
 		cached g = Git.configGet g cachekey ""
 		updatecache g u = when (g /= r) $ Annex.setConfig cachekey u
 		cachekey = "remote." ++ Git.repoRemoteName r ++ ".annex-uuid"
 
+getUncachedUUID :: Git.Repo -> UUID
+getUncachedUUID r = Git.configGet r "annex.uuid" ""
+
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
 prepUUID = do
diff --git a/debian/changelog b/debian/changelog
index 6bfc946442..42d45c3a3e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (0.20) UNRELEASED; urgency=low
   * unannex: Commit staged changes at end, to avoid some confusing behavior
     with the pre-commit hook, which would see some types of commits after
     an unannex as checking in of an unlocked file.
+  * map: New subcommand that uses graphviz to display a nice map of
+    the git repository network.
 
  -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 68a1672df0..d670d626ee 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -154,6 +154,23 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Note that the content is not removed from the backend it was previously in.
   Use `git annex unused` to find and remove such content. 
 
+* map
+
+  Helps you keep track of your repositories, and the connections between them,
+  by going out and looking at all the ones it can get to, and generating a
+  Graphviz file displaying it all. If the `dot` command is available, it is
+  used to display the file to your screen (using x11 backend).
+
+  Note that this only connects to hosts that the host it's run on can
+  directly connect to. It does not try to tunnel through intermediate hosts.
+  So it might not show all connections between the repositories in the network.
+
+  Also, if connecting to a host requires a password, you might have to enter
+  it several times as the map is being built.
+
+  Note that this subcommand can be used to graph any git repository; it
+  is not limited to git-annex repositories.
+
 * unannex [path ...]
 
   Use this to undo an accidental `git annex add` command. You can use

From 17829be0fd2ec090c2854f05856e91ca4359e71c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 22:20:55 -0400
Subject: [PATCH 0818/2835] map improvements

added uuid.log repos

group repos by host

avoid displaying most urls

display remote names on edges

still some bugs
---
 Command/Map.hs | 117 +++++++++++++++++++++++++++++++++++++++++--------
 GitRepo.hs     |  20 ++++++---
 Remotes.hs     |   8 ++--
 UUID.hs        |   3 +-
 4 files changed, 119 insertions(+), 29 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 753d6ebdcb..b89f8f89b4 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -10,6 +10,8 @@ module Command.Map where
 import Control.Monad.State (liftIO)
 import Control.Exception.Extensible
 import System.Cmd.Utils
+import qualified Data.Map as M
+import Data.List.Utils
 
 import Command
 import qualified Annex
@@ -18,6 +20,7 @@ import qualified Remotes
 import Messages
 import Types
 import Utility
+import UUID
 
 -- a link from the first repository to the second (its remote)
 data Link = Link Git.Repo Git.Repo
@@ -33,32 +36,110 @@ start = do
 	g <- Annex.gitRepo
 	rs <- spider g
 
-	liftIO $ writeFile file (dotGraph rs)
-	showLongNote $ "running: dot -Tx11 " ++ file ++ "\n"
+	umap <- uuidMap
+
+	liftIO $ writeFile file (drawMap rs umap)
+	showLongNote $ "running: dot -Tx11 " ++ file
+	showProgress
 	r <- liftIO $ boolSystem "dot" ["-Tx11", file]
 	return $ Just $ return $ Just $ return r
 	where
 		file = "map.dot"
 
-{- Generates a graph for dot(1). Each repository is displayed
- - as a node, and each of its remotes is represented as an edge
+{- Generates a graph for dot(1). Each repository, and any other uuids, are
+ - displayed as a node, and each of its remotes is represented as an edge
  - pointing at the node for the remote. -}
-dotGraph :: [Git.Repo] -> String
-dotGraph rs = unlines $ [header] ++ map dotGraphRepo rs ++ [footer]
+drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
+drawMap rs umap = dotGraph $ repos ++ others
+	where
+		repos = map (dotGraphRepo umap rs) rs
+		others = map uuidnode (M.keys umap)
+		uuidnode u = dotGraphNode u $ M.findWithDefault "" u umap
+
+dotGraphRepo :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
+dotGraphRepo umap fullinfo r = unlines $ node:edges
+	where
+		node = inhost $ dotGraphNode (nodeid r) (repoName umap r)
+		edges = map edge (Git.remotes r)
+
+		inhost a
+			| Git.repoIsUrl r = dotSubGraph hostname a
+			| otherwise = a
+
+		hostname = head $ split "." $ Git.urlHost r
+
+		edge to =
+			-- get the full info for the repo since its UUID
+			-- is in there
+			let to' = findfullinfo to
+			in dotGraphEdge
+				(nodeid r)
+				(nodeid $ makeabs r to')
+				(edgename to to')
+		
+		-- Only name an edge if the name is different than the name
+		-- that will be used for the destination node. (This
+		-- reduces visual clutter.)
+		edgename to to' =
+			case (Git.repoRemoteName to) of
+				Nothing -> Nothing
+				Just n ->
+					if (n == repoName umap to')
+						then Nothing
+						else Just n
+
+		nodeid n =
+			case (getUncachedUUID n) of
+				"" -> Git.repoLocation n
+				u -> u
+		findfullinfo n =
+			case (filter (same n) fullinfo) of
+				[] -> n
+				(n':_) -> n'
+
+repoName :: (M.Map UUID String) -> Git.Repo -> String
+repoName umap r
+	| null repouuid = fallback
+	| otherwise = M.findWithDefault fallback repouuid umap
+	where
+		repouuid = getUncachedUUID r
+		fallback =
+			case (Git.repoRemoteName r) of
+				Just n -> n
+				Nothing -> "unknown"
+
+dotGraphNode :: String -> String -> String
+dotGraphNode nodeid desc = dotLineLabeled desc $ dotQuote nodeid
+
+dotGraphEdge :: String -> String -> Maybe String -> String
+dotGraphEdge fromid toid d =
+	case d of
+		Nothing -> dotLine edge
+		Just desc -> dotLineLabeled desc edge
+	where
+		edge = dotQuote fromid ++ " -> " ++ dotQuote toid
+
+dotGraph :: [String] -> String
+dotGraph s = unlines $ [header] ++ s ++ [footer]
 	where
 		header = "digraph map {"
 		footer= "}"
 
-dotGraphRepo :: Git.Repo -> String
-dotGraphRepo r = unlines $ map dotline (node:edges)
+dotQuote :: String -> String
+dotQuote s = "\"" ++ s ++ "\""
+
+dotLine :: String -> String
+dotLine s = "\t" ++ s ++ ";"
+
+dotLineLabeled :: String -> String -> String
+dotLineLabeled label s = dotLine $ s ++ " [ label=" ++ dotQuote label ++ " ]"
+
+dotSubGraph :: String -> String -> String
+dotSubGraph label s = "subgraph " ++ name ++ "{ " ++ setlabel ++ s ++ " }"
 	where
-		node = nodename r ++
-			" [ label=" ++ dotquote (Git.repoDescribe r) ++ " ]"
-		edges = map edge (Git.remotes r)
-		edge e = nodename r ++ " -> " ++ nodename (makeabs r e)
-		nodename n = dotquote (Git.repoLocation n)
-		dotquote s = "\"" ++ s ++ "\""
-		dotline s = "\t" ++ s ++ ";"
+		-- the "cluster_" makes dot draw a box
+		name = dotQuote ("cluster_ " ++ label)
+		setlabel = dotLine $ "label=" ++ dotQuote label
 
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]
@@ -81,13 +162,13 @@ makeabs repo remote
 		where
 			combinedurl =
 				Git.urlScheme repo ++ "//" ++
-				Git.urlHost repo ++
+				Git.urlHostFull repo ++
 				Git.workTree remote
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
 same a b
-	| both Git.repoIsSsh = matching Git.urlHost && matching Git.workTree
+	| both Git.repoIsSsh = matching Git.urlHostFull && matching Git.workTree
 	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
 	| otherwise = False
 		
@@ -134,7 +215,7 @@ tryScan r
 				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
 				"git config --list"
 			liftIO $ pipedconfig "ssh" $
-				words sshoptions ++ [Git.urlHost r, sshcmd]
+				words sshoptions ++ [Git.urlHostFull r, sshcmd]
 
 		-- First, try sshing and running git config manually,
 		-- only fall back to git-annex-shell configlist if that
diff --git a/GitRepo.hs b/GitRepo.hs
index b5a94d4263..4b252868ff 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -22,6 +22,7 @@ module GitRepo (
 	relative,
 	urlPath,
 	urlHost,
+	urlHostFull,
 	urlScheme,
 	configGet,
 	configMap,
@@ -124,11 +125,11 @@ repoLocation Repo { location = Dir dir } = dir
 remotesAdd :: Repo -> [Repo] -> Repo
 remotesAdd repo rs = repo { remotes = rs }
 
-{- Returns the name of the remote that corresponds to the repo, if 
- - it is a remote. Otherwise, "" -}
-repoRemoteName :: Repo -> String
-repoRemoteName Repo { remoteName = Just name } = name
-repoRemoteName _ = ""
+{- Returns the name of the remote that corresponds to the repo, if
+ - it is a remote. -}
+repoRemoteName :: Repo -> Maybe String
+repoRemoteName Repo { remoteName = Just name } = Just name
+repoRemoteName _ = Nothing
 
 {- Some code needs to vary between URL and normal repos,
  - or bare and non-bare, these functions help with that. -}
@@ -209,11 +210,18 @@ urlScheme repo = assertUrl repo $ error "internal"
 
 {- Hostname of an URL repo. (May include a username and/or port too.) -}
 urlHost :: Repo -> String
-urlHost Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a
+urlHost Repo { location = Url u } = uriRegName a
 	where
 		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
 urlHost repo = assertUrl repo $ error "internal"
 
+{- Full hostname of an URL repo. (May include a username and/or port too.) -}
+urlHostFull :: Repo -> String
+urlHostFull Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a
+	where
+		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
+urlHostFull repo = assertUrl repo $ error "internal"
+
 {- Path of an URL repo. -}
 urlPath :: Repo -> String
 urlPath Repo { location = Url u } = uriPath u
diff --git a/Remotes.hs b/Remotes.hs
index 89b4032475..15f5185b91 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -205,7 +205,7 @@ repoNotIgnored r = do
 			name <- Annex.getState a
 			case name of
 				Nothing -> return False
-				Just n -> return $ n == Git.repoRemoteName r
+				n -> return $ n == Git.repoRemoteName r
 
 {- Checks if two repos are the same, by comparing their remote names. -}
 same :: Git.Repo -> Git.Repo -> Bool
@@ -217,7 +217,7 @@ byName "." = Annex.gitRepo -- special case to refer to current repository
 byName name = do
 	when (null name) $ error "no remote specified"
 	g <- Annex.gitRepo
-	let match = filter (\r -> name == Git.repoRemoteName r) $
+	let match = filter (\r -> Just name == Git.repoRemoteName r) $
 		Git.remotes g
 	when (null match) $ error $
 		"there is no git remote named \"" ++ name ++ "\""
@@ -309,7 +309,7 @@ git_annex_shell r command params
 	| Git.repoIsSsh r = do
 		sshoptions <- repoConfig r "ssh-options" ""
 		return $ Just $ ["ssh"] ++ words sshoptions ++ 
-			[Git.urlHost r, sshcmd]
+			[Git.urlHostFull r, sshcmd]
 	| otherwise = return Nothing
 	where
 		dir = Git.workTree r
@@ -325,5 +325,5 @@ repoConfig r key def = do
 	let def' = Git.configGet g global def
 	return $ Git.configGet g local def'
 	where
-		local = "remote." ++ Git.repoRemoteName r ++ ".annex-" ++ key
+		local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 		global = "annex." ++ key
diff --git a/UUID.hs b/UUID.hs
index 6c719b41ef..67cba30313 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -26,6 +26,7 @@ import Control.Monad.State
 import System.Cmd.Utils
 import System.IO
 import qualified Data.Map as M
+import Data.Maybe
 
 import qualified GitRepo as Git
 import Types
@@ -72,7 +73,7 @@ getUUID r = do
 	where
 		cached g = Git.configGet g cachekey ""
 		updatecache g u = when (g /= r) $ Annex.setConfig cachekey u
-		cachekey = "remote." ++ Git.repoRemoteName r ++ ".annex-uuid"
+		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
 getUncachedUUID r = Git.configGet r "annex.uuid" ""

From 1b1a37b7b1c9ee5c7f7d5c176222fc3c7e5e8ab4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 22:44:17 -0400
Subject: [PATCH 0819/2835] refactor

---
 Command/Map.hs | 52 ++++++++++----------------------------------------
 Dot.hs         | 47 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 42 deletions(-)
 create mode 100644 Dot.hs

diff --git a/Command/Map.hs b/Command/Map.hs
index b89f8f89b4..5b035e283f 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -21,6 +21,7 @@ import Messages
 import Types
 import Utility
 import UUID
+import qualified Dot
 
 -- a link from the first repository to the second (its remote)
 data Link = Link Git.Repo Git.Repo
@@ -50,41 +51,41 @@ start = do
  - displayed as a node, and each of its remotes is represented as an edge
  - pointing at the node for the remote. -}
 drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
-drawMap rs umap = dotGraph $ repos ++ others
+drawMap rs umap = Dot.graph $ repos ++ others
 	where
 		repos = map (dotGraphRepo umap rs) rs
 		others = map uuidnode (M.keys umap)
-		uuidnode u = dotGraphNode u $ M.findWithDefault "" u umap
+		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
 
 dotGraphRepo :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
 dotGraphRepo umap fullinfo r = unlines $ node:edges
 	where
-		node = inhost $ dotGraphNode (nodeid r) (repoName umap r)
+		node = inhost $ Dot.graphNode (nodeid r) (repoName umap r)
 		edges = map edge (Git.remotes r)
 
 		inhost a
-			| Git.repoIsUrl r = dotSubGraph hostname a
+			| Git.repoIsUrl r = Dot.subGraph (Git.urlHost r) (hostname r) a
 			| otherwise = a
 
-		hostname = head $ split "." $ Git.urlHost r
+		hostname n = head $ split "." $ Git.urlHost n
 
 		edge to =
 			-- get the full info for the repo since its UUID
 			-- is in there
 			let to' = findfullinfo to
-			in dotGraphEdge
+			in Dot.graphEdge
 				(nodeid r)
 				(nodeid $ makeabs r to')
 				(edgename to to')
 		
 		-- Only name an edge if the name is different than the name
-		-- that will be used for the destination node. (This
-		-- reduces visual clutter.)
+		-- that will be used for the destination node, and is
+		-- different from its hostname. (This reduces visual clutter.)
 		edgename to to' =
 			case (Git.repoRemoteName to) of
 				Nothing -> Nothing
 				Just n ->
-					if (n == repoName umap to')
+					if (n == repoName umap to' || n == hostname to')
 						then Nothing
 						else Just n
 
@@ -108,39 +109,6 @@ repoName umap r
 				Just n -> n
 				Nothing -> "unknown"
 
-dotGraphNode :: String -> String -> String
-dotGraphNode nodeid desc = dotLineLabeled desc $ dotQuote nodeid
-
-dotGraphEdge :: String -> String -> Maybe String -> String
-dotGraphEdge fromid toid d =
-	case d of
-		Nothing -> dotLine edge
-		Just desc -> dotLineLabeled desc edge
-	where
-		edge = dotQuote fromid ++ " -> " ++ dotQuote toid
-
-dotGraph :: [String] -> String
-dotGraph s = unlines $ [header] ++ s ++ [footer]
-	where
-		header = "digraph map {"
-		footer= "}"
-
-dotQuote :: String -> String
-dotQuote s = "\"" ++ s ++ "\""
-
-dotLine :: String -> String
-dotLine s = "\t" ++ s ++ ";"
-
-dotLineLabeled :: String -> String -> String
-dotLineLabeled label s = dotLine $ s ++ " [ label=" ++ dotQuote label ++ " ]"
-
-dotSubGraph :: String -> String -> String
-dotSubGraph label s = "subgraph " ++ name ++ "{ " ++ setlabel ++ s ++ " }"
-	where
-		-- the "cluster_" makes dot draw a box
-		name = dotQuote ("cluster_ " ++ label)
-		setlabel = dotLine $ "label=" ++ dotQuote label
-
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]
 spider r = spider' [r] []
diff --git a/Dot.hs b/Dot.hs
new file mode 100644
index 0000000000..1d9c29c532
--- /dev/null
+++ b/Dot.hs
@@ -0,0 +1,47 @@
+{- a simple graphviz / dot(1) digraph description generator library
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Dot where -- import qualified
+
+{- generates a graph description from a list of lines -}
+graph :: [String] -> String
+graph s = unlines $ [header] ++ s ++ [footer]
+	where
+		header = "digraph map {"
+		footer= "}"
+
+{- a node in the graph -}
+graphNode :: String -> String -> String
+graphNode nodeid desc = lineLabeled desc $ quote nodeid
+
+{- an edge between two nodes -}
+graphEdge :: String -> String -> Maybe String -> String
+graphEdge fromid toid d =
+	case d of
+		Nothing -> line edge
+		Just desc -> lineLabeled desc edge
+	where
+		edge = quote fromid ++ " -> " ++ quote toid
+
+quote :: String -> String
+quote s = "\"" ++ s ++ "\""
+
+line :: String -> String
+line s = "\t" ++ s ++ ";"
+
+{- a line with a label -}
+lineLabeled :: String -> String -> String
+lineLabeled label s = line $ s ++ " [ label=" ++ quote label ++ " ]"
+
+{- apply to graphNode to put the node in a labeled box -}
+subGraph :: String -> String -> String -> String
+subGraph subid label s = line $
+	"subgraph " ++ name ++ "{\n" ++ setlabel ++ "\n" ++ s ++ "\n}"
+	where
+		-- the "cluster_" makes dot draw a box
+		name = quote ("cluster_" ++ subid)
+		setlabel = line $ "label=" ++ quote label

From dff47d51e65fcf14566a06ebaae112c859d1824c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 23:23:16 -0400
Subject: [PATCH 0820/2835] cleanup

---
 Command/Map.hs | 88 +++++++++++++++++++++++++++-----------------------
 1 file changed, 47 insertions(+), 41 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 5b035e283f..1d38bc42fe 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -53,51 +53,20 @@ start = do
 drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
 drawMap rs umap = Dot.graph $ repos ++ others
 	where
-		repos = map (dotGraphRepo umap rs) rs
+		repos = map (node umap rs) rs
 		others = map uuidnode (M.keys umap)
 		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
 
-dotGraphRepo :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
-dotGraphRepo umap fullinfo r = unlines $ node:edges
-	where
-		node = inhost $ Dot.graphNode (nodeid r) (repoName umap r)
-		edges = map edge (Git.remotes r)
+hostname :: Git.Repo -> String
+hostname r
+	| Git.repoIsUrl r = Git.urlHost r
+	| otherwise = "localhost"
 
-		inhost a
-			| Git.repoIsUrl r = Dot.subGraph (Git.urlHost r) (hostname r) a
-			| otherwise = a
-
-		hostname n = head $ split "." $ Git.urlHost n
-
-		edge to =
-			-- get the full info for the repo since its UUID
-			-- is in there
-			let to' = findfullinfo to
-			in Dot.graphEdge
-				(nodeid r)
-				(nodeid $ makeabs r to')
-				(edgename to to')
-		
-		-- Only name an edge if the name is different than the name
-		-- that will be used for the destination node, and is
-		-- different from its hostname. (This reduces visual clutter.)
-		edgename to to' =
-			case (Git.repoRemoteName to) of
-				Nothing -> Nothing
-				Just n ->
-					if (n == repoName umap to' || n == hostname to')
-						then Nothing
-						else Just n
-
-		nodeid n =
-			case (getUncachedUUID n) of
-				"" -> Git.repoLocation n
-				u -> u
-		findfullinfo n =
-			case (filter (same n) fullinfo) of
-				[] -> n
-				(n':_) -> n'
+basehostname :: Git.Repo -> String
+basehostname r = head $ split "." $ hostname r
 
+{- A name to display for a repo. Uses the name from uuid.log if available,
+ - or the remote name if not. -}
 repoName :: (M.Map UUID String) -> Git.Repo -> String
 repoName umap r
 	| null repouuid = fallback
@@ -109,6 +78,43 @@ repoName umap r
 				Just n -> n
 				Nothing -> "unknown"
 
+{- A unique id for the node. Uses the annex.uuid if available. -}
+nodeId :: Git.Repo -> String
+nodeId r =
+	case (getUncachedUUID r) of
+		"" -> Git.repoLocation r
+		u -> u
+
+{- A node representing a repo. -}
+node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
+node umap fullinfo r = unlines $ n:edges
+	where
+		n = Dot.subGraph (hostname r) (basehostname r) $
+			Dot.graphNode (nodeId r) (repoName umap r)
+		edges = map (edge umap fullinfo r) (Git.remotes r)
+
+{- An edge between two repos. The second repo is a remote of the first. -}
+edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
+edge umap fullinfo from to =
+	Dot.graphEdge (nodeId from) (nodeId $ makeabs from fullto) edgename
+	where
+		-- get the full info for the remote, to get its UUID
+		fullto = findfullinfo to
+		findfullinfo n =
+			case (filter (same n) fullinfo) of
+				[] -> n
+				(n':_) -> n'
+		{- Only name an edge if the name is different than the name
+		 - that will be used for the destination node, and is
+		 - different from its hostname. (This reduces visual clutter.) -}
+		edgename =
+			case (Git.repoRemoteName to) of
+				Nothing -> Nothing
+				Just n ->
+					if (n == repoName umap fullto || n == hostname fullto)
+						then Nothing
+						else Just n
+
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]
 spider r = spider' [r] []
@@ -148,7 +154,7 @@ same a b
 {- reads the config of a remote, with progress display -}
 scan :: Git.Repo -> Annex Git.Repo
 scan r = do
-	showStart "map" (Git.repoDescribe r)
+	showStart "map" $ Git.repoDescribe r
 	v <- tryScan r
 	case v of
 		Just r' -> do

From 67c1facad150cfbc706721cbd9b482be22a31f4c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 23:23:36 -0400
Subject: [PATCH 0821/2835] fix infinite loop

Local repos with the same path are not different. :)
---
 Command/Map.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Command/Map.hs b/Command/Map.hs
index 1d38bc42fe..fa2a412536 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -144,6 +144,7 @@ same :: Git.Repo -> Git.Repo -> Bool
 same a b
 	| both Git.repoIsSsh = matching Git.urlHostFull && matching Git.workTree
 	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
+	| neither Git.repoIsSsh = matching Git.workTree
 	| otherwise = False
 		
 	where

From 0fd0e414ec593dcb965ca9a348798857be2bb3e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Feb 2011 00:06:23 -0400
Subject: [PATCH 0822/2835] color unreachable nodes

---
 Command/Map.hs | 17 +++++++++++++----
 Dot.hs         | 43 +++++++++++++++++++++++++++----------------
 2 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index fa2a412536..bc117d4790 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -51,11 +51,12 @@ start = do
  - displayed as a node, and each of its remotes is represented as an edge
  - pointing at the node for the remote. -}
 drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
-drawMap rs umap = Dot.graph $ repos ++ others
+drawMap rs umap = Dot.graph $ others ++ repos
 	where
 		repos = map (node umap rs) rs
 		others = map uuidnode (M.keys umap)
-		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
+		uuidnode u = unreachable $
+			Dot.graphNode u $ M.findWithDefault "" u umap
 
 hostname :: Git.Repo -> String
 hostname r
@@ -78,7 +79,7 @@ repoName umap r
 				Just n -> n
 				Nothing -> "unknown"
 
-{- A unique id for the node. Uses the annex.uuid if available. -}
+{- A unique id for the node for a repo. Uses the annex.uuid if available. -}
 nodeId :: Git.Repo -> String
 nodeId r =
 	case (getUncachedUUID r) of
@@ -90,8 +91,11 @@ node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
 node umap fullinfo r = unlines $ n:edges
 	where
 		n = Dot.subGraph (hostname r) (basehostname r) $
-			Dot.graphNode (nodeId r) (repoName umap r)
+			decorate $ Dot.graphNode (nodeId r) (repoName umap r)
 		edges = map (edge umap fullinfo r) (Git.remotes r)
+		decorate
+			| Git.configMap r == M.empty = unreachable
+			| otherwise = reachable
 
 {- An edge between two repos. The second repo is a remote of the first. -}
 edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
@@ -115,6 +119,11 @@ edge umap fullinfo from to =
 						then Nothing
 						else Just n
 
+unreachable :: String -> String
+unreachable s = Dot.fillColor "red" s
+reachable :: String -> String
+reachable s = Dot.fillColor "white" s
+
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]
 spider r = spider' [r] []
diff --git a/Dot.hs b/Dot.hs
index 1d9c29c532..0507c638ce 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -9,39 +9,50 @@ module Dot where -- import qualified
 
 {- generates a graph description from a list of lines -}
 graph :: [String] -> String
-graph s = unlines $ [header] ++ s ++ [footer]
+graph s = unlines $ [header] ++ map formatLine s ++ [footer]
 	where
 		header = "digraph map {"
 		footer= "}"
 
 {- a node in the graph -}
 graphNode :: String -> String -> String
-graphNode nodeid desc = lineLabeled desc $ quote nodeid
+graphNode nodeid desc = label desc $ quote nodeid
 
 {- an edge between two nodes -}
 graphEdge :: String -> String -> Maybe String -> String
-graphEdge fromid toid d =
-	case d of
-		Nothing -> line edge
-		Just desc -> lineLabeled desc edge
+graphEdge fromid toid desc =
+	case desc of
+		Nothing -> edge
+		Just d -> label d edge
 	where
 		edge = quote fromid ++ " -> " ++ quote toid
 
-quote :: String -> String
-quote s = "\"" ++ s ++ "\""
+{- adds a label to a node or edge -}
+label :: String -> String -> String
+label l s = attr "label" l s
 
-line :: String -> String
-line s = "\t" ++ s ++ ";"
+{- adds an attribute to a node or edge
+ - (can be called multiple times for multiple attributes) -}
+attr :: String -> String -> String -> String
+attr a v s = s ++ " [ " ++ a ++ "=" ++ quote v ++ " ]"
 
-{- a line with a label -}
-lineLabeled :: String -> String -> String
-lineLabeled label s = line $ s ++ " [ label=" ++ quote label ++ " ]"
+{- fills a node with a color -}
+fillColor :: String -> String -> String
+fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s
 
 {- apply to graphNode to put the node in a labeled box -}
 subGraph :: String -> String -> String -> String
-subGraph subid label s = line $
-	"subgraph " ++ name ++ "{\n" ++ setlabel ++ "\n" ++ s ++ "\n}"
+subGraph subid l s =
+	"subgraph " ++ name ++ "{\n\t" ++ setlabel ++ "\n\t\t" ++ s ++ "\n\t}"
 	where
 		-- the "cluster_" makes dot draw a box
 		name = quote ("cluster_" ++ subid)
-		setlabel = line $ "label=" ++ quote label
+		setlabel = formatLine $ "label=" ++ quote l
+
+formatLine :: String -> String	
+formatLine s = "\t" ++ s ++ ";"
+
+quote :: String -> String
+quote s = "\"" ++ s' ++ "\""
+	where
+		s' = filter (/= '"') s

From 926df3d91ec29f35e2cdf3df0d286721f40b9cbf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Feb 2011 00:13:47 -0400
Subject: [PATCH 0823/2835] node ordering

---
 Command/Map.hs | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index bc117d4790..83207d551c 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -49,12 +49,19 @@ start = do
 
 {- Generates a graph for dot(1). Each repository, and any other uuids, are
  - displayed as a node, and each of its remotes is represented as an edge
- - pointing at the node for the remote. -}
+ - pointing at the node for the remote.
+ -
+ - The order nodes are added to the graph matters, since dot will draw
+ - the first ones near to the top and left. So it looks better to put
+ - the repositories first, followed by uuids that were not matched
+ - to a repository.
+ -}
 drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
-drawMap rs umap = Dot.graph $ others ++ repos
+drawMap rs umap = Dot.graph $ repos ++ others
 	where
 		repos = map (node umap rs) rs
-		others = map uuidnode (M.keys umap)
+		ruuids = map getUncachedUUID rs
+		others = map uuidnode $ filter (`notElem` ruuids) (M.keys umap)
 		uuidnode u = unreachable $
 			Dot.graphNode u $ M.findWithDefault "" u umap
 
@@ -120,9 +127,9 @@ edge umap fullinfo from to =
 						else Just n
 
 unreachable :: String -> String
-unreachable s = Dot.fillColor "red" s
+unreachable = Dot.fillColor "red"
 reachable :: String -> String
-reachable s = Dot.fillColor "white" s
+reachable = Dot.fillColor "white"
 
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]

From 30869187f0890f9e742b4a5dbb4579b0fca6f7e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Feb 2011 00:36:36 -0400
Subject: [PATCH 0824/2835] improve output

---
 Dot.hs | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/Dot.hs b/Dot.hs
index 0507c638ce..fcd0c19cc2 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -9,7 +9,7 @@ module Dot where -- import qualified
 
 {- generates a graph description from a list of lines -}
 graph :: [String] -> String
-graph s = unlines $ [header] ++ map formatLine s ++ [footer]
+graph s = unlines $ [header] ++ map indent s ++ [footer]
 	where
 		header = "digraph map {"
 		footer= "}"
@@ -20,7 +20,7 @@ graphNode nodeid desc = label desc $ quote nodeid
 
 {- an edge between two nodes -}
 graphEdge :: String -> String -> Maybe String -> String
-graphEdge fromid toid desc =
+graphEdge fromid toid desc = indent $
 	case desc of
 		Nothing -> edge
 		Just d -> label d edge
@@ -43,14 +43,15 @@ fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s
 {- apply to graphNode to put the node in a labeled box -}
 subGraph :: String -> String -> String -> String
 subGraph subid l s =
-	"subgraph " ++ name ++ "{\n\t" ++ setlabel ++ "\n\t\t" ++ s ++ "\n\t}"
+	"subgraph " ++ name ++ " {\n" ++ ii setlabel ++ ii s ++ indent "}"
 	where
 		-- the "cluster_" makes dot draw a box
 		name = quote ("cluster_" ++ subid)
-		setlabel = formatLine $ "label=" ++ quote l
+		setlabel = "label=" ++ quote l
+		ii x = (indent $ indent x) ++ "\n"
 
-formatLine :: String -> String	
-formatLine s = "\t" ++ s ++ ";"
+indent ::String -> String
+indent s = "\t" ++ s
 
 quote :: String -> String
 quote s = "\"" ++ s' ++ "\""

From ef2d4f650edff99b67554be9288face87159131e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Feb 2011 01:56:45 -0400
Subject: [PATCH 0825/2835] fix absrepo data loss

it was dropping the config map for the repos it changed
---
 Command/Map.hs | 30 ++++++++++++++----------------
 GitRepo.hs     | 16 +++++++++++++++-
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 83207d551c..d8dd0e94cf 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -107,7 +107,7 @@ node umap fullinfo r = unlines $ n:edges
 {- An edge between two repos. The second repo is a remote of the first. -}
 edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
 edge umap fullinfo from to =
-	Dot.graphEdge (nodeId from) (nodeId $ makeabs from fullto) edgename
+	Dot.graphEdge (nodeId from) (nodeId $ absRepo from fullto) edgename
 	where
 		-- get the full info for the remote, to get its UUID
 		fullto = findfullinfo to
@@ -140,20 +140,13 @@ spider' (r:rs) known
 	| any (same r) known = spider' rs known
 	| otherwise = do
 		r' <- scan r
-		let remotes = map (makeabs r') (Git.remotes r')
+		let remotes = map (absRepo r') (Git.remotes r')
 		spider' (rs ++ remotes) (r':known)
 
-{- Makes a remote have an absolute url, rather than a host-local path. -}
-makeabs :: Git.Repo -> Git.Repo -> Git.Repo
-makeabs repo remote
-	| Git.repoIsUrl remote = remote
-	| not $ Git.repoIsUrl repo = remote
-	| otherwise = Git.repoFromUrl combinedurl
-		where
-			combinedurl =
-				Git.urlScheme repo ++ "//" ++
-				Git.urlHostFull repo ++
-				Git.workTree remote
+absRepo :: Git.Repo -> Git.Repo -> Git.Repo
+absRepo reference r
+	| Git.repoIsUrl reference = Git.localToUrl reference r
+	| otherwise = r
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
@@ -217,9 +210,14 @@ tryScan r
 		-- Secondly, configlist doesn't include information about
 		-- the remote's remotes.
 		sshscan = do
-			showNote "sshing..."
-			showProgress
+			sshnote
 			v <- manualconfiglist
 			case v of
-				Nothing -> configlist
+				Nothing -> do
+					sshnote
+					configlist
 				ok -> return ok
+
+		sshnote = do
+			showNote "sshing..."
+			showProgress
diff --git a/GitRepo.hs b/GitRepo.hs
index 4b252868ff..7bb20fc53c 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -13,6 +13,7 @@ module GitRepo (
 	repoFromCwd,
 	repoFromPath,
 	repoFromUrl,
+	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
 	repoDescribe,
@@ -109,6 +110,19 @@ repoFromUrl url
 				Just v -> v
 				Nothing -> error $ "bad url " ++ url
 
+{- Converts a Local Repo into a remote repo, using the reference repo
+ - which is assumed to be on the same host. -}
+localToUrl :: Repo -> Repo -> Repo
+localToUrl reference r
+	| not $ repoIsUrl reference = error "internal error; reference repo not url"
+	| repoIsUrl r = r
+	| otherwise = r { location = Url $ fromJust $ parseURI absurl }
+	where
+		absurl =
+			urlScheme reference ++ "//" ++
+			urlHostFull reference ++
+			workTree r
+
 {- User-visible description of a git repo. -}
 repoDescribe :: Repo -> String
 repoDescribe Repo { remoteName = Just name } = name
@@ -338,7 +352,7 @@ configStore :: Repo -> String -> Repo
 configStore repo s = r { remotes = configRemotes r }
 	where r = repo { config = configParse s }
 
-{- Checks if a string fron git config is a true value. -}
+{- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 

From c975384195467b5e5bca32d8199330e96be9feba Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
 
Date: Sun, 6 Feb 2011 06:02:59 +0000
Subject: [PATCH 0826/2835] Added a comment: Got it going!

---
 ..._1_34120e82331ace01a6a4960862d38f2d._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment b/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment
new file mode 100644
index 0000000000..a33fef7d99
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
+ nickname="Antoine"
+ subject="Got it going!"
+ date="2011-02-06T06:02:57Z"
+ content="""
+Thanks to your feedback, I got it going.  
+
+Maybe those two should be added to the 'OSX how-to' in the forum   
+
+[realizes pcre-light is needed but pcre not installed on my mac]  
+sudo port install pcre  
+sudo cabal install pcre-light  
+
+[tests are failing, need haskell's quickcheck]  
+sudo cabal install quickcheck  
+"""]]

From 413f50b82db47ca231fd36b73551b2633488a06e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 6 Feb 2011 17:39:53 +0000
Subject: [PATCH 0827/2835] Added a comment

---
 .../comment_2_cc53d1681d576186dbc868dd9801d551._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment b/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment
new file mode 100644
index 0000000000..91d3e89f06
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-02-06T17:39:52Z"
+ content="""
+Yes, I've moved it to [[install/OSX]] page where anyone can update it in this wiki, and added your improvements.
+"""]]

From 6c45f1123d0709787c7df15c46b8d97e9813eb83 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 6 Feb 2011 13:43:31 -0400
Subject: [PATCH 0828/2835] reord install pages; per-OS instructions

---
 doc/download.mdwn               |  6 ++----
 doc/forum/git-annex_on_OSX.mdwn | 22 +---------------------
 doc/install.mdwn                |  6 ++++++
 doc/install/Debian.mdwn         |  9 +++++++++
 doc/install/OSX.mdwn            | 17 +++++++++++++++++
 doc/install/Ubuntu.mdwn         |  5 +++++
 6 files changed, 40 insertions(+), 25 deletions(-)
 create mode 100644 doc/install/Debian.mdwn
 create mode 100644 doc/install/OSX.mdwn
 create mode 100644 doc/install/Ubuntu.mdwn

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 6c56c8e25a..748ac0ca34 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -1,7 +1,5 @@
 The main git repository for git-annex is `git://git.kitenet.net/git-annex`
 [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)]
 
-Users of Debian unstable/testing and Ubuntu natty can
-`apt-get install git-annex`
-
-Next: [[install]]
+Some operating systems include git-annex in easily prepackaged form and
+others need some manual work. See [[install]] for details.
diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn
index 302ad10994..a00548366a 100644
--- a/doc/forum/git-annex_on_OSX.mdwn
+++ b/doc/forum/git-annex_on_OSX.mdwn
@@ -1,21 +1 @@
-
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum
-
-[waits forever…]
-[finished]
-[realizes missingh isn't working in MacPorts]
-
-sudo cabal  update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo port install pcre
-sudo cabal install pcre-light
-
-git clone  git://git.kitenet.net/git-annex
-
-cd git-annex
-make
-sudo make install
-

-
-Originally posted by Jon at  --[[Joey]]
+See [[install/OSX]].
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 732660c507..3c7025fa60 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -12,3 +12,9 @@ To build and use git-annex, you will need:
 
 ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
 but that will be skipped if it is not installed.)
+
+OS-specific instructions:
+
+* [[OSX]]
+* [[Debian]]
+* [[Ubuntu]]
diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn
new file mode 100644
index 0000000000..90c9658a9c
--- /dev/null
+++ b/doc/install/Debian.mdwn
@@ -0,0 +1,9 @@
+If using Debian testing or unstable:
+
+	sudo apt-get install git-annex
+
+git-annex is not yet in a Debian stable release, but the source
+can be built from Debian stable. Just [[download]] it from git, and
+then in the `git-annex` directory, run `dpkg-checkbuilddeps` and install
+the necessary packages. You can also run `sudo debian/rules binary` to build
+a `git-annex.deb`.
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
new file mode 100644
index 0000000000..e107e48eb9
--- /dev/null
+++ b/doc/install/OSX.mdwn
@@ -0,0 +1,17 @@
+
+sudo port install haskell-platform git-core ossp-uuid md5sha1sum
+sudo cabal update
+sudo cabal install missingh
+sudo cabal install utf8-string
+sudo port install pcre
+sudo cabal install pcre-light
+sudo cabal install quickcheck  
+
+git clone  git://git.kitenet.net/git-annex
+
+cd git-annex
+make
+sudo make install
+

+
+Originally posted by Jon at  --[[Joey]]
diff --git a/doc/install/Ubuntu.mdwn b/doc/install/Ubuntu.mdwn
new file mode 100644
index 0000000000..ffc763ff31
--- /dev/null
+++ b/doc/install/Ubuntu.mdwn
@@ -0,0 +1,5 @@
+If using Ubuntu natty or newer:
+
+	sudo apt-get install git-annex
+
+Otherwise, see [[Debian]] manual installation instructions.

From 3093d2fad7c2dfd62b9e016a8e5a895568f4ce06 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 7 Feb 2011 12:43:44 +0000
Subject: [PATCH 0829/2835] Added a comment: tests fail with more recent
 installs of haskell platform

---
 ..._68f0f8ae953589ae26d57310b40c878d._comment | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment b/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment
new file mode 100644
index 0000000000..39f32c244f
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment
@@ -0,0 +1,57 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="tests fail with more recent installs of haskell platform"
+ date="2011-02-07T12:43:43Z"
+ content="""
+I'm running ghc 6.12.3 with the corresponding haskell-platform package from the HP site which I installed in preference to the macports version of haskell-platform (it's quite old). it seems when you install quickcheck, the version that is installed is of version 2.4.0.1 and not 1.2.0 which git-annex depends on for its tests.
+
+
+jtang@x00:~ $ cabal install quickcheck --reinstall               
+Resolving dependencies...
+Configuring QuickCheck-2.4.0.1...
+Preprocessing library QuickCheck-2.4.0.1...
+
+..
+and so on..
+..
+
+

+
+it fails with this
+
+
+[54 of 54] Compiling Main             ( test.hs, test.o )
+
+test.hs:56:3:
+    No instance for (QuickCheck-1.2.0.1:Test.QuickCheck.Arbitrary Char)
+      arising from a use of `qctest' at test.hs:56:3-64
+    Possible fix:
+      add an instance declaration for
+      (QuickCheck-1.2.0.1:Test.QuickCheck.Arbitrary Char)
+    In the expression:
+        qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode
+    In the first argument of `TestList', namely
+        `[qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode,
+          qctest \"prop_idempotent_fileKey\" Locations.prop_idempotent_fileKey,
+          qctest
+            \"prop_idempotent_key_read_show\"
+            BackendTypes.prop_idempotent_key_read_show,
+          qctest
+            \"prop_idempotent_shellEscape\" Utility.prop_idempotent_shellEscape,
+          ....]'
+    In the second argument of `($)', namely
+        `TestList
+           [qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode,
+            qctest \"prop_idempotent_fileKey\" Locations.prop_idempotent_fileKey,
+            qctest
+              \"prop_idempotent_key_read_show\"
+              BackendTypes.prop_idempotent_key_read_show,
+            qctest
+              \"prop_idempotent_shellEscape\" Utility.prop_idempotent_shellEscape,
+            ....]'
+

+
+I'd imagine if I could downgrade, it would compile and pass the tests (I hope)
+
+"""]]

From 82ff914492636869079a33b67b4d918e93213dc7 Mon Sep 17 00:00:00 2001
From: "http://ertai.myopenid.com/" 
Date: Mon, 7 Feb 2011 14:12:44 +0000
Subject: [PATCH 0830/2835] Added a comment: how to reproduce the package
 conflict issue

---
 ..._e552a6cc6d7d1882e14130edfc2d6b3b._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment

diff --git a/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment b/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment
new file mode 100644
index 0000000000..42f44bf9cf
--- /dev/null
+++ b/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="how to reproduce the package conflict issue"
+ date="2011-02-07T14:12:43Z"
+ content="""
+If you install the monads-fd package (with cabal install for instance), then you can no longer build git-annex:
+
+
+./configure
+  checking cp -a... yes
+  checking cp -p... yes
+  checking cp --reflink=auto... yes
+  checking uuid generator... uuid
+  checking xargs -0... yes
+  checking rsync... yes
+ghc -O2 -Wall --make git-annex
+
+Annex.hs:22:7:
+    Ambiguous module name `Control.Monad.State':
+      it was found in multiple packages: monads-fd-0.2.0.0 mtl-2.0.1.0
+make: *** [git-annex] Error 1
+

+"""]]

From 217e5de24483e30d8dc7f9167da1531ddc0c94a4 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 8 Feb 2011 19:00:14 +0000
Subject: [PATCH 0831/2835] Added a comment

---
 ...mment_4_c52be386f79f14c8570a8f1397c68581._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment b/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment
new file mode 100644
index 0000000000..e245e139fb
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-02-08T19:00:14Z"
+ content="""
+I doubt that git-annex can be used with QuickCheck 1.2.0. The QuickCheck I've tested it with is 2.1.0.3 actually.
+
+I suspect you have an old version of the TestPack haskell library on your system, that is linked against QuickCheck 1.2.0. Git-annex has been tested with TestPack 2.0.0, which uses QuickCheck 2.x.
+
+In any case, you don't have to run 'make test' to build git-annex, and my comments above should make the main program compile, I expect.
+"""]]

From 45387b3fcb0785d0e9957a638fdd8ffccb3eb40c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 15:11:49 -0400
Subject: [PATCH 0832/2835] Deal with the mtl/monads-fd conflict.

---
 Makefile                                   | 2 +-
 debian/changelog                           | 1 +
 doc/bugs/conflicting_haskell_packages.mdwn | 9 +++++++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 565edfb0e0..c888fc2151 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 PREFIX=/usr
-GHCFLAGS=-O2 -Wall
+GHCFLAGS=-O2 -Wall -ignore-package monads-fd
 GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell
diff --git a/debian/changelog b/debian/changelog
index 42d45c3a3e..3a5f60fefe 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,7 @@ git-annex (0.20) UNRELEASED; urgency=low
     an unannex as checking in of an unlocked file.
   * map: New subcommand that uses graphviz to display a nice map of
     the git repository network.
+  * Deal with the mtl/monads-fd conflict.
 
  -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
 
diff --git a/doc/bugs/conflicting_haskell_packages.mdwn b/doc/bugs/conflicting_haskell_packages.mdwn
index 6a619de90f..5528fad824 100644
--- a/doc/bugs/conflicting_haskell_packages.mdwn
+++ b/doc/bugs/conflicting_haskell_packages.mdwn
@@ -6,3 +6,12 @@ This can be done by the flags -hide-packages and then -package foo
 > 
 > Could you just show the build problem that you are suggesting I work
 > around? --[[Joey]]
+
+
+> Thanks npouillard, I see the problem now.
+> 
+> 
+> I've added "-ignore-package monads-fd" to GHCFLAGS. I hope I don't
+> really have to hide all packages and individually turn them back on;
+> surely this monads-fd/mtl conflict is an exception, and Haskell's module
+> system is not a mess of conflicting modules? --[[Joey]] [[done]]

From 3775dc96073d52dcae13a020a95186acad789d74 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 8 Feb 2011 19:56:56 +0000
Subject: [PATCH 0833/2835] Added a comment

---
 ..._7f1330a1e541b0f3e2192e596d7f7bee._comment | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment b/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment
new file mode 100644
index 0000000000..9c83feb32f
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment
@@ -0,0 +1,107 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2011-02-08T19:56:55Z"
+ content="""
+Ah, that gave me a good clue, my system just got pretty confused with a mixture of quickcheck and testpack installs. Would it be possible to put up a list of versions of the software you are using on your development environment? (at least the minimum tested version) 
+
+I guess it shouldn't matter to most users who are going to rely on packagers to sort these dependancy issues, but it's nice to know. 
+
+Anyway, the tests build now, and they seem to fail on my (rather messy) install of haskell platform + ghc 6.12 on osx 10.6.6.
+
+
+< output that passed some tests >
+Testing 1:blackbox:0:git-annex init
+Testing 1:blackbox:1:git-annex add:0
+Testing 1:blackbox:1:git-annex add:1
+Cases: 30  Tried: 9  Errors: 0  Failures: 0test: sha1sum: executeFile: does not exist (No such file or directory)
+  git-annex: : hGetLine: end of file
+### Failure in: 1:blackbox:1:git-annex add:1
+add with SHA1 failed
+Testing 1:blackbox:2:git-annex setkey/fromkey
+Cases: 30  Tried: 10  Errors: 0  Failures: 1(checksum...) test: sha1sum: executeFile: does not exist (No such file or directory)
+### Error in:   1:blackbox:2:git-annex setkey/fromkey
+: hGetLine: end of file
+Testing 1:blackbox:3:git-annex unannex:0:no content
+Cases: 30  Tried: 11  Errors: 1  Failures: 1chmod: -R: No such file or directory
+chmod: -R: No such file or directory
+Testing 1:blackbox:3:git-annex unannex:1:with content
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes
+Cases: 30  Tried: 13  Errors: 1  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:0:no remotes
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:4:git-annex drop:1:with remote
+Cases: 30  Tried: 14  Errors: 2  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:1:with remote
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote
+Cases: 30  Tried: 15  Errors: 3  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:2:untrusted remote
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:5:git-annex get
+Cases: 30  Tried: 16  Errors: 4  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:5:git-annex get
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:6:git-annex move
+Cases: 30  Tried: 17  Errors: 5  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:6:git-annex move
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:7:git-annex copy
+Cases: 30  Tried: 18  Errors: 6  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:7:git-annex copy
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:8:git-annex unlock/lock
+Cases: 30  Tried: 19  Errors: 7  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:8:git-annex unlock/lock
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:9:git-annex edit/commit:0
+Cases: 30  Tried: 20  Errors: 8  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:9:git-annex edit/commit:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:9:git-annex edit/commit:1
+Cases: 30  Tried: 21  Errors: 9  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:9:git-annex edit/commit:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:10:git-annex fix
+Cases: 30  Tried: 22  Errors: 10  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:10:git-annex fix
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust
+Cases: 30  Tried: 23  Errors: 11  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:11:git-annex trust/untrust/semitrust
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:0
+Cases: 30  Tried: 24  Errors: 12  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:1
+Cases: 30  Tried: 25  Errors: 13  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:2
+Cases: 30  Tried: 26  Errors: 14  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:2
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:13:git-annex migrate:0
+Cases: 30  Tried: 27  Errors: 15  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:13:git-annex migrate:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:13:git-annex migrate:1
+Cases: 30  Tried: 28  Errors: 16  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:13:git-annex migrate:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:14:git-annex unused/dropunused
+Cases: 30  Tried: 29  Errors: 17  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Cases: 30  Tried: 30  Errors: 18  Failures: 2
+chmod: -R: No such file or directory
+test: .t/repo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+make: *** [test] Error 1
+

+
+I assumed that since the tests built, then running them shouldn't be a problem. It looks like some argument isn't being passed about for the location of the .t directory that gets created. I will check the dependancies on my system again.
+"""]]

From 97d5e23430748487d9b6dea842cdba4e66ba39d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 17:46:52 -0400
Subject: [PATCH 0834/2835] map bugfix

Need to find the absolute repo path before looking up the full info for the
repo. Otherwise, it doesn't find the right full info.
---
 Command/Map.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index d8dd0e94cf..1b15e34f6a 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -107,10 +107,10 @@ node umap fullinfo r = unlines $ n:edges
 {- An edge between two repos. The second repo is a remote of the first. -}
 edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
 edge umap fullinfo from to =
-	Dot.graphEdge (nodeId from) (nodeId $ absRepo from fullto) edgename
+	Dot.graphEdge (nodeId from) (nodeId fullto) edgename
 	where
 		-- get the full info for the remote, to get its UUID
-		fullto = findfullinfo to
+		fullto = findfullinfo (absRepo from to)
 		findfullinfo n =
 			case (filter (same n) fullinfo) of
 				[] -> n

From 3ae654254da0449c5ddc4fe16600b76c82e828fe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 17:52:32 -0400
Subject: [PATCH 0835/2835] make remotes absolute while spidering

---
 Command/Map.hs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 1b15e34f6a..05c03feaf0 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -110,7 +110,7 @@ edge umap fullinfo from to =
 	Dot.graphEdge (nodeId from) (nodeId fullto) edgename
 	where
 		-- get the full info for the remote, to get its UUID
-		fullto = findfullinfo (absRepo from to)
+		fullto = findfullinfo to
 		findfullinfo n =
 			case (filter (same n) fullinfo) of
 				[] -> n
@@ -140,8 +140,13 @@ spider' (r:rs) known
 	| any (same r) known = spider' rs known
 	| otherwise = do
 		r' <- scan r
+
+		-- The remotes will be relative to r', and need to be
+		-- made absolute for later use.
 		let remotes = map (absRepo r') (Git.remotes r')
-		spider' (rs ++ remotes) (r':known)
+		let r'' = Git.remotesAdd r' remotes
+
+		spider' (rs ++ remotes) (r'':known)
 
 absRepo :: Git.Repo -> Git.Repo -> Git.Repo
 absRepo reference r

From c0ec5a35db54f45b14971ae61b42ccc64cb92f62 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 18:04:19 -0400
Subject: [PATCH 0836/2835] show trusted repos in green

---
 Command/Map.hs | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 05c03feaf0..05fa258857 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -21,6 +21,7 @@ import Messages
 import Types
 import Utility
 import UUID
+import Trust
 import qualified Dot
 
 -- a link from the first repository to the second (its remote)
@@ -38,8 +39,9 @@ start = do
 	rs <- spider g
 
 	umap <- uuidMap
+	trusted <- trustGet Trusted
 
-	liftIO $ writeFile file (drawMap rs umap)
+	liftIO $ writeFile file (drawMap rs umap trusted)
 	showLongNote $ "running: dot -Tx11 " ++ file
 	showProgress
 	r <- liftIO $ boolSystem "dot" ["-Tx11", file]
@@ -56,14 +58,15 @@ start = do
  - the repositories first, followed by uuids that were not matched
  - to a repository.
  -}
-drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
-drawMap rs umap = Dot.graph $ repos ++ others
+drawMap :: [Git.Repo] -> (M.Map UUID String) -> [UUID] -> String
+drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 	where
 		repos = map (node umap rs) rs
-		ruuids = map getUncachedUUID rs
-		others = map uuidnode $ filter (`notElem` ruuids) (M.keys umap)
-		uuidnode u = unreachable $
-			Dot.graphNode u $ M.findWithDefault "" u umap
+		ruuids = ts ++ map getUncachedUUID rs
+		others = map (unreachable . uuidnode) $
+			filter (`notElem` ruuids) (M.keys umap)
+		trusted = map (trustworthy . uuidnode) ts
+		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
 
 hostname :: Git.Repo -> String
 hostname r
@@ -130,6 +133,8 @@ unreachable :: String -> String
 unreachable = Dot.fillColor "red"
 reachable :: String -> String
 reachable = Dot.fillColor "white"
+trustworthy :: String -> String
+trustworthy = Dot.fillColor "green"
 
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]

From c1b69d1511cd9a6d63981f74f6d926d59dba7c8c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 18:17:46 -0400
Subject: [PATCH 0837/2835] fill color for host boxes

---
 Command/Map.hs |  2 +-
 Dot.hs         | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 05fa258857..74005b521d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -100,7 +100,7 @@ nodeId r =
 node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
 node umap fullinfo r = unlines $ n:edges
 	where
-		n = Dot.subGraph (hostname r) (basehostname r) $
+		n = Dot.subGraph (hostname r) (basehostname r) "grey" $
 			decorate $ Dot.graphNode (nodeId r) (repoName umap r)
 		edges = map (edge umap fullinfo r) (Git.remotes r)
 		decorate
diff --git a/Dot.hs b/Dot.hs
index fcd0c19cc2..a21d705365 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -41,13 +41,18 @@ fillColor :: String -> String -> String
 fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s
 
 {- apply to graphNode to put the node in a labeled box -}
-subGraph :: String -> String -> String -> String
-subGraph subid l s =
-	"subgraph " ++ name ++ " {\n" ++ ii setlabel ++ ii s ++ indent "}"
+subGraph :: String -> String -> String -> String -> String
+subGraph subid l color s =
+	"subgraph " ++ name ++ " {\n" ++
+		ii setlabel ++
+		ii setcolor ++
+		ii s ++
+		indent "}"
 	where
 		-- the "cluster_" makes dot draw a box
 		name = quote ("cluster_" ++ subid)
 		setlabel = "label=" ++ quote l
+		setcolor = "fillcolor=" ++ quote color
 		ii x = (indent $ indent x) ++ "\n"
 
 indent ::String -> String

From 81e045a539e5877d445abaa53783f94f4e17513a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 18:26:38 -0400
Subject: [PATCH 0838/2835] tweak

---
 Command/Map.hs | 2 +-
 Dot.hs         | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 74005b521d..0a3bb9fff0 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -100,7 +100,7 @@ nodeId r =
 node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
 node umap fullinfo r = unlines $ n:edges
 	where
-		n = Dot.subGraph (hostname r) (basehostname r) "grey" $
+		n = Dot.subGraph (hostname r) (basehostname r) "lightblue" $
 			decorate $ Dot.graphNode (nodeId r) (repoName umap r)
 		edges = map (edge umap fullinfo r) (Git.remotes r)
 		decorate
diff --git a/Dot.hs b/Dot.hs
index a21d705365..592b21f691 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -45,6 +45,7 @@ subGraph :: String -> String -> String -> String -> String
 subGraph subid l color s =
 	"subgraph " ++ name ++ " {\n" ++
 		ii setlabel ++
+		ii setfilled ++
 		ii setcolor ++
 		ii s ++
 		indent "}"
@@ -52,6 +53,7 @@ subGraph subid l color s =
 		-- the "cluster_" makes dot draw a box
 		name = quote ("cluster_" ++ subid)
 		setlabel = "label=" ++ quote l
+		setfilled = "style=" ++ quote "filled"
 		setcolor = "fillcolor=" ++ quote color
 		ii x = (indent $ indent x) ++ "\n"
 

From be8b42b40dba9f6ffebf4c3818fefb8dc5464137 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 8 Feb 2011 23:20:08 +0000
Subject: [PATCH 0839/2835] Added a comment

---
 .../comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment b/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment
new file mode 100644
index 0000000000..afc3088d4f
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-02-08T23:20:08Z"
+ content="""
+You're missing the sha1sum command, everything else is a followon error from that. Added a hint about this to [[install]],
+and in the next version configure will check for sha1sum.
+"""]]

From dd90d4a70e6f8e155902cd40dc21c138d4e1282c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 19:31:27 -0400
Subject: [PATCH 0840/2835] configure: Check for sha1sum.

---
 configure.hs     | 1 +
 debian/changelog | 5 +++--
 doc/install.mdwn | 1 +
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/configure.hs b/configure.hs
index 8d1c117a71..1451d7eaa3 100644
--- a/configure.hs
+++ b/configure.hs
@@ -10,6 +10,7 @@ tests = [
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
 	, TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"]
+	, TestCase "sha1sum" $ requireCmd "sha1sum" "sha1sum /dev/null"
 	]
diff --git a/debian/changelog b/debian/changelog
index 3a5f60fefe..b8d27f77e4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20) UNRELEASED; urgency=low
+git-annex (0.20) unstable; urgency=low
 
   * Preserve specified file ordering when instructed to act on multiple
     files or directories. For example, "git annex get a b" will now always
@@ -9,8 +9,9 @@ git-annex (0.20) UNRELEASED; urgency=low
   * map: New subcommand that uses graphviz to display a nice map of
     the git repository network.
   * Deal with the mtl/monads-fd conflict.
+  * configure: Check for sha1sum.
 
- -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
+ -- Joey Hess   Tue, 08 Feb 2011 18:57:24 -0400
 
 git-annex (0.19) unstable; urgency=low
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 3c7025fa60..fc9f28ae7b 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -8,6 +8,7 @@ To build and use git-annex, you will need:
   (or uuidgen from util-linux)
 * `xargs`: 
 * `rsync`: 
+* `sha1sum`: 
 * Then just [[download]] git-annex and run: `make; make install`
 
 ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,

From de1e33010232a8abad59a54758ec00808caa8a5c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 19:32:57 -0400
Subject: [PATCH 0841/2835] add news item for git-annex 0.20

---
 doc/news/version_0.15.mdwn | 15 ---------------
 doc/news/version_0.20.mdwn | 12 ++++++++++++
 2 files changed, 12 insertions(+), 15 deletions(-)
 delete mode 100644 doc/news/version_0.15.mdwn
 create mode 100644 doc/news/version_0.20.mdwn

diff --git a/doc/news/version_0.15.mdwn b/doc/news/version_0.15.mdwn
deleted file mode 100644
index c3d4d2c0f7..0000000000
--- a/doc/news/version_0.15.mdwn
+++ /dev/null
@@ -1,15 +0,0 @@
-git-annex 0.15 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Support scp-style urls for remotes (host:path).
-   * Support ssh urls containing "~".
-   * Add trust and untrust subcommands, to allow configuring repositories
-     that are trusted to retain files without explicit checking.
-   * Fix bug in numcopies handling when multiple remotes pointed to the
-     same repository.
-   * Introduce the git-annex-shell command. It's now possible to make
-     a user have it as a restricted login shell, similar to git-shell.
-   * Note that git-annex will always use git-annex-shell when accessing
-     a ssh remote, so all of your remotes need to be upgraded to this
-     version of git-annex at the same time.
-   * Now rsync is exclusively used for copying files to and from remotes.
-     scp is not longer supported."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20.mdwn b/doc/news/version_0.20.mdwn
new file mode 100644
index 0000000000..9b95b652e5
--- /dev/null
+++ b/doc/news/version_0.20.mdwn
@@ -0,0 +1,12 @@
+git-annex 0.20 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Preserve specified file ordering when instructed to act on multiple
+     files or directories. For example, "git annex get a b" will now always
+     get "a" before "b". Previously it could operate in either order.
+   * unannex: Commit staged changes at end, to avoid some confusing behavior
+     with the pre-commit hook, which would see some types of commits after
+     an unannex as checking in of an unlocked file.
+   * map: New subcommand that uses graphviz to display a nice map of
+     the git repository network.
+   * Deal with the mtl/monads-fd conflict.
+   * configure: Check for sha1sum."""]]
\ No newline at end of file

From 01a45b6be4063eddb980b87c816281f952338482 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 00:45:32 +0000
Subject: [PATCH 0842/2835] Added a comment

---
 ..._237a137cce58a28abcc736cbf2c420b0._comment | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment b/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment
new file mode 100644
index 0000000000..8d8aefcb20
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 7"
+ date="2011-02-09T00:45:31Z"
+ content="""
+That's odd, I have the md5sha1sum package installed and it still fails with pretty much the same error
+
+
+Testing 1:blackbox:0:git-annex init
+Cases: 30  Tried: 7  Errors: 0  Failures: 0chmod: -R: No such file or directory
+### Error in:   1:blackbox:0:git-annex init
+.t/repo/.git/annex/objects/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:1:git-annex add:0
+### Error in:   1:blackbox:1:git-annex add:0
+foo: openFile: permission denied (Permission denied)
+
+< and so on >
+

+
+the configure script finds sha1sum, builds and starts to run.
+"""]]

From 7a0826293e0ac6c0000f49a1618c1c613b909aa1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Feb 2011 00:12:45 -0400
Subject: [PATCH 0843/2835] test: Don't rely on chmod -R working.

Sometimes I wish I could give all my users a real OS.
---
 debian/changelog | 6 ++++++
 test.hs          | 8 +++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index b8d27f77e4..52a749178f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.21) UNRELEASED; urgency=low
+
+  * test: Don't rely on chmod -R working.
+
+ -- Joey Hess   Wed, 09 Feb 2011 00:12:11 -0400
+
 git-annex (0.20) unstable; urgency=low
 
   * Preserve specified file ordering when instructed to act on multiple
diff --git a/test.hs b/test.hs
index f7d3f7c53d..b68360e9dd 100644
--- a/test.hs
+++ b/test.hs
@@ -20,6 +20,8 @@ import Control.Exception (throw)
 import Control.Monad.State (liftIO)
 import Maybe
 import qualified Data.Map as M
+import System.Path (recurseDir)
+import System.IO.HVFS (SystemFS(..))
 
 import qualified Annex
 import qualified BackendList
@@ -536,8 +538,12 @@ cleanup dir = do
 	when e $ do
 		-- git-annex prevents annexed file content from being
 		-- removed via permissions bits; undo
-		_ <- Utility.boolSystem "chmod" ["+rw", "-R", dir]
+		recurseDir SystemFS dir >>= mapM_ fixmodes
 		removeDirectoryRecursive dir
+	where
+		fixmodes f = do
+			s <- getSymbolicLinkStatus f
+			unless (isSymbolicLink s) $ Content.allowWrite f
 	
 checklink :: FilePath -> Assertion
 checklink f = do

From c1bc8f31c50bc9d4dea180346adc874629c76b24 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 9 Feb 2011 04:10:27 +0000
Subject: [PATCH 0844/2835] Added a comment

---
 .../comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment b/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment
new file mode 100644
index 0000000000..9401bd453e
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-02-09T04:10:27Z"
+ content="""
+The chmod errors are because your chmod does not understand the -R argument. Only the test suite uses chmod -R. I've fixed it to modify modes manually.
+"""]]

From 2bd16f84ae3c6a26bff5eb5e4b39221577191def Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Feb 2011 00:25:22 -0400
Subject: [PATCH 0845/2835] more install notes

---
 doc/install.mdwn | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index fc9f28ae7b..f1305777c0 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -1,3 +1,11 @@
+## OS-specific instructions
+
+* [[OSX]]
+* [[Debian]]
+* [[Ubuntu]]
+
+## Generic instructions
+
 To build and use git-annex, you will need:
 
 * `git`: 
@@ -14,8 +22,7 @@ To build and use git-annex, you will need:
 ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
 but that will be skipped if it is not installed.)
 
-OS-specific instructions:
+Additionally, to run the test suite (via `make test`), you will need:
 
-* [[OSX]]
-* [[Debian]]
-* [[Ubuntu]]
+* `TestPack` 
+* `QuickCheck` 2 

From 62ebeb00d9f079f079c38798f8496dd666ae1fd5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Feb 2011 01:01:06 -0400
Subject: [PATCH 0846/2835] simpler approach

---
 test.hs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/test.hs b/test.hs
index b68360e9dd..b32a4abb04 100644
--- a/test.hs
+++ b/test.hs
@@ -11,7 +11,7 @@ import System.Directory
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
 import IO (bracket_, bracket)
-import Control.Monad (unless, when)
+import Control.Monad (unless, when, filterM)
 import Data.List
 import System.IO.Error
 import System.Posix.Env
@@ -537,13 +537,11 @@ cleanup dir = do
 	e <- doesDirectoryExist dir
 	when e $ do
 		-- git-annex prevents annexed file content from being
-		-- removed via permissions bits; undo
-		recurseDir SystemFS dir >>= mapM_ fixmodes
+		-- removed via directory permissions; undo
+		recurseDir SystemFS dir >>=
+			filterM doesDirectoryExist >>=
+			mapM_ Content.allowWrite
 		removeDirectoryRecursive dir
-	where
-		fixmodes f = do
-			s <- getSymbolicLinkStatus f
-			unless (isSymbolicLink s) $ Content.allowWrite f
 	
 checklink :: FilePath -> Assertion
 checklink f = do

From 0fb84912b611e03c47ebd4a3cdd3008dbb5fe417 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 09:09:44 +0000
Subject: [PATCH 0847/2835]

---
 ..._is_no_global_.gitconfig_for_the_user.mdwn | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn

diff --git a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
new file mode 100644
index 0000000000..2a26a67267
--- /dev/null
+++ b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
@@ -0,0 +1,25 @@
+Make test fails when git doesn't know what identity to give to commits
+
+
+
+Testing 1:blackbox:0:git-annex init
+Cases: 30  Tried: 7  Errors: 0  Failures: 0
+*** Please tell me who you are.
+
+Run
+
+  git config --global user.email "you@example.com"
+  git config --global user.name "Your Name"
+
+to set your account's default identity.
+Omit --global to set the identity only in this repository.
+
+fatal: empty ident   not allowed
+### Failure in: 1:blackbox:0:git-annex init
+init failed
+Testing 1:blackbox:1:git-annex add:0
+Cases: 30  Tried: 8  Errors: 0  Failures: 1
+*** Please tell me who you are.
+

+
+I guess most users or people testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop.

From 23ab9a228d31543a8f2ad8084d73dbc489b90054 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 09:12:53 +0000
Subject: [PATCH 0848/2835] Added a comment

---
 ..._cc283b485b3c95ba7eebc8f0c96969b3._comment | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment b/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment
new file mode 100644
index 0000000000..da6d7ca178
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment
@@ -0,0 +1,66 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 9"
+ date="2011-02-09T09:12:52Z"
+ content="""
+[a0826293][] fixed the last problem, there is coreutils available in macports, if they are installed you get the gnu equivalents but they are prefixed with a g (e.g. gchmod instead of chmod), I guess not everyone will have these install or prefer these on [[install/OSX]]
+
+Some more tests fail now...
+
+
+Testing 1:blackbox:3:git-annex unannex:1:with content
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Testing 1:blackbox:4:git-annex drop:1:with remote
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote
+Testing 1:blackbox:5:git-annex get
+Testing 1:blackbox:6:git-annex move
+Testing 1:blackbox:7:git-annex copy
+### Failure in: 1:blackbox:7:git-annex copy
+move --to of file already there failed
+Testing 1:blackbox:8:git-annex unlock/lock
+### Error in:   1:blackbox:8:git-annex unlock/lock
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:9:git-annex edit/commit:0
+### Error in:   1:blackbox:9:git-annex edit/commit:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:9:git-annex edit/commit:1
+### Error in:   1:blackbox:9:git-annex edit/commit:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:10:git-annex fix
+### Error in:   1:blackbox:10:git-annex fix
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust
+### Error in:   1:blackbox:11:git-annex trust/untrust/semitrust
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:0
+### Error in:   1:blackbox:12:git-annex fsck:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:1
+### Error in:   1:blackbox:12:git-annex fsck:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:2
+### Error in:   1:blackbox:12:git-annex fsck:2
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:13:git-annex migrate:0
+### Error in:   1:blackbox:13:git-annex migrate:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:13:git-annex migrate:1
+### Error in:   1:blackbox:13:git-annex migrate:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:14:git-annex unused/dropunused
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Cases: 30  Tried: 30  Errors: 11  Failures: 3
+test: failed
+make: *** [test] Error 1
+

+
+On a side note, I think I found another bug in the testing. I had tested in a virtual machine in archlinux (a very recent updated version) Please see the report here [[tests fail when there is no global .gitconfig for the user]]
+
+[a0826293]: http://git.kitenet.net/?p=git-annex;a=commit;h=7a0826293e0ac6c0000f49a1618c1c613b909aa1
+"""]]

From 077b46abc08ca93ef3cfbd1bd23aed8ebc3494bc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 09:13:37 +0000
Subject: [PATCH 0849/2835]

---
 ...ts_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
index 2a26a67267..855f19a67e 100644
--- a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
+++ b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
@@ -22,4 +22,4 @@ Cases: 30  Tried: 8  Errors: 0  Failures: 1
 *** Please tell me who you are.
 

 
-I guess most users or people testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop.
+I guess most users testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop.

From fdb2896958438e1069fa92e69c478f91b182b414 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 10:29:19 +0000
Subject: [PATCH 0850/2835]

---
 ..._is_no_global_.gitconfig_for_the_user.mdwn | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
index 855f19a67e..e9dda99167 100644
--- a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
+++ b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
@@ -23,3 +23,26 @@ Cases: 30  Tried: 8  Errors: 0  Failures: 1
I guess most users testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop. + +Update: I pulled the master on my rhel5 test machine and moved my .gitconfig out of the way, the tests passes and continues but I still get a "warning message" from git. + +
+Testing 1:blackbox:3:git-annex unannex:1:with content                         
+Cases: 30  Tried: 12  Errors: 0  Failures: 0[master fce0cde] content removed from git annex
+ Committer: Jimmy Tang 
+Your name and email address were configured automatically based
+on your username and hostname. Please check that they are accurate.
+You can suppress this message by setting them explicitly:
+
+    git config --global user.name "Your Name"
+    git config --global user.email you@example.com
+
+After doing this, you may fix the identity used for this commit with:
+
+    git commit --amend --reset-author
+
+ 2 files changed, 1 insertions(+), 2 deletions(-)
+ delete mode 120000 foo
+
+ +I guess it also depends a bit on how git figures out who it is is committing and how the machine in question is configured with hostnames and domain names. From 35a65068123f0215929e06e98bf4a6a67d2ae7dd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Feb 2011 11:02:21 -0400 Subject: [PATCH 0851/2835] unannex: Fix recently introduced bug when attempting to unannex more than one file at a time. --- Command/Unannex.hs | 13 ++++++++----- debian/changelog | 2 ++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Command/Unannex.hs b/Command/Unannex.hs index c663b29ab0..19cb1624e5 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -33,10 +33,13 @@ start file = isAnnexed file $ \(key, backend) -> do ishere <- inAnnex key if ishere then do - g <- Annex.gitRepo - staged <- liftIO $ Git.stagedFiles g [Git.workTree g] - unless (null staged) $ - error "This command cannot be run when there are already files staged for commit." + force <- Annex.getState Annex.force + unless force $ do + g <- Annex.gitRepo + staged <- liftIO $ Git.stagedFiles g [Git.workTree g] + unless (null staged) $ + error "This command cannot be run when there are already files staged for commit." + Annex.changeState $ \s -> s { Annex.force = True } showStart "unannex" file return $ Just $ perform file key backend @@ -65,6 +68,6 @@ cleanup file key = do -- Commit staged changes at end to avoid confusing the -- pre-commit hook if this file is later added back to -- git as a normal, non-annexed file. - Annex.queue "commit" ["-m", "content removed from git annex"] "--" + Annex.queue "commit" ["-m", "content removed from git annex"] "-a" return True diff --git a/debian/changelog b/debian/changelog index 52a749178f..072e5fc232 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.21) UNRELEASED; urgency=low * test: Don't rely on chmod -R working. + * unannex: Fix recently introduced bug when attempting to unannex more + than one file at a time. -- Joey Hess Wed, 09 Feb 2011 00:12:11 -0400 From df2312c165d5b019e761bbd80c062f6ff4629618 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 9 Feb 2011 15:04:50 +0000 Subject: [PATCH 0852/2835] Added a comment --- .../comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment b/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment new file mode 100644 index 0000000000..95a9773e2b --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 10" + date="2011-02-09T15:04:50Z" + content=""" +I don't know what these problems forking could be. Can you strace it? +"""]] From 82bc10d5eafa76ce06b294382e1597d67b06e911 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Feb 2011 11:17:26 -0400 Subject: [PATCH 0853/2835] test: Set git user name and email in case git can't guess values. --- debian/changelog | 1 + ...s_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn | 2 ++ test.hs | 3 +++ 3 files changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 072e5fc232..2e5b97c0f9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.21) UNRELEASED; urgency=low * test: Don't rely on chmod -R working. * unannex: Fix recently introduced bug when attempting to unannex more than one file at a time. + * test: Set git user name and email in case git can't guess values. -- Joey Hess Wed, 09 Feb 2011 00:12:11 -0400 diff --git a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn index e9dda99167..b90b501e31 100644 --- a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn +++ b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn @@ -46,3 +46,5 @@ After doing this, you may fix the identity used for this commit with: I guess it also depends a bit on how git figures out who it is is committing and how the machine in question is configured with hostnames and domain names. + +> Fixed that. [[done]] --[[Joey]] diff --git a/test.hs b/test.hs index b32a4abb04..309b541762 100644 --- a/test.hs +++ b/test.hs @@ -508,6 +508,9 @@ setuprepo dir = do cleanup dir ensuretmpdir Utility.boolSystem "git" ["init", "-q", dir] @? "git init failed" + indir dir $ do + Utility.boolSystem "git" ["config", "user.name", "Test User"] @? "git config failed" + Utility.boolSystem "git" ["config", "user.email", "test@example.com"] @? "git config failed" return dir copyrepo :: FilePath -> FilePath -> IO FilePath From cf27ae27532b525b41a1d79f2d6888fa6d5f7f97 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Feb 2011 11:19:19 -0400 Subject: [PATCH 0854/2835] regression test for multi-file unannex --- test.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.hs b/test.hs index 309b541762..1951b44260 100644 --- a/test.hs +++ b/test.hs @@ -139,7 +139,7 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] annexed_notpresent annexedfile withcopy = "with content" ~: intmpcopyrepo $ do annexed_present annexedfile - git_annex "unannex" ["-q", annexedfile] @? "unannex failed" + git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed" unannexed annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" unannexed annexedfile From 8cc289dca5fc89cc81a2c84531fffa3442a9a48b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 9 Feb 2011 19:35:48 +0000 Subject: [PATCH 0855/2835] Added a comment --- .../comment_11_56f1143fa191361d63b441741699e17f._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment b/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment new file mode 100644 index 0000000000..3fbe57ecd5 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 11" + date="2011-02-09T19:35:47Z" + content=""" +I got dtruss to give me a trace, the output is quite big to post here (~560kb gzip'd), do you mind if I emailed it or posted it somewhere else for you? +"""]] From 0885c886d05c9ce885096c43121a96f9a2e4c2aa Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 9 Feb 2011 19:47:30 +0000 Subject: [PATCH 0856/2835] Added a comment --- .../comment_12_ec5131624d0d2285d3b6880e47033f97._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment b/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment new file mode 100644 index 0000000000..beba5dc42c --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 12" + date="2011-02-09T19:47:30Z" + content=""" +joey@kitenet.net (hope I can make sense of dtruss output) +"""]] From fe84d45423b51ec69f7e82ebcff1947e278a14f5 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 9 Feb 2011 21:59:47 +0000 Subject: [PATCH 0857/2835] Added a comment --- ..._13_88ed095a448096bf8a69015a04e64df1._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment b/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment new file mode 100644 index 0000000000..dd25c3d0cb --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 13" + date="2011-02-09T21:59:47Z" + content=""" +The dtrace puzzlingly does not have the same errors shown above, but a set of mostly new errors. I don't know what to make of that. + +> git-annex: git-annex/.t/repo/.git/hooks/pre-commit: fileAccess: permission denied (Operation not permitted) + +This seems to be caused by it setting the execute bit on the file. I don't know why that would fail; it's just written the file and renamed it into place so clearly should be able to write to it. + +> was able to modify annexed file's sha1foo content + +This also suggests something breaking with permissions. +"""]] From fe55b4644e67bba60b35e07abcdd312b65c9d6f3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 10 Feb 2011 14:21:44 -0400 Subject: [PATCH 0858/2835] Fix display of unicode filenames. Internally, the filenames are stored as un-decoded unicode. I tried decoding them, but then haskell tries to access the wrong files. Hmm. So, I've unhappily chosen option "B", which is to decode filenames before they are displayed. --- Backend/File.hs | 6 ++--- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- Command/Find.hs | 3 ++- Command/PreCommit.hs | 2 +- Command/Unused.hs | 2 +- Content.hs | 2 +- Messages.hs | 9 +++++-- debian/changelog | 1 + doc/bugs/problems_with_utf8_names.mdwn | 22 ++++++++------- doc/bugs/unhappy_without_UTF8_locale.mdwn | 33 +++++++++++++++++++++++ 11 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 doc/bugs/unhappy_without_UTF8_locale.mdwn diff --git a/Backend/File.hs b/Backend/File.hs index d76cd29391..fca385a1e9 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -193,14 +193,14 @@ checkKeyNumCopies key file numcopies = do missingNote :: String -> Int -> Int -> String -> String missingNote file 0 _ [] = - "** No known copies of " ++ file ++ " exist!" + "** No known copies of " ++ showFile file ++ " exist!" missingNote file 0 _ untrusted = - "Only these untrusted locations may have copies of " ++ file ++ + "Only these untrusted locations may have copies of " ++ showFile file ++ "\n" ++ untrusted ++ "Back it up to trusted locations with git-annex copy." missingNote file present needed [] = "Only " ++ show present ++ " of " ++ show needed ++ - " trustworthy copies of " ++ file ++ " exist." ++ + " trustworthy copies of " ++ showFile file ++ " exist." ++ "\nBack it up with git-annex copy." missingNote file present needed untrusted = missingNote file present needed [] ++ diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 3d868dbd19..f1092492ed 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -58,5 +58,5 @@ checkKeySHA1 key = do then return True else do dest <- moveBad key - warning $ "Bad file content; moved to "++dest + warning $ "Bad file content; moved to " ++ showFile dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 20a81d8411..7f40a2acb5 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -67,5 +67,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - warning $ "Bad file size; moved to "++dest + warning $ "Bad file size; moved to " ++ showFile dest return False diff --git a/Command/Find.hs b/Command/Find.hs index 3ed15c1537..45156af051 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -12,6 +12,7 @@ import Control.Monad.State (liftIO) import Command import Content +import Messages command :: [Command] command = [Command "find" (paramOptional $ paramRepeating paramPath) seek @@ -24,5 +25,5 @@ seek = [withFilesInGit start] start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key - when exists $ liftIO $ putStrLn file + when exists $ liftIO $ putStrLn $ showFile file return Nothing diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 12e5ed806d..f22300a030 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -32,7 +32,7 @@ perform pair@(file, _) = do ok <- doCommand $ Command.Add.start pair if ok then return $ Just $ cleanup file - else error $ "failed to add " ++ file ++ "; canceling commit" + else error $ "failed to add " ++ showFile file ++ "; canceling commit" cleanup :: FilePath -> CommandCleanup cleanup file = do diff --git a/Command/Unused.hs b/Command/Unused.hs index d9f4e39783..2b390b9563 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -68,7 +68,7 @@ checkUnused = do dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"] table l = [" NUMBER KEY"] ++ map cols l - cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ (showFile . show) k pad n s = s ++ replicate (n - length s) ' ' number :: Int -> [a] -> [(Int, a)] diff --git a/Content.hs b/Content.hs index e16ad883c2..fae980bae0 100644 --- a/Content.hs +++ b/Content.hs @@ -49,7 +49,7 @@ calcGitLink file key = do cwd <- liftIO $ getCurrentDirectory let absfile = case absNormPath cwd file of Just f -> f - Nothing -> error $ "unable to normalize " ++ file + Nothing -> error $ "unable to normalize " ++ showFile file return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++ annexLocation key diff --git a/Messages.hs b/Messages.hs index 6f4ec1e626..12a836d3c5 100644 --- a/Messages.hs +++ b/Messages.hs @@ -11,6 +11,7 @@ import Control.Monad.State (liftIO) import System.IO import Control.Monad (unless) import Data.String.Utils +import Codec.Binary.UTF8.String as UTF8 import Types import qualified Annex @@ -25,7 +26,7 @@ showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")" showStart :: String -> String -> Annex () showStart command file = verbose $ do - liftIO $ putStr $ command ++ " " ++ file ++ " " + liftIO $ putStr $ command ++ " " ++ showFile file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () @@ -45,7 +46,6 @@ showEndOk = verbose $ liftIO $ putStrLn "ok" showEndFail :: Annex () showEndFail = verbose $ liftIO $ putStrLn "\nfailed" -{- Exception pretty-printing. -} showErr :: (Show a) => a -> Annex () showErr e = warning $ "git-annex: " ++ show e @@ -57,3 +57,8 @@ warning w = do indent :: String -> String indent s = join "\n" $ map (\l -> " " ++ l) $ lines s + +{- Prepares a filename for display. This is needed because strings are + - internally represented in git-annex is non-decoded form. -} +showFile :: String -> String +showFile = decodeString diff --git a/debian/changelog b/debian/changelog index 2e5b97c0f9..277869b020 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.21) UNRELEASED; urgency=low * unannex: Fix recently introduced bug when attempting to unannex more than one file at a time. * test: Set git user name and email in case git can't guess values. + * Fix display of unicode filenames. -- Joey Hess Wed, 09 Feb 2011 00:12:11 -0400 diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 30f3495f40..257f8dff21 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -37,10 +37,22 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > encoded in utf-8 (an archive could have historical filenames using > varying encodings), and you don't want which files are accessed to > depend on locale settings. +> > I tried to do this by making parts of GitRepo call +> > Codec.Binary.UTF8.String.decodeString when reading filenames from +> > git. This seemed to break attempts to operate on the files, +> > weirdly encoded strings were seen in syscalls in strace. > 1. Keep input and internal data un-decoded, but decode it when > outputting a filename (assuming the filename is encoded using the > user's configured encoding), and allow haskell's output encoding to then > encode it according to the user's locale configuration. +> > This is now [[implemented|done]]. I'm not very happy that I have to watch +> > out for any place that a filename is output and call `showFile` +> > on it, but there are really not too many such places in git-annex. +> > +> > Note that this only affects filenames apparently. +> > (Names of files in the annex, and also some places where names +> > of keys are displayed.) Utf-8 in the uuid.map file etc seems +> > to be handled cleanly. > 1. Avoid encodings entirely. Mostly what I'm doing now; probably > could find a way to disable encoding of console output. Then the raw > filename would be displayed, which should work ok. git-annex does @@ -50,13 +62,3 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > One other possible > issue would be that this could cause problems if git-annex were > translated. -> -> BTW, for more fun, try unsetting LANG, and then you can see -> stuff like this: - - joey@gnu:~/tmp/aa>git annex add ./Üa - add add add add git-annex: : commitAndReleaseBuffer: invalid - argument (Invalid or incomplete multibyte or wide character) - -> (Add -q to work around this; once it doesn't need to print the filename, -> it can act on it ok!) diff --git a/doc/bugs/unhappy_without_UTF8_locale.mdwn b/doc/bugs/unhappy_without_UTF8_locale.mdwn new file mode 100644 index 0000000000..6f1df4fab7 --- /dev/null +++ b/doc/bugs/unhappy_without_UTF8_locale.mdwn @@ -0,0 +1,33 @@ +Try unsetting LANG and passing git-annex unicode filenames. + + joey@gnu:~/tmp/aa>git annex add ./Üa + add add add add git-annex: : commitAndReleaseBuffer: invalid + argument (Invalid or incomplete multibyte or wide character) + +The same problem can be seen with a simple haskell program: + + import System.Environment + import Codec.Binary.UTF8.String + main = do + args <- getArgs + putStrLn $ decodeString $ args !! 0 + + joey@gnu:~/src/git-annex>LANG= runghc ~/foo.hs Ü + foo.hs: : hPutChar: invalid argument (Invalid or incomplete multibyte or wide character) + +(The call to `decodeString` is necessary to make the input +unicode string be displayed properly in a utf8 locale, but +does not contribute to this problem.) + +I guess that haskell is setting the IO encoding to latin1, which +is [documented](http://haskell.org/ghc/docs/latest/html/libraries/base/System-IO.html#v:latin1) +to error out on characters > 255. + +So this program doesn't have the problem -- but may output garbage +on non-utf-8 capable terminals: + + import System.IO + main = do + hSetEncoding stdout utf8 + args <- getArgs + putStrLn $ decodeString $ args !! 0 From 52fa424faef63690d5b44cc4b6fd11aa67f1e619 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 10 Feb 2011 14:45:35 -0400 Subject: [PATCH 0859/2835] update --- doc/bugs/problems_with_utf8_names.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 257f8dff21..58eb5ad1c0 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -58,7 +58,8 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > filename would be displayed, which should work ok. git-annex does > not really need to pull apart filenames; they are almost entirely > opaque blobs. I guess that the `--exclude` option is the exception -> to that, but it is currently not unicode safe anyway. +> to that, but it is currently not unicode safe anyway. (Update: tried +> `--exclude` again, seems it is unicode clean..) > One other possible > issue would be that this could cause problems if git-annex were > translated. From 0fef480bcaee9ae87f17d008a3787871d3bc9786 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 10 Feb 2011 14:58:09 -0400 Subject: [PATCH 0860/2835] update --- doc/bugs/problems_with_utf8_names.mdwn | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 58eb5ad1c0..29213e1ec9 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -63,3 +63,39 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > One other possible > issue would be that this could cause problems if git-annex were > translated. + +---- + +Simpler test case: + +
+import Codec.Binary.UTF8.String
+import System.Environment
+
+main = do
+        args <- getArgs
+        let file = decodeString $ head args
+        putStrLn $ "file is: " ++ file
+        putStr =<< readFile file
+
+ +If I pass this a filename like 'ü', it will fail, and notice +the bad encoding of the filename in the error message: + +
+$ echo hi > ü; runghc foo.hs ü
+file is: ü
+foo.hs: �: openFile: does not exist (No such file or directory)
+
+ +On the other hand, if I remove the decodeString, it prints the filename +wrong, while accessing it right: + +
+$ runghc foo.hs ü
+file is: üa
+hi
+
+ +The only way that seems to consistently work is to delay decoding the +filename to places where it's output. But then it's easy to miss some. From 75e507d0af707f9773f4b93c17c76c5161d2a195 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 10 Feb 2011 16:53:35 -0400 Subject: [PATCH 0861/2835] foo --- Messages.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Messages.hs b/Messages.hs index 12a836d3c5..ab5992a6c6 100644 --- a/Messages.hs +++ b/Messages.hs @@ -60,5 +60,5 @@ indent s = join "\n" $ map (\l -> " " ++ l) $ lines s {- Prepares a filename for display. This is needed because strings are - internally represented in git-annex is non-decoded form. -} -showFile :: String -> String +showFile :: FilePath -> String showFile = decodeString From 285fb2bb08c7da534c111ebfeee5911e850570cc Mon Sep 17 00:00:00 2001 From: Michael Kenney Date: Thu, 10 Feb 2011 15:45:34 -0800 Subject: [PATCH 0862/2835] Fixed missing import of Messages module --- Command/PreCommit.hs | 1 + Content.hs | 1 + 2 files changed, 2 insertions(+) diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index f22300a030..76aab3855a 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -14,6 +14,7 @@ import qualified Annex import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix +import Messages command :: [Command] command = [Command "pre-commit" paramPath seek "run by git pre-commit hook"] diff --git a/Content.hs b/Content.hs index fae980bae0..188a38787e 100644 --- a/Content.hs +++ b/Content.hs @@ -34,6 +34,7 @@ import UUID import qualified GitRepo as Git import qualified Annex import Utility +import Messages {- Checks if a given key is currently present in the gitAnnexLocation. -} inAnnex :: Key -> Annex Bool From 5a50a7cf137997a9d940b9a89a0968452a1ac411 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 11 Feb 2011 15:37:37 -0400 Subject: [PATCH 0863/2835] update unicode FilePath handling Based on http://hackage.haskell.org/trac/ghc/ticket/3307 , whether FilePath contains decoded unicode varies by OS. So, add a configure check for it. Also, renamed showFile to filePathToString --- Backend/File.hs | 6 +++--- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- Command/Find.hs | 2 +- Command/PreCommit.hs | 2 +- Command/Unused.hs | 2 +- Content.hs | 2 +- Messages.hs | 14 ++++++++------ configure.hs | 15 +++++++++++++++ doc/bugs/problems_with_utf8_names.mdwn | 2 +- testdata/unicode-test-ö | 1 + 11 files changed, 34 insertions(+), 16 deletions(-) create mode 100644 testdata/unicode-test-ö diff --git a/Backend/File.hs b/Backend/File.hs index fca385a1e9..d5691595a8 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -193,14 +193,14 @@ checkKeyNumCopies key file numcopies = do missingNote :: String -> Int -> Int -> String -> String missingNote file 0 _ [] = - "** No known copies of " ++ showFile file ++ " exist!" + "** No known copies of " ++ filePathToString file ++ " exist!" missingNote file 0 _ untrusted = - "Only these untrusted locations may have copies of " ++ showFile file ++ + "Only these untrusted locations may have copies of " ++ filePathToString file ++ "\n" ++ untrusted ++ "Back it up to trusted locations with git-annex copy." missingNote file present needed [] = "Only " ++ show present ++ " of " ++ show needed ++ - " trustworthy copies of " ++ showFile file ++ " exist." ++ + " trustworthy copies of " ++ filePathToString file ++ " exist." ++ "\nBack it up with git-annex copy." missingNote file present needed untrusted = missingNote file present needed [] ++ diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index f1092492ed..9636787f0d 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -58,5 +58,5 @@ checkKeySHA1 key = do then return True else do dest <- moveBad key - warning $ "Bad file content; moved to " ++ showFile dest + warning $ "Bad file content; moved to " ++ filePathToString dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 7f40a2acb5..92fe5a2d4c 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -67,5 +67,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - warning $ "Bad file size; moved to " ++ showFile dest + warning $ "Bad file size; moved to " ++ filePathToString dest return False diff --git a/Command/Find.hs b/Command/Find.hs index 45156af051..3e9125b9a6 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -25,5 +25,5 @@ seek = [withFilesInGit start] start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key - when exists $ liftIO $ putStrLn $ showFile file + when exists $ liftIO $ putStrLn $ filePathToString file return Nothing diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 76aab3855a..750997f540 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -33,7 +33,7 @@ perform pair@(file, _) = do ok <- doCommand $ Command.Add.start pair if ok then return $ Just $ cleanup file - else error $ "failed to add " ++ showFile file ++ "; canceling commit" + else error $ "failed to add " ++ filePathToString file ++ "; canceling commit" cleanup :: FilePath -> CommandCleanup cleanup file = do diff --git a/Command/Unused.hs b/Command/Unused.hs index 2b390b9563..67a2272371 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -68,7 +68,7 @@ checkUnused = do dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"] table l = [" NUMBER KEY"] ++ map cols l - cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ (showFile . show) k + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ (filePathToString . show) k pad n s = s ++ replicate (n - length s) ' ' number :: Int -> [a] -> [(Int, a)] diff --git a/Content.hs b/Content.hs index 188a38787e..345599dbad 100644 --- a/Content.hs +++ b/Content.hs @@ -50,7 +50,7 @@ calcGitLink file key = do cwd <- liftIO $ getCurrentDirectory let absfile = case absNormPath cwd file of Just f -> f - Nothing -> error $ "unable to normalize " ++ showFile file + Nothing -> error $ "unable to normalize " ++ filePathToString file return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++ annexLocation key diff --git a/Messages.hs b/Messages.hs index ab5992a6c6..80b53e5cd5 100644 --- a/Messages.hs +++ b/Messages.hs @@ -11,10 +11,11 @@ import Control.Monad.State (liftIO) import System.IO import Control.Monad (unless) import Data.String.Utils -import Codec.Binary.UTF8.String as UTF8 +import qualified Codec.Binary.UTF8.String as UTF8 import Types import qualified Annex +import SysConfig verbose :: Annex () -> Annex () verbose a = do @@ -26,7 +27,7 @@ showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")" showStart :: String -> String -> Annex () showStart command file = verbose $ do - liftIO $ putStr $ command ++ " " ++ showFile file ++ " " + liftIO $ putStr $ command ++ " " ++ filePathToString file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () @@ -58,7 +59,8 @@ warning w = do indent :: String -> String indent s = join "\n" $ map (\l -> " " ++ l) $ lines s -{- Prepares a filename for display. This is needed because strings are - - internally represented in git-annex is non-decoded form. -} -showFile :: FilePath -> String -showFile = decodeString +{- Prepares a filename for display. This is needed because on many + - platforms (eg, unix), FilePaths are internally stored in + - non-decoded form. -} +filePathToString :: FilePath -> String +filePathToString = if unicodefilepath then id else UTF8.decodeString diff --git a/configure.hs b/configure.hs index 1451d7eaa3..b5437ec1a5 100644 --- a/configure.hs +++ b/configure.hs @@ -1,6 +1,7 @@ {- Checks system configuration and generates SysConfig.hs. -} import System.Directory +import Data.List import TestConfig @@ -13,6 +14,7 @@ tests = [ , TestCase "sha1sum" $ requireCmd "sha1sum" "sha1sum /dev/null" + , TestCase "unicode FilePath support" $ unicodeFilePath ] tmpDir :: String @@ -27,6 +29,19 @@ testCp k option = TestCase cmd $ testCmd k run cmd = "cp " ++ option run = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new" +{- Checks if FilePaths contain decoded unicode, or not. The testdata + - directory contains a "unicode-test-ü" file; try to find the file, + - and see if the "ü" is encoded correctly. + - + - Note that the file is shipped with git-annex, rather than created, + - to avoid other potential unicode issues. + -} +unicodeFilePath :: Test +unicodeFilePath = do + fs <- getDirectoryContents "testdata" + let file = head $ filter (isInfixOf "unicode-test") fs + return $ Config "unicodefilepath" (BoolConfig $ isInfixOf "ü" file) + setup :: IO () setup = do createDirectoryIfMissing True tmpDir diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 29213e1ec9..efde1c9a3a 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -46,7 +46,7 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > user's configured encoding), and allow haskell's output encoding to then > encode it according to the user's locale configuration. > > This is now [[implemented|done]]. I'm not very happy that I have to watch -> > out for any place that a filename is output and call `showFile` +> > out for any place that a filename is output and call `filePathToString` > > on it, but there are really not too many such places in git-annex. > > > > Note that this only affects filenames apparently. diff --git a/testdata/unicode-test-ö b/testdata/unicode-test-ö new file mode 100644 index 0000000000..45b983be36 --- /dev/null +++ b/testdata/unicode-test-ö @@ -0,0 +1 @@ +hi From 735076c767f09c740e0c99201d341a12d7b7df2c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 11 Feb 2011 15:49:59 -0400 Subject: [PATCH 0864/2835] qualified import --- Messages.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Messages.hs b/Messages.hs index 80b53e5cd5..90857280a5 100644 --- a/Messages.hs +++ b/Messages.hs @@ -15,7 +15,7 @@ import qualified Codec.Binary.UTF8.String as UTF8 import Types import qualified Annex -import SysConfig +import qualified SysConfig verbose :: Annex () -> Annex () verbose a = do @@ -63,4 +63,4 @@ indent s = join "\n" $ map (\l -> " " ++ l) $ lines s - platforms (eg, unix), FilePaths are internally stored in - non-decoded form. -} filePathToString :: FilePath -> String -filePathToString = if unicodefilepath then id else UTF8.decodeString +filePathToString = if SysConfig.unicodefilepath then id else UTF8.decodeString From ddd305aa10b2ee63c493aee05ab30d06b9424139 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 11 Feb 2011 23:23:36 -0400 Subject: [PATCH 0865/2835] releasing version 0.21 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 277869b020..11a745f00d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.21) UNRELEASED; urgency=low +git-annex (0.21) unstable; urgency=low * test: Don't rely on chmod -R working. * unannex: Fix recently introduced bug when attempting to unannex more @@ -6,7 +6,7 @@ git-annex (0.21) UNRELEASED; urgency=low * test: Set git user name and email in case git can't guess values. * Fix display of unicode filenames. - -- Joey Hess Wed, 09 Feb 2011 00:12:11 -0400 + -- Joey Hess Fri, 11 Feb 2011 23:21:08 -0400 git-annex (0.20) unstable; urgency=low From 9806af73686259bf400222df1636cf3987b1ff76 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 11 Feb 2011 23:23:53 -0400 Subject: [PATCH 0866/2835] add news item for git-annex 0.21 --- doc/news/version_0.16.mdwn | 13 ------------- doc/news/version_0.21.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 doc/news/version_0.16.mdwn create mode 100644 doc/news/version_0.21.mdwn diff --git a/doc/news/version_0.16.mdwn b/doc/news/version_0.16.mdwn deleted file mode 100644 index 3247e302dd..0000000000 --- a/doc/news/version_0.16.mdwn +++ /dev/null @@ -1,13 +0,0 @@ -git-annex 0.16 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * git-annex-shell: Avoid exposing any git repo config except for the - annex.uuid when doing configlist. - * bugfix: Running `move --to` with a remote whose UUID was not yet known - could result in git-annex not recording on the local side where the - file was moved to. This could not result in data loss, or even a - significant problem, since the remote *did* record that it had the file. - * Also, add a general guard to detect attempts to record information - about repositories with missing UUIDs. - * bugfix: Running `move --to` with a non-ssh remote failed. - * bugfix: Running `copy --to` with a non-ssh remote actually did a move. - * Many test suite improvements. Current top-level test coverage: 65%"""]] \ No newline at end of file diff --git a/doc/news/version_0.21.mdwn b/doc/news/version_0.21.mdwn new file mode 100644 index 0000000000..996474910e --- /dev/null +++ b/doc/news/version_0.21.mdwn @@ -0,0 +1,7 @@ +git-annex 0.21 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * test: Don't rely on chmod -R working. + * unannex: Fix recently introduced bug when attempting to unannex more + than one file at a time. + * test: Set git user name and email in case git can't guess values. + * Fix display of unicode filenames."""]] \ No newline at end of file From 4f5f29c146a162c10a29c7744a923b9b4499846c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sat, 12 Feb 2011 21:19:26 +0000 Subject: [PATCH 0867/2835] Added a comment --- .../comment_14_89a960b6706ed703b390a81a8bc4e311._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment b/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment new file mode 100644 index 0000000000..724fe5505a --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 14" + date="2011-02-12T21:19:24Z" + content=""" +I've been trying to dig around the trace and code, and used google to see if the forkProcess issue was a haskell thing or an OSX thing. It seems that someone may have ran into a similar issue, though I am not sure if its related. +"""]] From 44d015974911ed1d5beb3ddde49ff6d59d533ca1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 13 Feb 2011 02:45:52 +0000 Subject: [PATCH 0868/2835] Added a comment --- ..._6b8867b8e48bf807c955779c9f8f0909._comment | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment b/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment new file mode 100644 index 0000000000..733ec997a6 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment @@ -0,0 +1,71 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 15" + date="2011-02-13T02:45:51Z" + content=""" +It may be possible that OSX has some low resource limits, for user processes (266 per user I think) doing a + + sudo sysctl -w kern.maxproc=2048 + sudo sysctl -w kern.maxprocperuid=1024 + sudo echo \"limit maxfiles 1024 unlimited\" >> /etc/launchd.conf + sudo echo \"limit maxproc 1024 2048\" >> /etc/launchd.conf + +seems to change the behaviour of the tests abit... + +
+Testing 1:blackbox:3:git-annex unannex:1:with content                         
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes                              
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Testing 1:blackbox:4:git-annex drop:1:with remote                             
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote                        
+Testing 1:blackbox:5:git-annex get                                            
+Testing 1:blackbox:6:git-annex move                                           
+Testing 1:blackbox:7:git-annex copy                                           
+Testing 1:blackbox:8:git-annex unlock/lock                                    
+Testing 1:blackbox:9:git-annex edit/commit:0                                  
+Cases: 30  Tried: 20  Errors: 0  Failures: 2add foo ok
+ok
+Testing 1:blackbox:9:git-annex edit/commit:1                                  
+Testing 1:blackbox:10:git-annex fix                                           
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust                       
+Testing 1:blackbox:12:git-annex fsck:0                                        
+Cases: 30  Tried: 24  Errors: 0  Failures: 2  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  Bad file size; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/WORM:1297565141:20:foo
+  Bad file content; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa
+Testing 1:blackbox:12:git-annex fsck:1                                        
+### Failure in: 1:blackbox:12:git-annex fsck:1
+fsck failed to fail with content only available in untrusted (current) repository
+Testing 1:blackbox:12:git-annex fsck:2                                        
+Cases: 30  Tried: 26  Errors: 0  Failures: 3  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	58e831c2-371b-11e0-bc1f-47d738dc52ee  -- test repo
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	58e831c2-371b-11e0-bc1f-47d738dc52ee  -- test repo
+Testing 1:blackbox:13:git-annex migrate:0                                     
+Cases: 30  Tried: 27  Errors: 0  Failures: 3  git-annex: user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))
+### Failure in: 1:blackbox:13:git-annex migrate:0
+migrate annexedfile failed
+Testing 1:blackbox:13:git-annex migrate:1                                     
+### Error in:   1:blackbox:13:git-annex migrate:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:14:git-annex unused/dropunused                             
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Cases: 30  Tried: 30  Errors: 2  Failures: 4
+test: failed
+
+ + +the number of failures vary as I change the values of the maxprocs, I think I have narrowed it down to OSX just being stupid with limits thus causing the tests to fail. + +"""]] From c319a336a3829b2f613a86725603c4b2e8837476 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Feb 2011 00:50:09 -0400 Subject: [PATCH 0869/2835] Fix test suite to reap zombies. I had not taken into account that the code was written to run git and leave zombies, for performance/laziness reasons, when I wrote the test suite. So rather than the typical 1 zombie process that git-annex develops, test developed dozens. Caused problems on system with low process limits. Added a reap function to GitRepo, that waits for any zombie child processes. --- CmdLine.hs | 3 +++ GitRepo.hs | 37 ++++++++++++++++++------------------- debian/changelog | 6 ++++++ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index d65739791f..475ca99e78 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -101,4 +101,7 @@ shutdown = do unless (q == GitQueue.empty) $ do showSideAction "Recording state in git..." Annex.queueRun + + liftIO $ Git.reap + return True diff --git a/GitRepo.hs b/GitRepo.hs index 7bb20fc53c..7cf0891eda 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -34,7 +34,6 @@ module GitRepo ( gitCommandLine, run, pipeRead, - hPipeRead, attributes, remotes, remotesAdd, @@ -50,6 +49,7 @@ module GitRepo ( typeChangedFiles, typeChangedStagedFiles, absDir, + reap, prop_idempotent_deencode ) where @@ -58,6 +58,7 @@ import Control.Monad (unless) import System.Directory import System.Posix.Directory import System.Posix.User +import System.Posix.Process import System.Path import System.Cmd.Utils import IO (bracket_) @@ -254,22 +255,24 @@ run repo params = assertLocal repo $ do ok <- boolSystem "git" (gitCommandLine repo params) unless ok $ error $ "git " ++ show params ++ " failed" -{- Runs a git subcommand and returns its output. -} +{- Runs a git subcommand and returns it output, lazily. + - + - Note that this leaves the git process running, and so zombies will + - result unless reap is called. + -} pipeRead :: Repo -> [String] -> IO String pipeRead repo params = assertLocal repo $ do - pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - hGetContentsStrict h + (_, s) <- pipeFrom "git" (gitCommandLine repo params) + return s -{- Like pipeRead, but does not read output strictly; recommended - - for git commands that produce a lot of output that will be processed - - lazily. - - - - ONLY AFTER the string has been read completely, You must call either - - getProcessStatus or forceSuccess on the PipeHandle. Zombies will result - - otherwise.-} -hPipeRead :: Repo -> [String] -> IO (PipeHandle, String) -hPipeRead repo params = assertLocal repo $ do - pipeFrom "git" (gitCommandLine repo params) +{- Reaps any zombie git processes. -} +reap :: IO () +reap = do + -- throws an exception when there are no child processes + r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing) + case r of + Nothing -> return () + Just _ -> reap {- Scans for files that are checked into git at the specified locations. -} inRepo :: Repo -> [FilePath] -> IO [FilePath] @@ -322,9 +325,7 @@ typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end - parameter), and splits it into a list of files. -} pipeNullSplit :: Repo -> [String] -> IO [FilePath] pipeNullSplit repo params = do - -- XXX handle is left open, this is ok for git-annex, but may need - -- to be cleaned up for other uses. - (_, fs0) <- hPipeRead repo params + fs0 <- pipeRead repo params return $ split0 fs0 where split0 s = filter (not . null) $ split "\0" s @@ -410,8 +411,6 @@ checkAttr repo attr files = do (_, s) <- pipeBoth "git" params $ join "\0" absfiles cwd <- getCurrentDirectory return $ map (topair $ cwd++"/") $ lines s - -- XXX handle is left open, this is ok for git-annex, but may need - -- to be cleaned up for other uses. where params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] topair cwd l = (relfile, value) diff --git a/debian/changelog b/debian/changelog index 11a745f00d..9fc2e559a7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.22) UNRELEASED; urgency=low + + * Fix test suite to reap zombies. + + -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 + git-annex (0.21) unstable; urgency=low * test: Don't rely on chmod -R working. From 123f6a571d2b6aff097cd03e64769756a900bc6a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Feb 2011 00:51:20 -0400 Subject: [PATCH 0870/2835] heh --- debian/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian/changelog b/debian/changelog index 9fc2e559a7..fd038a2dab 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.22) UNRELEASED; urgency=low * Fix test suite to reap zombies. + (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang + for his help eliminating the infestation... for now.) -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 From c52bd2f10b2e4317c4c9cbc451f273dc4dfec79e Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 13 Feb 2011 04:52:26 +0000 Subject: [PATCH 0871/2835] Added a comment --- .../comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment b/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment new file mode 100644 index 0000000000..ca1b8e8cd5 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 16" + date="2011-02-13T04:52:26Z" + content=""" +I've fixed the test suite to not accumulate all those zombie processes. Now only 2 or 3 processes should run max. Am curious to see if that clears up all the problems. +"""]] From ed15a4e184a7d962d8f45dbadf640c24e397af0e Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 13 Feb 2011 10:46:55 +0000 Subject: [PATCH 0872/2835] Added a comment --- ..._62fccb04b0e4b695312f7a3f32fb96ee._comment | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment b/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment new file mode 100644 index 0000000000..7c7200fb9e --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 17" + date="2011-02-13T10:46:54Z" + content=""" +Yeap, that did the trick. I just tested a few separate OSX 10.6.6 systems and the tests are better behaved now, only 3 failures now. + +So the tests behave better (at least we don't get resource fork errors any more) + + * after the commit c319a3 without modifying the system limits (of 266 procs per user) + * without the commit c319a3 and when I increase the system process limits to as much as OSX allows + +On all the systems I tested on, I'm down to 3 failures now. + +
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Cases: 30  Tried: 20  Errors: 0  Failures: 2add foo ok
+ok
+Cases: 30  Tried: 24  Errors: 0  Failures: 2  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  Bad file size; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/WORM:1297594011:20:foo
+  Bad file content; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa
+### Failure in: 1:blackbox:12:git-annex fsck:1
+fsck failed to fail with content only available in untrusted (current) repository
+Cases: 30  Tried: 26  Errors: 0  Failures: 3  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	90d63906-375e-11e0-8867-abb8a6368269  -- test repo
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	90d63906-375e-11e0-8867-abb8a6368269  -- test repo
+Cases: 30  Tried: 30  Errors: 0  Failures: 3
+
+ +It's the same set of failures across all the OSX systems that I have tested on. Now I just need to figure out why there are still these three failures. +"""]] From 65d30ffc375259a0d09c914abf15057e8ef639c5 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 13 Feb 2011 15:12:11 +0000 Subject: [PATCH 0873/2835] Added a comment: maybe killed another osx bug in the test. --- ..._64fab50d95de619eb2e8f08f90237de1._comment | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment b/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment new file mode 100644 index 0000000000..df76bb3017 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="maybe killed another osx bug in the test." + date="2011-02-13T15:12:10Z" + content=""" +I think I have figured out why + + ### Failure in: 1:blackbox:3:git-annex unannex:1:with content + foo is not a symlink + +It goes back to the this piece of code (in test.hs) + + copyrepo :: FilePath -> FilePath -> IO FilePath + copyrepo old new = do + cleanup new + ensuretmpdir + Utility.boolSystem \"cp\" [\"-pr\", old, new] @? \"cp -pr failed\" + +It seems that on OSX it does not preserve the symbolic link information, basically cp is not gnu cp on OSX, doing a \"cp -a SOURCE DEST\" seem's to the right thing on OSX. I tried it out on my archlinux workstation by replacing *-pr* with just *-a* and all the tests passed on archlinux. + +I'm not sure what the implications would be with changing the test with changing the cp command. + +"""]] From 02711df66e77c8eb1686e5d4d50c93da475daa7c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 13 Feb 2011 15:55:48 +0000 Subject: [PATCH 0874/2835] Added a comment --- ...mment_19_4253988ed178054c8b6400beeed68a29._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment b/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment new file mode 100644 index 0000000000..090c991c3a --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 19" + date="2011-02-13T15:55:47Z" + content=""" +On second thought and after some messing (trying most of the options and combinations of options on OSX for).... I tried replacing cp with gnu cp from coreutils on my OSX install, and all the tests passed. *sigh* cp -a is preserving some permissions and attributes but not all, its not behaving in the same way as the gnu cp does... the closet thing that I have found on OSX that behaves in the same way as gnu \"cp -pr\" is to use \"ditto\". + +Just doing a \"ditto SOURCE DEST\" in the tests passes everything. I'm not sure if its a good idea to use this even though it works. Though this is just the tests, does it affect CopyFile.hs where \"cp\" is called? + +"""]] From e2cf7389be5158b1016ef81d41d235b4e7cf7c2d Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 13 Feb 2011 17:54:09 +0000 Subject: [PATCH 0875/2835] Added a comment --- ...omment_20_7db27d1a22666c831848bc6c06d66a84._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment b/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment new file mode 100644 index 0000000000..b617da9261 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 20" + date="2011-02-13T17:54:09Z" + content=""" +Outside the test suite, git-annex's actual use of cp puts fairly low demands on it. It tries to use cp -a or cp -p if available just to preserve whatever attributes it can preserve, but the worst case if that you have a symlink pointing to a file that doesn't have the original timestamp or whatever. And there's little expectation git preserves that stuff anyway. + +I will probably try to make the test suite entirely use git clone rather than cp. +"""]] From 33901834008bef4e260d0be3b9e344b8f82b5856 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Feb 2011 14:17:02 -0400 Subject: [PATCH 0876/2835] Make test suite not rely on a working cp -pr. (The Unix wars are still ON!) --- debian/changelog | 2 ++ doc/bugs/Problems_running_make_on_osx.mdwn | 6 +++++ test.hs | 31 ++++++++++------------ 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/debian/changelog b/debian/changelog index fd038a2dab..173c441b03 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,8 @@ git-annex (0.22) UNRELEASED; urgency=low * Fix test suite to reap zombies. (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang for his help eliminating the infestation... for now.) + * Make test suite not rely on a working cp -pr. + (The Unix wars are still ON!) -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn index bd0f2e64a8..83b75fb544 100644 --- a/doc/bugs/Problems_running_make_on_osx.mdwn +++ b/doc/bugs/Problems_running_make_on_osx.mdwn @@ -41,3 +41,9 @@ My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary > entire end of the file from line 68, "for quickcheck" down. This code > is only used by the test suite (so "make test" will fail), > but it should get it to build. --[[Joey]] + +--- + +Closing this bug because the above problem now has a solution documented on +the install page, and the below test suite failure problems should all be +resolved on OSX. [[done]] --[[Joey]] diff --git a/test.hs b/test.hs index 1951b44260..9b50bcb2ef 100644 --- a/test.hs +++ b/test.hs @@ -137,7 +137,10 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] annexed_notpresent annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed with no copy" annexed_notpresent annexedfile - withcopy = "with content" ~: intmpcopyrepo $ do + withcopy = "with content" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get failed" + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "state changed"] + @? "git commit of state failed" annexed_present annexedfile git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed" unannexed annexedfile @@ -149,7 +152,12 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] test_drop :: Test test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote] where - noremote = "no remotes" ~: TestCase $ intmpcopyrepo $ do + noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get failed" + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "state changed"] + @? "git commit of state failed" + Utility.boolSystem "git" ["remote", "rm", "origin"] + @? "git remote rm origin failed" r <- git_annex "drop" ["-q", annexedfile] not r @? "drop wrongly succeeded with no known copy of file" annexed_present annexedfile @@ -347,11 +355,13 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed" corrupt annexedfile corrupt sha1annexedfile - withlocaluntrusted = TestCase $ intmpcopyrepo $ do + withlocaluntrusted = TestCase $ intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get failed" + git_annex "untrust" ["-q", "origin"] @? "untrust of origin repo failed" git_annex "untrust" ["-q", "."] @? "untrust of current repo failed" fsck_should_fail "content only available in untrusted (current) repository" git_annex "trust" ["-q", "."] @? "trust of current repo failed" - git_annex "fsck" ["-q"] @? "fsck failed on trusted repo" + git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo" withremoteuntrusted = TestCase $ intmpclonerepo $ do Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed" git_annex "get" ["-q", annexedfile] @? "get failed" @@ -475,15 +485,9 @@ innewrepo a = withgitrepo $ \r -> indir r a inmainrepo :: Assertion -> Assertion inmainrepo a = indir mainrepodir a -intmpcopyrepo :: Assertion -> Assertion -intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a - intmpclonerepo :: Assertion -> Assertion intmpclonerepo a = withtmpclonerepo $ \r -> indir r a -withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion -withtmpcopyrepo = bracket (copyrepo mainrepodir tmprepodir) cleanup - withtmpclonerepo :: (FilePath -> Assertion) -> Assertion withtmpclonerepo = bracket (clonerepo mainrepodir tmprepodir) cleanup @@ -513,13 +517,6 @@ setuprepo dir = do Utility.boolSystem "git" ["config", "user.email", "test@example.com"] @? "git config failed" return dir -copyrepo :: FilePath -> FilePath -> IO FilePath -copyrepo old new = do - cleanup new - ensuretmpdir - Utility.boolSystem "cp" ["-pr", old, new] @? "cp -pr failed" - return new - -- clones are always done as local clones; we cannot test ssh clones clonerepo :: FilePath -> FilePath -> IO FilePath clonerepo old new = do From 68db7bba4390fb54ffeb9ce5d72644d2e8f4790a Mon Sep 17 00:00:00 2001 From: "http://dieter-be.myopenid.com/" Date: Mon, 14 Feb 2011 21:06:55 +0000 Subject: [PATCH 0877/2835] --- doc/forum/can_git-annex_replace_ddm__63__.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/can_git-annex_replace_ddm__63__.mdwn diff --git a/doc/forum/can_git-annex_replace_ddm__63__.mdwn b/doc/forum/can_git-annex_replace_ddm__63__.mdwn new file mode 100644 index 0000000000..8d49652c3d --- /dev/null +++ b/doc/forum/can_git-annex_replace_ddm__63__.mdwn @@ -0,0 +1,13 @@ +Hi, +a few years ago I wrote a tool called 'ddm'. The code is overengineered and the script is more complicated then it should be, +but I think it demonstrates some good use cases, and I wonder how well git-annex can fulfill the requirements for those use cases - maybe I should remove ddm and start hacking with git-annex instead. + +To answer this question, you should read the section about the possible dataset types on http://dieter.plaetinck.be/ddm_a_distributed_data_manager.html, and the example at the bottom of that page. it demonstrates the idea behind the "selection" dataset to always try to keep a subset (the most appropriate, based on the output of some script) of files "checked out". +the introduction section on https://github.com/Dieterbe/ddm/raw/358f7cf92c0ba7b336dc97638351d4e324461afa/MANUAL should further clarify things, as well as give some more good use cases (as you can see it's a bit more about [semi-]automated workflows then purely tracking what's where) + +So I'm not sure, maybe the way to go for me is to make git-annex my "housekeeping about which data is where" backend and make ddm into a set of policies and tools on top of git-annex. + +Any input? + +Thanks, +Dieter From 208fb142d40e80da4f3dd9744e06027b3c2bbc46 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 14 Feb 2011 22:08:54 +0000 Subject: [PATCH 0878/2835] Added a comment --- ...mment_1_aa05008dfe800474ff76678a400099e1._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment new file mode 100644 index 0000000000..eb824971f6 --- /dev/null +++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-02-14T22:08:54Z" + content=""" +Yes, there is value in layering something over git-annex to use a policy to choose what goes where. + +I use [mr](http://kitenet.net/~joey/code/mr/) to update and manage all my repositories, and since mr can be made to run arbitrary commands when doing eg, an update, I use its config file as such a policy layer. For example, my podcasts are pulled into my sound repository in a subdirectory; boxes that consume podcasts run \"git pull; git annex get podcasts --exclude=\"*/out/*\"; git annex drop podcasts/*/out\". I move podcasts to \"out\" directories once done with them (I have yet to teach mpd to do that for me..), and the next time I run \"mr update\" to update everything, it pulls down new ones and removes old ones. + +I don't see any obstacle to doing what you want. May be that you'd need better querying facilities in git-annex (so the policy layer can know what is available where), or finer control (--exclude is a good enough hammer for me, but maybe not for you). +"""]] From 1760ad71d85bcb66841dfcd45ba64f29fd809637 Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:25:05 +0000 Subject: [PATCH 0879/2835] Typo fixes in index.mdwn --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index a279e3e5fb..503c858018 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -51,7 +51,7 @@ files with git. * [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage * [[location_tracking]] reminds you where git-annex has seen files -* git-annex prevents accidential data loss by [[tracking copies|copies]] +* git-annex prevents accidental data loss by [[tracking copies|copies]] of your files * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. From 399be585380839eda2ccd77cffaeab289ecc9ffa Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:42:20 +0000 Subject: [PATCH 0880/2835] Typo fixes in git-annex.mdwn --- doc/git-annex.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d670d626ee..2f25f9bf81 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -219,7 +219,7 @@ Many git-annex commands will stage changes for later `git commit` by you. * fromkey file - This can be used to maually set up a file to link to a specified key + This can be used to manually set up a file to link to a specified key in the key-value backend. How you determine an existing key in the backend varies. For the URL backend, the key is just a URL to the content. @@ -244,7 +244,7 @@ Many git-annex commands will stage changes for later `git commit` by you. * setkey file - This plumbing-level command sets the annxed data for a key to the content of + This plumbing-level command sets the annexed data for a key to the content of the specified file, and then removes the file. A backend will typically need to be specified with --backend. If none @@ -380,7 +380,7 @@ These files are used by git-annex, in your git repository: available. Annexed files in your git repository symlink to that content. `.git-annex/uuid.log` is used to map between repository UUID and -decscriptions. +descriptions. `.git-annex/trust.log` is used to indicate which repositories are trusted and untrusted. From cf595181748bac6791970f3a3bf2e43196c7422a Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:47:47 +0000 Subject: [PATCH 0881/2835] Typo fixes in location_tracking.mdwn --- doc/location_tracking.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn index 3791029f8f..301282b6ff 100644 --- a/doc/location_tracking.mdwn +++ b/doc/location_tracking.mdwn @@ -27,5 +27,5 @@ descriptions to help you with finding them: c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 e1938fee-d95b-11df-96cc-002170d25c55 -In certian cases you may want to configure git-annex to [[trust]] +In certain cases you may want to configure git-annex to [[trust]] that location tracking information is always correct for a repository. From c9348f8e55f4fd2c5f63e10262434771d6aff359 Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:50:54 +0000 Subject: [PATCH 0882/2835] Typo fixes in walkthrough.mdwn --- doc/walkthrough.mdwn | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 231c3e543c..d08b247f73 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -84,7 +84,7 @@ can get them. ## transferring files: When things go wrong -After a while, you'll have serveral annexes, with different file contents. +After a while, you'll have several annexes, with different file contents. You don't have to try to keep all that straight; git-annex does [[location_tracking]] for you. If you ask it to get a file and the drive or file server is not accessible, it will let you know what it needs to get @@ -146,7 +146,7 @@ That's a good thing, because it might be the only copy, you wouldn't want to lose it in a fumblefingered mistake. # echo oops > my_cool_big_file - bash: my_cool_big_file: Permission deined + bash: my_cool_big_file: Permission denied In order to modify a file, it should first be unlocked. @@ -176,7 +176,7 @@ There is one problem with using `git commit` like this: Git wants to first stage the entire contents of the file in its index. That can be slow for big files (sorta why git-annex exists in the first place). So, the automatic handling on commit is a nice safety feature, since it prevents -the file content being accidentially commited into git. But when working with +the file content being accidentally committed into git. But when working with big files, it's faster to explicitly add them to the annex yourself before committing. @@ -267,12 +267,12 @@ that the URL is stable; no local backup is kept. Another handy alternative to the default [[backend|backends]] is the SHA1 backend. This backend provides more git-style assurance that your data -has not been damanged. And the checksum means that when you add the same +has not been damaged. And the checksum means that when you add the same content to the annex twice, only one copy need be stored in the backend. The only reason it's not the default is that it needs to checksum files when they're added to the annex, and this can slow things down -significantly for really big files. To make SHA1 the detault, just +significantly for really big files. To make SHA1 the default, just add something like this to `.gitattributes`: * annex.backend=SHA1 @@ -292,7 +292,7 @@ files will be skipped. After migrating a file to a new backend, the old content in the old backend will still be present. That is necessary because multiple files -can point to the same content. The `git annex unused` sucommand can be +can point to the same content. The `git annex unused` subcommand can be used to clear up that detritus later. Note that hard links are used, to avoid wasting disk space. @@ -342,7 +342,7 @@ setting is satisfied for all files. fsck my_cool_big_file (checksum...) ok ... -You can also specifiy the files to check. This is particularly useful if +You can also specify the files to check. This is particularly useful if you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file @@ -367,7 +367,7 @@ might say about a badly messed up annex: ## backups git-annex can be configured to require more than one copy of a file exists, -as a simple backup for your data. This is controled by the "annex.numcopies" +as a simple backup for your data. This is controlled by the "annex.numcopies" setting, which defaults to 1 copy. Let's change that to require 2 copies, and send a copy of every file to a USB drive. @@ -394,9 +394,9 @@ For more details about the numcopies setting, see [[copies]]. ## untrusted repositories -Suppose you have a USB thunb drive and are using it as a git annex +Suppose you have a USB thumb drive and are using it as a git annex repository. You don't trust the drive, because you could lose it, or -accidentially run it through the laundry. Or, maybe you have a drive that +accidentally run it through the laundry. Or, maybe you have a drive that you know is dying, and you'd like to be warned if there are any files on it not backed up somewhere else. Maybe the drive has already died or been lost. From 36e13cfce64b43a47c2812be6b17b7a8af22f786 Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:54:03 +0000 Subject: [PATCH 0883/2835] Typo fix in not.mdwn --- doc/not.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/not.mdwn b/doc/not.mdwn index c9c5754d7e..80c0acafaf 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -26,7 +26,7 @@ I only learned of git-media after writing git-annex, but I probably would have still written git-annex instead of using it. Currently, git-media has the advantage of using git smudge filters rather than - git-annex's pile of symlinks, and it may be a tighter fit for certian + git-annex's pile of symlinks, and it may be a tighter fit for certain situations. It lacks git-annex's support for widely distributed storage, using only a single backend data store. It also does not support partial checkouts of file contents, like git-annex does. From 000d9fcf334436dc085d52655dc07f69cd465980 Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:56:24 +0000 Subject: [PATCH 0884/2835] Typo fixes in trust.mdwn --- doc/trust.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/trust.mdwn b/doc/trust.mdwn index f843d3858c..317e4b541f 100644 --- a/doc/trust.mdwn +++ b/doc/trust.mdwn @@ -11,12 +11,12 @@ information. When removing content, it will directly check that other repositories have enough [[copies]]. Generally that explicit checking is a good idea. Consider that the current -[[location_tracking]] information for a remote may not yet have propigated +[[location_tracking]] information for a remote may not yet have propagated out. Or, a remote may have suffered a catastrophic loss of data, or itself been lost. There is still some trust involved here. A semitrusted repository is -dependended on to retain a copy of the file content; possibly the only +depended on to retain a copy of the file content; possibly the only [[copy|copies]]. (Being semitrusted is the default. The `git annex semitrust` command From 982d0962b3ff4e82e83b715d8cc3fc93ea71783c Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:58:23 +0000 Subject: [PATCH 0885/2835] Typo fixes in use_cases/Bob.mdwn --- doc/use_case/Bob.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index a5dc01b373..573d9cde97 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -6,13 +6,13 @@ safe place. With git-annex, Bob has a single directory tree that includes all his files, even if their content is being stored offline. He can reorganize his files using that tree, committing new versions to git, -without worry about accidentially deleting anything. +without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. -Run in a cron job, git-annex adds new files to achival drives at night. It +Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. From dd0f662849fa24ded0d9ecb43000ac0ab8b1f7e7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 19 Feb 2011 17:00:40 -0400 Subject: [PATCH 0886/2835] hello, liftM --- Annex.hs | 4 +--- Command.hs | 28 +++++++++------------------- Utility.hs | 6 ++---- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/Annex.hs b/Annex.hs index 4a1b89dcf2..5496ada67c 100644 --- a/Annex.hs +++ b/Annex.hs @@ -75,9 +75,7 @@ eval state action = evalStateT action state {- Gets a value from the internal state, selected by the passed value - constructor. -} getState :: (AnnexState -> a) -> Annex a -getState c = do - state <- get - return (c state) +getState c = liftM c get {- Applies a state mutation function to change the internal state. - diff --git a/Command.hs b/Command.hs index 601b584642..86da454195 100644 --- a/Command.hs +++ b/Command.hs @@ -10,7 +10,7 @@ module Command where import Control.Monad.State (liftIO) import System.Directory import System.Posix.Files -import Control.Monad (filterM) +import Control.Monad (filterM, liftM) import System.Path.WildMatch import Text.Regex.PCRE.Light.Char8 import Data.List @@ -110,15 +110,13 @@ withFilesInGit :: CommandSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ runPreserveOrder (Git.inRepo repo) params - files' <- filterFiles files - return $ map a files' + liftM (map a) $ filterFiles files withAttrFilesInGit :: String -> CommandSeekAttrFiles withAttrFilesInGit attr a params = do repo <- Annex.gitRepo files <- liftIO $ runPreserveOrder (Git.inRepo repo) params files' <- filterFiles files - pairs <- liftIO $ Git.checkAttr repo attr files' - return $ map a pairs + liftM (map a) $ liftIO $ Git.checkAttr repo attr files' withBackendFilesInGit :: CommandSeekBackendFiles withBackendFilesInGit a params = do repo <- Annex.gitRepo @@ -128,8 +126,7 @@ withBackendFilesInGit a params = do withFilesMissing :: CommandSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params - files' <- filterFiles files - return $ map a files' + liftM (map a) $ filterFiles files where missing f = do e <- doesFileExist f @@ -148,8 +145,7 @@ withFilesToBeCommitted :: CommandSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo tocommit <- liftIO $ runPreserveOrder (Git.stagedFilesNotDeleted repo) params - tocommit' <- filterFiles tocommit - return $ map a tocommit' + liftM (map a) $ filterFiles tocommit withFilesUnlocked :: CommandSeekBackendFiles withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles @@ -172,9 +168,7 @@ withNothing a [] = return [a] withNothing _ _ = return [] backendPairs :: CommandSeekBackendFiles -backendPairs a files = do - pairs <- Backend.chooseBackends files - return $ map a pairs +backendPairs a files = liftM (map a) $ Backend.chooseBackends files {- Filter out files from the state directory, and those matching the - exclude glob pattern, if it was specified. -} @@ -201,9 +195,7 @@ wildsRegex' (w:ws) c = wildsRegex' ws (c ++ "|" ++ wildToRegex w) {- filter out symlinks -} notSymlink :: FilePath -> IO Bool -notSymlink f = do - s <- liftIO $ getSymbolicLinkStatus f - return $ not $ isSymbolicLink s +notSymlink f = liftM (not . isSymbolicLink) $ liftIO $ getSymbolicLinkStatus f {- Descriptions of params used in usage messages. -} paramRepeating :: String -> String @@ -260,10 +252,8 @@ preserveOrder orig new = collect orig new - its return list preserves order. - - This assumes that it's cheaper to call preserveOrder on the result, - - than it would be to run the action once with each param. In the case + - than it would be to run the action separately with each param. In the case - of git file list commands, that assumption tends to hold. -} runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath] -runPreserveOrder a files = do - r <- a files - return $ preserveOrder files r +runPreserveOrder a files = liftM (preserveOrder files) (a files) diff --git a/Utility.hs b/Utility.hs index 2bb623532d..89e129181a 100644 --- a/Utility.hs +++ b/Utility.hs @@ -38,6 +38,7 @@ import System.FilePath import System.Directory import Foreign (complement) import Data.List +import Control.Monad (liftM2) {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} @@ -95,10 +96,7 @@ absPath file = do - relPathCwdToDir "/tmp/foo/bar" == "" -} relPathCwdToDir :: FilePath -> IO FilePath -relPathCwdToDir dir = do - cwd <- getCurrentDirectory - a <- absPath dir - return $ relPathDirToDir cwd a +relPathCwdToDir dir = liftM2 relPathDirToDir getCurrentDirectory (absPath dir) {- Constructs a relative path from one directory to another. - From d8f65ced83a8ad35e72ba9eff3539638cc0c9f6c Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Sun, 20 Feb 2011 15:38:22 +0000 Subject: [PATCH 0887/2835] describe myself and what i think of git-annex --- doc/forum/chrysn.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/chrysn.mdwn diff --git a/doc/forum/chrysn.mdwn b/doc/forum/chrysn.mdwn new file mode 100644 index 0000000000..f5c07b88b3 --- /dev/null +++ b/doc/forum/chrysn.mdwn @@ -0,0 +1,5 @@ +* **name**: chrysn +* **website**: +* **uses git-annex for**: managing the family's photos (and possibly videos and music in the future) +* **likes git-annex because**: it adds a layer of commit semantics over a regular file system without keeping everything in duplicate locally +* **would like git-annex to**: not be required any more as git itself learns to use cow filesystems to avoid abundant disk usage and gets better with sparser checkouts (git-annex might then still be a simpler tool that watches over what can be safely dropped for a sparser checkout) From c215c68aaa16cfbd173881e24db49e920749bbb9 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Sun, 20 Feb 2011 16:24:41 +0000 Subject: [PATCH 0888/2835] feeling lucky with proposing deeper git-annex changes --- doc/forum/relying_on_git_for_numcopies.mdwn | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/forum/relying_on_git_for_numcopies.mdwn diff --git a/doc/forum/relying_on_git_for_numcopies.mdwn b/doc/forum/relying_on_git_for_numcopies.mdwn new file mode 100644 index 0000000000..5097c9e7e8 --- /dev/null +++ b/doc/forum/relying_on_git_for_numcopies.mdwn @@ -0,0 +1,45 @@ +This is a rough sketch of a modification of git-annex to rely more on git commit semantics. It might be flawed due to my lack of understanding of git-annex internals. --[[chrysn]] + +Summary +========= + +Currently, the location tracking is only used for informational purposes unless a repository is [[trust]]ed, in which case there is no checking at all. It is proposed to use the location tracking information as a commitment to keep track of a file without any promise that it might not be dropped if another repository takes over responsibility. + +git's semantics for atomic commits are proposed to be used, which makes sure that before files are actually deleted, another repository has accepted the deletion. + +Modified git-annex-drop behavior +========================== + +The most important (if not only) git-annex command that is affected by this is `git annex drop`. Currently, for dropping a large number of files, every file is checked with another (or multiple, if so configured) host if it's safe to delete. + +The new behavior would be to + +* decrement the location tracking counter for all files to be dropped, +* commit that change, +* try to push it to at least as many repositories that the numcopies constraints are met, +* revert if that fails, +* otherwise really drop the files from the backend. + +Unlike explicit checking, this never looks at the remote backend if the file is really present -- otoh, git-annex already relies on the files in the backend to not be touched by anyone but git-annex, and git-annex would only drop them if they were derefed and committed, in which case git would not accept the push. (git by itself would accept a merged push, but even if the reverting step failed due to a power outage or similar, git-annex would, before really deleting files from the backend, check again if the numcopies restraint is still met, and revert its own delete commit as the files are still present anyway.) + +Implications for trust +============== + +The proposed change also changes the semantics of trust. Trust can now be controlled in a finer-grained way between untrusted and semi-trusted, as best illustrated by a use case: + +> Alice takes her netbook with her on a trip through Spain, and will fill most of its disk up with pictures she takes. As she expects to meet some old friends during the first days, she wants to take older pictures with her, which are safely backed up at home. +> +> She tells her netbook's repository to dereference the old images (but not other parts of the repository she has not copied anywhere yet) and pushes to the server before leaving. When she adds pictures from her camera to the repository, git-annex can now free up space as needed. + +Dereferencing could be implemented as `git annex drop --not-yet`, freeing space is similar to `dropunused`. + +A trusted repository with the new semantics would mean that the repository would not accept dropping anything, just as before. + +Advantages / Disadvantages +===================== + +The advantage of this proposal is that the round trips required for dropping something could be greatly reduced. + +There should also be simplifications in the `git annex drop` command as it doesn't need to take care of locking any more (git should already do that between checking if HEAD is a parent of the pushed commit and replacing HEAD). + +Besides being a major change in git-annex (with the requirement to track hosts' git-annex versions for migration, as the new trust system is incompatible with the old one), no disadvantages of that stragegy are known to the author (hoping for discussion below). From 39dfdea642a3d0d0a6a8d5b51f1f1bc41b16e961 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Tue, 22 Feb 2011 10:17:46 +0000 Subject: [PATCH 0889/2835] better wording at some places --- doc/forum/relying_on_git_for_numcopies.mdwn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/forum/relying_on_git_for_numcopies.mdwn b/doc/forum/relying_on_git_for_numcopies.mdwn index 5097c9e7e8..b7ebba805a 100644 --- a/doc/forum/relying_on_git_for_numcopies.mdwn +++ b/doc/forum/relying_on_git_for_numcopies.mdwn @@ -3,7 +3,7 @@ This is a rough sketch of a modification of git-annex to rely more on git commit Summary ========= -Currently, the location tracking is only used for informational purposes unless a repository is [[trust]]ed, in which case there is no checking at all. It is proposed to use the location tracking information as a commitment to keep track of a file without any promise that it might not be dropped if another repository takes over responsibility. +Currently, [[location tracking]] is only used for informational purposes unless a repository is [[trust]]ed, in which case there is no checking at all. It is proposed to use the location tracking information as a commitment to keep track of a file until another repository takes over responsibility. git's semantics for atomic commits are proposed to be used, which makes sure that before files are actually deleted, another repository has accepted the deletion. @@ -20,18 +20,18 @@ The new behavior would be to * revert if that fails, * otherwise really drop the files from the backend. -Unlike explicit checking, this never looks at the remote backend if the file is really present -- otoh, git-annex already relies on the files in the backend to not be touched by anyone but git-annex, and git-annex would only drop them if they were derefed and committed, in which case git would not accept the push. (git by itself would accept a merged push, but even if the reverting step failed due to a power outage or similar, git-annex would, before really deleting files from the backend, check again if the numcopies restraint is still met, and revert its own delete commit as the files are still present anyway.) +Unlike explicit checking, this never looks at the remote backend if the file is really present -- otoh, git-annex already relies on the files in the backend to not be touched by anyone but git-annex itself, and git-annex would only drop them if they were derefed and committed, in which case git would not accept the push. (git by itself would accept a merged push, but even if the reverting step failed due to a power outage or similar, git-annex would, before really deleting files from the backend, check again if the numcopies restraint is still met, and revert its own delete commit as the files are still present anyway.) Implications for trust ============== The proposed change also changes the semantics of trust. Trust can now be controlled in a finer-grained way between untrusted and semi-trusted, as best illustrated by a use case: -> Alice takes her netbook with her on a trip through Spain, and will fill most of its disk up with pictures she takes. As she expects to meet some old friends during the first days, she wants to take older pictures with her, which are safely backed up at home. +> Alice takes her netbook with her on a trip through Spain, and will fill most of its disk up with pictures she takes. As she expects to meet some old friends during the first days, she wants to take older pictures with her, which are safely backed up at home, so they can be deleted on demand. > > She tells her netbook's repository to dereference the old images (but not other parts of the repository she has not copied anywhere yet) and pushes to the server before leaving. When she adds pictures from her camera to the repository, git-annex can now free up space as needed. -Dereferencing could be implemented as `git annex drop --not-yet`, freeing space is similar to `dropunused`. +Dereferencing could be implemented as `git annex drop --no-rm` (or `move --no-rm`), freeing space is similar to `dropunused`. A trusted repository with the new semantics would mean that the repository would not accept dropping anything, just as before. From 1f44ccb7026e0ef033f9415130f62d35604590d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 22 Feb 2011 14:44:17 -0400 Subject: [PATCH 0890/2835] add --- doc/todo/hidden_files.mdwn | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/todo/hidden_files.mdwn diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn new file mode 100644 index 0000000000..2e5ec4d9e8 --- /dev/null +++ b/doc/todo/hidden_files.mdwn @@ -0,0 +1,20 @@ +Add a `git annex hide $file` that behaves like drop, checking counter info +and updating location log to say the current repo no longer has a file -- +but does not actually remove the content. + +Then `git annex unused` can be used to clean it up later. And in the +meantime, it's still locally accessible. This can be useful if you're +planning to need to free up space later, but want to hold onto the content +for a while. Possibly you'll be disconnected later, so it's easier to push +out that intent now. + +-- + +TODO: + +* Make 100% sure this is safe. Drop, etc should never check content files + are present on other repos if the location log doesn't say the repo + has the content. + +* What will `git annex get` do if it's asked to get a file that has been + hidden? From 725c9b1ecb3ac126bbfb238730a11d9d474ddd53 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 22 Feb 2011 18:44:28 +0000 Subject: [PATCH 0891/2835] Added a comment --- ..._8ad3cccd7f66f6423341d71241ba89fc._comment | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment diff --git a/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment new file mode 100644 index 0000000000..219bb60b9a --- /dev/null +++ b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment @@ -0,0 +1,36 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-02-22T18:44:28Z" + content=""" +I see the following problems with this scheme: + +- Disallows removal of files when disconnected. It's currently safe to force that, as long as + git-annex tells you enough other repos are belived to have the file. Just as long as you + only force on one machine (say your laptop). With your scheme, if you drop a file while + disconnected, any other host could see that the counter is still at N, because your + laptop had the file last time it was online, and can decide to drop the file, and lose the last +version. + +- pushing a changed counter commit to other repos is tricky, because they're not bare, and + the network topology to get the commit pulled into the other repo could vary. + +- Merging counter files issues. If the counter file doesn't automerge, two repos dropping the same file will conflict. But, if it does automerge, it breaks the counter conflict detection. + +- Needing to revert commits is going to be annoying. An actual git revert + could probably not reliably be done. It's need to construct a revert + and commit it as a new commit. And then try to push that to remotes, and + what if *that* push conflicts? + +- I do like the pre-removal dropping somewhat as an alternative to + trust checking. I think that can be done with current git-annex though, + just remove the files from the location log, but keep them in-annex. + Dropping a file only looks at repos that the location log says have a + file; so other repos can have retained a copy of a file secretly like + this, and can safely remove it at any time. I'd need to look into this a bit more to be 100% sure it's safe, but have started [[todo/hidden_files]]. + +- I don't see any reduced round trips. It still has to contact N other + repos on drop. Now, rather than checking that they have a file, it needs + to push a change to them. +"""]] From 0d699b7a322bfa2977cba1f29bd8f44fabb88aa6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 22 Feb 2011 14:49:36 -0400 Subject: [PATCH 0892/2835] meh --- .../comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment index 219bb60b9a..83a908da8c 100644 --- a/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment +++ b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment @@ -28,7 +28,7 @@ version. just remove the files from the location log, but keep them in-annex. Dropping a file only looks at repos that the location log says have a file; so other repos can have retained a copy of a file secretly like - this, and can safely remove it at any time. I'd need to look into this a bit more to be 100% sure it's safe, but have started [[todo/hidden_files]]. + this, and can safely remove it at any time. I'd need to look into this a bit more to be 100% sure it's safe, but have started [[todo/hidden_files]]. - I don't see any reduced round trips. It still has to contact N other repos on drop. Now, rather than checking that they have a file, it needs From 4c761b0b08f904e3bce78e979bb855f4c78507aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 22 Feb 2011 14:52:22 -0400 Subject: [PATCH 0893/2835] move --- doc/{forum => users}/chrysn.mdwn | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{forum => users}/chrysn.mdwn (100%) diff --git a/doc/forum/chrysn.mdwn b/doc/users/chrysn.mdwn similarity index 100% rename from doc/forum/chrysn.mdwn rename to doc/users/chrysn.mdwn From 5b96961957678765480a4e540f3f82e28d921d5d Mon Sep 17 00:00:00 2001 From: tyger Date: Wed, 23 Feb 2011 09:49:34 +0000 Subject: [PATCH 0894/2835] --- .../migrate_existing_git_repository_to_git-annex.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/forum/migrate_existing_git_repository_to_git-annex.mdwn diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn new file mode 100644 index 0000000000..fe26f69523 --- /dev/null +++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn @@ -0,0 +1,7 @@ +I have a large git repository with binary files scattered over different branches. I want to switch to git-annex mainly for performance reasons, but I don't want to loose my history. + +I tried to rewrite the (cloned) repository with git-filter-branch but failed miserably for several reasons: + * --tree-filter performs its operations in a temporary directory (.git-rewrite/t/) so the symlinks point to the wrong destination (../../.git/annex/). + * annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them + +Any suggestions how to proceed? From 0573ccecc51792c68d3d179601361c3ac97d1a0b Mon Sep 17 00:00:00 2001 From: tyger Date: Wed, 23 Feb 2011 15:15:03 +0000 Subject: [PATCH 0895/2835] reformat list --- doc/forum/migrate_existing_git_repository_to_git-annex.mdwn | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn index fe26f69523..94c5f1d133 100644 --- a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn +++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn @@ -1,7 +1,8 @@ I have a large git repository with binary files scattered over different branches. I want to switch to git-annex mainly for performance reasons, but I don't want to loose my history. I tried to rewrite the (cloned) repository with git-filter-branch but failed miserably for several reasons: - * --tree-filter performs its operations in a temporary directory (.git-rewrite/t/) so the symlinks point to the wrong destination (../../.git/annex/). - * annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them + +* --tree-filter performs its operations in a temporary directory (.git-rewrite/t/) so the symlinks point to the wrong destination (../../.git/annex/). +* annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them Any suggestions how to proceed? From c5ffefcde1e2056d0c0257cbc84c5a6e8e046b62 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 23 Feb 2011 16:44:01 +0000 Subject: [PATCH 0896/2835] Added a comment --- ...2_be6acbc26008a9cb54e7b8f498f2c2a2._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment diff --git a/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment b/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment new file mode 100644 index 0000000000..d9ce8b50e0 --- /dev/null +++ b/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="comment 2" + date="2011-02-23T16:43:59Z" + content=""" +i'll comment on each of the points separately, well aware that even a single little leftover issue can show that my plan is faulty: + +* force removal: well, yes -- but the file that is currently force-removed on the laptop could just as well be the last of its kind itself. i see the problem, but am not sure if it's fatal (after all, if we rely on out-of-band knowledge when forcing something, we could just as well ask a little more) +* non-bare repos: pushing is tricky with non-bare repos now just as well; a post-commit hook could auto-accept counter changes. (but pushing causes problems with counters anyway, doesn't it?) +* merging: i'd have them auto-merge. git-annex will have to check the validity of the current state anyway, and a situation in which a counter-decrementing commit is not a fast-forward one would be reverted in the next step (or upon discovery, in case the next step never took place). +* reverting: my wording was bad as \"revert\" is already taken in git-lingo. the correct term for what i was thinking of is \"reset\". (as the commit could not be pushed, it would be rolled back completely). + * we might have to resort to reverting, though, if the commit has already been pused to a first server of many. +* [[todo/hidden files]]: yes, this solves pre-removal dropping :-) +* round trips: it's not the number of servers, it's the number of files (up to 30k in my case). it seems to me that an individual request was made for every single file i wanted to drop (that would be N*M roundtrips for N affected servers and M files, and N roundtrips with git managed numcopies) + +all together, it seems to be a bit more complicated than i imagined, although not completely impossible. a combination of [[todo/hidden files]] and maybe a simpler reduction of the number of requests might though achieve the important goals as well. +"""]] From bcd3a9ea34763b4b8f40eef75c05c6e9ea27d178 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 23 Feb 2011 21:48:15 +0000 Subject: [PATCH 0897/2835] Added a comment: relation to [[todo/branching]] --- .../comment_3_43d8e1513eb9947f8a503f094c03f307._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment diff --git a/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment b/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment new file mode 100644 index 0000000000..27076a877f --- /dev/null +++ b/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="relation to [[todo/branching]]" + date="2011-02-23T21:48:14Z" + content=""" +the non-bare repository issue would go away if this was combined with the \"alternate\" approach to [[todo/branching]]. (with the \"fleshed out proposal\" of branching, this would not work at all for lack of shared commits.) +"""]] From e61b47bc8a51447f9fbde6463e09d4beef8d5a7f Mon Sep 17 00:00:00 2001 From: "http://jandd.myopenid.com/" Date: Thu, 24 Feb 2011 10:11:36 +0000 Subject: [PATCH 0898/2835] --- .../error_with_file_names_starting_with_dash.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/error_with_file_names_starting_with_dash.mdwn diff --git a/doc/bugs/error_with_file_names_starting_with_dash.mdwn b/doc/bugs/error_with_file_names_starting_with_dash.mdwn new file mode 100644 index 0000000000..7a4d33911f --- /dev/null +++ b/doc/bugs/error_with_file_names_starting_with_dash.mdwn @@ -0,0 +1,12 @@ +git annex add has problems if items start with dashes, example: + +-wut-a-directory-name-/file1 + +leads to + +[[!format bash """ +add -wut-a-directory-name-/file1 (checksum...) sha1sum: invalid option -- 'u' +„sha1sum --help“ gibt weitere Informationen. + + git-annex: : hGetLine: end of file +"""]] From 836e71297b8e3b5bd6f89f7eb1198f59af985b0b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 25 Feb 2011 01:13:01 -0400 Subject: [PATCH 0899/2835] Support filenames that start with a dash; when such a file is passed to a utility it will be escaped to avoid it being interpreted as an option. --- Backend/SHA1.hs | 3 ++- Backend/URL.hs | 2 +- Command/SetKey.hs | 3 ++- CopyFile.hs | 10 ++++++---- Remotes.hs | 2 +- RsyncFile.hs | 5 +++-- Utility.hs | 8 ++++++++ debian/changelog | 3 +++ doc/bugs/error_with_file_names_starting_with_dash.mdwn | 3 +++ 9 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 9636787f0d..e1830bc134 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -20,6 +20,7 @@ import qualified Annex import Locations import Content import Types +import Utility backend :: Backend Annex backend = Backend.File.backend { @@ -31,7 +32,7 @@ backend = Backend.File.backend { sha1 :: FilePath -> Annex String sha1 file = do showNote "checksum..." - liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do + liftIO $ pOpen ReadFromPipe "sha1sum" [utilityEscape file] $ \h -> do line <- hGetLine h let bits = split " " line if null bits diff --git a/Backend/URL.hs b/Backend/URL.hs index 38954e5a31..15cc88d644 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -51,6 +51,6 @@ downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" showProgress -- make way for curl progress bar - liftIO $ boolSystem "curl" ["-#", "-o", file, url] + liftIO $ boolSystem "curl" ["-#", "-o", utilityEscape file, url] where url = join ":" $ drop 1 $ split ":" $ show key diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 388392cd60..025fb74d62 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -35,7 +35,8 @@ perform file = do -- rather than simply calling moveToObjectDir ok <- getViaTmp key $ \dest -> do if dest /= file - then liftIO $ boolSystem "mv" [file, dest] + then liftIO $ + boolSystem "mv" [utilityEscape file, utilityEscape dest] else return True if ok then return $ Just $ cleanup diff --git a/CopyFile.hs b/CopyFile.hs index e913aa070d..73d911a291 100644 --- a/CopyFile.hs +++ b/CopyFile.hs @@ -23,9 +23,11 @@ copyFile src dest = do boolSystem "cp" opts where opts = if SysConfig.cp_reflink_auto - then ["--reflink=auto", src, dest] + then ["--reflink=auto", src', dest'] else if SysConfig.cp_a - then ["-a", src, dest] + then ["-a", src', dest'] else if SysConfig.cp_p - then ["-p", src, dest] - else [src, dest] + then ["-p", src', dest'] + else [src', dest'] + src' = utilityEscape src + dest' = utilityEscape dest diff --git a/Remotes.hs b/Remotes.hs index 15f5185b91..c7e69aad8b 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -282,7 +282,7 @@ rsyncParams r sending key file = do -- inplace makes rsync resume partial files options = ["-p", "--progress", "--inplace"] -- the rsync shell parameter controls where rsync - -- does, so the source/dest parameter can be a dummy value, + -- goes, so the source/dest parameter can be a dummy value, -- that just enables remote rsync mode. dummy = ":" diff --git a/RsyncFile.hs b/RsyncFile.hs index 274e66151b..9de2e2c594 100644 --- a/RsyncFile.hs +++ b/RsyncFile.hs @@ -24,11 +24,12 @@ rsyncShell command = ["-e", unwords $ map escape command] {- Runs rsync in server mode to send a file, and exits. -} rsyncServerSend :: FilePath -> IO () -rsyncServerSend file = rsyncExec $ rsyncServerParams ++ ["--sender", file] +rsyncServerSend file = rsyncExec $ + rsyncServerParams ++ ["--sender", utilityEscape file] {- Runs rsync in server mode to receive a file. -} rsyncServerReceive :: FilePath -> IO Bool -rsyncServerReceive file = rsync $ rsyncServerParams ++ [file] +rsyncServerReceive file = rsync $ rsyncServerParams ++ [utilityEscape file] rsyncServerParams :: [String] rsyncServerParams = diff --git a/Utility.hs b/Utility.hs index 89e129181a..6a90e3cd50 100644 --- a/Utility.hs +++ b/Utility.hs @@ -15,6 +15,7 @@ module Utility ( boolSystem, shellEscape, shellUnEscape, + utilityEscape, unsetFileMode, readMaybe, safeWriteFile, @@ -179,6 +180,13 @@ shellUnEscape s = word:(shellUnEscape rest) | c == q = findword w cs | otherwise = inquote q (w++[c]) cs +{- Ensures that a filename is safe to pass to a utility program. In particular + - since utilities tend to interpret things starting with a dash as + - an option, relative filenames starting with a dash are escaped. -} +utilityEscape :: FilePath -> FilePath +utilityEscape ('-':s) = "./-" ++ s +utilityEscape s = s + {- For quickcheck. -} prop_idempotent_shellEscape :: String -> Bool prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s) diff --git a/debian/changelog b/debian/changelog index 173c441b03..d634223245 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,9 @@ git-annex (0.22) UNRELEASED; urgency=low for his help eliminating the infestation... for now.) * Make test suite not rely on a working cp -pr. (The Unix wars are still ON!) + * Support filenames that start with a dash; when such a file is passed + to a utility it will be escaped to avoid it being interpreted as an + option. -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 diff --git a/doc/bugs/error_with_file_names_starting_with_dash.mdwn b/doc/bugs/error_with_file_names_starting_with_dash.mdwn index 7a4d33911f..84bf1cfa07 100644 --- a/doc/bugs/error_with_file_names_starting_with_dash.mdwn +++ b/doc/bugs/error_with_file_names_starting_with_dash.mdwn @@ -10,3 +10,6 @@ add -wut-a-directory-name-/file1 (checksum...) sha1sum: invalid option -- 'u' git-annex: : hGetLine: end of file """]] + +> This is fixed in git, at least I think I've found all cases where +> filenames are passed to programs and escaped them. --[[Joey]] [[done]] From d35c11de7a83e135636749deb563de2ef3fee3a6 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 05:16:48 +0000 Subject: [PATCH 0900/2835] Added a comment --- ...comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment new file mode 100644 index 0000000000..e88794d621 --- /dev/null +++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-02-25T05:16:48Z" + content=""" +I don't know how to approach this yet, but I support the idea -- it would be great if there was a tool that could punch files out of git history and put them in the annex. (Of course with typical git history rewriting caveats.) + +Sounds like it might be enough to add a switch to git-annex that overrides where it considers the top of the git repository to be? +"""]] From fa37ae1d9669a76d8e8e9a08cf044836f7eb8c0b Mon Sep 17 00:00:00 2001 From: jbd Date: Fri, 25 Feb 2011 09:12:45 +0000 Subject: [PATCH 0901/2835] --- ...list:___34__git_annex_add__34___multiple_processes.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn new file mode 100644 index 0000000000..f3d3837015 --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn @@ -0,0 +1,7 @@ +Hello, + +i'm in the process of managing my music collection with git-annex. The initial "git annex add" using the sha1 banckend is quite long an i was wondering that it could be nice to launch multiple "sha1sum" processes in parallel to speed up things. + +Anyway, thanks for this wonderful piece of software. + +Jean-Baptiste From 5cb24bebf4479f5fc83980e328bce511b36b05fd Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 19:12:42 +0000 Subject: [PATCH 0902/2835] Added a comment --- ...comment_1_85b14478411a33e6186a64bd41f0910d._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment new file mode 100644 index 0000000000..2364b7fb83 --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-02-25T19:12:42Z" + content=""" +I'd expect the checksumming to be disk bound, not CPU bound, on most systems. + +I suggest you start off on the WORM backend, and then you can run a job later to [[migrate|walkthrough#index14h2]] to the SHA1 backend. +"""]] From 3f7c0b6970f58c1484615df7b3ec40f3ee44c9df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 25 Feb 2011 15:50:17 -0400 Subject: [PATCH 0903/2835] further investigation --- doc/todo/smudge.mdwn | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn index f51f45a391..c51662b282 100644 --- a/doc/todo/smudge.mdwn +++ b/doc/todo/smudge.mdwn @@ -1,14 +1,34 @@ git-annex should use smudge/clean filters. -The trick is doing it efficiently. Since git a2b665d, 2011-01-05, +The clean filter is run when files are staged for commit. So a user could copy +any file into the annex, git add it, and git-annex's clean filter causes +the file's key to be staged, while its value is added to the annex. + +The smudge filter is run when files are checked out. Since git annex +repos have partial content, this would not git annex get the file content. +Instead, if the content is not currently available, it would need to do +something like return empty file content. (Sadly, it cannot create a +symlink, as git still wants to write the file afterwards. + +So the nice current behavior of unavailable files being clearly missing due +to dangling symlinks, would be lost when using smudge/clean filters. +(Contact git developers to get an interface to do this?) + +Instead, we get the nice behavior of not having to remeber to `git annex +add` files, and just being able to use `git add` or `git commit -a`, +and have it use git-annex when .gitattributes says to. Also, annexed +files can be directly modified without having to `git annex unlock`. + +### efficiency + +The trick is doing it efficiently. Since git a2b665d, v1.7.4.1, something like this works to provide a filename to the clean script: git config --global filter.huge.clean huge-clean %f This avoids it needing to read all the current file content from stdin when doing eg, a git status or git commit. Instead it is passed the -filename that git is operating on, I think that's from the working -directory. +filename that git is operating on, in the working directory. So, WORM could just look at that file and easily tell if it is one it already knows (same mtime and size). If so, it can short-circuit and @@ -32,12 +52,6 @@ I've a demo implementation of this technique in the scripts below. ---- -It may further be possible to use the %f with the smudge filter -(docs say it's supported), and instead of outputting the dummy content, -it could create a dangling symlink, which would be more like git-annex's -behavior now, and makes it easy to tell what content is not available -with `ls`. - ### test files huge-smudge: From 42b181a04b6e3cc4f2019aa8b5b7dfa7ffb5ce51 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 19:54:29 +0000 Subject: [PATCH 0904/2835] Added a comment --- .../comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment new file mode 100644 index 0000000000..9b8240658b --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-02-25T19:54:28Z" + content=""" +But, see [[todo/parallel_possibilities]] +"""]] From 8aebf9a337e26328c5cb406300fcef5df42bc7b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 25 Feb 2011 15:54:29 -0400 Subject: [PATCH 0905/2835] update --- doc/todo/parallel_possibilities.mdwn | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/todo/parallel_possibilities.mdwn b/doc/todo/parallel_possibilities.mdwn index 15e5171ca3..9c0e69e294 100644 --- a/doc/todo/parallel_possibilities.mdwn +++ b/doc/todo/parallel_possibilities.mdwn @@ -6,7 +6,8 @@ Anyway, each git-annex command is broken down into a series of independant actions, which has some potential for parallelism. Each action has 3 distinct phases, basically "check", "perform", and -"cleanup". The perform actions are not parellizable; the cleanup may be, -and the check should be easily parallelizable, although they may access the -disk or run minor git query commands, so would probably not want to run -too many of them at once. +"cleanup". The perform actions are probably parellizable; the cleanup may be +(but not if it has to run git commands to stage state; it can queue +commands though); the check should be easily parallelizable, although they +may access the disk or run minor git query commands, so would probably not +want to run too many of them at once. From 2d1e1e414d4c2fbaa5c1a43cc751cebad937d898 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 19:54:31 +0000 Subject: [PATCH 0906/2835] Added a comment --- .../comment_3_bd17c490fce8de85352938976eeb92d7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment new file mode 100644 index 0000000000..e8ac2d266b --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-02-25T19:54:31Z" + content=""" +But, see [[todo/parallel_possibilities]] +"""]] From 82d6545b6f92792f81f2de33a99b5cb5403485d1 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 19:54:48 +0000 Subject: [PATCH 0907/2835] removed --- .../comment_3_bd17c490fce8de85352938976eeb92d7._comment | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment deleted file mode 100644 index e8ac2d266b..0000000000 --- a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="http://joey.kitenet.net/" - nickname="joey" - subject="comment 3" - date="2011-02-25T19:54:31Z" - content=""" -But, see [[todo/parallel_possibilities]] -"""]] From 307685c3494e59c6e92b378e3a171bb0f6a11c95 Mon Sep 17 00:00:00 2001 From: jbd Date: Sat, 26 Feb 2011 10:26:13 +0000 Subject: [PATCH 0908/2835] Added a comment --- .../comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment new file mode 100644 index 0000000000..ee769f0ddd --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="jbd" + ip="89.158.228.148" + subject="comment 3" + date="2011-02-26T10:26:12Z" + content=""" +Thank your for your answer and the link ! +"""]] From 9e74d3bfc341c142b6ea14a173640831dca14450 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Sat, 26 Feb 2011 21:43:22 +0000 Subject: [PATCH 0909/2835] Added a comment: git-add instead of git-annex-add --- .../comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment diff --git a/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment b/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment new file mode 100644 index 0000000000..a4eb3cf235 --- /dev/null +++ b/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="git-add instead of git-annex-add" + date="2011-02-26T21:43:21Z" + content=""" +would, with these modifications in place, there still be a way to *really* git-add a file? (my main repository contains both normal git and git-annex files.) +"""]] From 715faa86afcaca5ffc8f0af24a06cf9cabf54619 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkju1sxeJoVVa04plfuhH4Dp8KJOA-Gu_g" Date: Sun, 27 Feb 2011 13:59:42 +0000 Subject: [PATCH 0910/2835] --- ...ge_argument_to___34__git_annex_dropunused__34___.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn diff --git a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn new file mode 100644 index 0000000000..97b04e2926 --- /dev/null +++ b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn @@ -0,0 +1,9 @@ +The command `git annex dropunused` currently takes a number, as referenced in output of last `git annex unused` command. + +When you want to drop all, or a range, this may be annoying, as you have to specify each number on the command line. + +A range argument, such as `1-1845`, possibly combined with other argument types (Cf. many print dialogues: `1,3,5-7,9`) would be great. + +I work around this lack as I want to drop all unused files anyway by something like this: + + git annex unused | grep -o -P "^ [0-9]+" | xargs git annex dropunused From 22c8dfcd3fb8e052b2c3ae10da6ebc862d6e6efc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkju1sxeJoVVa04plfuhH4Dp8KJOA-Gu_g" Date: Sun, 27 Feb 2011 14:01:37 +0000 Subject: [PATCH 0911/2835] --- .../git_annex_migrate_leaves_old_backend_versions_around.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn new file mode 100644 index 0000000000..2fed00be7e --- /dev/null +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -0,0 +1 @@ +`git annex migrate` leaves old, unlinked backend versions lying around. It would be great if these were purged automatically somehow. From b14ad6338150243b7f60d0c59aeebf7b0c6e1d03 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:17:19 -0400 Subject: [PATCH 0912/2835] responses --- ...t_to___34__git_annex_dropunused__34___.mdwn | 9 +++++++++ ...ate_leaves_old_backend_versions_around.mdwn | 18 +++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn index 97b04e2926..dc9b7acda5 100644 --- a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn +++ b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn @@ -7,3 +7,12 @@ A range argument, such as `1-1845`, possibly combined with other argument types I work around this lack as I want to drop all unused files anyway by something like this: git annex unused | grep -o -P "^ [0-9]+" | xargs git annex dropunused + +> It's designed to be used with `seq`. There's an example in the +> [[walkthrough]], and of course multiple seq calls can be used to +> specifiy multiple ranges. So: + + git annex dropunused `seq 1 9` `seq 11 1845` + +> I don't see adding my own range operations to be an improvement worth +> making; it'd arguably only be a complication. --[[Joey]] [[done]] diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn index 2fed00be7e..c32b763a2d 100644 --- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -1 +1,17 @@ -`git annex migrate` leaves old, unlinked backend versions lying around. It would be great if these were purged automatically somehow. +`git annex migrate` leaves old, unlinked backend versions lying around. It +would be great if these were purged automatically somehow. + +> Yes, this is an issue mentioned in the [[walkthrough#index14h2]]. +> +> Since multiple files can point to the same content, it could be that +> only one file has been migrated, and the content is still used. So +> the content either has to be retained, or an operation as expensive +> as `git annex unused` used to find it something else still uses it. +> +> Rather than adding such an +> expensive operation to each call to migrate, I focused on hard-linking +> the values for the old and new keys, so that the old keys don't actually +> use any additional resources (beyond an extra inode). +> +> This way a lot of migrations can be done, and only when you're done you +> can do the more expensive cleanup pass if you want to. --[[Joey]] From 9e49a71282def0b6d6f7507d59eb0f805c6e0073 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:19:28 -0400 Subject: [PATCH 0913/2835] typo --- .../git_annex_migrate_leaves_old_backend_versions_around.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn index c32b763a2d..fc8551ddb4 100644 --- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -6,7 +6,7 @@ would be great if these were purged automatically somehow. > Since multiple files can point to the same content, it could be that > only one file has been migrated, and the content is still used. So > the content either has to be retained, or an operation as expensive -> as `git annex unused` used to find it something else still uses it. +> as `git annex unused` used to find if something else still uses it. > > Rather than adding such an > expensive operation to each call to migrate, I focused on hard-linking From 98e246b49b3c4fed319fe7bc1e900ba20ebfc9e1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:45:48 -0400 Subject: [PATCH 0914/2835] split the walkthrough and inline back together --- doc/walkthrough.mdwn | 441 +----------------- doc/walkthrough/adding_a_remote.mdwn | 19 + doc/walkthrough/adding_files.mdwn | 11 + doc/walkthrough/backups.mdwn | 25 + doc/walkthrough/creating_a_repository.mdwn | 6 + .../fsck:_verifying_your_data.mdwn | 16 + .../fsck:_when_things_go_wrong.mdwn | 13 + doc/walkthrough/getting_file_content.mdwn | 16 + .../migrating_data_to_a_new_backend.mdwn | 16 + doc/walkthrough/modifying_annexed_files.mdwn | 43 ++ ...ing_file_content_between_repositories.mdwn | 13 + doc/walkthrough/removing_files.mdwn | 6 + .../removing_files:_When_things_go_wrong.mdwn | 24 + doc/walkthrough/renaming_files.mdwn | 13 + ...nsferring_files:_When_things_go_wrong.mdwn | 18 + doc/walkthrough/untrusted_repositories.mdwn | 28 ++ doc/walkthrough/unused_data.mdwn | 30 ++ doc/walkthrough/using_ssh_remotes.mdwn | 33 ++ doc/walkthrough/using_the_SHA1_backend.mdwn | 11 + doc/walkthrough/using_the_URL_backend.mdwn | 24 + 20 files changed, 386 insertions(+), 420 deletions(-) create mode 100644 doc/walkthrough/adding_a_remote.mdwn create mode 100644 doc/walkthrough/adding_files.mdwn create mode 100644 doc/walkthrough/backups.mdwn create mode 100644 doc/walkthrough/creating_a_repository.mdwn create mode 100644 doc/walkthrough/fsck:_verifying_your_data.mdwn create mode 100644 doc/walkthrough/fsck:_when_things_go_wrong.mdwn create mode 100644 doc/walkthrough/getting_file_content.mdwn create mode 100644 doc/walkthrough/migrating_data_to_a_new_backend.mdwn create mode 100644 doc/walkthrough/modifying_annexed_files.mdwn create mode 100644 doc/walkthrough/moving_file_content_between_repositories.mdwn create mode 100644 doc/walkthrough/removing_files.mdwn create mode 100644 doc/walkthrough/removing_files:_When_things_go_wrong.mdwn create mode 100644 doc/walkthrough/renaming_files.mdwn create mode 100644 doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn create mode 100644 doc/walkthrough/untrusted_repositories.mdwn create mode 100644 doc/walkthrough/unused_data.mdwn create mode 100644 doc/walkthrough/using_ssh_remotes.mdwn create mode 100644 doc/walkthrough/using_the_SHA1_backend.mdwn create mode 100644 doc/walkthrough/using_the_URL_backend.mdwn diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index d08b247f73..896b560ec6 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -2,423 +2,24 @@ A walkthrough of the basic features of git-annex. [[!toc]] -## creating a repository - -This is very straightforward. Just tell it a description of the repository. - - # mkdir ~/annex - # cd ~/annex - # git init - # git annex init "my laptop" - -## adding a remote - -Like any other git repository, git-annex repositories have remotes. -Let's start by adding a USB drive as a remote. - - # sudo mount /media/usb - # cd /media/usb - # git clone ~/annex - # cd annex - # git annex init "portable USB drive" - # git remote add laptop ~/annex - # cd ~/annex - # git remote add usbdrive /media/usb - -This is all standard ad-hoc distributed git repository setup. -The only git-annex specific part is telling it the name -of the new repository created on the USB drive. - -Notice that both repos are set up as remotes of one another. This lets -either get annexed files from the other. You'll want to do that even -if you are using git in a more centralized fashion. - -## adding files - - # cd ~/annex - # cp /tmp/big_file . - # cp /tmp/debian.iso . - # git annex add . - add big_file ok - add debian.iso ok - # git commit -a -m added - -When you add a file to the annex and commit it, only a symlink to -the annexed content is committed. The content itself is stored in -git-annex's backend. - -## renaming files - - # cd ~/annex - # git mv big_file my_cool_big_file - # mkdir iso - # git mv debian.iso iso/ - # git commit -m moved - -You can use any normal git operations to move files around, or even -make copies or delete them. - -Notice that, since annexed files are represented by symlinks, -the symlink will break when the file is moved into a subdirectory. -But, git-annex will fix this up for you when you commit -- -it has a pre-commit hook that watches for and corrects broken symlinks. - -## getting file content - -A repository does not always have all annexed file contents available. -When you need the content of a file, you can use "git annex get" to -make it available. - -We can use this to copy everything in the laptop's annex to the -USB drive. - - # cd /media/usb/annex - # git pull laptop master - # git annex get . - get my_cool_big_file (copying from laptop...) ok - get iso/debian.iso (copying from laptop...) ok - -Notice that you had to git pull from laptop first, this lets git-annex know -what has changed in laptop, and so it knows about the files present there and -can get them. - -## transferring files: When things go wrong - -After a while, you'll have several annexes, with different file contents. -You don't have to try to keep all that straight; git-annex does -[[location_tracking]] for you. If you ask it to get a file and the drive -or file server is not accessible, it will let you know what it needs to get -it: - - # git annex get video/hackity_hack_and_kaxxt.mov - get video/_why_hackity_hack_and_kaxxt.mov (not available) - Unable to access these remotes: usbdrive, server - Try making some of these repositories available: - 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server - 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive - ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive - failed - # sudo mount /media/usb - # git annex get video/hackity_hack_and_kaxxt.mov - get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok - # git commit -a -m "got a video I want to rewatch on the plane" - -## removing files - -You can always drop files safely. Git-annex checks that some other annex -has the file before removing it. - - # git annex drop iso/debian.iso - drop iso/Debian_5.0.iso ok - # git commit -a -m "freed up space" - -## removing files: When things go wrong - -Before dropping a file, git-annex wants to be able to look at other -remotes, and verify that they still have a file. After all, it could -have been dropped from them too. If the remotes are not mounted/available, -you'll see something like this. - - # git annex drop important_file other.iso - drop important_file (unsafe) - Could only verify the existence of 0 out of 1 necessary copies - Unable to access these remotes: usbdrive - Try making some of these repositories available: - 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive - ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive - (Use --force to override this check, or adjust annex.numcopies.) - failed - drop other.iso (unsafe) - Could only verify the existence of 0 out of 1 necessary copies - No other repository is known to contain the file. - (Use --force to override this check, or adjust annex.numcopies.) - failed - -Here you might --force it to drop `important_file` if you [[trust]] your backup. -But `other.iso` looks to have never been copied to anywhere else, so if -it's something you want to hold onto, you'd need to transfer it to -some other repository before dropping it. - -## modifying annexed files - -Normally, the content of files in the annex is prevented from being modified. -That's a good thing, because it might be the only copy, you wouldn't -want to lose it in a fumblefingered mistake. - - # echo oops > my_cool_big_file - bash: my_cool_big_file: Permission denied - -In order to modify a file, it should first be unlocked. - - # git annex unlock my_cool_big_file - unlock my_cool_big_file (copying...) ok - -That replaces the symlink that normally points at its content with a copy -of the content. You can then modify the file like any regular file. Because -it is a regular file. - -(If you decide you don't need to modify the file after all, or want to discard -modifications, just use `git annex lock`.) - -When you `git commit`, git-annex's pre-commit hook will automatically -notice that you are committing an unlocked file, and add its new content -to the annex. The file will be replaced with a symlink to the new content, -and this symlink is what gets committed to git in the end. - - # echo "now smaller, but even cooler" > my_cool_big_file - # git commit my_cool_big_file -m "changed an annexed file" - add my_cool_big_file ok - [master 64cda67] changed an annexed file - 2 files changed, 2 insertions(+), 1 deletions(-) - create mode 100644 .git-annex/WORM:1289672605:30:file.log - -There is one problem with using `git commit` like this: Git wants to first -stage the entire contents of the file in its index. That can be slow for -big files (sorta why git-annex exists in the first place). So, the -automatic handling on commit is a nice safety feature, since it prevents -the file content being accidentally committed into git. But when working with -big files, it's faster to explicitly add them to the annex yourself -before committing. - - # echo "now smaller, but even cooler yet" > my_cool_big_file - # git annex add my_cool_big_file - add my_cool_big_file ok - # git commit my_cool_big_file -m "changed an annexed file" - -## using ssh remotes - -So far in this walkthrough, git-annex has been used with a remote -repository on a USB drive. But it can also be used with a git remote -that is truely remote, a host accessed by ssh. - -Say you have a desktop on the same network as your laptop and want -to clone the laptop's annex to it: - - # git clone ssh://mylaptop/home/me/annex ~/annex - # cd ~/annex - # git annex init "my desktop" - -Now you can get files and they will be transferred (using `rsync`): - - # git annex get my_cool_big_file - get my_cool_big_file (getting UUID for origin...) (copying from origin...) - WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 - ok - -When you drop files, git-annex will ssh over to the remote and make -sure the file's content is still there before removing it locally: - - # git annex drop my_cool_big_file - drop my_cool_big_file (checking origin..) ok - -Note that normally git-annex prefers to use non-ssh remotes, like -a USB drive, before ssh remotes. They are assumed to be faster/cheaper to -access, if available. There is a annex-cost setting you can configure in -`.git/config` to adjust which repositories it prefers. See -[[the_man_page|git-annex]] for details. - -Also, note that you need full shell access for this to work -- -git-annex needs to be able to ssh in and run commands. - -## moving file content between repositories - -Often you will want to move some file contents from a repository to some -other one. For example, your laptop's disk is getting full; time to move -some files to an external disk before moving another file from a file -server to your laptop. Doing that by hand (by using `git annex get` and -`git annex drop`) is possible, but a bit of a pain. `git annex move` -makes it very easy. - - # git annex move my_cool_big_file --to usbdrive - move my_cool_big_file (moving to usbdrive...) ok - # git annex move video/hackity_hack_and_kaxxt.mov --from fileserver - move video/hackity_hack_and_kaxxt.mov (moving from fileserver...) - WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 - ok - -## using the URL backend - -git-annex has multiple key-value [[backends]]. So far this walkthrough has -demonstrated the default, WORM (Write Once, Read Many) backend. - -Another handy backend is the URL backend, which can fetch file's content -from remote URLs. Here's how to set up some files in your repository -that use this backend: - - # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile - fromkey somefile ok - # git commit -m "added a file from the Internet Archive" - -Now you if you ask git-annex to get that file, it will download it, -and cache it locally. - - # git annex get somefile - get somefile (downloading) - #########################################################################100.0% - ok - -You can always drop files downloaded by the URL backend. It is assumed -that the URL is stable; no local backup is kept. - - # git annex drop somefile - drop somefile (ok) - -## using the SHA1 backend - -Another handy alternative to the default [[backend|backends]] is the -SHA1 backend. This backend provides more git-style assurance that your data -has not been damaged. And the checksum means that when you add the same -content to the annex twice, only one copy need be stored in the backend. - -The only reason it's not the default is that it needs to checksum -files when they're added to the annex, and this can slow things down -significantly for really big files. To make SHA1 the default, just -add something like this to `.gitattributes`: - - * annex.backend=SHA1 - -## migrating data to a new backend - -Maybe you started out using the WORM backend, and have now configured -git-annex to use SHA1. But files you added to the annex before still -use the WORM backend. There is a simple command that can migrate that -data: - - # git annex migrate my_cool_big_file - migrate my_cool_big_file (checksum...) ok - -You can only migrate files whose content is currently available. Other -files will be skipped. - -After migrating a file to a new backend, the old content in the old backend -will still be present. That is necessary because multiple files -can point to the same content. The `git annex unused` subcommand can be -used to clear up that detritus later. Note that hard links are used, -to avoid wasting disk space. - -## unused data - -It's possible for data to accumulate in the annex that no files point to -anymore. One way it can happen is if you `git rm` a file without -first calling `git annex drop`. And, when you modify an annexed file, the old -content of the file remains in the annex. Another way is when migrating -between backends. - -This might be historical data you want to preserve, so git-annex defaults to -preserving it. So from time to time, you may want to check for such data and -eliminate it to save space. - - # git annex unused - unused (checking for unused data...) - Some annexed data is no longer pointed to by any files in the repository. - NUMBER KEY - 1 WORM:1289672605:3:file - 2 WORM:1289672605:14:file - (To see where data was previously used, try: git log --stat -S'KEY') - (To remove unwanted data: git-annex dropunused NUMBER) - ok - -After running `git annex unused`, you can follow the instructions to examine -the history of files that used the data, and if you decide you don't need that -data anymore, you can easily remove it: - - # git annex dropunused 1 - dropunused 1 ok - -Hint: To drop a lot of unused data, use a command like this: - - # git annex dropunused `seq 1 1000` - -## fsck: verifying your data - -You can use the fsck subcommand to check for problems in your data. -What can be checked depends on the [[backend|backends]] you've used to store -the data. For example, when you use the SHA1 backend, fsck will verify that -the checksums of your files are good. Fsck also checks that the annex.numcopies -setting is satisfied for all files. - - # git annex fsck - unused (checking for unused data...) ok - fsck my_cool_big_file (checksum...) ok - ... - -You can also specify the files to check. This is particularly useful if -you're using sha1 and don't want to spend a long time checksumming everything. - - # git annex fsck my_cool_big_file - fsck my_cool_big_file (checksum...) ok - -## fsck: When things go wrong - -Fsck never deletes possibly bad data; instead it will be moved to -`.git/annex/bad/` for you to recover. Here is a sample of what fsck -might say about a badly messed up annex: - - # git annex fsck - fsck my_cool_big_file (checksum...) - git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 - git-annex: ** No known copies of the file exist! - failed - fsck important_file - git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. - failed - git-annex: 2 failed - -## backups - -git-annex can be configured to require more than one copy of a file exists, -as a simple backup for your data. This is controlled by the "annex.numcopies" -setting, which defaults to 1 copy. Let's change that to require 2 copies, -and send a copy of every file to a USB drive. - - # echo "* annex.numcopies=2" >> .gitattributes - # git annex copy . --to usbdrive - -Now when we try to `git annex drop` a file, it will verify that it -knows of 2 other repositories that have a copy before removing its -content from the current repository. - -You can also vary the number of copies needed, depending on the file name. -So, if you want 3 copies of all your flac files, but only 1 copy of oggs: - - # echo "*.ogg annex.numcopies=1" >> .gitattributes - # echo "*.flac annex.numcopies=3" >> .gitattributes - -Or, you might want to make a directory for important stuff, and configure -it so anything put in there is backed up more thoroughly: - - # mkdir important_stuff - # echo "* annex.numcopies=3" > important_stuff/.gitattributes - -For more details about the numcopies setting, see [[copies]]. - -## untrusted repositories - -Suppose you have a USB thumb drive and are using it as a git annex -repository. You don't trust the drive, because you could lose it, or -accidentally run it through the laundry. Or, maybe you have a drive that -you know is dying, and you'd like to be warned if there are any files -on it not backed up somewhere else. Maybe the drive has already died -or been lost. - -You can let git-annex know that you don't trust a repository, and it will -adjust its behavior to avoid relying on that repositories's continued -availability. - - # git annex untrust usbdrive - untrust usbdrive ok - -Now when you do a fsck, you'll be warned appropriately: - - # git annex fsck . - fsck my_big_file - Only these untrusted locations may have copies of this file! - 05e296c4-2989-11e0-bf40-bad1535567fe -- portable USB drive - Back it up to trusted locations with git-annex copy. - failed - -Also, git-annex will refuse to drop a file from elsewhere just because -it can see a copy on the untrusted repository. - -It's also possible to tell git-annex that you have an unusually high -level of trust for a repository. See [[trust]] for details. +[[!inline feeds=no pagenames=""" + creating_a_repository + adding_a_remote + adding_files + renaming_files + getting_file_content + transferring_files:_When_things_go_wrong + removing_files + removing_files:_When_things_go_wrong + modifying_annexed_files + using_ssh_remotes + moving_file_content_between_repositories + using_the_URL_backend + using_the_SHA1_backend + migrating_data_to_a_new_backend + unused_data + fsck:_verifying_your_data + fsck:_when_things_go_wrong + backups + untrusted_repositories +"""]] diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn new file mode 100644 index 0000000000..be8e8e7fe5 --- /dev/null +++ b/doc/walkthrough/adding_a_remote.mdwn @@ -0,0 +1,19 @@ +Like any other git repository, git-annex repositories have remotes. +Let's start by adding a USB drive as a remote. + + # sudo mount /media/usb + # cd /media/usb + # git clone ~/annex + # cd annex + # git annex init "portable USB drive" + # git remote add laptop ~/annex + # cd ~/annex + # git remote add usbdrive /media/usb + +This is all standard ad-hoc distributed git repository setup. +The only git-annex specific part is telling it the name +of the new repository created on the USB drive. + +Notice that both repos are set up as remotes of one another. This lets +either get annexed files from the other. You'll want to do that even +if you are using git in a more centralized fashion. diff --git a/doc/walkthrough/adding_files.mdwn b/doc/walkthrough/adding_files.mdwn new file mode 100644 index 0000000000..77a7fbc154 --- /dev/null +++ b/doc/walkthrough/adding_files.mdwn @@ -0,0 +1,11 @@ + # cd ~/annex + # cp /tmp/big_file . + # cp /tmp/debian.iso . + # git annex add . + add big_file ok + add debian.iso ok + # git commit -a -m added + +When you add a file to the annex and commit it, only a symlink to +the annexed content is committed. The content itself is stored in +git-annex's backend. diff --git a/doc/walkthrough/backups.mdwn b/doc/walkthrough/backups.mdwn new file mode 100644 index 0000000000..9723022b4c --- /dev/null +++ b/doc/walkthrough/backups.mdwn @@ -0,0 +1,25 @@ +git-annex can be configured to require more than one copy of a file exists, +as a simple backup for your data. This is controlled by the "annex.numcopies" +setting, which defaults to 1 copy. Let's change that to require 2 copies, +and send a copy of every file to a USB drive. + + # echo "* annex.numcopies=2" >> .gitattributes + # git annex copy . --to usbdrive + +Now when we try to `git annex drop` a file, it will verify that it +knows of 2 other repositories that have a copy before removing its +content from the current repository. + +You can also vary the number of copies needed, depending on the file name. +So, if you want 3 copies of all your flac files, but only 1 copy of oggs: + + # echo "*.ogg annex.numcopies=1" >> .gitattributes + # echo "*.flac annex.numcopies=3" >> .gitattributes + +Or, you might want to make a directory for important stuff, and configure +it so anything put in there is backed up more thoroughly: + + # mkdir important_stuff + # echo "* annex.numcopies=3" > important_stuff/.gitattributes + +For more details about the numcopies setting, see [[copies]]. diff --git a/doc/walkthrough/creating_a_repository.mdwn b/doc/walkthrough/creating_a_repository.mdwn new file mode 100644 index 0000000000..51ff1c72b3 --- /dev/null +++ b/doc/walkthrough/creating_a_repository.mdwn @@ -0,0 +1,6 @@ +This is very straightforward. Just tell it a description of the repository. + + # mkdir ~/annex + # cd ~/annex + # git init + # git annex init "my laptop" diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn new file mode 100644 index 0000000000..cd3a47a8a9 --- /dev/null +++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn @@ -0,0 +1,16 @@ +You can use the fsck subcommand to check for problems in your data. +What can be checked depends on the [[backend|backends]] you've used to store +the data. For example, when you use the SHA1 backend, fsck will verify that +the checksums of your files are good. Fsck also checks that the annex.numcopies +setting is satisfied for all files. + + # git annex fsck + unused (checking for unused data...) ok + fsck my_cool_big_file (checksum...) ok + ... + +You can also specify the files to check. This is particularly useful if +you're using sha1 and don't want to spend a long time checksumming everything. + + # git annex fsck my_cool_big_file + fsck my_cool_big_file (checksum...) ok diff --git a/doc/walkthrough/fsck:_when_things_go_wrong.mdwn b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn new file mode 100644 index 0000000000..05b9f385c0 --- /dev/null +++ b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn @@ -0,0 +1,13 @@ +Fsck never deletes possibly bad data; instead it will be moved to +`.git/annex/bad/` for you to recover. Here is a sample of what fsck +might say about a badly messed up annex: + + # git annex fsck + fsck my_cool_big_file (checksum...) + git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 + git-annex: ** No known copies of the file exist! + failed + fsck important_file + git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. + failed + git-annex: 2 failed diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn new file mode 100644 index 0000000000..a863303cef --- /dev/null +++ b/doc/walkthrough/getting_file_content.mdwn @@ -0,0 +1,16 @@ +A repository does not always have all annexed file contents available. +When you need the content of a file, you can use "git annex get" to +make it available. + +We can use this to copy everything in the laptop's annex to the +USB drive. + + # cd /media/usb/annex + # git pull laptop master + # git annex get . + get my_cool_big_file (copying from laptop...) ok + get iso/debian.iso (copying from laptop...) ok + +Notice that you had to git pull from laptop first, this lets git-annex know +what has changed in laptop, and so it knows about the files present there and +can get them. diff --git a/doc/walkthrough/migrating_data_to_a_new_backend.mdwn b/doc/walkthrough/migrating_data_to_a_new_backend.mdwn new file mode 100644 index 0000000000..b9acb8bd15 --- /dev/null +++ b/doc/walkthrough/migrating_data_to_a_new_backend.mdwn @@ -0,0 +1,16 @@ +Maybe you started out using the WORM backend, and have now configured +git-annex to use SHA1. But files you added to the annex before still +use the WORM backend. There is a simple command that can migrate that +data: + + # git annex migrate my_cool_big_file + migrate my_cool_big_file (checksum...) ok + +You can only migrate files whose content is currently available. Other +files will be skipped. + +After migrating a file to a new backend, the old content in the old backend +will still be present. That is necessary because multiple files +can point to the same content. The `git annex unused` subcommand can be +used to clear up that detritus later. Note that hard links are used, +to avoid wasting disk space. diff --git a/doc/walkthrough/modifying_annexed_files.mdwn b/doc/walkthrough/modifying_annexed_files.mdwn new file mode 100644 index 0000000000..3ad4e82eab --- /dev/null +++ b/doc/walkthrough/modifying_annexed_files.mdwn @@ -0,0 +1,43 @@ +Normally, the content of files in the annex is prevented from being modified. +That's a good thing, because it might be the only copy, you wouldn't +want to lose it in a fumblefingered mistake. + + # echo oops > my_cool_big_file + bash: my_cool_big_file: Permission denied + +In order to modify a file, it should first be unlocked. + + # git annex unlock my_cool_big_file + unlock my_cool_big_file (copying...) ok + +That replaces the symlink that normally points at its content with a copy +of the content. You can then modify the file like any regular file. Because +it is a regular file. + +(If you decide you don't need to modify the file after all, or want to discard +modifications, just use `git annex lock`.) + +When you `git commit`, git-annex's pre-commit hook will automatically +notice that you are committing an unlocked file, and add its new content +to the annex. The file will be replaced with a symlink to the new content, +and this symlink is what gets committed to git in the end. + + # echo "now smaller, but even cooler" > my_cool_big_file + # git commit my_cool_big_file -m "changed an annexed file" + add my_cool_big_file ok + [master 64cda67] changed an annexed file + 2 files changed, 2 insertions(+), 1 deletions(-) + create mode 100644 .git-annex/WORM:1289672605:30:file.log + +There is one problem with using `git commit` like this: Git wants to first +stage the entire contents of the file in its index. That can be slow for +big files (sorta why git-annex exists in the first place). So, the +automatic handling on commit is a nice safety feature, since it prevents +the file content being accidentally committed into git. But when working with +big files, it's faster to explicitly add them to the annex yourself +before committing. + + # echo "now smaller, but even cooler yet" > my_cool_big_file + # git annex add my_cool_big_file + add my_cool_big_file ok + # git commit my_cool_big_file -m "changed an annexed file" diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn new file mode 100644 index 0000000000..d7150f109a --- /dev/null +++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn @@ -0,0 +1,13 @@ +Often you will want to move some file contents from a repository to some +other one. For example, your laptop's disk is getting full; time to move +some files to an external disk before moving another file from a file +server to your laptop. Doing that by hand (by using `git annex get` and +`git annex drop`) is possible, but a bit of a pain. `git annex move` +makes it very easy. + + # git annex move my_cool_big_file --to usbdrive + move my_cool_big_file (moving to usbdrive...) ok + # git annex move video/hackity_hack_and_kaxxt.mov --from fileserver + move video/hackity_hack_and_kaxxt.mov (moving from fileserver...) + WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 + ok diff --git a/doc/walkthrough/removing_files.mdwn b/doc/walkthrough/removing_files.mdwn new file mode 100644 index 0000000000..85a7d50a6b --- /dev/null +++ b/doc/walkthrough/removing_files.mdwn @@ -0,0 +1,6 @@ +You can always drop files safely. Git-annex checks that some other annex +has the file before removing it. + + # git annex drop iso/debian.iso + drop iso/Debian_5.0.iso ok + # git commit -a -m "freed up space" diff --git a/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn b/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn new file mode 100644 index 0000000000..2d3c0cde08 --- /dev/null +++ b/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn @@ -0,0 +1,24 @@ +Before dropping a file, git-annex wants to be able to look at other +remotes, and verify that they still have a file. After all, it could +have been dropped from them too. If the remotes are not mounted/available, +you'll see something like this. + + # git annex drop important_file other.iso + drop important_file (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + Unable to access these remotes: usbdrive + Try making some of these repositories available: + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + (Use --force to override this check, or adjust annex.numcopies.) + failed + drop other.iso (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + No other repository is known to contain the file. + (Use --force to override this check, or adjust annex.numcopies.) + failed + +Here you might --force it to drop `important_file` if you [[trust]] your backup. +But `other.iso` looks to have never been copied to anywhere else, so if +it's something you want to hold onto, you'd need to transfer it to +some other repository before dropping it. diff --git a/doc/walkthrough/renaming_files.mdwn b/doc/walkthrough/renaming_files.mdwn new file mode 100644 index 0000000000..85964d1ea5 --- /dev/null +++ b/doc/walkthrough/renaming_files.mdwn @@ -0,0 +1,13 @@ + # cd ~/annex + # git mv big_file my_cool_big_file + # mkdir iso + # git mv debian.iso iso/ + # git commit -m moved + +You can use any normal git operations to move files around, or even +make copies or delete them. + +Notice that, since annexed files are represented by symlinks, +the symlink will break when the file is moved into a subdirectory. +But, git-annex will fix this up for you when you commit -- +it has a pre-commit hook that watches for and corrects broken symlinks. diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn new file mode 100644 index 0000000000..d8f0a19bd6 --- /dev/null +++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn @@ -0,0 +1,18 @@ +After a while, you'll have several annexes, with different file contents. +You don't have to try to keep all that straight; git-annex does +[[location_tracking]] for you. If you ask it to get a file and the drive +or file server is not accessible, it will let you know what it needs to get +it: + + # git annex get video/hackity_hack_and_kaxxt.mov + get video/_why_hackity_hack_and_kaxxt.mov (not available) + Unable to access these remotes: usbdrive, server + Try making some of these repositories available: + 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + failed + # sudo mount /media/usb + # git annex get video/hackity_hack_and_kaxxt.mov + get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok + # git commit -a -m "got a video I want to rewatch on the plane" diff --git a/doc/walkthrough/untrusted_repositories.mdwn b/doc/walkthrough/untrusted_repositories.mdwn new file mode 100644 index 0000000000..cdb5da7c3d --- /dev/null +++ b/doc/walkthrough/untrusted_repositories.mdwn @@ -0,0 +1,28 @@ +Suppose you have a USB thumb drive and are using it as a git annex +repository. You don't trust the drive, because you could lose it, or +accidentally run it through the laundry. Or, maybe you have a drive that +you know is dying, and you'd like to be warned if there are any files +on it not backed up somewhere else. Maybe the drive has already died +or been lost. + +You can let git-annex know that you don't trust a repository, and it will +adjust its behavior to avoid relying on that repositories's continued +availability. + + # git annex untrust usbdrive + untrust usbdrive ok + +Now when you do a fsck, you'll be warned appropriately: + + # git annex fsck . + fsck my_big_file + Only these untrusted locations may have copies of this file! + 05e296c4-2989-11e0-bf40-bad1535567fe -- portable USB drive + Back it up to trusted locations with git-annex copy. + failed + +Also, git-annex will refuse to drop a file from elsewhere just because +it can see a copy on the untrusted repository. + +It's also possible to tell git-annex that you have an unusually high +level of trust for a repository. See [[trust]] for details. diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn new file mode 100644 index 0000000000..69a581fe1e --- /dev/null +++ b/doc/walkthrough/unused_data.mdwn @@ -0,0 +1,30 @@ +It's possible for data to accumulate in the annex that no files point to +anymore. One way it can happen is if you `git rm` a file without +first calling `git annex drop`. And, when you modify an annexed file, the old +content of the file remains in the annex. Another way is when migrating +between backends. + +This might be historical data you want to preserve, so git-annex defaults to +preserving it. So from time to time, you may want to check for such data and +eliminate it to save space. + + # git annex unused + unused (checking for unused data...) + Some annexed data is no longer pointed to by any files in the repository. + NUMBER KEY + 1 WORM:1289672605:3:file + 2 WORM:1289672605:14:file + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused NUMBER) + ok + +After running `git annex unused`, you can follow the instructions to examine +the history of files that used the data, and if you decide you don't need that +data anymore, you can easily remove it: + + # git annex dropunused 1 + dropunused 1 ok + +Hint: To drop a lot of unused data, use a command like this: + + # git annex dropunused `seq 1 1000` diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn new file mode 100644 index 0000000000..831746ac0e --- /dev/null +++ b/doc/walkthrough/using_ssh_remotes.mdwn @@ -0,0 +1,33 @@ +So far in this walkthrough, git-annex has been used with a remote +repository on a USB drive. But it can also be used with a git remote +that is truely remote, a host accessed by ssh. + +Say you have a desktop on the same network as your laptop and want +to clone the laptop's annex to it: + + # git clone ssh://mylaptop/home/me/annex ~/annex + # cd ~/annex + # git annex init "my desktop" + +Now you can get files and they will be transferred (using `rsync`): + + # git annex get my_cool_big_file + get my_cool_big_file (getting UUID for origin...) (copying from origin...) + WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 + ok + +When you drop files, git-annex will ssh over to the remote and make +sure the file's content is still there before removing it locally: + + # git annex drop my_cool_big_file + drop my_cool_big_file (checking origin..) ok + +Note that normally git-annex prefers to use non-ssh remotes, like +a USB drive, before ssh remotes. They are assumed to be faster/cheaper to +access, if available. There is a annex-cost setting you can configure in +`.git/config` to adjust which repositories it prefers. See +[[the_man_page|git-annex]] for details. + +Also, note that you need full shell access for this to work -- +git-annex needs to be able to ssh in and run commands. Or at least, +your shell needs to be able to run the [[git-annex-shell]] command. diff --git a/doc/walkthrough/using_the_SHA1_backend.mdwn b/doc/walkthrough/using_the_SHA1_backend.mdwn new file mode 100644 index 0000000000..c04729e2cd --- /dev/null +++ b/doc/walkthrough/using_the_SHA1_backend.mdwn @@ -0,0 +1,11 @@ +Another handy alternative to the default [[backend|backends]] is the +SHA1 backend. This backend provides more git-style assurance that your data +has not been damaged. And the checksum means that when you add the same +content to the annex twice, only one copy need be stored in the backend. + +The only reason it's not the default is that it needs to checksum +files when they're added to the annex, and this can slow things down +significantly for really big files. To make SHA1 the default, just +add something like this to `.gitattributes`: + + * annex.backend=SHA1 diff --git a/doc/walkthrough/using_the_URL_backend.mdwn b/doc/walkthrough/using_the_URL_backend.mdwn new file mode 100644 index 0000000000..fe79a6be2e --- /dev/null +++ b/doc/walkthrough/using_the_URL_backend.mdwn @@ -0,0 +1,24 @@ +git-annex has multiple key-value [[backends]]. So far this walkthrough has +demonstrated the default, WORM (Write Once, Read Many) backend. + +Another handy backend is the URL backend, which can fetch file's content +from remote URLs. Here's how to set up some files in your repository +that use this backend: + + # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + fromkey somefile ok + # git commit -m "added a file from the Internet Archive" + +Now you if you ask git-annex to get that file, it will download it, +and cache it locally. + + # git annex get somefile + get somefile (downloading) + #########################################################################100.0% + ok + +You can always drop files downloaded by the URL backend. It is assumed +that the URL is stable; no local backup is kept. + + # git annex drop somefile + drop somefile (ok) From 78fa8c1b439ef6a5ea4da04b6cff1d7174818c7e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:47:43 -0400 Subject: [PATCH 0915/2835] walkthrough deep links --- ...dd_range_argument_to___34__git_annex_dropunused__34___.mdwn | 2 +- .../git_annex_migrate_leaves_old_backend_versions_around.mdwn | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn index dc9b7acda5..bbe6007a87 100644 --- a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn +++ b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn @@ -9,7 +9,7 @@ I work around this lack as I want to drop all unused files anyway by something l git annex unused | grep -o -P "^ [0-9]+" | xargs git annex dropunused > It's designed to be used with `seq`. There's an example in the -> [[walkthrough]], and of course multiple seq calls can be used to +> [[walkthrough|walkthrough/unused_data]], and of course multiple seq calls can be used to > specifiy multiple ranges. So: git annex dropunused `seq 1 9` `seq 11 1845` diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn index fc8551ddb4..e37ee06bb3 100644 --- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -1,7 +1,8 @@ `git annex migrate` leaves old, unlinked backend versions lying around. It would be great if these were purged automatically somehow. -> Yes, this is an issue mentioned in the [[walkthrough#index14h2]]. +> Yes, this is an issue mentioned in the +> [[walkthrough|walkthrough/migrating_data_to_a_new_backend]]. > > Since multiple files can point to the same content, it could be that > only one file has been migrated, and the content is still used. So From 9a4e54a5e7200a0ff6e3b3741174b4c10a3a3b44 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:52:27 -0400 Subject: [PATCH 0916/2835] custom template for walkthrough inline --- doc/templates/walkthrough.tmpl | 2 ++ doc/walkthrough.mdwn | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 doc/templates/walkthrough.tmpl diff --git a/doc/templates/walkthrough.tmpl b/doc/templates/walkthrough.tmpl new file mode 100644 index 0000000000..a500a5a865 --- /dev/null +++ b/doc/templates/walkthrough.tmpl @@ -0,0 +1,2 @@ +

+ diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 896b560ec6..ea2ee63546 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -2,7 +2,7 @@ A walkthrough of the basic features of git-annex. [[!toc]] -[[!inline feeds=no pagenames=""" +[[!inline feeds=no template=walkthrough pagenames=""" creating_a_repository adding_a_remote adding_files From 1bf6ed740e2f9fce319ec822585a60a114029b3d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:55:13 -0400 Subject: [PATCH 0917/2835] show all --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index ea2ee63546..f400191c61 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -2,7 +2,7 @@ A walkthrough of the basic features of git-annex. [[!toc]] -[[!inline feeds=no template=walkthrough pagenames=""" +[[!inline feeds=no show=0 template=walkthrough pagenames=""" creating_a_repository adding_a_remote adding_files From 7e5678bcf7cd78bd04520117201be37dc9d4d544 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 28 Feb 2011 16:09:17 -0400 Subject: [PATCH 0918/2835] add ShellParam type, for type-checked shell params --- Utility.hs | 145 +++++++++++++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 66 deletions(-) diff --git a/Utility.hs b/Utility.hs index 6a90e3cd50..b5c0dd617e 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,6 +6,8 @@ -} module Utility ( + ShellParam(..), + toShell, hGetContentsStrict, readFileStrict, parentDir, @@ -15,7 +17,6 @@ module Utility ( boolSystem, shellEscape, shellUnEscape, - utilityEscape, unsetFileMode, readMaybe, safeWriteFile, @@ -41,6 +42,83 @@ import Foreign (complement) import Data.List import Control.Monad (liftM2) +{- A type for parameters passed to a shell command. A command can + - be passed either some Params (multiple parameters can be included, + - whitespace-separated, or a single Param (for when parameters contain + - whitespace), or a File. + -} +data ShellParam = Params String | Param String | File FilePath + deriving (Eq, Show, Ord) + +{- When converting ShellParam to a String in preparation for passing to + - a shell command, Files that start with a dash are modified to avoid + - the shell command interpreting them as options. -} +toShell :: [ShellParam] -> [String] +toShell l = concat $ map unwrap l + where + unwrap (Param s) = [s] + unwrap (Params s) = filter (not . null) (split " " s) + unwrap (File ('-':s)) = ["./-" ++ s] + unwrap (File (s)) = [s] + +{- Run a system command, and returns True or False + - if it succeeded or failed. + - + - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. + -} +boolSystem :: FilePath -> [ShellParam] -> IO Bool +boolSystem command params = do + -- Going low-level because all the high-level system functions + -- block SIGINT etc. We need to block SIGCHLD, but allow + -- SIGINT to do its default program termination. + let sigset = addSignal sigCHLD emptySignalSet + oldint <- installHandler sigINT Default Nothing + oldset <- getSignalMask + blockSignals sigset + childpid <- forkProcess $ childaction oldint oldset + mps <- getProcessStatus True False childpid + restoresignals oldint oldset + case mps of + Just (Exited ExitSuccess) -> return True + _ -> return False + where + restoresignals oldint oldset = do + _ <- installHandler sigINT oldint Nothing + setSignalMask oldset + childaction oldint oldset = do + restoresignals oldint oldset + executeFile command True (toShell params) Nothing + +{- Escapes a filename to be safely able to be exposed to the shell. -} +shellEscape :: FilePath -> String +shellEscape f = "'" ++ escaped ++ "'" + where + -- replace ' with '"'"' + escaped = join "'\"'\"'" $ split "'" f + +{- Unescapes a set of shellEscaped words or filenames. -} +shellUnEscape :: String -> [String] +shellUnEscape [] = [] +shellUnEscape s = word:(shellUnEscape rest) + where + (word, rest) = findword "" s + findword w [] = (w, "") + findword w (c:cs) + | c == ' ' = (w, cs) + | c == '\'' = inquote c w cs + | c == '"' = inquote c w cs + | otherwise = findword (w++[c]) cs + inquote _ w [] = (w, "") + inquote q w (c:cs) + | c == q = findword w cs + | otherwise = inquote q (w++[c]) cs + +{- For quickcheck. -} +prop_idempotent_shellEscape :: String -> Bool +prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s) +prop_idempotent_shellEscape_multiword :: [String] -> Bool +prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) + {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} hGetContentsStrict :: Handle -> IO String @@ -128,71 +206,6 @@ prop_relPathDirToDir_basics from to where r = relPathDirToDir from to -{- Run a system command, and returns True or False - - if it succeeded or failed. - - - - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. - -} -boolSystem :: FilePath -> [String] -> IO Bool -boolSystem command params = do - -- Going low-level because all the high-level system functions - -- block SIGINT etc. We need to block SIGCHLD, but allow - -- SIGINT to do its default program termination. - let sigset = addSignal sigCHLD emptySignalSet - oldint <- installHandler sigINT Default Nothing - oldset <- getSignalMask - blockSignals sigset - childpid <- forkProcess $ childaction oldint oldset - mps <- getProcessStatus True False childpid - restoresignals oldint oldset - case mps of - Just (Exited ExitSuccess) -> return True - _ -> return False - where - restoresignals oldint oldset = do - _ <- installHandler sigINT oldint Nothing - setSignalMask oldset - childaction oldint oldset = do - restoresignals oldint oldset - executeFile command True params Nothing - -{- Escapes a filename to be safely able to be exposed to the shell. -} -shellEscape :: FilePath -> String -shellEscape f = "'" ++ escaped ++ "'" - where - -- replace ' with '"'"' - escaped = join "'\"'\"'" $ split "'" f - -{- Unescapes a set of shellEscaped words or filenames. -} -shellUnEscape :: String -> [String] -shellUnEscape [] = [] -shellUnEscape s = word:(shellUnEscape rest) - where - (word, rest) = findword "" s - findword w [] = (w, "") - findword w (c:cs) - | c == ' ' = (w, cs) - | c == '\'' = inquote c w cs - | c == '"' = inquote c w cs - | otherwise = findword (w++[c]) cs - inquote _ w [] = (w, "") - inquote q w (c:cs) - | c == q = findword w cs - | otherwise = inquote q (w++[c]) cs - -{- Ensures that a filename is safe to pass to a utility program. In particular - - since utilities tend to interpret things starting with a dash as - - an option, relative filenames starting with a dash are escaped. -} -utilityEscape :: FilePath -> FilePath -utilityEscape ('-':s) = "./-" ++ s -utilityEscape s = s - -{- For quickcheck. -} -prop_idempotent_shellEscape :: String -> Bool -prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s) -prop_idempotent_shellEscape_multiword :: [String] -> Bool -prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) - {- Removes a FileMode from a file. - For example, call with otherWriteMode to chmod o-w -} unsetFileMode :: FilePath -> FileMode -> IO () From fcdc4797a9ab2b792a9bb20f2ca9802b8f6d5a1e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 28 Feb 2011 16:10:16 -0400 Subject: [PATCH 0919/2835] use ShellParam type So, I have a type checked safe handling of filenames starting with dashes, throughout the code. --- Annex.hs | 5 +++-- Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Command/Add.hs | 3 ++- Command/Fix.hs | 2 +- Command/FromKey.hs | 2 +- Command/Init.hs | 17 ++++++++++----- Command/Lock.hs | 5 +++-- Command/Map.hs | 9 ++++---- Command/Move.hs | 9 ++++---- Command/PreCommit.hs | 3 ++- Command/SetKey.hs | 2 +- Command/Unannex.hs | 4 ++-- Content.hs | 2 +- CopyFile.hs | 14 ++++++------- GitQueue.hs | 11 +++++----- GitRepo.hs | 44 +++++++++++++++++++------------------- Remotes.hs | 50 ++++++++++++++++++++++++-------------------- RsyncFile.hs | 29 +++++++++++++------------ Trust.hs | 8 +++++-- Upgrade.hs | 3 ++- Utility.hs | 11 +++++----- git-annex-shell.hs | 2 +- test.hs | 36 +++++++++++++++---------------- 24 files changed, 151 insertions(+), 124 deletions(-) diff --git a/Annex.hs b/Annex.hs index 5496ada67c..cb662a1307 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,6 +24,7 @@ import Control.Monad.State import qualified GitRepo as Git import qualified GitQueue import qualified BackendTypes +import Utility -- git-annex's monad type Annex = StateT AnnexState IO @@ -91,7 +92,7 @@ gitRepo :: Annex Git.Repo gitRepo = getState repo {- Adds a git command to the queue. -} -queue :: String -> [String] -> FilePath -> Annex () +queue :: String -> [ShellParam] -> FilePath -> Annex () queue command params file = do state <- get let q = repoqueue state @@ -110,7 +111,7 @@ queueRun = do setConfig :: String -> String -> Annex () setConfig k value = do g <- Annex.gitRepo - liftIO $ Git.run g ["config", k, value] + liftIO $ Git.run g "config" [Param k, Param value] -- re-read git config and update the repo's state g' <- liftIO $ Git.configRead g Annex.changeState $ \s -> s { Annex.repo = g' } diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index e1830bc134..a7f592b73e 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -32,7 +32,7 @@ backend = Backend.File.backend { sha1 :: FilePath -> Annex String sha1 file = do showNote "checksum..." - liftIO $ pOpen ReadFromPipe "sha1sum" [utilityEscape file] $ \h -> do + liftIO $ pOpen ReadFromPipe "sha1sum" (toShell [File file]) $ \h -> do line <- hGetLine h let bits = split " " line if null bits diff --git a/Backend/URL.hs b/Backend/URL.hs index 15cc88d644..864c793010 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -51,6 +51,6 @@ downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" showProgress -- make way for curl progress bar - liftIO $ boolSystem "curl" ["-#", "-o", utilityEscape file, url] + liftIO $ boolSystem "curl" [Params "-# -o", File file, File url] where url = join ":" $ drop 1 $ split ":" $ show key diff --git a/Command/Add.hs b/Command/Add.hs index 4b49297fc6..26e7fa258d 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -17,6 +17,7 @@ import LocationLog import Types import Content import Messages +import Utility command :: [Command] command = [Command "add" paramPath seek "add files to annex"] @@ -52,5 +53,5 @@ cleanup file key = do link <- calcGitLink file key liftIO $ createSymbolicLink link file - Annex.queue "add" ["--"] file + Annex.queue "add" [Param "--"] file return True diff --git a/Command/Fix.hs b/Command/Fix.hs index d67eca164c..0047548715 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -44,5 +44,5 @@ perform file link = do cleanup :: FilePath -> CommandCleanup cleanup file = do - Annex.queue "add" ["--"] file + Annex.queue "add" [Param "--"] file return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 8817942580..d16eff8466 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -47,5 +47,5 @@ perform file = do cleanup :: FilePath -> CommandCleanup cleanup file = do - Annex.queue "add" ["--"] file + Annex.queue "add" [Param "--"] file return True diff --git a/Command/Init.hs b/Command/Init.hs index 2976b988d1..1074d100ea 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -51,8 +51,12 @@ cleanup :: CommandCleanup cleanup = do g <- Annex.gitRepo logfile <- uuidLog - liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-q", "-m", "git annex init", logfile] + liftIO $ Git.run g "add" [File logfile] + liftIO $ Git.run g "commit" + [ Params "-q -m" + , Param "git annex init" + , File logfile + ] return True {- configure git to use union merge driver on state files, if it is not @@ -72,9 +76,12 @@ gitAttributesWrite repo = do where attributes = Git.attributes repo commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-q", "-m", "git-annex setup", - attributes] + Git.run repo "add" [Param attributes] + Git.run repo "commit" + [ Params "-q -m" + , Param "git-annex setup" + , Param attributes + ] attrLine :: String attrLine = stateDir "*.log merge=union" diff --git a/Command/Lock.hs b/Command/Lock.hs index 00a553e956..a3a39a9078 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -14,6 +14,7 @@ import Command import Messages import qualified Annex import qualified GitRepo as Git +import Utility command :: [Command] command = [Command "lock" paramPath seek "undo unlock command"] @@ -32,7 +33,7 @@ perform file = do liftIO $ removeFile file g <- Annex.gitRepo -- first reset the file to drop any changes checked into the index - liftIO $ Git.run g ["reset", "-q", "--", file] + liftIO $ Git.run g "reset" [Params "-q --", File file] -- checkout the symlink - liftIO $ Git.run g ["checkout", "--", file] + liftIO $ Git.run g "checkout" [Param "--", File file] return $ Just $ return True -- no cleanup needed diff --git a/Command/Map.hs b/Command/Map.hs index 0a3bb9fff0..00b5fc21b2 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -44,7 +44,7 @@ start = do liftIO $ writeFile file (drawMap rs umap trusted) showLongNote $ "running: dot -Tx11 " ++ file showProgress - r <- liftIO $ boolSystem "dot" ["-Tx11", file] + r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file] return $ Just $ return $ Just $ return r where file = "map.dot" @@ -198,7 +198,7 @@ tryScan r Left _ -> return Nothing Right r' -> return $ Just r' pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd params $ + pOpen ReadFromPipe cmd (toShell params) $ Git.hConfigRead r configlist = @@ -208,8 +208,9 @@ tryScan r let sshcmd = "cd " ++ shellEscape(Git.workTree r) ++ " && " ++ "git config --list" - liftIO $ pipedconfig "ssh" $ - words sshoptions ++ [Git.urlHostFull r, sshcmd] + liftIO $ pipedconfig "ssh" $ map Param $ + words sshoptions ++ + [Git.urlHostFull r, sshcmd] -- First, try sshing and running git config manually, -- only fall back to git-annex-shell configlist if that diff --git a/Command/Move.hs b/Command/Move.hs index 6dc2e48746..8c19539fba 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -56,7 +56,7 @@ remoteHasKey remote key present = do g <- Annex.gitRepo remoteuuid <- getUUID remote logfile <- liftIO $ logChange g key remoteuuid status - Annex.queue "add" ["--"] logfile + Annex.queue "add" [Param "--"] logfile where status = if present then ValuePresent else ValueMissing @@ -130,9 +130,10 @@ fromPerform src move key = do fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup fromCleanup src True key = do ok <- Remotes.onRemote src (boolSystem, False) "dropkey" - ["--quiet", "--force", - "--backend=" ++ backendName key, - keyName key] + [ Params "--quiet --force" + , Param $ "--backend=" ++ backendName key + , Param $ keyName key + ] -- better safe than sorry: assume the src dropped the key -- even if it seemed to fail; the failure could have occurred -- after it really dropped it diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 750997f540..d2f6964343 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -15,6 +15,7 @@ import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix import Messages +import Utility command :: [Command] command = [Command "pre-commit" paramPath seek "run by git pre-commit hook"] @@ -41,6 +42,6 @@ cleanup file = do -- drop that and run command queued by Add.state to -- stage the symlink g <- Annex.gitRepo - liftIO $ Git.run g ["reset", "-q", "--", file] + liftIO $ Git.run g "reset" [Params "-q --", File file] Annex.queueRun return True diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 025fb74d62..fdda1c3bee 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -36,7 +36,7 @@ perform file = do ok <- getViaTmp key $ \dest -> do if dest /= file then liftIO $ - boolSystem "mv" [utilityEscape file, utilityEscape dest] + boolSystem "mv" [File file, File dest] else return True if ok then return $ Just $ cleanup diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 19cb1624e5..42dc1fb0ab 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -58,7 +58,7 @@ cleanup file key = do g <- Annex.gitRepo liftIO $ removeFile file - liftIO $ Git.run g ["rm", "--quiet", "--", file] + liftIO $ Git.run g "rm" [Params "--quiet --", File file] -- git rm deletes empty directories; put them back liftIO $ createDirectoryIfMissing True (parentDir file) @@ -68,6 +68,6 @@ cleanup file key = do -- Commit staged changes at end to avoid confusing the -- pre-commit hook if this file is later added back to -- git as a normal, non-annexed file. - Annex.queue "commit" ["-m", "content removed from git annex"] "-a" + Annex.queue "commit" [Params "-a -m", Param "content removed from git annex"] "-a" return True diff --git a/Content.hs b/Content.hs index 345599dbad..cb954f4a0b 100644 --- a/Content.hs +++ b/Content.hs @@ -60,7 +60,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g logfile <- liftIO $ logChange g key u status - Annex.queue "add" ["--"] logfile + Annex.queue "add" [Param "--"] logfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into diff --git a/CopyFile.hs b/CopyFile.hs index 73d911a291..4575fb08ad 100644 --- a/CopyFile.hs +++ b/CopyFile.hs @@ -20,14 +20,12 @@ copyFile src dest = do e <- doesFileExist dest when e $ removeFile dest - boolSystem "cp" opts + boolSystem "cp" [params, File src, File dest] where - opts = if SysConfig.cp_reflink_auto - then ["--reflink=auto", src', dest'] + params = if SysConfig.cp_reflink_auto + then Params "--reflink=auto" else if SysConfig.cp_a - then ["-a", src', dest'] + then Params "-a" else if SysConfig.cp_p - then ["-p", src', dest'] - else [src', dest'] - src' = utilityEscape src - dest' = utilityEscape dest + then Params "-p" + else Params "" diff --git a/GitQueue.hs b/GitQueue.hs index 4a777af4db..328243fa00 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -17,6 +17,7 @@ import System.IO import System.Cmd.Utils import Data.String.Utils import Control.Monad (unless, forM_) +import Utility import qualified GitRepo as Git @@ -24,7 +25,7 @@ import qualified GitRepo as Git - is not included, and must be able to be appended after the params. -} data Action = Action { getSubcommand :: String, - getParams :: [String] + getParams :: [ShellParam] } deriving (Show, Eq, Ord) {- A queue of actions to perform (in any order) on a git repository, @@ -37,7 +38,7 @@ empty :: Queue empty = M.empty {- Adds an action to a queue. -} -add :: Queue -> String -> [String] -> FilePath -> Queue +add :: Queue -> String -> [ShellParam] -> FilePath -> Queue add queue subcommand params file = M.insertWith (++) action [file] queue where action = Action subcommand params @@ -55,7 +56,7 @@ runAction :: Git.Repo -> Action -> [FilePath] -> IO () runAction repo action files = do unless (null files) runxargs where - runxargs = pOpen WriteToPipe "xargs" ("-0":gitcmd) feedxargs - gitcmd = "git" : Git.gitCommandLine repo - (getSubcommand action:getParams action) + runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs + params = toShell $ Git.gitCommandLine repo + (Param (getSubcommand action):getParams action) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index 7cf0891eda..3f2acdcf4f 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -243,16 +243,18 @@ urlPath Repo { location = Url u } = uriPath u urlPath repo = assertUrl repo $ error "internal" {- Constructs a git command line operating on the specified repo. -} -gitCommandLine :: Repo -> [String] -> [String] +gitCommandLine :: Repo -> [ShellParam] -> [ShellParam] gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir=" ++ d ++ "/" ++ gitDir repo, "--work-tree=" ++ d] ++ params + [ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo) + , Param ("--work-tree=" ++ d) + ] ++ params gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo, throwing an error if it fails. -} -run :: Repo -> [String] -> IO () -run repo params = assertLocal repo $ do - ok <- boolSystem "git" (gitCommandLine repo params) +run :: Repo -> String -> [ShellParam] -> IO () +run repo subcommand params = assertLocal repo $ do + ok <- boolSystem "git" (gitCommandLine repo ((Param subcommand):params)) unless ok $ error $ "git " ++ show params ++ " failed" {- Runs a git subcommand and returns it output, lazily. @@ -260,9 +262,9 @@ run repo params = assertLocal repo $ do - Note that this leaves the git process running, and so zombies will - result unless reap is called. -} -pipeRead :: Repo -> [String] -> IO String +pipeRead :: Repo -> [ShellParam] -> IO String pipeRead repo params = assertLocal repo $ do - (_, s) <- pipeFrom "git" (gitCommandLine repo params) + (_, s) <- pipeFrom "git" $ toShell $ gitCommandLine repo params return s {- Reaps any zombie git processes. -} @@ -277,13 +279,13 @@ reap = do {- Scans for files that are checked into git at the specified locations. -} inRepo :: Repo -> [FilePath] -> IO [FilePath] inRepo repo l = pipeNullSplit repo $ - ["ls-files", "--cached", "--exclude-standard", "-z", "--"] ++ l + [Params "ls-files --cached --exclude-standard -z --"] ++ map File l {- Scans for files at the specified locations that are not checked into git, - and not gitignored. -} notInRepo :: Repo -> [FilePath] -> IO [FilePath] notInRepo repo l = pipeNullSplit repo $ - ["ls-files", "--others", "--exclude-standard", "-z", "--"] ++ l + [Params "ls-files --others --exclude-standard -z --"] ++ map File l {- Returns a list of all files that are staged for commit. -} stagedFiles :: Repo -> [FilePath] -> IO [FilePath] @@ -292,38 +294,38 @@ stagedFiles repo l = stagedFiles' repo l [] {- Returns a list of the files, staged for commit, that are being added, - moved, or changed (but not deleted), from the specified locations. -} stagedFilesNotDeleted :: Repo -> [FilePath] -> IO [FilePath] -stagedFilesNotDeleted repo l = stagedFiles' repo l ["--diff-filter=ACMRT"] +stagedFilesNotDeleted repo l = stagedFiles' repo l [Param "--diff-filter=ACMRT"] -stagedFiles' :: Repo -> [FilePath] -> [String] -> IO [FilePath] +stagedFiles' :: Repo -> [FilePath] -> [ShellParam] -> IO [FilePath] stagedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where - start = ["diff", "--cached", "--name-only", "-z"] - end = ["--"] ++ l + start = [Params "diff --cached --name-only -z"] + end = [Param "--"] ++ map File l {- Returns a list of files that have unstaged changes. -} changedUnstagedFiles :: Repo -> [FilePath] -> IO [FilePath] changedUnstagedFiles repo l = pipeNullSplit repo $ - ["diff", "--name-only", "-z", "--"] ++ l + [Params "diff --name-only -z --"] ++ map File l {- Returns a list of the files in the specified locations that are staged - for commit, and whose type has changed. -} typeChangedStagedFiles :: Repo -> [FilePath] -> IO [FilePath] -typeChangedStagedFiles repo l = typeChangedFiles' repo l ["--cached"] +typeChangedStagedFiles repo l = typeChangedFiles' repo l [Param "--cached"] {- Returns a list of the files in the specified locations whose type has - changed. Files only staged for commit will not be included. -} typeChangedFiles :: Repo -> [FilePath] -> IO [FilePath] typeChangedFiles repo l = typeChangedFiles' repo l [] -typeChangedFiles' :: Repo -> [FilePath] -> [String] -> IO [FilePath] +typeChangedFiles' :: Repo -> [FilePath] -> [ShellParam] -> IO [FilePath] typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where - start = ["diff", "--name-only", "--diff-filter=T", "-z"] - end = ["--"] ++ l + start = [Params "diff --name-only --diff-filter=T -z"] + end = [Param "--"] ++ map File l {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} -pipeNullSplit :: Repo -> [String] -> IO [FilePath] +pipeNullSplit :: Repo -> [ShellParam] -> IO [FilePath] pipeNullSplit repo params = do fs0 <- pipeRead repo params return $ split0 fs0 @@ -408,11 +410,11 @@ checkAttr repo attr files = do -- directory. Convert to absolute, and then convert the filenames -- in its output back to relative. absfiles <- mapM absPath files - (_, s) <- pipeBoth "git" params $ join "\0" absfiles + (_, s) <- pipeBoth "git" (toShell params) $ join "\0" absfiles cwd <- getCurrentDirectory return $ map (topair $ cwd++"/") $ lines s where - params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] + params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"] topair cwd l = (relfile, value) where relfile diff --git a/Remotes.hs b/Remotes.hs index c7e69aad8b..1523e67509 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -64,7 +64,7 @@ tryGitConfigRead r Left _ -> return r Right r' -> return r' pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd params $ + pOpen ReadFromPipe cmd (toShell params) $ Git.hConfigRead r store a = do r' <- a @@ -154,7 +154,7 @@ inAnnex r key = if Git.repoIsUrl r checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- onRemote r (boolSystem, False) "inannex" - ["--backend=" ++ backendName key, keyName key] + [Param ("--backend=" ++ backendName key), Param (keyName key)] return $ Right inannex {- Cost Ordered list of remotes. -} @@ -263,28 +263,31 @@ rsynchelper r sending key file = do {- Generates rsync parameters that ssh to the remote and asks it - to either receive or send the key's content. -} -rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [String] +rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [ShellParam] rsyncParams r sending key file = do - -- Note that the command is terminated with "--", because - -- rsync will tack on its own options to this command, - -- and they need to be ignored. - shellcmd <- git_annex_shell r + Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") - ["--backend=" ++ backendName key, keyName key, "--"] + [ Param $ "--backend=" ++ backendName key + , Param $ keyName key + -- Command is terminated with "--", because + -- rsync will tack on its own options afterwards, + -- and they need to be ignored. + , Param "--" + ] -- Convert the ssh command into rsync command line. - let eparam = rsyncShell $ fromJust shellcmd + let eparam = rsyncShell (Param shellcmd:shellparams) o <- repoConfig r "rsync-options" "" - let base = options ++ words o ++ eparam + let base = options ++ map Param (words o) ++ eparam if sending - then return $ base ++ [dummy, file] - else return $ base ++ [file, dummy] + then return $ base ++ [dummy, File file] + else return $ base ++ [File file, dummy] where -- inplace makes rsync resume partial files - options = ["-p", "--progress", "--inplace"] + options = [Params "-p --progress --inplace"] -- the rsync shell parameter controls where rsync -- goes, so the source/dest parameter can be a dummy value, -- that just enables remote rsync mode. - dummy = ":" + dummy = Param ":" {- Uses a supplied function to run a git-annex-shell command on a remote. - @@ -292,30 +295,31 @@ rsyncParams r sending key file = do - a specified error value. -} onRemote :: Git.Repo - -> (String -> [String] -> IO a, a) + -> (FilePath -> [ShellParam] -> IO a, a) -> String - -> [String] + -> [ShellParam] -> Annex a onRemote r (with, errorval) command params = do s <- git_annex_shell r command params case s of - Just shellcmd -> liftIO $ with (shellcmd !! 0) (tail shellcmd) + Just (c, ps) -> liftIO $ with c ps Nothing -> return errorval {- Generates parameters to run a git-annex-shell command on a remote. -} -git_annex_shell :: Git.Repo -> String -> [String] -> Annex (Maybe [String]) +git_annex_shell :: Git.Repo -> String -> [ShellParam] -> Annex (Maybe (FilePath, [ShellParam])) git_annex_shell r command params - | not $ Git.repoIsUrl r = return $ Just (shellcmd:shellopts) + | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) | Git.repoIsSsh r = do sshoptions <- repoConfig r "ssh-options" "" - return $ Just $ ["ssh"] ++ words sshoptions ++ - [Git.urlHostFull r, sshcmd] + return $ Just ("ssh", map Param (words sshoptions) ++ + [Param (Git.urlHostFull r), Param sshcmd]) | otherwise = return Nothing where dir = Git.workTree r shellcmd = "git-annex-shell" - shellopts = command:dir:params - sshcmd = shellcmd ++ " " ++ unwords (map shellEscape shellopts) + shellopts = (Param command):(File dir):params + sshcmd = shellcmd ++ " " ++ + unwords (map shellEscape $ toShell shellopts) {- Looks up a per-remote config option in git config. - Failing that, tries looking for a global config option. -} diff --git a/RsyncFile.hs b/RsyncFile.hs index 9de2e2c594..149b45b11b 100644 --- a/RsyncFile.hs +++ b/RsyncFile.hs @@ -14,8 +14,8 @@ import Utility {- Generates parameters to make rsync use a specified command as its remote - shell. -} -rsyncShell :: [String] -> [String] -rsyncShell command = ["-e", unwords $ map escape command] +rsyncShell :: [ShellParam] -> [ShellParam] +rsyncShell command = [Param "-e", Param $ unwords $ map escape (toShell command)] where {- rsync requires some weird, non-shell like quoting in - here. A doubled single quote inside the single quoted @@ -25,22 +25,25 @@ rsyncShell command = ["-e", unwords $ map escape command] {- Runs rsync in server mode to send a file, and exits. -} rsyncServerSend :: FilePath -> IO () rsyncServerSend file = rsyncExec $ - rsyncServerParams ++ ["--sender", utilityEscape file] + rsyncServerParams ++ [Param "--sender", File file] {- Runs rsync in server mode to receive a file. -} rsyncServerReceive :: FilePath -> IO Bool -rsyncServerReceive file = rsync $ rsyncServerParams ++ [utilityEscape file] +rsyncServerReceive file = rsync $ rsyncServerParams ++ [File file] -rsyncServerParams :: [String] +rsyncServerParams :: [ShellParam] rsyncServerParams = - [ "--server" - , "-p" -- preserve permissions - , "--inplace" -- allow resuming of transfers of big files - , "-e.Lsf", "." -- other options rsync normally uses in server mode + [ Param "--server" + -- preserve permissions + , Param "-p" + -- allow resuming of transfers of big files + , Param "--inplace" + -- other options rsync normally uses in server mode + , Params "-e.Lsf ." ] -rsync :: [String] -> IO Bool -rsync params = boolSystem "rsync" params +rsync :: [ShellParam] -> IO Bool +rsync = boolSystem "rsync" -rsyncExec :: [String] -> IO () -rsyncExec params = executeFile "rsync" True params Nothing +rsyncExec :: [ShellParam] -> IO () +rsyncExec params = executeFile "rsync" True (toShell params) Nothing diff --git a/Trust.hs b/Trust.hs index 695059a932..7b2cf9ff88 100644 --- a/Trust.hs +++ b/Trust.hs @@ -81,8 +81,12 @@ trustSet uuid level = do logfile <- trustLog liftIO $ safeWriteFile logfile (serialize m') g <- Annex.gitRepo - liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-q", "-m", "git annex trust change", logfile] + liftIO $ Git.run g "add" [File logfile] + liftIO $ Git.run g "commit" + [ Params "-q -m" + , Param "git annex trust change" + , File logfile + ] where serialize m = unlines $ map showpair $ M.toList m showpair (u, t) = u ++ " " ++ show t diff --git a/Upgrade.hs b/Upgrade.hs index b584b2666f..3c16bcc862 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -22,6 +22,7 @@ import qualified Annex import qualified Backend import Messages import Version +import Utility {- Uses the annex.version git config setting to automate upgrades. -} upgrade :: Annex Bool @@ -62,7 +63,7 @@ upgradeFrom0 = do link <- calcGitLink f k liftIO $ removeFile f liftIO $ createSymbolicLink link f - Annex.queue "add" ["--"] f + Annex.queue "add" [Param "--"] f fixlinks fs getKeysPresent0' :: FilePath -> Annex [Key] diff --git a/Utility.hs b/Utility.hs index b5c0dd617e..90494a0c44 100644 --- a/Utility.hs +++ b/Utility.hs @@ -1,6 +1,6 @@ {- git-annex utility functions - - - Copyright 2010 Joey Hess + - Copyright 2010-2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -50,16 +50,17 @@ import Control.Monad (liftM2) data ShellParam = Params String | Param String | File FilePath deriving (Eq, Show, Ord) -{- When converting ShellParam to a String in preparation for passing to - - a shell command, Files that start with a dash are modified to avoid - - the shell command interpreting them as options. -} +{- Used to pass a list of ShellParams to a function that runs + - a shell command and expects Strings. -} toShell :: [ShellParam] -> [String] toShell l = concat $ map unwrap l where unwrap (Param s) = [s] unwrap (Params s) = filter (not . null) (split " " s) + -- Files that start with a dash are modified to avoid + -- the shell command interpreting them as options. unwrap (File ('-':s)) = ["./-" ++ s] - unwrap (File (s)) = [s] + unwrap (File s) = [s] {- Run a system command, and returns True or False - if it succeeded or failed. diff --git a/git-annex-shell.hs b/git-annex-shell.hs index fee4091ef8..aeaadcbf85 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -66,7 +66,7 @@ builtin cmd dir params = do external :: [String] -> IO () external params = do - ret <- boolSystem "git-shell" ("-c":(filterparams params)) + ret <- boolSystem "git-shell" $ map Param $ ("-c":filterparams params) when (not ret) $ error "git-shell failed" diff --git a/test.hs b/test.hs index 9b50bcb2ef..1bae3bd83e 100644 --- a/test.hs +++ b/test.hs @@ -105,8 +105,8 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup] git_annex "add" ["-q", annexedfile] @? "add failed" annexed_present annexedfile writeFile ingitfile $ content ingitfile - Utility.boolSystem "git" ["add", ingitfile] @? "git add failed" - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" + Utility.boolSystem "git" [Utility.Param "add", Utility.File ingitfile] @? "git add failed" + Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed" git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op" unannexed ingitfile sha1dup = TestCase $ intmpclonerepo $ do @@ -125,7 +125,7 @@ test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do let sha1 = BackendTypes.keyName $ fromJust r git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" + Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed" annexed_present sha1annexedfile where tmp = "tmpfile" @@ -139,7 +139,7 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] annexed_notpresent annexedfile withcopy = "with content" ~: intmpclonerepo $ do git_annex "get" ["-q", annexedfile] @? "get failed" - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "state changed"] + Utility.boolSystem "git" [Utility.Params "commit -q -a -m statechanged"] @? "git commit of state failed" annexed_present annexedfile git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed" @@ -154,9 +154,9 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote] where noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do git_annex "get" ["-q", annexedfile] @? "get failed" - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "state changed"] + Utility.boolSystem "git" [Utility.Params "commit -q -a -m statechanged"] @? "git commit of state failed" - Utility.boolSystem "git" ["remote", "rm", "origin"] + Utility.boolSystem "git" [Utility.Params "remote rm origin"] @? "git remote rm origin failed" r <- git_annex "drop" ["-q", annexedfile] not r @? "drop wrongly succeeded with no known copy of file" @@ -287,12 +287,12 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True] then do -- pre-commit depends on the file being -- staged, normally git commit does this - Utility.boolSystem "git" ["add", annexedfile] + Utility.boolSystem "git" [Utility.Param "add", Utility.File annexedfile] @? "git add of edited file failed" git_annex "pre-commit" ["-q"] @? "pre-commit failed" else do - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "content changed"] + Utility.boolSystem "git" [Utility.Params "commit -q -a -m contentchanged"] @? "git commit of edited file failed" runchecks [checklink, checkunwritable] annexedfile c <- readFile annexedfile @@ -310,7 +310,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do git_annex "fix" ["-q", annexedfile] @? "fix of present file failed" annexed_present annexedfile createDirectory subdir - Utility.boolSystem "git" ["mv", annexedfile, subdir] + Utility.boolSystem "git" [Utility.Param "mv", Utility.File annexedfile, Utility.File subdir] @? "git mv failed" git_annex "fix" ["-q", newfile] @? "fix of moved file failed" runchecks [checklink, checkunwritable] newfile @@ -350,9 +350,9 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem where basicfsck = TestCase $ intmpclonerepo $ do git_annex "fsck" ["-q"] @? "fsck failed" - Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed" fsck_should_fail "numcopies unsatisfied" - Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config annex.numcopies 1"] @? "git config failed" corrupt annexedfile corrupt sha1annexedfile withlocaluntrusted = TestCase $ intmpclonerepo $ do @@ -363,7 +363,7 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem git_annex "trust" ["-q", "."] @? "trust of current repo failed" git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo" withremoteuntrusted = TestCase $ intmpclonerepo $ do - Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed" git_annex "get" ["-q", annexedfile] @? "get failed" git_annex "get" ["-q", sha1annexedfile] @? "get failed" git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies" @@ -433,9 +433,9 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do git_annex "get" ["-q", annexedfile] @? "get of file failed" git_annex "get" ["-q", sha1annexedfile] @? "get of file failed" checkunused [] - Utility.boolSystem "git" ["rm", "-q", annexedfile] @? "git rm failed" + Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File annexedfile] @? "git rm failed" checkunused [annexedfilekey] - Utility.boolSystem "git" ["rm", "-q", sha1annexedfile] @? "git rm failed" + Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File sha1annexedfile] @? "git rm failed" checkunused [annexedfilekey, sha1annexedfilekey] -- good opportunity to test dropkey also @@ -511,10 +511,10 @@ setuprepo :: FilePath -> IO FilePath setuprepo dir = do cleanup dir ensuretmpdir - Utility.boolSystem "git" ["init", "-q", dir] @? "git init failed" + Utility.boolSystem "git" [Utility.Params "init -q", Utility.File dir] @? "git init failed" indir dir $ do - Utility.boolSystem "git" ["config", "user.name", "Test User"] @? "git config failed" - Utility.boolSystem "git" ["config", "user.email", "test@example.com"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config user.name", Utility.Param "Test User"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config user.email test@example.com"] @? "git config failed" return dir -- clones are always done as local clones; we cannot test ssh clones @@ -522,7 +522,7 @@ clonerepo :: FilePath -> FilePath -> IO FilePath clonerepo old new = do cleanup new ensuretmpdir - Utility.boolSystem "git" ["clone", "-q", old, new] @? "git clone failed" + Utility.boolSystem "git" [Utility.Params "clone -q", Utility.File old, Utility.File new] @? "git clone failed" indir new $ git_annex "init" ["-q", new] @? "git annex init failed" return new From 4cd96ad2db0867ef7450215d3de7afcf748d7088 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 28 Feb 2011 16:25:31 -0400 Subject: [PATCH 0920/2835] rename --- Annex.hs | 2 +- Backend/SHA1.hs | 2 +- Command/Map.hs | 2 +- GitQueue.hs | 6 +++--- GitRepo.hs | 16 ++++++++-------- Remotes.hs | 12 ++++++------ RsyncFile.hs | 12 ++++++------ Utility.hs | 20 ++++++++++---------- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Annex.hs b/Annex.hs index cb662a1307..62e7e023d5 100644 --- a/Annex.hs +++ b/Annex.hs @@ -92,7 +92,7 @@ gitRepo :: Annex Git.Repo gitRepo = getState repo {- Adds a git command to the queue. -} -queue :: String -> [ShellParam] -> FilePath -> Annex () +queue :: String -> [CommandParam] -> FilePath -> Annex () queue command params file = do state <- get let q = repoqueue state diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index a7f592b73e..22bc493b77 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -32,7 +32,7 @@ backend = Backend.File.backend { sha1 :: FilePath -> Annex String sha1 file = do showNote "checksum..." - liftIO $ pOpen ReadFromPipe "sha1sum" (toShell [File file]) $ \h -> do + liftIO $ pOpen ReadFromPipe "sha1sum" (toCommand [File file]) $ \h -> do line <- hGetLine h let bits = split " " line if null bits diff --git a/Command/Map.hs b/Command/Map.hs index 00b5fc21b2..4d0f900038 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -198,7 +198,7 @@ tryScan r Left _ -> return Nothing Right r' -> return $ Just r' pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd (toShell params) $ + pOpen ReadFromPipe cmd (toCommand params) $ Git.hConfigRead r configlist = diff --git a/GitQueue.hs b/GitQueue.hs index 328243fa00..07cf9f62fc 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -25,7 +25,7 @@ import qualified GitRepo as Git - is not included, and must be able to be appended after the params. -} data Action = Action { getSubcommand :: String, - getParams :: [ShellParam] + getParams :: [CommandParam] } deriving (Show, Eq, Ord) {- A queue of actions to perform (in any order) on a git repository, @@ -38,7 +38,7 @@ empty :: Queue empty = M.empty {- Adds an action to a queue. -} -add :: Queue -> String -> [ShellParam] -> FilePath -> Queue +add :: Queue -> String -> [CommandParam] -> FilePath -> Queue add queue subcommand params file = M.insertWith (++) action [file] queue where action = Action subcommand params @@ -57,6 +57,6 @@ runAction repo action files = do unless (null files) runxargs where runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs - params = toShell $ Git.gitCommandLine repo + params = toCommand $ Git.gitCommandLine repo (Param (getSubcommand action):getParams action) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index 3f2acdcf4f..04a0c2d540 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -243,7 +243,7 @@ urlPath Repo { location = Url u } = uriPath u urlPath repo = assertUrl repo $ error "internal" {- Constructs a git command line operating on the specified repo. -} -gitCommandLine :: Repo -> [ShellParam] -> [ShellParam] +gitCommandLine :: Repo -> [CommandParam] -> [CommandParam] gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree [ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo) @@ -252,7 +252,7 @@ gitCommandLine repo@(Repo { location = Dir d} ) params = gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo, throwing an error if it fails. -} -run :: Repo -> String -> [ShellParam] -> IO () +run :: Repo -> String -> [CommandParam] -> IO () run repo subcommand params = assertLocal repo $ do ok <- boolSystem "git" (gitCommandLine repo ((Param subcommand):params)) unless ok $ error $ "git " ++ show params ++ " failed" @@ -262,9 +262,9 @@ run repo subcommand params = assertLocal repo $ do - Note that this leaves the git process running, and so zombies will - result unless reap is called. -} -pipeRead :: Repo -> [ShellParam] -> IO String +pipeRead :: Repo -> [CommandParam] -> IO String pipeRead repo params = assertLocal repo $ do - (_, s) <- pipeFrom "git" $ toShell $ gitCommandLine repo params + (_, s) <- pipeFrom "git" $ toCommand $ gitCommandLine repo params return s {- Reaps any zombie git processes. -} @@ -296,7 +296,7 @@ stagedFiles repo l = stagedFiles' repo l [] stagedFilesNotDeleted :: Repo -> [FilePath] -> IO [FilePath] stagedFilesNotDeleted repo l = stagedFiles' repo l [Param "--diff-filter=ACMRT"] -stagedFiles' :: Repo -> [FilePath] -> [ShellParam] -> IO [FilePath] +stagedFiles' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath] stagedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where start = [Params "diff --cached --name-only -z"] @@ -317,7 +317,7 @@ typeChangedStagedFiles repo l = typeChangedFiles' repo l [Param "--cached"] typeChangedFiles :: Repo -> [FilePath] -> IO [FilePath] typeChangedFiles repo l = typeChangedFiles' repo l [] -typeChangedFiles' :: Repo -> [FilePath] -> [ShellParam] -> IO [FilePath] +typeChangedFiles' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath] typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where start = [Params "diff --name-only --diff-filter=T -z"] @@ -325,7 +325,7 @@ typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} -pipeNullSplit :: Repo -> [ShellParam] -> IO [FilePath] +pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath] pipeNullSplit repo params = do fs0 <- pipeRead repo params return $ split0 fs0 @@ -410,7 +410,7 @@ checkAttr repo attr files = do -- directory. Convert to absolute, and then convert the filenames -- in its output back to relative. absfiles <- mapM absPath files - (_, s) <- pipeBoth "git" (toShell params) $ join "\0" absfiles + (_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles cwd <- getCurrentDirectory return $ map (topair $ cwd++"/") $ lines s where diff --git a/Remotes.hs b/Remotes.hs index 1523e67509..4dcc4c9adf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -64,7 +64,7 @@ tryGitConfigRead r Left _ -> return r Right r' -> return r' pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd (toShell params) $ + pOpen ReadFromPipe cmd (toCommand params) $ Git.hConfigRead r store a = do r' <- a @@ -263,7 +263,7 @@ rsynchelper r sending key file = do {- Generates rsync parameters that ssh to the remote and asks it - to either receive or send the key's content. -} -rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [ShellParam] +rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam] rsyncParams r sending key file = do Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") @@ -295,9 +295,9 @@ rsyncParams r sending key file = do - a specified error value. -} onRemote :: Git.Repo - -> (FilePath -> [ShellParam] -> IO a, a) + -> (FilePath -> [CommandParam] -> IO a, a) -> String - -> [ShellParam] + -> [CommandParam] -> Annex a onRemote r (with, errorval) command params = do s <- git_annex_shell r command params @@ -306,7 +306,7 @@ onRemote r (with, errorval) command params = do Nothing -> return errorval {- Generates parameters to run a git-annex-shell command on a remote. -} -git_annex_shell :: Git.Repo -> String -> [ShellParam] -> Annex (Maybe (FilePath, [ShellParam])) +git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam])) git_annex_shell r command params | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) | Git.repoIsSsh r = do @@ -319,7 +319,7 @@ git_annex_shell r command params shellcmd = "git-annex-shell" shellopts = (Param command):(File dir):params sshcmd = shellcmd ++ " " ++ - unwords (map shellEscape $ toShell shellopts) + unwords (map shellEscape $ toCommand shellopts) {- Looks up a per-remote config option in git config. - Failing that, tries looking for a global config option. -} diff --git a/RsyncFile.hs b/RsyncFile.hs index 149b45b11b..afff46c0ce 100644 --- a/RsyncFile.hs +++ b/RsyncFile.hs @@ -14,8 +14,8 @@ import Utility {- Generates parameters to make rsync use a specified command as its remote - shell. -} -rsyncShell :: [ShellParam] -> [ShellParam] -rsyncShell command = [Param "-e", Param $ unwords $ map escape (toShell command)] +rsyncShell :: [CommandParam] -> [CommandParam] +rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand command)] where {- rsync requires some weird, non-shell like quoting in - here. A doubled single quote inside the single quoted @@ -31,7 +31,7 @@ rsyncServerSend file = rsyncExec $ rsyncServerReceive :: FilePath -> IO Bool rsyncServerReceive file = rsync $ rsyncServerParams ++ [File file] -rsyncServerParams :: [ShellParam] +rsyncServerParams :: [CommandParam] rsyncServerParams = [ Param "--server" -- preserve permissions @@ -42,8 +42,8 @@ rsyncServerParams = , Params "-e.Lsf ." ] -rsync :: [ShellParam] -> IO Bool +rsync :: [CommandParam] -> IO Bool rsync = boolSystem "rsync" -rsyncExec :: [ShellParam] -> IO () -rsyncExec params = executeFile "rsync" True (toShell params) Nothing +rsyncExec :: [CommandParam] -> IO () +rsyncExec params = executeFile "rsync" True (toCommand params) Nothing diff --git a/Utility.hs b/Utility.hs index 90494a0c44..e63fa1f6be 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,8 +6,8 @@ -} module Utility ( - ShellParam(..), - toShell, + CommandParam(..), + toCommand, hGetContentsStrict, readFileStrict, parentDir, @@ -47,18 +47,18 @@ import Control.Monad (liftM2) - whitespace-separated, or a single Param (for when parameters contain - whitespace), or a File. -} -data ShellParam = Params String | Param String | File FilePath +data CommandParam = Params String | Param String | File FilePath deriving (Eq, Show, Ord) -{- Used to pass a list of ShellParams to a function that runs - - a shell command and expects Strings. -} -toShell :: [ShellParam] -> [String] -toShell l = concat $ map unwrap l +{- Used to pass a list of CommandParams to a function that runs + - a command and expects Strings. -} +toCommand :: [CommandParam] -> [String] +toCommand l = concat $ map unwrap l where unwrap (Param s) = [s] unwrap (Params s) = filter (not . null) (split " " s) -- Files that start with a dash are modified to avoid - -- the shell command interpreting them as options. + -- the command interpreting them as options. unwrap (File ('-':s)) = ["./-" ++ s] unwrap (File s) = [s] @@ -67,7 +67,7 @@ toShell l = concat $ map unwrap l - - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -} -boolSystem :: FilePath -> [ShellParam] -> IO Bool +boolSystem :: FilePath -> [CommandParam] -> IO Bool boolSystem command params = do -- Going low-level because all the high-level system functions -- block SIGINT etc. We need to block SIGCHLD, but allow @@ -88,7 +88,7 @@ boolSystem command params = do setSignalMask oldset childaction oldint oldset = do restoresignals oldint oldset - executeFile command True (toShell params) Nothing + executeFile command True (toCommand params) Nothing {- Escapes a filename to be safely able to be exposed to the shell. -} shellEscape :: FilePath -> String From e2e4096a28042327ccb905b80ec0cbf550f88c61 Mon Sep 17 00:00:00 2001 From: tyger Date: Tue, 1 Mar 2011 14:07:52 +0000 Subject: [PATCH 0921/2835] Added a comment --- ..._5f08da5e21c0b3b5a8d1e4408c0d6405._comment | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment new file mode 100644 index 0000000000..71a40ad8cb --- /dev/null +++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment @@ -0,0 +1,60 @@ +[[!comment format=mdwn + username="tyger" + ip="80.66.20.180" + subject="comment 2" + date="2011-03-01T14:07:50Z" + content=""" +My current workflow looks like this (I'm still experimenting): + +### Create backup clone for migration + + git clone original migrate + cd migrate + for branch in $(git branch -a | grep remotes/origin | grep -v HEAD); do git checkout --track $branch; done + +### Inject git annex initialization at repository base + + git symbolic-ref HEAD refs/heads/newroot + git rm --cached *.rpm + git clean -f -d + git annex init master + git cherry-pick $(git rev-list --reverse master | head -1) + git rebase --onto newroot newroot master + git rebase master mybranch # how to automate this for all branches? + git branch -d newroot + +### Start migration with tree filter + + echo \*.rpm annex.backend=SHA1 > .git/info/attributes + MYWORKDIR=$(pwd) git filter-branch --tree-filter ' \ + if [ ! -d .git-annex ]; then \ + mkdir .git-annex; \ + cp ${MYWORKDIR}/.git-annex/uuid.log .git-annex/; \ + cp ${MYWORKDIR}/.gitattributes .; \ + fi + for rpm in $(git ls-files | grep \"\.rpm$\"); do \ + echo; \ + git annex add $rpm; \ + annexdest=$(readlink $rpm); \ + if [ -e .git-annex/$(basename $annexdest).log ]; then \ + echo \"FOUND $(basename $annexdest).log\"; \ + else \ + echo \"COPY $(basename $annexdest).log\"; \ + cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log .git-annex/; \ + fi; \ + ln -sf ${annexdest#../../} $rpm; \ + done; \ + git reset HEAD .git-rewrite; \ + : \ + ' -- $(git branch | cut -c 3-) + rm -rf .temp + git reset --hard + + +There are still some drawbacks: + +* git history shows that git annex log files are modified with each checkin +* branches have to be rebased manually before starting migration + + +"""]] From be4af85f98c45888a2ab3436642850bdfb3db721 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 13:23:44 -0400 Subject: [PATCH 0922/2835] a slightly annoying thing --- doc/bugs/git_rename_detection_on_file_move.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git_rename_detection_on_file_move.mdwn diff --git a/doc/bugs/git_rename_detection_on_file_move.mdwn b/doc/bugs/git_rename_detection_on_file_move.mdwn new file mode 100644 index 0000000000..76f1e098e5 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move.mdwn @@ -0,0 +1,13 @@ +It's unfortunate that git-annex sorta defeats git's rename detection. + +When an annexed file is moved to a different directory (specifically, a +directory that is shallower or deeper than the old directory), +the symlink often has to change. And so git log cannot --follow back +through the rename history, since all it has to go on is that symlink, +which it effectively sees as a one line file containing the symlink target. + +One way to fix this might be to do the `git annex fix` *after* the rename +is committed. This would mean that a commit would result in new staged +changes for another commit, which is perhaps startling behavior. + +The other way to fix it is to stop using symlinks, see [[todo/smudge]]. From d140c01bfd09669c632c3973b7e5e342f0ad7de5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 13:37:24 -0400 Subject: [PATCH 0923/2835] bug --- doc/bugs/weird_local_clone_confuses.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/weird_local_clone_confuses.mdwn diff --git a/doc/bugs/weird_local_clone_confuses.mdwn b/doc/bugs/weird_local_clone_confuses.mdwn new file mode 100644 index 0000000000..fc652ae5eb --- /dev/null +++ b/doc/bugs/weird_local_clone_confuses.mdwn @@ -0,0 +1,9 @@ +See + + +If a local repo is cloned with "git clone orig/.git new", then git-annex in +new cannot see origin. + +the .git/config has "url=/.../orig/.git". Apparently git is ok with that +weird construction; probably it treats it as a bare git repo. But git-annex +just sees a directory w/o a .git subdir, and gives up. From b7f4801801aa8b8e20ea82261b193a73b7fec799 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 16:50:53 -0400 Subject: [PATCH 0924/2835] generic SHA size support --- Backend/SHA.hs | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ Backend/SHA1.hs | 55 +++----------------------------------- 2 files changed, 74 insertions(+), 52 deletions(-) create mode 100644 Backend/SHA.hs diff --git a/Backend/SHA.hs b/Backend/SHA.hs new file mode 100644 index 0000000000..d779e80553 --- /dev/null +++ b/Backend/SHA.hs @@ -0,0 +1,71 @@ +{- git-annex SHA abstract backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA (genBackend) where + +import Control.Monad.State +import Data.String.Utils +import System.Cmd.Utils +import System.IO +import System.Directory + +import qualified Backend.File +import BackendTypes +import Messages +import qualified Annex +import Locations +import Content +import Types +import Utility + +type SHASize = Int + +-- Constructor for Backends using a given SHASize. +genBackend :: SHASize -> Backend Annex +genBackend size = Backend.File.backend + { name = shaName size + , getKey = keyValue size + , fsckKey = Backend.File.checkKey $ checkKeyChecksum size + } + +shaName :: SHASize -> String +shaName size = "SHA" ++ show size + +shaN :: SHASize -> FilePath -> Annex String +shaN size file = do + showNote "checksum..." + liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do + line <- hGetLine h + let bits = split " " line + if null bits + then error $ command ++ " parse error" + else return $ head bits + where + command = "sha" ++ (show size) ++ "sum" + +-- A key is a checksum of its contents. +keyValue :: SHASize -> FilePath -> Annex (Maybe Key) +keyValue size file = do + s <- shaN size file + return $ Just $ Key (shaName size, s) + +-- A key's checksum is checked during fsck. +checkKeyChecksum :: SHASize -> Key -> Annex Bool +checkKeyChecksum size key = do + g <- Annex.gitRepo + let file = gitAnnexLocation g key + present <- liftIO $ doesFileExist file + if not present + then return True + else do + s <- shaN size file + if s == keyName key + then return True + else do + dest <- moveBad key + warning $ "Bad file content; moved to " ++ filePathToString dest + return False diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 22bc493b77..76d2af69e3 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -1,63 +1,14 @@ {- git-annex "SHA1" backend - - - Copyright 2010 Joey Hess + - Copyright 2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} module Backend.SHA1 (backend) where -import Control.Monad.State -import Data.String.Utils -import System.Cmd.Utils -import System.IO -import System.Directory - -import qualified Backend.File -import BackendTypes -import Messages -import qualified Annex -import Locations -import Content import Types -import Utility +import Backend.SHA backend :: Backend Annex -backend = Backend.File.backend { - name = "SHA1", - getKey = keyValue, - fsckKey = Backend.File.checkKey checkKeySHA1 -} - -sha1 :: FilePath -> Annex String -sha1 file = do - showNote "checksum..." - liftIO $ pOpen ReadFromPipe "sha1sum" (toCommand [File file]) $ \h -> do - line <- hGetLine h - let bits = split " " line - if null bits - then error "sha1sum parse error" - else return $ head bits - --- A key is a sha1 of its contents. -keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = do - s <- sha1 file - return $ Just $ Key (name backend, s) - --- A key's sha1 is checked during fsck. -checkKeySHA1 :: Key -> Annex Bool -checkKeySHA1 key = do - g <- Annex.gitRepo - let file = gitAnnexLocation g key - present <- liftIO $ doesFileExist file - if not present - then return True - else do - s <- sha1 file - if s == keyName key - then return True - else do - dest <- moveBad key - warning $ "Bad file content; moved to " ++ filePathToString dest - return False +backend = genBackend 1 From 1b9c4477fb542cddbb05012a52c602eb203b2d83 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 17:07:15 -0400 Subject: [PATCH 0925/2835] New backends: SHA512 SHA384 SHA256 SHA224 --- Backend/SHA224.hs | 14 ++++++++++++++ Backend/SHA256.hs | 14 ++++++++++++++ Backend/SHA384.hs | 14 ++++++++++++++ Backend/SHA512.hs | 14 ++++++++++++++ BackendList.hs | 8 ++++++++ configure.hs | 10 ++++++++-- debian/changelog | 1 + doc/backends.mdwn | 3 +++ 8 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 Backend/SHA224.hs create mode 100644 Backend/SHA256.hs create mode 100644 Backend/SHA384.hs create mode 100644 Backend/SHA512.hs diff --git a/Backend/SHA224.hs b/Backend/SHA224.hs new file mode 100644 index 0000000000..614f31c4b8 --- /dev/null +++ b/Backend/SHA224.hs @@ -0,0 +1,14 @@ +{- git-annex "SHA224" backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA224 (backend) where + +import Types +import Backend.SHA + +backend :: Backend Annex +backend = genBackend 224 diff --git a/Backend/SHA256.hs b/Backend/SHA256.hs new file mode 100644 index 0000000000..42d3d48c7d --- /dev/null +++ b/Backend/SHA256.hs @@ -0,0 +1,14 @@ +{- git-annex "SHA256" backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA256 (backend) where + +import Types +import Backend.SHA + +backend :: Backend Annex +backend = genBackend 256 diff --git a/Backend/SHA384.hs b/Backend/SHA384.hs new file mode 100644 index 0000000000..0cf77ea647 --- /dev/null +++ b/Backend/SHA384.hs @@ -0,0 +1,14 @@ +{- git-annex "SHA384" backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA384 (backend) where + +import Types +import Backend.SHA + +backend :: Backend Annex +backend = genBackend 384 diff --git a/Backend/SHA512.hs b/Backend/SHA512.hs new file mode 100644 index 0000000000..aed8bbcedf --- /dev/null +++ b/Backend/SHA512.hs @@ -0,0 +1,14 @@ +{- git-annex "SHA512" backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA512 (backend) where + +import Types +import Backend.SHA + +backend :: Backend Annex +backend = genBackend 512 diff --git a/BackendList.hs b/BackendList.hs index 5ae78bcc7f..4279b69fc9 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -10,6 +10,10 @@ module BackendList (allBackends) where -- When adding a new backend, import it here and add it to the list. import qualified Backend.WORM import qualified Backend.SHA1 +import qualified Backend.SHA256 +import qualified Backend.SHA512 +import qualified Backend.SHA224 +import qualified Backend.SHA384 import qualified Backend.URL import Types @@ -17,5 +21,9 @@ allBackends :: [Backend Annex] allBackends = [ Backend.WORM.backend , Backend.SHA1.backend + , Backend.SHA256.backend + , Backend.SHA512.backend + , Backend.SHA224.backend + , Backend.SHA384.backend , Backend.URL.backend ] diff --git a/configure.hs b/configure.hs index b5437ec1a5..772ba54899 100644 --- a/configure.hs +++ b/configure.hs @@ -11,11 +11,17 @@ tests = [ , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] - , TestCase "sha1sum" $ requireCmd "sha1sum" "sha1sum /dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath - ] + ] ++ shaTestCases [1, 256, 512, 224, 384] + +shaTestCases :: [Int] -> [TestCase] +shaTestCases l = map make l + where + make n = + let cmd = "sha" ++ show n ++ "sum" + in TestCase cmd $ requireCmd cmd (cmd ++ " Sun, 13 Feb 2011 00:48:02 -0400 diff --git a/doc/backends.mdwn b/doc/backends.mdwn index 3e605e4b15..5d02ad3a1c 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -19,6 +19,9 @@ can use different backends for different files. allows modifications of files to be tracked. Its need to generate checksums can make it slower for large files. for use. +* `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger + checksums. Mostly useful for the very paranoid, or anyone who is + researching checksum collisions and wants to annex their colliding data. ;) * `URL` -- This backend downloads the file's content from an external URL. The `annex.backends` git-config setting can be used to list the backends From fa92c29764d0deaf0f016729b90c6fdc4d4b7390 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 21:32:28 -0400 Subject: [PATCH 0926/2835] add internals page --- doc/index.mdwn | 1 + doc/internals.mdwn | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 doc/internals.mdwn diff --git a/doc/index.mdwn b/doc/index.mdwn index 503c858018..a6d072e093 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -53,6 +53,7 @@ files with git. * [[location_tracking]] reminds you where git-annex has seen files * git-annex prevents accidental data loss by [[tracking copies|copies]] of your files +* [[internals]] * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. diff --git a/doc/internals.mdwn b/doc/internals.mdwn new file mode 100644 index 0000000000..fe50681870 --- /dev/null +++ b/doc/internals.mdwn @@ -0,0 +1,56 @@ +In the world of git, we're not scared about internal implementation +details, and sometimes we like to dive in and tweak things by hand. Here's +some documentation to that end. + +## .git/annex/objects/*/* + +This is where locally available file contents are actually stored. +Files added to the annex get a symlink checked into git that points +to the file content. + +Each subdirectory has the name of a key. The file inside also has the name +of the key. This two-level structure is used because it allows the write +bit to be removed from the subdirectories as well as from the files. +That prevents accidentially deleting or changing the file contents. + +## .git-annex/uuid.log + +Records the UUIDs of known repositories, and associates them with a +description of the repository. This allows git-annex to display something +more useful than a UUID when it refers to a repository that does not have +a configured git remote pointing at it. + +The file format is simply one line per repository, with the uuid followed by a +space and then the description through to the end of the line. Example: + + e605dca6-446a-11e0-8b2a-002170d25c55 laptop + 26339d22-446b-11e0-9101-002170d25c55 usb disk + +## .git-annex/trust.log + +Records the [[trust]] information for repositories. Does not exist unless +[[trust]] values are configured. + +The file format is one line per repository, with the uuid followed by a +space, and then either 1 (trusted), 0 (untrusted), or ? (semi-trusted). +Repositories not listed are semi-trusted. + +Example: + + e605dca6-446a-11e0-8b2a-002170d25c55 1 + 26339d22-446b-11e0-9101-002170d25c55 ? + +## .git-annex/*.log + +The remainder of the log files record [[location_tracking]] information +for file contents. The name of the key is the filename, and the content +consists of a timestamp, either 1 (present) or 0 (not present), and +the UUID of the repository that has or lacks the file content. + +Example: + + 1287290776.765152s 1 e605dca6-446a-11e0-8b2a-002170d25c55 + 1287290767.478634s 0 26339d22-446b-11e0-9101-002170d25c55 + +These files are designed to be auto-merged using git's union merge driver. +The timestamps allow the most recent information to be identified. From d6be2e222f2c8ffa0f6cf5e46a19795846d5ba93 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 21:37:27 -0400 Subject: [PATCH 0927/2835] format --- doc/internals.mdwn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/internals.mdwn b/doc/internals.mdwn index fe50681870..f69ee6f8b9 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -2,7 +2,7 @@ In the world of git, we're not scared about internal implementation details, and sometimes we like to dive in and tweak things by hand. Here's some documentation to that end. -## .git/annex/objects/*/* +## `.git/annex/objects/*/*` This is where locally available file contents are actually stored. Files added to the annex get a symlink checked into git that points @@ -13,7 +13,7 @@ of the key. This two-level structure is used because it allows the write bit to be removed from the subdirectories as well as from the files. That prevents accidentially deleting or changing the file contents. -## .git-annex/uuid.log +## `.git-annex/uuid.log` Records the UUIDs of known repositories, and associates them with a description of the repository. This allows git-annex to display something @@ -26,7 +26,7 @@ space and then the description through to the end of the line. Example: e605dca6-446a-11e0-8b2a-002170d25c55 laptop 26339d22-446b-11e0-9101-002170d25c55 usb disk -## .git-annex/trust.log +## `.git-annex/trust.log` Records the [[trust]] information for repositories. Does not exist unless [[trust]] values are configured. @@ -40,7 +40,7 @@ Example: e605dca6-446a-11e0-8b2a-002170d25c55 1 26339d22-446b-11e0-9101-002170d25c55 ? -## .git-annex/*.log +## `.git-annex/*.log` The remainder of the log files record [[location_tracking]] information for file contents. The name of the key is the filename, and the content From 1072683a47efe384a85280fa68afa4a8d31cfd7a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 21:38:47 -0400 Subject: [PATCH 0928/2835] link --- doc/internals.mdwn | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/internals.mdwn b/doc/internals.mdwn index f69ee6f8b9..3f680dd8f2 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -8,10 +8,11 @@ This is where locally available file contents are actually stored. Files added to the annex get a symlink checked into git that points to the file content. -Each subdirectory has the name of a key. The file inside also has the name -of the key. This two-level structure is used because it allows the write -bit to be removed from the subdirectories as well as from the files. -That prevents accidentially deleting or changing the file contents. +Each subdirectory has the name of a key in one of the +[[key-value_backends|backends]]. The file inside also has the name of the key. +This two-level structure is used because it allows the write bit to be removed +from the subdirectories as well as from the files. That prevents accidentially +deleting or changing the file contents. ## `.git-annex/uuid.log` From 3db25aaa3bcb243b3ba5c5638af1894e1aae9ed0 Mon Sep 17 00:00:00 2001 From: tyger Date: Wed, 2 Mar 2011 08:15:37 +0000 Subject: [PATCH 0929/2835] Added a comment --- ...mment_3_f483038c006cf7dcccf1014fa771744f._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment new file mode 100644 index 0000000000..90bf23b6cf --- /dev/null +++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="tyger" + ip="80.66.20.180" + subject="comment 3" + date="2011-03-02T08:15:37Z" + content=""" +> Sounds like it might be enough to add a switch to git-annex that overrides where it considers the top of the git repository to be? + +It should sufficient to honor GIT_DIR/GIT_WORK_TREE/GIT_INDEX_FILE environment variables. git filter-branch sets GIT_WORK_TREE to ., but this can be mitigated by starting the filter script with 'GIT_WORK_TREE=$(pwd $GIT_WORK_TREE)'. E.g. GIT_DIR=/home/tyger/repo/.git, GIT_WORK_TREE=/home/tyger/repo/.git-rewrite/t, then git annex should be able to compute the correct relative path or maybe use absolute pathes in symlinks. + +Another problem I observed is that git annex add automatically commits the symlink; this behaviour doesn't work well with filter-tree. git annex commits the wrong path (.git-rewrite/t/LINK instead of LINK). Also filter-tree doesn't expect that the filter script commmits anything; new files in the temporary work tree will be committed by filter-tree on each iteration of the filter script (missing files will be removed). +"""]] From 1c08b8bf8aae73238f3571b643d4ed9d8301cfae Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 2 Mar 2011 10:36:33 +0000 Subject: [PATCH 0930/2835] --- doc/install/OSX.mdwn | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn index e107e48eb9..af078e363d 100644 --- a/doc/install/OSX.mdwn +++ b/doc/install/OSX.mdwn @@ -1,5 +1,5 @@ 
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum
+sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string
@@ -7,6 +7,9 @@ sudo port install pcre
 sudo cabal install pcre-light
 sudo cabal install quickcheck  
 
+# this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
+export PATH=$PATH:/opt/local/libexec/gnubin
+
 git clone  git://git.kitenet.net/git-annex
 
 cd git-annex

From af6a7c4b81527d3798c48c0ef0f3a317882bc651 Mon Sep 17 00:00:00 2001
From: tyger 
Date: Wed, 2 Mar 2011 13:08:46 +0000
Subject: [PATCH 0931/2835] Found temporary solution

---
 ..._existing_git_repository_to_git-annex.mdwn | 56 +++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
index 94c5f1d133..93ff4972de 100644
--- a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
@@ -6,3 +6,59 @@ I tried to rewrite the (cloned) repository with git-filter-branch but failed mis
 * annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them
 
 Any suggestions how to proceed?
+
+EDIT 3/2/2010
+I finally got it working for my purposes. Hardest part was preserving the branches while injecting the new `git annex setup` base commit.
+
+#### Clone repository
+    git clone original migrate
+    cd migrate
+    git checkout mybranch
+    git checkout master
+    git remote rm origin
+
+#### Inject `git annex setup` base commit and repair branches
+    git symbolic-ref HEAD refs/heads/newroot
+    git rm --cached *
+    git clean -f -d
+    git annex init master
+    echo \*.rpm annex.backend=SHA1 >> .gitattributes
+    git commit -m "store rpms in git annex" .gitattributes
+    git cherry-pick $(git rev-list --reverse master | head -1)
+    git rebase --onto newroot newroot master
+    git rebase --onto master mybranch~1 mybranch
+    git branch -d newroot
+
+#### Migrate repository
+    mkdir .temp
+    cp .git-annex/* .temp/
+    MYWORKDIR=$(pwd) git filter-branch --tree-filter '
+        mkdir -p .git-annex;
+        cp ${MYWORKDIR}/.temp/* .git-annex/;
+        for rpm in $(git ls-files | grep "\.rpm$"); do
+            echo;
+            git annex add $rpm;
+            annexdest=$(readlink $rpm);
+            if [ -e .git-annex/$(basename $annexdest).log ]; then
+                echo "FOUND $(basename $annexdest).log";
+            else
+                echo "COPY $(basename $annexdest).log";
+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log .git-annex/;
+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log ${MYWORKDIR}/.temp/;
+            fi;
+            ln -sf ${annexdest#../../} $rpm;
+        done;
+        git reset HEAD .git-rewrite;
+        :
+        ' -- $(git branch | cut -c 3-)
+    rm -rf .temp
+    git reset --hard
+
+
+TODO:
+
+* Find a way to repair branches automatically (detect branch points and run appropriate `git rebase` commands)
+
+I'll be happy to try any suggestions to improve this migration script.
+
+P.S. Is there a way to edit comments?

From 70a6eb6d73b3e302b9b9c02342581fada25dcad9 Mon Sep 17 00:00:00 2001
From: tyger 
Date: Wed, 2 Mar 2011 13:18:10 +0000
Subject: [PATCH 0932/2835] don't forget to rewrite tags

---
 doc/forum/migrate_existing_git_repository_to_git-annex.mdwn | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
index 93ff4972de..f673de765b 100644
--- a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
@@ -32,7 +32,9 @@ I finally got it working for my purposes. Hardest part was preserving the branch
 #### Migrate repository
     mkdir .temp
     cp .git-annex/* .temp/
-    MYWORKDIR=$(pwd) git filter-branch --tree-filter '
+    MYWORKDIR=$(pwd) git filter-branch \
+     --tag-name-filter cat \
+     --tree-filter '
         mkdir -p .git-annex;
         cp ${MYWORKDIR}/.temp/* .git-annex/;
         for rpm in $(git ls-files | grep "\.rpm$"); do

From a3daac8a8b06bbe2f35ca16cc1b27e21cad8a0e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Mar 2011 13:47:45 -0400
Subject: [PATCH 0933/2835] only enable SHA backends that configure finds
 support for

---
 Backend.hs           |  3 ++-
 Backend/SHA.hs       | 32 ++++++++++++++++++++++++--------
 Backend/SHA1.hs      | 14 --------------
 Backend/SHA224.hs    | 14 --------------
 Backend/SHA384.hs    | 14 --------------
 Backend/SHA512.hs    | 14 --------------
 Backend/URL.hs       |  5 ++++-
 Backend/WORM.hs      |  5 ++++-
 BackendList.hs       | 18 +++++-------------
 doc/install.mdwn     |  2 +-
 doc/install/OSX.mdwn |  2 +-
 test.hs              | 23 ++++++++++++++++-------
 12 files changed, 57 insertions(+), 89 deletions(-)
 delete mode 100644 Backend/SHA1.hs
 delete mode 100644 Backend/SHA224.hs
 delete mode 100644 Backend/SHA384.hs
 delete mode 100644 Backend/SHA512.hs

diff --git a/Backend.hs b/Backend.hs
index 74f71f8bea..df23e80a33 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -26,7 +26,8 @@ module Backend (
 	fsckKey,
 	lookupFile,
 	chooseBackends,
-	keyBackend
+	keyBackend,
+	lookupBackendName
 ) where
 
 import Control.Monad.State
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index d779e80553..c074ab48a2 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -5,13 +5,14 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Backend.SHA (genBackend) where
+module Backend.SHA (backends) where
 
 import Control.Monad.State
 import Data.String.Utils
 import System.Cmd.Utils
 import System.IO
 import System.Directory
+import Data.Maybe
 
 import qualified Backend.File
 import BackendTypes
@@ -21,16 +22,31 @@ import Locations
 import Content
 import Types
 import Utility
+import qualified SysConfig
 
 type SHASize = Int
 
--- Constructor for Backends using a given SHASize.
-genBackend :: SHASize -> Backend Annex
-genBackend size = Backend.File.backend 
-	{ name = shaName size
-	, getKey = keyValue size
-	, fsckKey = Backend.File.checkKey $ checkKeyChecksum size
-	}
+backends :: [Backend Annex]
+-- order is slightly significant; want sha1 first ,and more general
+-- sizes earlier
+backends = catMaybes $ map genBackend [1, 256, 512, 224, 384]
+
+genBackend :: SHASize -> Maybe (Backend Annex)
+genBackend size
+	| supported size = Just b 
+	| otherwise = Nothing
+	where
+		b = Backend.File.backend 
+			{ name = shaName size
+			, getKey = keyValue size
+			, fsckKey = Backend.File.checkKey $ checkKeyChecksum size
+			}
+		supported 1 = SysConfig.sha1sum
+		supported 256 = SysConfig.sha256sum
+		supported 224 = SysConfig.sha224sum
+		supported 384 = SysConfig.sha384sum
+		supported 512 = SysConfig.sha512sum
+		supported _ = False
 
 shaName :: SHASize -> String
 shaName size = "SHA" ++ show size
diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs
deleted file mode 100644
index 76d2af69e3..0000000000
--- a/Backend/SHA1.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-{- git-annex "SHA1" backend
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.SHA1 (backend) where
-
-import Types
-import Backend.SHA
-
-backend :: Backend Annex
-backend = genBackend 1
diff --git a/Backend/SHA224.hs b/Backend/SHA224.hs
deleted file mode 100644
index 614f31c4b8..0000000000
--- a/Backend/SHA224.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-{- git-annex "SHA224" backend
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.SHA224 (backend) where
-
-import Types
-import Backend.SHA
-
-backend :: Backend Annex
-backend = genBackend 224
diff --git a/Backend/SHA384.hs b/Backend/SHA384.hs
deleted file mode 100644
index 0cf77ea647..0000000000
--- a/Backend/SHA384.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-{- git-annex "SHA384" backend
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.SHA384 (backend) where
-
-import Types
-import Backend.SHA
-
-backend :: Backend Annex
-backend = genBackend 384
diff --git a/Backend/SHA512.hs b/Backend/SHA512.hs
deleted file mode 100644
index aed8bbcedf..0000000000
--- a/Backend/SHA512.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-{- git-annex "SHA512" backend
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.SHA512 (backend) where
-
-import Types
-import Backend.SHA
-
-backend :: Backend Annex
-backend = genBackend 512
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 864c793010..29dc8fefa7 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Backend.URL (backend) where
+module Backend.URL (backends) where
 
 import Control.Monad.State (liftIO)
 import Data.String.Utils
@@ -15,6 +15,9 @@ import BackendTypes
 import Utility
 import Messages
 
+backends :: [Backend Annex]
+backends = [backend]
+
 backend :: Backend Annex
 backend = Backend {
 	name = "URL",
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 92fe5a2d4c..8a6412eb11 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Backend.WORM (backend) where
+module Backend.WORM (backends) where
 
 import Control.Monad.State
 import System.FilePath
@@ -22,6 +22,9 @@ import Content
 import Messages
 import Types
 
+backends :: [Backend Annex]
+backends = [backend]
+
 backend :: Backend Annex
 backend = Backend.File.backend {
 	name = "WORM",
diff --git a/BackendList.hs b/BackendList.hs
index 4279b69fc9..bc3fd83142 100644
--- a/BackendList.hs
+++ b/BackendList.hs
@@ -9,21 +9,13 @@ module BackendList (allBackends) where
 
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
-import qualified Backend.SHA1
-import qualified Backend.SHA256
-import qualified Backend.SHA512
-import qualified Backend.SHA224
-import qualified Backend.SHA384
+import qualified Backend.SHA
 import qualified Backend.URL
 import Types
 
 allBackends :: [Backend Annex]
-allBackends = 
-	[ Backend.WORM.backend
-	, Backend.SHA1.backend
-	, Backend.SHA256.backend
-	, Backend.SHA512.backend
-	, Backend.SHA224.backend
-	, Backend.SHA384.backend
-	, Backend.URL.backend
+allBackends = concat 
+	[ Backend.WORM.backends
+	, Backend.SHA.backends
+	, Backend.URL.backends
 	]
diff --git a/doc/install.mdwn b/doc/install.mdwn
index f1305777c0..9eb8bbacf7 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -16,7 +16,7 @@ To build and use git-annex, you will need:
   (or uuidgen from util-linux)
 * `xargs`: 
 * `rsync`: 
-* `sha1sum`: 
+* `sha1sum`:  (optional, but recommended)
 * Then just [[download]] git-annex and run: `make; make install`
 
 ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index af078e363d..3c29fc101c 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,7 +7,7 @@ sudo port install pcre
 sudo cabal install pcre-light
 sudo cabal install quickcheck  
 
-# this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
+# optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
 git clone  git://git.kitenet.net/git-annex
diff --git a/test.hs b/test.hs
index 1bae3bd83e..6e35eaee59 100644
--- a/test.hs
+++ b/test.hs
@@ -37,8 +37,8 @@ import qualified UUID
 import qualified Trust
 import qualified Remotes
 import qualified Content
-import qualified Backend.SHA1
-import qualified Backend.WORM
+import qualified BackendList
+import qualified Backend
 import qualified Command.DropUnused
 
 main :: IO ()
@@ -121,7 +121,7 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup]
 test_setkey :: Test
 test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
 	writeFile tmp $ content sha1annexedfile
-	r <- annexeval $ BackendTypes.getKey Backend.SHA1.backend tmp
+	r <- annexeval $ BackendTypes.getKey backendSHA1 tmp
 	let sha1 = BackendTypes.keyName $ fromJust r
 	git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed"
 	git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed"
@@ -405,8 +405,8 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 					@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
-		checkbackend annexedfile Backend.SHA1.backend
-		checkbackend sha1annexedfile Backend.SHA1.backend
+		checkbackend annexedfile backendSHA1
+		checkbackend sha1annexedfile backendSHA1
 
 		-- check that reversing a migration works
 		writeFile ".gitattributes" $ "* annex.backend=WORM"
@@ -416,8 +416,8 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 			@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
-		checkbackend annexedfile Backend.WORM.backend
-		checkbackend sha1annexedfile Backend.WORM.backend
+		checkbackend annexedfile backendWORM
+		checkbackend sha1annexedfile backendWORM
 		
 		where
 			checkbackend file expected = do
@@ -682,3 +682,12 @@ changecontent f = writeFile f $ changedcontent f
 
 changedcontent :: FilePath -> String
 changedcontent f = (content f) ++ " (modified)"
+
+backendSHA1 :: Types.Backend Types.Annex
+backendSHA1 = backend_ "SHA1"
+
+backendWORM :: Types.Backend Types.Annex
+backendWORM = backend_ "WORM"
+
+backend_ :: String -> Types.Backend Types.Annex
+backend_ name = Backend.lookupBackendName BackendList.allBackends name

From 6206b46e60803b9d15c08062780d153ebfe4a9ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Mar 2011 14:30:36 -0400
Subject: [PATCH 0934/2835] fsck: Check for and repair location log damage.

---
 Command/Fsck.hs  | 44 +++++++++++++++++++++++++++++++++++++++++---
 debian/changelog |  1 +
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index b6f330d4c2..f8c957053d 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,11 +7,17 @@
 
 module Command.Fsck where
 
+import Control.Monad.State (liftIO)
+
 import Command
 import qualified Backend
+import qualified Annex
+import UUID
 import Types
 import Messages
 import Utility
+import Content
+import LocationLog
 
 command :: [Command]
 command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek
@@ -20,7 +26,6 @@ command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek
 seek :: [CommandSeek]
 seek = [withAttrFilesInGit "annex.numcopies" start]
 
-{- Checks a file's backend data for problems. -}
 start :: CommandStartAttrFile
 start (file, attr) = isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
@@ -30,7 +35,40 @@ start (file, attr) = isAnnexed file $ \(key, backend) -> do
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
 perform key file backend numcopies = do
-	success <- Backend.fsckKey backend key (Just file) numcopies
-	if success
+	-- the location log is checked first, so that if it has bad data
+	-- that gets corrected
+	locationlogok <- verifyLocationLog key file
+	backendok <- Backend.fsckKey backend key (Just file) numcopies
+	if locationlogok && backendok
 		then return $ Just $ return True
 		else return Nothing
+
+{- Checks that the location log reflects the current status of the key,
+   in this repository only. -}
+verifyLocationLog :: Key -> FilePath -> Annex Bool
+verifyLocationLog key file = do
+	present <- inAnnex key
+	
+	g <- Annex.gitRepo
+	u <- getUUID g
+        uuids <- liftIO $ keyLocations g key
+
+	case (present, u `elem` uuids) of
+		(True, False) -> do
+				fix g u ValuePresent
+				-- There is no data loss, so do not fail.
+				return True
+		(False, True) -> do
+				fix g u ValueMissing
+				warning $
+					"** Based on the location log, " ++ file
+					++ "\n** was expected to be present, " ++
+					"but its content is missing."
+				return False
+		_ -> return True
+	
+	where
+		fix g u s = do
+			showNote "fixing location log"
+			_ <- liftIO $ logChange g key u s
+			return ()
diff --git a/debian/changelog b/debian/changelog
index 8d7dd46e5c..29f2fd134c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,7 @@ git-annex (0.22) UNRELEASED; urgency=low
     to a utility it will be escaped to avoid it being interpreted as an
     option.
   * New backends: SHA512 SHA384 SHA256 SHA224
+  * fsck: Check for and repair location log damage.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 

From f9da8625003b29ec47baaed272bca1c772186b10 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Thu, 3 Mar 2011 10:31:47 +0000
Subject: [PATCH 0935/2835]

---
 ...nused_seems_to_check_for_current_path.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
new file mode 100644
index 0000000000..14caee022c
--- /dev/null
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -0,0 +1,21 @@
+When I run `git annex unused` from my repository's root it shows everything ok:
+
+    ~/annex$ git annex unused
+    unused  (checking for unused data...) ok
+
+But... When I run it from a subdirectory, it shows a lot:
+
+    ~/annex/Software$ git annex unused
+    unused  (checking for unused data...) 
+      Some annexed data is no longer pointed to by any files in the repository:
+        NUMBER  KEY
+        1       SHA1:########################################
+    ...
+        921     SHA1:########################################
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      (To remove unwanted data: git-annex dropunused NUMBER)
+      ok
+
+Is this a bug or by design? By removing these "unused" files with `dropunused` I've just lost the only copy of 160 files.
+
+I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b

From 1bf3be5d6cf24e6a04066e68cfe8d92dd58859f6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 13:25:17 -0400
Subject: [PATCH 0936/2835] doesn't seem possible..

---
 ...git_annex_unused_seems_to_check_for_current_path.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
index 14caee022c..76a0fc1ec7 100644
--- a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -19,3 +19,12 @@ But... When I run it from a subdirectory, it shows a lot:
 Is this a bug or by design? By removing these "unused" files with `dropunused` I've just lost the only copy of 160 files.
 
 I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b
+
+> I'm very sorry you lost data.
+> 
+> But, git annex unused absolutely does not let the current directory
+> influence what it does. It always scans the entire repo from the top.
+> And I've tested it just now to make sure that in a subdirectory
+> it does the same thing as at the top. The only way I could explain
+> what you show above is if "Software" were a separate git repository
+> than "~/annex". --[[Joey]] 

From f93b8a66971b9a8cb036258e14da6614264c800d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 13:26:53 -0400
Subject: [PATCH 0937/2835] amend

---
 .../git_annex_unused_seems_to_check_for_current_path.mdwn     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
index 76a0fc1ec7..ff318d616e 100644
--- a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -27,4 +27,6 @@ I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b
 > And I've tested it just now to make sure that in a subdirectory
 > it does the same thing as at the top. The only way I could explain
 > what you show above is if "Software" were a separate git repository
-> than "~/annex". --[[Joey]] 
+> than "~/annex". Or if the symlinks to the content are somehow broken
+> when looked at from within Software, but unbroken when looked at from the
+> parent directory. I can't think how that would happen. --[[Joey]] 

From 7ea9f52c2859a72f8e46522338c1c8a112549d84 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 13:37:46 -0400
Subject: [PATCH 0938/2835] cannot be broken symlinks after all.. one other
 idea

---
 ...ex_unused_seems_to_check_for_current_path.mdwn | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
index ff318d616e..df0fb50cc1 100644
--- a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -25,8 +25,13 @@ I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b
 > But, git annex unused absolutely does not let the current directory
 > influence what it does. It always scans the entire repo from the top.
 > And I've tested it just now to make sure that in a subdirectory
-> it does the same thing as at the top. The only way I could explain
-> what you show above is if "Software" were a separate git repository
-> than "~/annex". Or if the symlinks to the content are somehow broken
-> when looked at from within Software, but unbroken when looked at from the
-> parent directory. I can't think how that would happen. --[[Joey]] 
+> it does the same thing as at the top. 
+> 
+> There are only two ways this could happen that I can think of:
+> 
+> 1. If "Software" were a separate git repository than "~/annex".
+> 2. If gitignores or something made `git ls-files`
+>    not list the files when ran in the subdir. This seems *possible*,
+>    but I don't know how to construct such an ignore.
+> 
+> --[[Joey]] 

From d651d4985b9fca2fbeeab2f1b700e6423994c400 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 14:51:57 -0400
Subject: [PATCH 0939/2835] some initial support for local bare repos

This relies on git-annex's behavior of reading the config of local repos.
That allows repoIsLocalBare to examine the git config for core.bare.

Hopefully, gitAnnexLocation, gitAnnexDir, and gitAnnexObjectDir
are only used on local repos. But, I have not audited fully, since
they're probably not (see for example copyToRemote). And so,
the functions fall back to their old non-bare-aware behavior for
non-local repos.
---
 Content.hs   |  4 ++--
 GitRepo.hs   | 18 ++++++++++++++----
 Locations.hs | 33 +++++++++++++++++++--------------
 3 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/Content.hs b/Content.hs
index cb954f4a0b..658d9a8611 100644
--- a/Content.hs
+++ b/Content.hs
@@ -51,8 +51,8 @@ calcGitLink file key = do
 	let absfile = case absNormPath cwd file of
 		Just f -> f
 		Nothing -> error $ "unable to normalize " ++ filePathToString file
-	return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++
-		annexLocation key
+	return $ relPathDirToDir (parentDir absfile) 
+			(Git.workTree g)  ".git"  annexLocation key
 
 {- Updates the LocationLog when a key's presence changes. -}
 logStatus :: Key -> LogStatus -> Annex ()
diff --git a/GitRepo.hs b/GitRepo.hs
index 04a0c2d540..ba01251218 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -16,6 +16,8 @@ module GitRepo (
 	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
+	repoIsLocalBare,
+	repoIsLocalFull,
 	repoDescribe,
 	repoLocation,
 	workTree,
@@ -161,6 +163,14 @@ repoIsSsh Repo { location = Url url }
 	| otherwise = False
 repoIsSsh _ = False
 
+repoIsLocalBare :: Repo -> Bool
+repoIsLocalBare r@(Repo { location = Dir _ }) = configBare r
+repoIsLocalBare _ = False
+
+repoIsLocalFull :: Repo -> Bool
+repoIsLocalFull r@(Repo { location = Dir _ }) = not $ configBare r
+repoIsLocalFull _ = False
+
 assertLocal :: Repo -> a -> a
 assertLocal repo action = 
 	if not $ repoIsUrl repo
@@ -174,8 +184,8 @@ assertUrl repo action =
 		else error $ "acting on local git repo " ++  repoDescribe repo ++ 
 				" not supported"
 
-bare :: Repo -> Bool
-bare repo = case Map.lookup "core.bare" $ config repo of
+configBare :: Repo -> Bool
+configBare repo = case Map.lookup "core.bare" $ config repo of
 	Just v -> configTrue v
 	Nothing -> error $ "it is not known if git repo " ++
 			repoDescribe repo ++
@@ -184,13 +194,13 @@ bare repo = case Map.lookup "core.bare" $ config repo of
 {- Path to a repository's gitattributes file. -}
 attributes :: Repo -> String
 attributes repo
-	| bare repo = workTree repo ++ "/info/.gitattributes"
+	| configBare repo = workTree repo ++ "/info/.gitattributes"
 	| otherwise = workTree repo ++ "/.gitattributes"
 
 {- Path to a repository's .git directory, relative to its workTree. -}
 gitDir :: Repo -> String
 gitDir repo
-	| bare repo = ""
+	| configBare repo = ""
 	| otherwise = ".git"
 
 {- Path to a repository's --work-tree, that is, its top.
diff --git a/Locations.hs b/Locations.hs
index d30ceb1367..908d5b74ed 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -50,35 +50,40 @@ stateDir = addTrailingPathSeparator $ ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
 gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
 
-{- Annexed content is stored in .git/annex/objects; .git/annex is used
- - for other temporary storage also. -}
+{- The directory git annex uses for local state, relative to the .git
+ - directory -}
 annexDir :: FilePath
-annexDir = addTrailingPathSeparator $ ".git/annex"
+annexDir = addTrailingPathSeparator $ "annex"
+
+{- The directory git annex uses for locally available object content,
+ - relative to the .git directory -}
 objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
-{- Annexed file's location relative to git's working tree. 
- -
- - Note: Assumes repo is NOT bare.-}
+{- Annexed file's location relative to the .git directory. -}
 annexLocation :: Key -> FilePath
-annexLocation key = ".git/annex/objects"  f  f
+annexLocation key = objectDir  f  f
 	where
 		f = keyFile key
 
 {- Annexed file's absolute location in a repository. -}
 gitAnnexLocation :: Git.Repo -> Key -> FilePath
-gitAnnexLocation r key = Git.workTree r  annexLocation key
+gitAnnexLocation r key
+	| Git.repoIsLocalBare r = Git.workTree r  annexLocation key
+	| otherwise = Git.workTree r  ".git"  annexLocation key
 
-{- The annex directory of a repository.
- -
- - Note: Assumes repo is NOT bare. -}
+{- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
-gitAnnexDir r = addTrailingPathSeparator $ Git.workTree r  annexDir
+gitAnnexDir r
+	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  annexDir
+	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  annexDir
 
 {- The part of the annex directory where file contents are stored.
  -}
 gitAnnexObjectDir :: Git.Repo -> FilePath
-gitAnnexObjectDir r = addTrailingPathSeparator $ Git.workTree r  objectDir
+gitAnnexObjectDir r
+	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  objectDir
+	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  objectDir
 
 {- .git-annex/tmp/ is used for temp files -}
 gitAnnexTmpDir :: Git.Repo -> FilePath
@@ -98,7 +103,7 @@ gitAnnexUnusedLog r = gitAnnexDir r  "unused"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = ("/" ++ objectDir) `isInfixOf` s
+isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
 {- Converts a key into a filename fragment.
  -

From d25a8540854fed30567868799322bbdf4e947c2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 14:59:45 -0400
Subject: [PATCH 0940/2835] remove redundant imports

---
 test.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test.hs b/test.hs
index 6e35eaee59..936e4da175 100644
--- a/test.hs
+++ b/test.hs
@@ -37,8 +37,6 @@ import qualified UUID
 import qualified Trust
 import qualified Remotes
 import qualified Content
-import qualified BackendList
-import qualified Backend
 import qualified Command.DropUnused
 
 main :: IO ()

From 9f20aee2192bcc5f2c0ae1f59db88f6eadeb7335 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 15:22:53 -0400
Subject: [PATCH 0941/2835] avoid logging to location log when in a bare repo

This assumes that changes to content in bare repos are made from some
non-bare repo, and that the location log is updated on that side.

That's true for move --from and move --to.

It's *not* true for dropkey and setkey and recvkey. But those are plumbing
level commands, so I guess it's ok to assume that someone running those
in a bare repo knows what they're doing. And git-annex-shell is used to
run those, and if the bare repo is non-local, it needs to be able to use
them even though they cannot update the location log. So this seems
unavoidable.
---
 Command/Move.hs |  5 +++++
 Content.hs      | 16 +++++++++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 8c19539fba..3774ccbe9d 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -51,6 +51,11 @@ showAction :: Bool -> FilePath -> Annex ()
 showAction True file = showStart "move" file
 showAction False file = showStart "copy" file
 
+{- Used to log a change in a remote's having a key. The change is logged
+ - in the local repo, not on the remote. The process of transferring the
+ - key to the remote, or removing the key from it *may* log the change
+ - on the remote, but this cannot be relied on. For example, it's not done
+ - for bare repos. -}
 remoteHasKey :: Git.Repo -> Key -> Bool -> Annex ()
 remoteHasKey remote key present	= do
 	g <- Annex.gitRepo
diff --git a/Content.hs b/Content.hs
index 658d9a8611..6b8316c082 100644
--- a/Content.hs
+++ b/Content.hs
@@ -23,7 +23,7 @@ import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import System.Path
-import Control.Monad (when, filterM)
+import Control.Monad (when, unless, filterM)
 import System.Posix.Files
 import System.FilePath
 
@@ -54,13 +54,19 @@ calcGitLink file key = do
 	return $ relPathDirToDir (parentDir absfile) 
 			(Git.workTree g)  ".git"  annexLocation key
 
-{- Updates the LocationLog when a key's presence changes. -}
+{- Updates the LocationLog when a key's presence changes.
+ -
+ - Note that the LocationLog is not updated in bare repositories.
+ - Operations that change a bare repository should be done from
+ - a non-bare repository, and the LocationLog in that repository be
+ - updated instead. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
 	g <- Annex.gitRepo
-	u <- getUUID g
-	logfile <- liftIO $ logChange g key u status
-	Annex.queue "add" [Param "--"] logfile
+	unless (Git.repoIsLocalBare g) $ do
+		u <- getUUID g
+		logfile <- liftIO $ logChange g key u status
+		Annex.queue "add" [Param "--"] logfile
 
 {- Runs an action, passing it a temporary filename to download,
  - and if the action succeeds, moves the temp file into 

From d28d659a241fb1780264416e4317e65691201f50 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 15:44:01 -0400
Subject: [PATCH 0942/2835] initial documantation/todo list for bare
 repositories

---
 doc/bare_repositories.mdwn | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/bare_repositories.mdwn

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
new file mode 100644
index 0000000000..eb48dfa10b
--- /dev/null
+++ b/doc/bare_repositories.mdwn
@@ -0,0 +1,29 @@
+Due to popular demand, git-annex can now be used with bare repositories.
+
+**This is still an experimental feature!** 
+
+Known to work ok, so far for local bare repositories only:
+
+* `git annex move --to` and `--from`, when pointed at a bare repository.
+* `git annex copy` ditto.
+* `git annex drop` can check that a bare repository has a copy of data
+  that is being dropped.
+* `git annex get` can transfer data from a bare repository.
+
+There are a few caveats to keep in mind:
+
+* Using non-local bare repositories is not tested and probably broken.
+* `git annex init` can be run in a bare repository, but it cannot
+  store the name you gave the repository in .git-annex/uuid.log (because
+  the bare repository has no such file to commit to).
+* `git annex trust` cannot be used in a bare repository, and currently
+  does something pointless. Same for `untrust` and `semitrust`.
+* `git annex fromkey` does something pointless in a bare repository.
+* `git annex fsck` cannot detect any problems in a bare repository.
+* `git annex unused` will think everything stored in a bare repository
+  is unused.
+* `git annex setkey` is a plumbing-level command, and using it manually
+  to add content to a bare repository is not recommended, since there
+  will be no record accessible by other repositories that the content
+  is stored there.
+* `git-annex-shell inannex` fails in a bare repository

From 486f882471829f7438b49f8f3d9791e4b8b926d6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 15:49:53 -0400
Subject: [PATCH 0943/2835] wow, non-local bare repos just worked

---
 doc/bare_repositories.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index eb48dfa10b..dee2ba69e7 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -2,7 +2,7 @@ Due to popular demand, git-annex can now be used with bare repositories.
 
 **This is still an experimental feature!** 
 
-Known to work ok, so far for local bare repositories only:
+Known to work ok:
 
 * `git annex move --to` and `--from`, when pointed at a bare repository.
 * `git annex copy` ditto.
@@ -26,4 +26,3 @@ There are a few caveats to keep in mind:
   to add content to a bare repository is not recommended, since there
   will be no record accessible by other repositories that the content
   is stored there.
-* `git-annex-shell inannex` fails in a bare repository

From a9d0538da559c29509c50b664e31ea3b23b7bc14 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 15:59:16 -0400
Subject: [PATCH 0944/2835] updates for bare repo support

---
 debian/changelog                         | 2 ++
 doc/bare_repositories.mdwn               | 7 +++++++
 doc/bugs/bare_git_repos.mdwn             | 3 +++
 doc/bugs/fat_support.mdwn                | 4 ++--
 doc/bugs/weird_local_clone_confuses.mdwn | 5 +++++
 doc/git-annex.mdwn                       | 8 ++++----
 6 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 29f2fd134c..3d96934a55 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,8 @@ git-annex (0.22) UNRELEASED; urgency=low
     option.
   * New backends: SHA512 SHA384 SHA256 SHA224
   * fsck: Check for and repair location log damage.
+  * Git annexes can now be attached to bare git repositories. Due to popular
+    demand.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index dee2ba69e7..f08cf605d1 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -1,5 +1,12 @@
 Due to popular demand, git-annex can now be used with bare repositories.
 
+So, for example, you can stash a file away in your
+repos's origin: `git annex move mybigfile --to origin`
+
+Of course, for that to work, the bare repository has to be on a system with
+[[git-annex-shell]] installed. If "origin" is on gitweb, you still can't
+use git-annex to there.
+
 **This is still an experimental feature!** 
 
 Known to work ok:
diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn
index 67917db8a6..f219840e71 100644
--- a/doc/bugs/bare_git_repos.mdwn
+++ b/doc/bugs/bare_git_repos.mdwn
@@ -24,3 +24,6 @@ A possible other approach to the state recording repo is to not
 record state changes on the remote in that case. Git-annex already
 records remote state changes locally whenever it modifies the state of a
 remote. --[[Joey]]
+
+> And... [[done]]! See [[/bare_repositories]] for current status
+> and gotchas. --[[Joey]] 
diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn
index 2a998cf1ad..2a7187a7f6 100644
--- a/doc/bugs/fat_support.mdwn
+++ b/doc/bugs/fat_support.mdwn
@@ -3,8 +3,8 @@ git-annex from being used on USB keys, that would typically
 be VFAT formatted:
 
 - Use of symlinks, which VFAT does not support. Very hard to fix.
-  One possibility is to add [[bare_git_repos]] support, then
-  a git repo on a thumb drive could be used to transfer data.
+  Instead, just use [[/bare_repositories]] on the key,
+  they're supported now.
 - Use of ":" in filenames of object files, also not supported.
   Could easily be fixed by reorganizing the object directory.
 
diff --git a/doc/bugs/weird_local_clone_confuses.mdwn b/doc/bugs/weird_local_clone_confuses.mdwn
index fc652ae5eb..d209dd80de 100644
--- a/doc/bugs/weird_local_clone_confuses.mdwn
+++ b/doc/bugs/weird_local_clone_confuses.mdwn
@@ -7,3 +7,8 @@ new cannot see origin.
 the .git/config has "url=/.../orig/.git". Apparently git is ok with that
 weird construction; probably it treats it as a bare git repo. But git-annex
 just sees a directory w/o a .git subdir, and gives up.
+
+---
+
+Just tested, and the new support for bare repositories didn't solve this.
+--[[Joey]] 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 2f25f9bf81..5a0c710593 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -327,10 +327,10 @@ Here are all the supported configuration settings.
   from using this remote by default. (You can still request it be used
   by the --from and --to options.)
 
-  This is, for example, useful if the remote is a bare repository,
-  which git-annex does not currently support. Or, it could be used
-  if the network connection between two repositories is too slow
-  to be used normally.
+  This is, for example, useful if the remote is located somewhere
+  without [[git-annex-shell]]. (For example, if it's on GitHub).
+  Or, it could be used if the network connection between two
+  repositories is too slow to be used normally.
 
 * `remote..annex-uuid`
 

From b88637fff10d4d845404882e4ec95cfc071dcac0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 16:22:53 -0400
Subject: [PATCH 0945/2835] prevent trust commands from trying to do things in
 a bare repo

Since they need to stage changes, they would actually, if allowed to run,
succeed, but wipe out existing trust.log content.
---
 Command.hs                 | 9 ++++++++-
 Command/Semitrust.hs       | 2 +-
 Command/Trust.hs           | 2 +-
 Command/Untrust.hs         | 2 +-
 doc/bare_repositories.mdwn | 3 ---
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/Command.hs b/Command.hs
index 86da454195..d54a7052e9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -10,7 +10,7 @@ module Command where
 import Control.Monad.State (liftIO)
 import System.Directory
 import System.Posix.Files
-import Control.Monad (filterM, liftM)
+import Control.Monad (filterM, liftM, when)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
@@ -104,6 +104,13 @@ isAnnexed file a = do
 		Just v -> a v
 		Nothing -> return Nothing
 
+notBareRepo :: Annex a -> Annex a
+notBareRepo a = do
+	g <- Annex.gitRepo
+	when (Git.repoIsLocalBare g) $ do
+		error "You cannot run this subcommand in a bare repository."
+	a
+
 {- These functions find appropriate files or other things based on a
    user's parameters, and run a specified action on them. -}
 withFilesInGit :: CommandSeekStrings
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index a91d25359c..13c6847e17 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -22,7 +22,7 @@ seek :: [CommandSeek]
 seek = [withString start]
 
 start :: CommandStartString
-start name = do
+start name = notBareRepo $ do
 	showStart "semitrust" name
 	Remotes.readConfigs
 	r <- Remotes.byName name
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 3fbff68b89..ea661da2a6 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -22,7 +22,7 @@ seek :: [CommandSeek]
 seek = [withString start]
 
 start :: CommandStartString
-start name = do
+start name = notBareRepo $ do
 	showStart "trust" name
 	Remotes.readConfigs
 	r <- Remotes.byName name
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 69d0ab391a..fdf9a83dec 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -22,7 +22,7 @@ seek :: [CommandSeek]
 seek = [withString start]
 
 start :: CommandStartString
-start name = do
+start name = notBareRepo $ do
 	showStart "untrust" name
 	Remotes.readConfigs
 	r <- Remotes.byName name
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index f08cf605d1..e3c82d0194 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -19,12 +19,9 @@ Known to work ok:
 
 There are a few caveats to keep in mind:
 
-* Using non-local bare repositories is not tested and probably broken.
 * `git annex init` can be run in a bare repository, but it cannot
   store the name you gave the repository in .git-annex/uuid.log (because
   the bare repository has no such file to commit to).
-* `git annex trust` cannot be used in a bare repository, and currently
-  does something pointless. Same for `untrust` and `semitrust`.
 * `git annex fromkey` does something pointless in a bare repository.
 * `git annex fsck` cannot detect any problems in a bare repository.
 * `git annex unused` will think everything stored in a bare repository

From b5b78f26ecabdb74c05f8200de5f9d054da5cbae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 16:40:55 -0400
Subject: [PATCH 0946/2835] fix up commands that are trouble on bare repos

Most will just abort. init does a basic init and gives a command to
run elsewhere to finish it.
---
 Command/DropUnused.hs      |  2 +-
 Command/FromKey.hs         |  3 +--
 Command/Fsck.hs            |  2 +-
 Command/Init.hs            | 16 ++++++++++++----
 Command/Unused.hs          |  2 +-
 debian/changelog           |  3 ++-
 doc/bare_repositories.mdwn | 25 ++++++++++++-------------
 doc/index.mdwn             |  1 +
 8 files changed, 31 insertions(+), 23 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 63216ce4f1..594564cb72 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -29,7 +29,7 @@ seek = [withStrings start]
 
 {- Drops unused content by number. -} 
 start :: CommandStartString
-start s = do
+start s = notBareRepo $ do
 	m <- readUnusedLog
 	case M.lookup s m of
 		Nothing -> return Nothing
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index d16eff8466..717d528bc9 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -27,9 +27,8 @@ command = [Command "fromkey" paramPath seek
 seek :: [CommandSeek]
 seek = [withFilesMissing start]
 
-{- Adds a file pointing at a manually-specified key -}
 start :: CommandStartString
-start file = do
+start file = notBareRepo $ do
 	key <- cmdlineKey
 	inbackend <- Backend.hasKey key
 	unless inbackend $ error $
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index f8c957053d..76d0e38b4b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -27,7 +27,7 @@ seek :: [CommandSeek]
 seek = [withAttrFilesInGit "annex.numcopies" start]
 
 start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, backend) -> do
+start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
 	return $ Just $ perform key file backend numcopies
 	where
diff --git a/Command/Init.hs b/Command/Init.hs
index 1074d100ea..509c9e51c0 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -41,11 +41,19 @@ perform :: String -> CommandPerform
 perform description = do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	describeUUID u description
 	setVersion
-	liftIO $ gitAttributesWrite g
-	gitPreCommitHookWrite g
-	return $ Just cleanup
+	if Git.repoIsLocalBare g
+		then do
+			showLongNote $
+				"This is a bare repository, so its description cannot be committed.\n" ++
+				"To record the description, run this command in a clone of this repository:\n" ++
+				"   git annex describe " ++ (show u) ++ " '" ++ description ++ "'\n\n"
+			return $ Just $ return True
+		else do
+			describeUUID u description
+			liftIO $ gitAttributesWrite g
+			gitPreCommitHookWrite g
+			return $ Just cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 67a2272371..9f3881d595 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -32,7 +32,7 @@ seek = [withNothing start]
 
 {- Finds unused content in the annex. -} 
 start :: CommandStartNothing
-start = do
+start = notBareRepo $ do
 	showStart "unused" ""
 	return $ Just perform
 
diff --git a/debian/changelog b/debian/changelog
index 3d96934a55..4405ee2a4a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -11,7 +11,8 @@ git-annex (0.22) UNRELEASED; urgency=low
   * New backends: SHA512 SHA384 SHA256 SHA224
   * fsck: Check for and repair location log damage.
   * Git annexes can now be attached to bare git repositories. Due to popular
-    demand.
+    demand. Both the local and remote host must have this version of git-annex
+    installed for it to work.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index e3c82d0194..b7ae0b316f 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -1,13 +1,13 @@
+**This is still an experimental feature!** Use with caution.
+
 Due to popular demand, git-annex can now be used with bare repositories.
 
 So, for example, you can stash a file away in your
 repos's origin: `git annex move mybigfile --to origin`
 
 Of course, for that to work, the bare repository has to be on a system with
-[[git-annex-shell]] installed. If "origin" is on gitweb, you still can't
-use git-annex to there.
-
-**This is still an experimental feature!** 
+[[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't
+use git-annex to store stuff there.
 
 Known to work ok:
 
@@ -17,16 +17,15 @@ Known to work ok:
   that is being dropped.
 * `git annex get` can transfer data from a bare repository.
 
-There are a few caveats to keep in mind:
+There are a few caveats to keep in mind when using bare repositories:
 
 * `git annex init` can be run in a bare repository, but it cannot
   store the name you gave the repository in .git-annex/uuid.log (because
-  the bare repository has no such file to commit to).
-* `git annex fromkey` does something pointless in a bare repository.
-* `git annex fsck` cannot detect any problems in a bare repository.
-* `git annex unused` will think everything stored in a bare repository
-  is unused.
-* `git annex setkey` is a plumbing-level command, and using it manually
+  the bare repository has no such file to commit to). Instead, it will
+  tell you a command to run in some non-bare clone of the repository.
+* Some subcommands, like `fsck`, `trust`, `unused` and `fromkey`, 
+  cannot be run in a bare repository. Those subcommands will
+  refuse to do anything.
+* `git annex setkey` is a plumbing-level command; using it manually
   to add content to a bare repository is not recommended, since there
-  will be no record accessible by other repositories that the content
-  is stored there.
+  will be no record that the content is stored there.
diff --git a/doc/index.mdwn b/doc/index.mdwn
index a6d072e093..00a3315cbb 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -54,6 +54,7 @@ files with git.
 * git-annex prevents accidental data loss by [[tracking copies|copies]]
   of your files
 * [[internals]]
+* [[bare_repositories]]
 * [[what git annex is not|not]]
 * git-annex is Free Software, licensed under the [[GPL]].
 

From 1de12a291891463c6d532a10c74cbda1872c8b9b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 16:58:52 -0400
Subject: [PATCH 0947/2835] document describe command

---
 debian/changelog                              |  5 ++-
 doc/git-annex.mdwn                            | 40 +++++++++++--------
 ...what_to_do_when_you_lose_a_repository.mdwn | 18 +++++++++
 3 files changed, 46 insertions(+), 17 deletions(-)
 create mode 100644 doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn

diff --git a/debian/changelog b/debian/changelog
index 4405ee2a4a..ca1c51c4bf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,7 +12,10 @@ git-annex (0.22) UNRELEASED; urgency=low
   * fsck: Check for and repair location log damage.
   * Git annexes can now be attached to bare git repositories. Due to popular
     demand. Both the local and remote host must have this version of git-annex
-    installed for it to work.
+    installed for it to work. This is still a semi-experimental feature;
+    use caution!
+  * describe: New subcommand that can set or change the description of
+    a repository.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 5a0c710593..76f1a577b9 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -83,19 +83,6 @@ Many git-annex commands will stage changes for later `git commit` by you.
   git-annex may refuse to drop content if the backend does not think
   it is safe to do so, typically because of the setting of annex.numcopies.
 
-* unlock [path ...]
-
-  Normally, the content of annexed files is protected from being changed.
-  Unlocking a annexed file allows it to be modified. This replaces the
-  symlink for each specified file with a copy of the file's content.
-  You can then modify it and `git annex add` (or `git commit`) to inject
-  it back into the annex.
-
-* edit [path ...]
-
-  This is an alias for the unlock command. May be easier to remember,
-  if you think of this as allowing you to edit an annexed file.
-
 * move [path ...]
 
   When used with the --to option, moves the content of annexed files from
@@ -112,16 +99,37 @@ Many git-annex commands will stage changes for later `git commit` by you.
   When used with the --from option, copies the content of annexed files
   from the specified repository to the current one.
 
-* init description
+* unlock [path ...]
 
-  Initializes git-annex with a description of the git repository,
-  and sets up `.gitattributes` and the pre-commit hook.
+  Normally, the content of annexed files is protected from being changed.
+  Unlocking a annexed file allows it to be modified. This replaces the
+  symlink for each specified file with a copy of the file's content.
+  You can then modify it and `git annex add` (or `git commit`) to inject
+  it back into the annex.
+
+* edit [path ...]
+
+  This is an alias for the unlock command. May be easier to remember,
+  if you think of this as allowing you to edit an annexed file.
 
 * lock [path ...]
 
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
+* init description
+
+  Initializes git-annex with a description of the git repository,
+  and sets up `.gitattributes` and the pre-commit hook.
+
+* describe repository description
+
+  Changes the description of a git repository.
+
+  The repository to describe can be specified by git remote name or
+  by uuid. To change the description of the current repository, use
+  "."
+
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
diff --git a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
new file mode 100644
index 0000000000..c914c1bb3d
--- /dev/null
+++ b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
@@ -0,0 +1,18 @@
+So you lost a thumb drive containing a git-annex repository. Or a hard
+drive died or some other misfortune has befallen your data.
+
+Unless you configured backups, git-annex can't get your data back. But it
+can help you deal with the loss.
+
+First, go somewhere that knows about the lost repository, and mark it as
+untrusted.
+
+# git annex untrust usbdrive
+
+To remind yourself later what happened, you can change its description, too:
+
+# git annex describe usbdrive "USB drive lost in Timbuktu. Probably gone forever."
+
+This retains the [[location_tracking]] information for the repository.
+Maybe you'll find the drive later. Maybe that's impossible. Either way,
+this lets git-annex tell you why a file is no longer accessible.

From 657395b628b46b378eda77d5dc6edcdde7183b38 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:21:00 -0400
Subject: [PATCH 0948/2835] add describe subcommand

---
 Command.hs          |  2 ++
 Command/Describe.hs | 41 +++++++++++++++++++++++++++++++++++++++++
 Command/Init.hs     |  2 +-
 GitAnnex.hs         |  2 ++
 Remotes.hs          | 12 +++++++++---
 5 files changed, 55 insertions(+), 4 deletions(-)
 create mode 100644 Command/Describe.hs

diff --git a/Command.hs b/Command.hs
index d54a7052e9..09adc09491 100644
--- a/Command.hs
+++ b/Command.hs
@@ -209,6 +209,8 @@ paramRepeating :: String -> String
 paramRepeating s = s ++ " ..."
 paramOptional :: String -> String
 paramOptional s = "[" ++ s ++ "]"
+paramPair :: String -> String -> String
+paramPair a b = a ++ " " ++ b
 paramPath :: String
 paramPath = "PATH"
 paramKey :: String
diff --git a/Command/Describe.hs b/Command/Describe.hs
new file mode 100644
index 0000000000..643ca04718
--- /dev/null
+++ b/Command/Describe.hs
@@ -0,0 +1,41 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Describe where
+
+
+import Command
+import qualified GitRepo as Git
+import qualified Remotes
+import UUID
+import Messages
+import qualified Command.Init
+
+command :: [Command]
+command = [Command "describe" (paramPair paramRemote paramDesc) seek
+	"change description of a repository"]
+
+seek :: [CommandSeek]
+seek = [withString start]
+
+start :: CommandStartString
+start params = notBareRepo $ do
+	let (name, description) =
+		case (words params) of
+			(n:d) -> (n,unwords d)
+			_ -> error "Specify a repository and a description."
+	
+	showStart "describe" name
+	Remotes.readConfigs
+	r <- Remotes.byName name
+	return $ Just $ perform r description
+
+perform :: Git.Repo -> String -> CommandPerform
+perform repo description = do
+	u <- getUUID repo
+	describeUUID u description
+	return $ Just $ Command.Init.cleanup
diff --git a/Command/Init.hs b/Command/Init.hs
index 509c9e51c0..2c5fdc2fc9 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -62,7 +62,7 @@ cleanup = do
 	liftIO $ Git.run g "add" [File logfile]
 	liftIO $ Git.run g "commit" 
 		[ Params "-q -m"
-		, Param "git annex init"
+		, Param "git annex repository description"
 		, File logfile
 		]
 	return True
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 3be222874f..b26714a598 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -26,6 +26,7 @@ import qualified Command.DropKey
 import qualified Command.SetKey
 import qualified Command.Fix
 import qualified Command.Init
+import qualified Command.Describe
 import qualified Command.Fsck
 import qualified Command.Unused
 import qualified Command.DropUnused
@@ -50,6 +51,7 @@ cmds = concat
 	, Command.Unlock.command
 	, Command.Lock.command
 	, Command.Init.command
+	, Command.Describe.command
 	, Command.Unannex.command
 	, Command.Uninit.command
 	, Command.PreCommit.command
diff --git a/Remotes.hs b/Remotes.hs
index 4dcc4c9adf..a7d6be67df 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -211,17 +211,23 @@ repoNotIgnored r = do
 same :: Git.Repo -> Git.Repo -> Bool
 same a b = Git.repoRemoteName a == Git.repoRemoteName b
 
-{- Looks up a remote by name. -}
+{- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex Git.Repo
 byName "." = Annex.gitRepo -- special case to refer to current repository
 byName name = do
 	when (null name) $ error "no remote specified"
 	g <- Annex.gitRepo
-	let match = filter (\r -> Just name == Git.repoRemoteName r) $
-		Git.remotes g
+	match <- filterM matching $ Git.remotes g
 	when (null match) $ error $
 		"there is no git remote named \"" ++ name ++ "\""
 	return $ head match
+	where
+		matching r = do
+			if Just name == Git.repoRemoteName r
+				then return True
+				else do
+					u <- getUUID r
+					return $ (name == u)
 
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool

From 7b2762fb9226ca76dce6510dd821bfe432c1cfa2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:22:17 -0400
Subject: [PATCH 0949/2835] better quoting of description via show

---
 Command/Init.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 2c5fdc2fc9..6618351699 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -47,7 +47,7 @@ perform description = do
 			showLongNote $
 				"This is a bare repository, so its description cannot be committed.\n" ++
 				"To record the description, run this command in a clone of this repository:\n" ++
-				"   git annex describe " ++ (show u) ++ " '" ++ description ++ "'\n\n"
+				"   git annex describe " ++ show u ++ " " ++ show description ++ "\n\n"
 			return $ Just $ return True
 		else do
 			describeUUID u description

From ea4ea466f550b71ab389d6651c90bf521e30a1c1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:23:44 -0400
Subject: [PATCH 0950/2835] update

---
 doc/walkthrough.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index f400191c61..1e8ad7e988 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -22,4 +22,5 @@ A walkthrough of the basic features of git-annex.
 	fsck:_when_things_go_wrong
 	backups
 	untrusted_repositories
+	what_to_do_when_you_lose_a_repository
 """]]

From eeadc2e3e041b77e409bfe258ebe6a778f70ccb5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:24:46 -0400
Subject: [PATCH 0951/2835] formatting

---
 doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
index c914c1bb3d..1159b22171 100644
--- a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
+++ b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
@@ -7,11 +7,11 @@ can help you deal with the loss.
 First, go somewhere that knows about the lost repository, and mark it as
 untrusted.
 
-# git annex untrust usbdrive
+	git annex untrust usbdrive
 
 To remind yourself later what happened, you can change its description, too:
 
-# git annex describe usbdrive "USB drive lost in Timbuktu. Probably gone forever."
+	git annex describe usbdrive "USB drive lost in Timbuktu. Probably gone forever."
 
 This retains the [[location_tracking]] information for the repository.
 Maybe you'll find the drive later. Maybe that's impossible. Either way,

From e49e6a403b16cd12594239e18fa708aab61243a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:33:15 -0400
Subject: [PATCH 0952/2835] avoid failing if a local repo is not currently
 available

The added check if a repo is bare means its config needs to be read, but
in this case it cannot be. That means that a repo currently not available
is assumed to be non-bare.
---
 GitRepo.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index ba01251218..92539eb826 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -163,12 +163,15 @@ repoIsSsh Repo { location = Url url }
 	| otherwise = False
 repoIsSsh _ = False
 
+configAvail ::Repo -> Bool
+configAvail Repo { config = c } = c /= Map.empty
+
 repoIsLocalBare :: Repo -> Bool
-repoIsLocalBare r@(Repo { location = Dir _ }) = configBare r
+repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r
 repoIsLocalBare _ = False
 
 repoIsLocalFull :: Repo -> Bool
-repoIsLocalFull r@(Repo { location = Dir _ }) = not $ configBare r
+repoIsLocalFull r@(Repo { location = Dir _ }) = configAvail r && not (configBare r)
 repoIsLocalFull _ = False
 
 assertLocal :: Repo -> a -> a

From b27b0d5cd463afcce8a95730a49288edd89eb8b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:35:44 -0400
Subject: [PATCH 0953/2835] remove unused code

---
 GitRepo.hs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 92539eb826..8e3f14ee5a 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -17,7 +17,6 @@ module GitRepo (
 	repoIsUrl,
 	repoIsSsh,
 	repoIsLocalBare,
-	repoIsLocalFull,
 	repoDescribe,
 	repoLocation,
 	workTree,
@@ -170,10 +169,6 @@ repoIsLocalBare :: Repo -> Bool
 repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r
 repoIsLocalBare _ = False
 
-repoIsLocalFull :: Repo -> Bool
-repoIsLocalFull r@(Repo { location = Dir _ }) = configAvail r && not (configBare r)
-repoIsLocalFull _ = False
-
 assertLocal :: Repo -> a -> a
 assertLocal repo action = 
 	if not $ repoIsUrl repo

From 42259eee9200588f69af2b56557d5d191d426ad0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:02:29 -0400
Subject: [PATCH 0954/2835] support git funky remote syntaxes

* Look for dir.git directories the same as git does.
* Support remote urls specified as relative paths.
* Support non-ssh remote paths that contain tilde expansions.
---
 GitRepo.hs         | 114 ++++++++++++++++++++++++++++-----------------
 debian/changelog   |   3 ++
 git-annex-shell.hs |   4 +-
 3 files changed, 77 insertions(+), 44 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 8e3f14ee5a..bb0e3d5b7a 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -11,7 +11,7 @@
 module GitRepo (
 	Repo,
 	repoFromCwd,
-	repoFromPath,
+	repoFromAbsPath,
 	repoFromUrl,
 	localToUrl,
 	repoIsUrl,
@@ -49,7 +49,7 @@ module GitRepo (
 	encodeGitFile,
 	typeChangedFiles,
 	typeChangedStagedFiles,
-	absDir,
+	repoAbsPath,
 	reap,
 
 	prop_idempotent_deencode
@@ -57,6 +57,7 @@ module GitRepo (
 
 import Control.Monad (unless)
 import System.Directory
+import System.FilePath
 import System.Posix.Directory
 import System.Posix.User
 import System.Posix.Process
@@ -98,15 +99,23 @@ newFrom l =
 		remoteName = Nothing
 	}
 
-{- Local Repo constructor. -}
-repoFromPath :: FilePath -> Repo
-repoFromPath dir = newFrom $ Dir dir
+{- Local Repo constructor, requires an absolute path to the repo be
+ - specified. -}
+repoFromAbsPath :: FilePath -> IO Repo
+repoFromAbsPath dir
+	| "/" `isPrefixOf` dir = do
+ 		-- Git always looks for "dir.git" in preference to
+		-- to "dir", even if dir ends in a "/".
+		let dir' = (dropTrailingPathSeparator dir) ++ ".git"
+		e <- doesDirectoryExist dir'
+		return $ newFrom $ Dir $ if e then dir' else dir
+	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
 
 {- Remote Repo constructor. Throws exception on invalid url. -}
-repoFromUrl :: String -> Repo
+repoFromUrl :: String -> IO Repo
 repoFromUrl url
-	| startswith "file://" url = repoFromPath $ uriPath u
-	| otherwise = newFrom $ Url u
+	| startswith "file://" url = repoFromAbsPath $ uriPath u
+	| otherwise = return $ newFrom $ Url u
 		where
 			u = case (parseURI url) of
 				Just v -> v
@@ -356,31 +365,35 @@ configRead r = assertLocal r $ error "internal"
 hConfigRead :: Repo -> Handle -> IO Repo
 hConfigRead repo h = do
 	val <- hGetContentsStrict h
-	return $ configStore repo val
+	configStore repo val
 
 {- Parses a git config and returns a version of the repo using it. -}
-configStore :: Repo -> String -> Repo
-configStore repo s = r { remotes = configRemotes r }
-	where r = repo { config = configParse s }
+configStore :: Repo -> String -> IO Repo
+configStore repo s = do
+	rs <- configRemotes r
+	return $ r { remotes = rs }
+	where
+		r = repo { config = configParse s }
 
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
-configRemotes :: Repo -> [Repo]
-configRemotes repo = map construct remotepairs
+configRemotes :: Repo -> IO [Repo]
+configRemotes repo = mapM construct remotepairs
 	where
 		remotepairs = Map.toList $ filterremotes $ config repo
 		filterremotes = Map.filterWithKey (\k _ -> isremote k)
 		isremote k = startswith "remote." k && endswith ".url" k
 		remotename k = split "." k !! 1
-		construct (k,v) = (gen v) { remoteName = Just $ remotename k }
+		construct (k,v) = do
+			r <- gen v
+			return $ r { remoteName = Just $ remotename k }
 		gen v	| scpstyle v = repoFromUrl $ scptourl v
 			| isURI v = repoFromUrl v
-			| otherwise = repoFromPath v
+			| otherwise = repoFromRemotePath v repo
 		-- git remotes can be written scp style -- [user@]host:dir
-		-- where dir is relative to the user's home directory.
 		scpstyle v = ":" `isInfixOf` v && (not $ "//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
 			where
@@ -389,6 +402,7 @@ configRemotes repo = map construct remotepairs
 				dir = join ":" $ drop 1 bits
 				slash d	| d == "" = "/~/" ++ dir
 					| d !! 0 == '/' = dir
+					| d !! 0 == '~' = '/':dir
 					| otherwise = "/~/" ++ dir
 
 {- Parses git config --list output into a config map. -}
@@ -503,37 +517,51 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""
 				e_utf c = concat $ map showoctal $
 						(encode [c] :: [Word8])
 
-
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
 prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)
 
-{- Git ssh remotes can have a directory that is specified relative
- - to a home directory. This converts such a directory to an absolute path.
- - Note that it has to run on the remote system.
+{- Constructs a Repo from the path specified in the git remotes of
+ - another Repo. -}
+repoFromRemotePath :: FilePath -> Repo -> IO Repo
+repoFromRemotePath dir repo = do
+	dir' <- expandTilde dir
+	repoFromAbsPath $ workTree repo  dir'
+
+{- Git remotes can have a directory that is specified relative
+ - to the user's home directory, or that contains tilde expansions.
+ - This converts such a directory to an absolute path.
+ - Note that it has to run on the system where the remote is.
  -}
-absDir :: String -> IO String
-absDir d
-	| "/" `isPrefixOf` d = expandt d
-	| otherwise = do
-		h <- myhomedir
-		return $ h ++ d
+repoAbsPath :: FilePath -> IO FilePath
+repoAbsPath d = do
+	d' <- expandTilde d
+	h <- myHomeDir
+	hPutStrLn stderr $ "repoAbsPath " ++ d
+	return $ h  d'
+
+myHomeDir :: IO FilePath
+myHomeDir = do
+	uid <- getEffectiveUserID
+	u <- getUserEntryForID uid
+	return $ homeDirectory u
+
+expandTilde :: FilePath -> IO FilePath
+expandTilde = expandt True
 	where
-		homedir u = (homeDirectory u) ++ "/"
-		myhomedir = do
-			uid <- getEffectiveUserID
-			u <- getUserEntryForID uid
-			return $ homedir u
-		expandt [] = return ""
-		expandt ('/':'~':'/':cs) = do
-			h <- myhomedir
-			return $ h ++ cs
-		expandt ('/':'~':cs) = do
+		expandt _ [] = return ""
+		expandt _ ('/':cs) = do
+			v <- expandt True cs
+			return ('/':v)
+		expandt True ('~':'/':cs) = do
+			h <- myHomeDir
+			return $ h  cs
+		expandt True ('~':cs) = do
 			let (name, rest) = findname "" cs
 			u <- getUserEntryForName name
-			return $ homedir u ++ rest
-		expandt (c:cs) = do
-			v <- expandt cs
+			return $ homeDirectory u  rest
+		expandt _ (c:cs) = do
+			v <- expandt False cs
 			return (c:v)
 		findname n [] = (n, "")
 		findname n (c:cs)
@@ -546,10 +574,12 @@ repoFromCwd = do
 	cwd <- getCurrentDirectory
 	top <- seekUp cwd isRepoTop
 	case top of
-		(Just dir) -> return $ repoFromPath dir
+		-- repoFromAbsPath is not used to avoid looking for
+		-- "dir.git" directories.
+		(Just dir) -> return $ newFrom $ Dir dir
 		Nothing -> error "Not in a git repository."
 
-seekUp :: String -> (String -> IO Bool) -> IO (Maybe String)
+seekUp :: FilePath -> (FilePath -> IO Bool) -> IO (Maybe FilePath)
 seekUp dir want = do
 	ok <- want dir
 	if ok
diff --git a/debian/changelog b/debian/changelog
index ca1c51c4bf..a854235ce4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -16,6 +16,9 @@ git-annex (0.22) UNRELEASED; urgency=low
     use caution!
   * describe: New subcommand that can set or change the description of
     a repository.
+  * Look for dir.git directories the same as git does.
+  * Support remote urls specified as relative paths.
+  * Support non-ssh remote paths that contain tilde expansions.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index aeaadcbf85..e8a744748b 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -60,8 +60,8 @@ builtins = map cmdname cmds
 
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
-	dir' <- Git.absDir dir
-	let gitrepo = Git.repoFromPath dir'
+	dir' <- Git.repoAbsPath dir
+	gitrepo <- Git.repoFromAbsPath dir'
 	dispatch gitrepo (cmd:(filterparams params)) cmds commonOptions header
 
 external :: [String] -> IO ()

From 14e0d01b58c504e889b636e952e165f3cf8ab6c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:19:01 -0400
Subject: [PATCH 0955/2835] remove debugging printf

---
 GitRepo.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index bb0e3d5b7a..ef8ad25baa 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -537,7 +537,6 @@ repoAbsPath :: FilePath -> IO FilePath
 repoAbsPath d = do
 	d' <- expandTilde d
 	h <- myHomeDir
-	hPutStrLn stderr $ "repoAbsPath " ++ d
 	return $ h  d'
 
 myHomeDir :: IO FilePath

From bc2df77642490be830cde9a46428d25051120893 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:34:30 -0400
Subject: [PATCH 0956/2835] Bugfix: When fsck detected and moved away corrupt
 file content, it did not update the location log.

---
 Content.hs       | 1 +
 debian/changelog | 2 ++
 test.hs          | 3 ++-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Content.hs b/Content.hs
index 6b8316c082..bcd4ac0e13 100644
--- a/Content.hs
+++ b/Content.hs
@@ -147,6 +147,7 @@ moveBad key = do
 	liftIO $ allowWrite (parentDir src)
 	liftIO $ renameFile src dest
 	liftIO $ removeDirectory (parentDir src)
+	logStatus key ValueMissing
 	return dest
 
 {- List of keys whose content exists in .git/annex/objects/ -}
diff --git a/debian/changelog b/debian/changelog
index a854235ce4..ab9aa3a272 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,6 +19,8 @@ git-annex (0.22) UNRELEASED; urgency=low
   * Look for dir.git directories the same as git does.
   * Support remote urls specified as relative paths.
   * Support non-ssh remote paths that contain tilde expansions.
+  * Bugfix: When fsck detected and moved away corrupt file content, it did
+    not update the location log.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/test.hs b/test.hs
index 936e4da175..31960bb2e2 100644
--- a/test.hs
+++ b/test.hs
@@ -367,13 +367,14 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
 			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
 			fsck_should_fail "content not replicated to enough non-untrusted repositories"
+
 		corrupt f = do
 			git_annex "get" ["-q", f] @? "get of file failed"
 			Content.allowWrite f
 			writeFile f (changedcontent f)
 			r <- git_annex "fsck" ["-q"]
 			not r @? "fsck failed to fail with corrupted file content"
-			git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem"
+			git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f
 		fsck_should_fail m = do
 			r <- git_annex "fsck" ["-q"]
 			not r @? "fsck failed to fail with " ++ m

From 8514a2a98783d520e7115635a27428923b81ffb9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:55:56 -0400
Subject: [PATCH 0957/2835] updates

---
 doc/git-annex-shell.mdwn | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 1231ef668d..b32114a74d 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -11,11 +11,15 @@ git-annex-shell [-c] command [params ...]
 git-annex-shell is a restricted shell, similar to git-shell, which
 can be used as a login shell for SSH accounts.
 
+Since its syntax is identical to git-shell's, it can be used as a drop-in
+replacement anywhere git-shell is used. For example it can be used as a 
+user's restricted login shell.
+
 # COMMANDS
 
 * configlist directory
 
-  This outputs the git configuration, in the same form as
+  This outputs a subset of the git configuration, in the same form as
   `git config --list`
 
 * inannex directory [key ...]

From c5c7eaf00917d8654c82de5a4b17465b354f8fbf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:56:03 -0400
Subject: [PATCH 0958/2835] prep for release

---
 debian/changelog           | 13 +++++++------
 doc/bare_repositories.mdwn |  6 ++----
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ab9aa3a272..79be983679 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,20 +5,21 @@ git-annex (0.22) UNRELEASED; urgency=low
     for his help eliminating the infestation... for now.)
   * Make test suite not rely on a working cp -pr.
     (The Unix wars are still ON!)
+  * Git annexes can now be attached to bare git repositories.
+    (Both the local and remote host must have this version of git-annex
+    installed for it to work.)
   * Support filenames that start with a dash; when such a file is passed
     to a utility it will be escaped to avoid it being interpreted as an
-    option.
+    option. (I went a little overboard and got the type checker involved
+    in this, so such files are rather comprehensively supported now.)
   * New backends: SHA512 SHA384 SHA256 SHA224
-  * fsck: Check for and repair location log damage.
-  * Git annexes can now be attached to bare git repositories. Due to popular
-    demand. Both the local and remote host must have this version of git-annex
-    installed for it to work. This is still a semi-experimental feature;
-    use caution!
+    (Supported on systems where corresponding shaNsum commands are available.)
   * describe: New subcommand that can set or change the description of
     a repository.
   * Look for dir.git directories the same as git does.
   * Support remote urls specified as relative paths.
   * Support non-ssh remote paths that contain tilde expansions.
+  * fsck: Check for and repair location log damage.
   * Bugfix: When fsck detected and moved away corrupt file content, it did
     not update the location log.
 
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index b7ae0b316f..a9ccab8d1b 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -1,9 +1,7 @@
-**This is still an experimental feature!** Use with caution.
-
 Due to popular demand, git-annex can now be used with bare repositories.
 
-So, for example, you can stash a file away in your
-repos's origin: `git annex move mybigfile --to origin`
+So, for example, you can stash a file away in the origin:
+`git annex move mybigfile --to origin`
 
 Of course, for that to work, the bare repository has to be on a system with
 [[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't

From 69c14d130bc7a754e3a4fa184ff317690ad48ca6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Mar 2011 12:31:01 -0400
Subject: [PATCH 0959/2835] update

---
 doc/distributed_version_control.mdwn |  13 +++++++++++++
 doc/future_proofing.mdwn             |  24 ++++++++++++++++++++++++
 doc/not.mdwn                         |   8 ++++++++
 doc/repomap.png                      | Bin 0 -> 129065 bytes
 doc/transferring_data.mdwn           |  14 ++++++++++++++
 doc/use_case/Alice.mdwn              |   6 ++++--
 doc/use_case/Bob.mdwn                |   7 +++++++
 7 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/distributed_version_control.mdwn
 create mode 100644 doc/future_proofing.mdwn
 create mode 100644 doc/repomap.png
 create mode 100644 doc/transferring_data.mdwn

diff --git a/doc/distributed_version_control.mdwn b/doc/distributed_version_control.mdwn
new file mode 100644
index 0000000000..f9cdb7e99e
--- /dev/null
+++ b/doc/distributed_version_control.mdwn
@@ -0,0 +1,13 @@
+In git, there can be multiple clones of a repository, each clone can 
+be independently modified, and clones can push or pull changes to
+one-another to get back in sync.
+
+git-annex preserves that fundamental distributed nature of git, while
+dropping the requirement that, once in sync, each clone contains all the data
+that was committed to each other clone. Instead of storing the content
+of a file in the repository, git-annex stores a pointer to the content.
+
+Each git-annex repository is responsible for storing some of the content,
+and can copy it to or from other repositories. [[Location_tracking]]
+information is committed to git, to let repositories inform other
+repositories what file contents they have available.
diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn
new file mode 100644
index 0000000000..4d4939b5a9
--- /dev/null
+++ b/doc/future_proofing.mdwn
@@ -0,0 +1,24 @@
+Imagine putting a git-annex drive in a time capsule. In 20, or 50, or 100
+years, you'd like its contents to be as accessible as possible to whoever
+digs it up.
+
+This is a hard problem. git-annex cannot completly solve it, but it does
+its best to not contribute to the problem. Here are some aspects of the
+problem:
+
+* How are files accessed? Git-annex carefully adds minimal complexity
+  to access files in a repository. Nothing needs to be done to extract
+  files from the repository; they are there on disk in the usual way,
+  with just some symlinks pointing at the annexed file contents.
+  Neither git-annex nor git is needed to get at the file contents.
+
+* What file formats are used? Will they still be readable? To deal with
+  this, it's best to stick to plain text files, and the most common
+  image, sound, etc formats. Consider storing the same content in multiple
+  formats.
+
+* What filesystem is used on the drive? Will that filesystem still be
+  available?
+
+* What is the hardware interface of the drive? Will hardware still exist
+  to talk to it?
diff --git a/doc/not.mdwn b/doc/not.mdwn
index 80c0acafaf..fe6e1b37d9 100644
--- a/doc/not.mdwn
+++ b/doc/not.mdwn
@@ -30,3 +30,11 @@
   situations. It lacks git-annex's support for widely distributed storage,
   using only a single backend data store. It also does not support
   partial checkouts of file contents, like git-annex does.
+
+* git-annex is also not [boar](http://code.google.com/p/boar/),
+  although it shares many of its goals and characteristics. Boar implements
+  its own version control system, rather than simply embarcing and
+  extending git. And while boar supports distributed clones of a repository,
+  it does not support keeping different files in different clones of the
+  same repository, which git-annex does, and is an important feature for
+  large-scale archiving.
diff --git a/doc/repomap.png b/doc/repomap.png
new file mode 100644
index 0000000000000000000000000000000000000000..4d334aec947e33900920351ccbd094b22c4aed3a
GIT binary patch
literal 129065
zcmce8c{G;o+wMcANM;!`Bua#Y2J=uPN~uhlr9{S3GLMNAl~h7Xip(h#GAAOEF(Q>Q
zQOXqdarM5x{e5eHYyY>`w%&K8Jf7#iui-q8>AEBL8}4RiJg{4nlHDXQg1^w&>hIRZpZJ>7dLDp3R=Vk$dQvD%9OQp0%B@5m
z{3E@Wo`DX1&k7n=?v=EcXkSq%!W2Di4db)jtF{^vK^|Ic3C
zrdeM=GrdJqOKV_gs7{lW%_uYPc#G&+AD?_yZEbvVR`X!pW%tjo*v_6mU%;cS-S&x=
zp5AVHxJ8PFjZH4};Mmh+Z}_-_E?h8E_CLN?TU(mBE|i1q@0Z;F_=Zhu+dn`0@89UF
z|JQA{eST3|YTg#7)4U@&{!9${!QZ=o6DmZ_ieLSoZ`l$R>$vj5@KFi&NH<(14IQ1R
zii#k`+R5o+&>v;cuV7TQT!>U!Cmz2kkrI=sI
zcrKYgb=FWH_NxbmzoZxm3nwFN9{YQ?}#jV%xpO{!p
zDLV6w@2N{qc+WG}LEKWpHh0mb*+G^MFV1mDN=lmKT2sAv@dEE39U4mW=SPD~t6fQP
zvCW4f_uOaCBJ%UMT)cSE4IkwEu}qMjWv%Z3>sk(u5NY$LDJMI!H%Lo|B_|67hlEVk
z2?gkC%{C+kSX&EIteXAoo80PyxKl7Zzq3^tRphx3XkTk
zOYG~}r^h<>m6Qf7FI~|Un;p%}aT{q}zc+E4QMY6%e!5Cv+y0WA(o&VO`O!@y&8n)Z
z+kB=swC7k|{`ASV?frYb@8@SH2bhw39FzURvyVQfrg(XI<+=@M($Led+rEA7dXv@e
z-SnoWrbV7Zs~H&?txud-DIp1bSG
zhKAXC&2Q@KSC%jS=Ku8RliT+`73H&&>U^)1T>GMP%9n1j2;;qsgmU
zFh}=V{5o>1Tyys=cI4*f4z0~8EA*ZCP%JAi9}*l)P3dpCv(t6)&sa$5y?g7#4HGZO
zl+99~|2m6LP&aI+eIO-@d(S=n%r9xJY^w>equ^83sqN}Uk}Y5!SyrQAX4fmp-+uVUBz
zyT|SA?MZQ=T0v)x;=;;6v(HlY%a=U&!Cil_wu{^bs9RfGg*R-(5)7@
zkY`u7=kmzd7?X~U4yJI$+}xZqvxKOqsFUWFxFLf9{pnRpOG^n`92m{3F)D{9fB&u>
zsEUv~_rsn$1O=HUW+izwzHAx(&u>J--TwK}fBz;vkW4SWP$K^M`rqG>Isg0hKfn3^
z^6H1*uAt0c7(R6U#*KT~*{dBK99Sc@A7*Fo($S%HI(ag!2W#;D0=Cq>2M-=p$c>JU
z?l(1EN!gF8l9*@_vcSd3ney=AhDB2>I_e+_D)p5cHzF=xWaQ-HO6&Rj`EyD}#%uZr
z+#|WehYsvfH5)c;xYEbS#AIY_9NOA?;M(=;dgbi_R%T`wRW}Mr@&)BIJTh
zOc?S^q!tAa9Xd2TI;yF!&$MI5j<+o>8g(&(ceAp3`yV`dv@%GYA}uYAn@~S=hzlQy
zm4%Nu)jlSP7N;wwX==**x}m}7;K7j8RM89CqMU2i=;w@$n4{brICzkJ#@+oFa4U2b
zrAsBFIlXV+vd+)X%WmCz&0j{Qd3gs}5Lh3>cf{EvCugVZMx@uRTd{rn_E+`w8kUx8
zF#{9|8V6a;Qc_Z+-5xx6aCmI2>eVZT@bGY1S=qq6Jon7urlUeXe*OAoHOKSi%a>P;
zjoOBWtMdyAVy|Bh8y|Nbd5Y$E|G|SjRV9gu0@Ksed~F$i3%^O>DlXnmLrs+&{Vzyp
zJKWaRCcJ$+FN=_>Giymfe!ehwln5&WspBV4o;-W*Tt#K2rq4-SkJstbrq$^WGcqc>
zyZPGN+XJ3k;fitBvA1qrympQA{{8#?E%$aaFfmo#)rt^ij}(@WpkKFc-3ehOC8db#
z*SVymrSGPsytE~C0&``gf7sG8=JRKJF)=aWjT=`+Mn>xD>1iB2D!6s)R%;iROPi0D
zSbu&Mn_E~Ih6aV-?&9lNg_}u9Pmfru;mNUUZ0+Dc%g@j6HQpI}wGJOmI#Nr|wVO91J3CFPYimVSRo8CZxDo9U
z)1-|zlRMEW=JmZ#(9qB@_sJ9c3Nr?*3KXW!t}ca=@3=+M>#$JW7Jj=301{yOu3W#a
zv+(;{OPc^-s?
z@Bv3lyn+e}DT7v-BL0vYu2nO
zDk`#``c#9#C4<(4Zs9r9G!P|I7Kqp3a^7MP0Fv)vD@zIdJ6u%iC^wp#nrJzp!^6i>
z9c&sC3U81EZsiKt0^i|Lo_7-y}C}(o$Ea
z7$zx40!^;s+sc5kz6#u`Xr_T%zW
zL~_6quU7b~iqG{|x1T{HdPr+qt{|FvV?VZ7>G>Jk(Y8k@OO+1{ZeNLw4RYyuDi`?9
z+|JH!g_@cgY0iuDzsRuQ=iw+6cT-b?Y5#43PDe4vhIe*^Z9Mp(e|qGd$@4SPYn9Ii
z(lT+Y8yH-oP30zAwydm7ch8>6(f66|KR)a2O;YAfH@p+9EouXj^S3#emHqkS``9Fp
z+TY(lLFK$6mRD$+{tb`D8P8X*Ub%G@KC@s#eUdWe%OY@x{Nk=j657GQ*
z{@Ogk!omhZSX2qVtE{Zd1%kl($dg_5BP;mnQ$Uv$qIS`&Y86S{mmZ&
zDn6W>nD65Zrm@Nf9Lpd?j|>ZcX&DTeoh__J^?SH#3`Woap)0gFc++^UEnAA%Wb}ogJqu
zvEn`qcLsFO{&6UYD4!@+UVQ$meCDy;+8tgDeSLjzTU&$Cd{EhPedj!|^3{I2-oT=`
zcUe{Q`I&FRGBV4rPct$z*E~7#4xoz>VC>AdZkx9$8o*hanP>eM=I3p3CzC&aR^g`R
zSdst{n6Jj>HVc>11Ox=^y}5<9qM|~Iu24@wK_NlmBscJ3#J8Sj&mMYmFoq3x7jDAs
z@;M91+0qy$t1?rVOM*Hg+E2WDFwkESaw%qJ;d{9qM)}h3fc5~U0Pl}c;|p7cet6rI
zmt`)rJvun>{rhsG-}tw01j8Obethe;K5Qs=tWdf-pF2BL#!5iT`ch8MIgI{(AZ_-9
zL1JH0bzkXubfxa`@)NGEt^%sQJXa<5ePu$$C}eO3VDow1Rp2PX9hK+5IM3~UieJ%L
z{qSLK3=!+vwK1ckPN5MI%XTNf0;#%BebPbk71^{&pKrtBpFe%G3NKz2D+`*8Y0bPF
zY()8I!WfvCnDkdia)7y#DHC0{ZeU;_2t>>G=NqLVW>w@y6++K)iqm?`+$P7Om>SS%}y&wHP7k#S9p59cUg
zV_OUWj3&()M8NFxT-yEn6pFHn3cvI6SaZOaiHW{`M-C1Siu#EYBmCvzxE6F)ih7P^
zDId*;bn}L>lfRy*`pv)iUA*|a$2DQ=iB?ao(|}=_K20;SMdjSX^W@d(%NQIk7onnpF5jDE^
z@t~2>xo@84C`~9<+#p0C(<%A+?Cg;=Si@OAifxV^Yws`K_ph4Zy4O#4VQjox8r6iB
zA}l5KxZ-#D(w_`qA&T-y_~H|=@6xH0S>J1=xuf`d9bH{n`t_n_t?HtePyac4P=D`U
znn`=*!b_K!uU)%FNqN7zWuv4dH3it{2Q8rk1mV5eqNo`D;DI=R0)~uPNL7F$8?fZ7
zy=#{yU<<$i6SvgyxWSz}c2HAPmuBq5OnDi}A;DMZ)FH!<
zB_u1b&5d&6#0hXZozthu6HjL#gAOAGjbvtqufifHKh74e7Tm!}kv;Wc8$NNXiG=Uh#+#GBe$_xC!j-Q|xP4no
zn3eSKr%#_I$UCmiwkq$F8u~R5Aw+$)Rk{?76x_Z7dw$vd#X0@cll>axit_V)cY@**
zrBWMxFqjoAm$$=6>3sxEe`~!3->-lxB?@X^Zh2rA!tz*X$N_wvAM^ykc
z4Pn-ojg6}^kL1*%WZ+s)L0N$=q=kl?TpV5qJq55bbZW4!s=As6oaXiG*N|fB{>=Sk
zTe}vQS%_Nv*tCESgb3IleR;*<9Lr09m(NOu=D(LO$3W?^`&1qICjyIUn$A6Pi_b=j
z)aX9i-7x|cXxeOie2nz;5!?Y~YRiA*kG;7?i-KiQ=6jPxc<~;#1w=wyfIA+gQx><&
z4sSLBFp*|92YXr~vYVTm+e;Ktwl!-yQkz^L7?KtSx^HA*5nUC*GB|iyjV@48=)uDi
z)z#J3DCxuPS^Tvzf(^m{nc<^bwrrXG^RpQXo({!=hKA-zg6Gb2Kg5)kl~eBDrvbYG
zi{=75b~|&XHgUV>Y=6tg9iCrT%}%&!i3;Pp_^V@ba`NTm
zKotRc#)l7={rb7%{BJ4nZ3e(Gtb!D~x)_tE&b-0FP;^sX(#Iq{N=>EyIn-Q7u+qp#
zEbvu6QO
zm?6p7P*8RWp?4C5l#qAs=P`0q1d^6`jRap-ooB$ll3HE3fhdCGT?H+LCy#&{eCly6
zuL1hLo1R|x>E979+-NdM28zw8RU-F}2Pt8f@UCrV!Q!
z)(Fl{&@cI{Cr^Cu9(DouBB+jO&E{&15vzbeG
zl!B2YJC_j7%*=)EfFt-cf{xLr_nVkp=ud6j1Lo%ZglnL@?}s=-q;oI;lYIM?zJ7i!
zceUtOtyShpN=hP><>pOZp|{?hSXZbg7vH|!pSaDP8n|LVga?qm!;ejZuz=LHv_iuE
zh3P)>U?6EC3(&Gq&8xAPFkZCh&YgSW`f)AT`sCD95D)<%QJ>TCjp!8G+BEo8R?tKU
zGs7bzHk17oZeCv6XU-@R8IZ65v?G+k;dkjQC?5S^zJ!4WXc!vCu=w&DA2|~7-23|{
z0DeOFF|k*Yl2)UtcujmTofkQP#fP0h7&3?$I?)P{Y0ztXHV7bprO=rD8=T+f-??@z
z6lex2>BC2lDqg>41hT5Vz_yNXc76Q@I*znbX38}wEtHy8DkzYr6%3$61>VWEwy>t(iey9k9>R8)lAwfoSa>(k!2h1j^bVDuHyty?((
zEBN^M2oi?6|F-nJcQzGlr?!i)h)>m^F7q1u=HP?98c=zoGz=bKVX&-KKJJ$3x^nB5
zEt(Ol1#y4tOQC!fCU`c#dUfTg*GCAM0AaFFN^oNzK78o^{CT?dYM!1LyR!cu9>CI*
zve~7NAFqPOKK!9r9u=VF(LssM?(S6VQ?l4^-1ljKU`%)wfGW6QKN=wJs}?s8y=nz&
zgV$)=O03FI;1*{W7n@V3R`c@ST(jdhx|{o0yR=2QzaX{~B*NAOpr(QT{y@CNz4J2@
z23yKC%m==GyM%gY_v3TD6y4v7qqVK{{KcrK*eh287iPZeukeN*M98GAtthhuntyJ7
zzLWcy%aOx}MR)Ar12DzzC891UYzqdhsJOV|{rf`G5=MP7cKn{OoE#ewfea0I`S|Pr
z$M{EzdW)^m+0&zc($U6-iiwFS;pWY}UoTdMoPl?A;J|?^n3ntZ#Xgh=2zh_+tHBkD
zKzc&M0Pq16scLOy$MP>a_fs5lJw0G5%nvqTYX7V{Z7J?`_)KUBnkFVZ>>M0MN00Uj
zU+d}VxreuMb9dJOpW>wr|wC*=sr7Y2`w
zvhP@>l7d3mqemMd1*Iv$yCA4+hd)NgfAQB53or;!P98OXOVux!oUyVGmb#IgR!^`ANObgRiR2M3YO
zn^$9lqPQ-WFR!wHH}R0+Hu
z2rOhJhxpbxv0yac3(?Wq$B%EA=zSgmv`*K7%cEssVSzW6YY^c^fSP3VKp@&aFW&>j27%aof9WE?%A`4FZ8z$
z{@4#M6EhCr1n2KAY60{T@Mm>l)~#mEW|##d=`poy8#ix0HEppl_7wdP0yPKN`sm?9b(At<&;cL7SA?mQ
zl9?GrJSi|6EL*;(@7vl^?%t(xaQMPK*9k}rO29D-y^xG86upYNy7Vsd^cq_^VmCr_
z(1tB7FMn`eBMS4fckkY}sNTdz&(6-yI%7%oUyG~-Eu{6`JO0oK6bm#)l)H+)zO_&W
zKn?=a(i}3K1Ywfb{u=8b0t(*u9yAA3TQ_g-i099DqQhGre=yqq6(Fj!^t=+q8kXhR
z<;9tvw$ppW*r&hstf9z)OiCRsd2-0o3do44TLfHUjQ%3QeMujjr3ahiAnQZ1z#NFI
zTX(Ob=*PU=BoUqQ84z#G$LaYA2n1hSHPDv&arPq$FzM6TU$Axwz{$#z2Dqk%uaD|o
z-Xi;!6Whnu&dz)4EnVH>ar?r=bScHUx|k_ToDii{-k_rTa#6b#8Sj)v2{upA*C00_}SAjvz`
zTcX(Y+G-y>a_m5q3Ic2KS-I8~!H3UDV-b?VOmtxosDTcvfEo~fS@;jpf;*pf)n%v|
zU~R6TUtJBXn%3jt>6uolhYnAJsy99D1;#VD+-LEsK?~~|W)|@;z;ki`w6H69059g}
zxc))R!^P+ttjHEXQ2$WuNe|PMSb0E%C1<`}tiLL8Pyfbd43JA+)LZUU2|wG`55U
zp6a@~b<)zzxKQGa?b*YKOKI;XN-KCdN?}|~R|l$9%wUeyr>wGHR*(xy3{b@=_sgw>
z=Et7bHsIGVdZ;VS7Lqa0I9=?#y?i!~$MogD&*1`qk}mx+G(>#%)vH$%EQf!xIe8Ms
zam``BEC3+@TP?H&;DWm#&TvV=CsRP`q%(Go-}XR^V*Eq#!5wnl0ZTuHpkREzQ}i3c
z`T*$TrTC+_e5+)+kHj?e=*7#kb+JM@)d0cv1jm6VhaTd=H5
z72ept`~x(ts1`Bpbq;8<9d+jj1}
zcO=Jha(dc%N}n(^t!+qJ$!~%m9XWEp6%_w@2oYKjMXI*_h)wSGTM9)h}o&W7m=>zi`*(c(`VD2+>t_X?#kCG
znfPTTZIVmj5Z`8;j%V$?0+VjlKs9R<>k{1|v?L_6h#Lr42JS))9{>oLs0>6j+HrccZPdJE
z?0S6sC(>f!FXF~lLXAMdhDb#?8_1vUf&O}vWvIpLfY*=?dieNp^-X!l+%wBHJNO3V2H#gyKN3ty{U}m7F
zg#hr`wWR7m+d6()|DHlRmi`|I)Scx4YAANO=Vwkqt^Gu*@)|4ODMLCpicFhBXLB
R0`k<2u)YQVZ7(r_m8Xf2&VJ*
zFMD!*am3t@={&f09Kvj1Fu;?j8$-nbK+_z+>o9qoCJUDq=6ec$4-|R*^>=+%0K2ep
zafM;yL4v`qVZm^Lrd0g+(lGGpQ|s5@Y+%xY=r`{)zrE&8ZvZ5Ds5hY@AzHUf{J}g{LGi^V4l6HL
z^PeA)NK`q`4aW}(;#ak#fcdwOn|z@x
zO)nsv0GCA1wRu#Lg@R>&N%OqlrUk2K9RXHCe+sexv$K~BrK)tb+ioGRRLXf~x9bjk
zers1L%t{0h32gd@H*$fVyJVZ&flm^p;n(OPLIKGR_x9cmof-~p_XD$;+Xv7aR|{+l
zwDO;$CF%mk3fgEmi8<9=brb-ggO73{*FpieOqaTVIJ}ei(Awd}rMX44S-QDxxEBp!@|VC9RE@v5H|)
z`!gA4_N1YqfuH8r{P(ePZfh;WL<;s-|%*;Aqr&uS5b5VJF6WV+io7ly}H0#Net&o;^AyT!T{5?7OS2`RY
zZ&W(BEDRosn3pz+%dzI9OF*O=FvQShh#-%}FxBOlTzJtT<5tf@qkE`5Qj5i9n|?zd
zlSL2;Vr03e0!Z&m6pYxbSBcpSS1|3p%)ZW6&DnQCzRrCwc*#abCS>Z%4J%tLrZ@98
zPA)E4TNB~>-$TFrhrzhWZeAL3A)z#3tL2~X^s7}2fZ6$4lUEPzqo1GWo^uSFoj*0>
z>xY2W1=v2fVojuKu|0|Gg|0%1Hj)(Bp@BI$GJq=G;u2H!uU}VwX}H0LcmRn1@slU-
z?MqfAzAUP4UzdA^%stQvwZ&-;0Na(g8gj98baaS1@GZ{`B@@9GBI6MMhvghqi`PI2
z*d|0B1oY}39i7KM@2@{Q1r2%6b3>~Vua)}oQbBM>l4T7V;Tb@zuEHA1LxQ(4NrfMF
zHjLDe^mNCcFCwqr*iE`6kV?mn7+ohdrN9gToBq1B-m;wxD3qqoiPEn)AJw
zqZ_RWi-@qpUL{o-S|TEHFX?7nyYhiGX=g{$lRK~WZ^ZBcS=68z%i2~mJbd_280rHO
zy??!&t`Y_sYRJqf0n1CBd3N+fk*7jP59=FL@GD0;Z&C!-@QS6y?*Vr8aqEYBpDU^U
znbdl0RbEz;y?rw)!~K(4<);#ITy1i`?;$g6HTL{n`u+P`yNAvl9S2ED;%yt#nlG4m
zhuZxQqu)dZ8Svrs^dh~epO>G>3@BOfGcsa>Jiy-z!K;0^
z2-U4seaUNE!2H5cZ!=45Bs8Rlw0ipbRgi>8wgt72*aY`>UDEtm=GQas(~kDYuySPu
zS=@l%xIKU_>(={vsPQ%pp=yvOUb}MY0W<#F!k6{
z9Ba20swNveYB+Max)kjAmMn9nKQM+m%lvjO{rS1i_eZ@1i8s05s(=W8StEqbW~4QZ
zNB-EVV~q(^YHH=)OImk_%H*!!2>5pR9H;=+CZ;4MH+O%@2aCm-eXi%HhmArOAh+-d
z?n6q=$54jj(;;Ky*+BLdd9-Dpjfn}&See?ox(v9&2w-}A{WYyRay~N2xW41d*^Y+1
zM#fSbpcJwOxyom*Rw!_#o*XI)>wz2zpawkQL3&1Yes~=mR@I*dUl50mMgRlV_NuhJ
z%)o8eG7Tt(s17x@e{?)NwvZSIO2gjUip;3sCpZXxZBon83C-!Z?Dou3>V?wqO((?AW~2xQB^K3{~1yvEb8Ni
z7vcy8$U8RmW7s3&J?06aOhXodbl4_=h~)7dg6b-#)$%W93R)wgltf&L{ufsVb1*
z;Y)Ax_{@MdWP@lP8XGHAW7mSCD}|5+BJpEcSyr%KqD_Box)X){n73fhQ4jSCnIdvj
z0H%)}l1u?Tw}}l2CQ$>Ml}FS{185vdN)Tc@C`7r!0WkZbpv}GP?G+sB$f*Ncl>-Zh
zBYblc@75In!~}gnuTbEYu_R$109F|NC1MHYP;U
zrv{d;Xr`p>fHV)_#t^Kmf~|wFd7$^16X{<;Tp|FpZLzIo6%=aF4=jhb7kYoUB>WF_
zS?bfK@Vyv!#8Tu#mzlA7X6NR}k%-Rc-ZBKg!B6pcd{$mX0w%I2DFVpTklKrmR^QR%
zeFg9Y;e4yJU#?+-aX&=wCQKQ0?Z1Mc)z#WWRmWH$Hrx8~AhayZY4oK_HZZ9GJkaK=
z+S)j4Yio(@j96*Zor+{qq?q;h?W+UD3CFw=47qN1Of@d9=d;8s)Fje~$h~7E2{l0t
zCDALg0Xam?@mW|{kVSzoH8~|Dyi7t0fDU9!
zAx1``h@T3YRYQOqquy2TI5U0$OP|CHNIacvd6efo+3D_4iIA}m#Xmnzpv8m({gL@a
zfEeaAnR-b6&{SiQY#kpTUuFVL0Z6?@_Y*vjfF6KP0_wxUYUCw&)|132Wnamig@uJd
zXqO-yl`yMikH1Y(oLTuD&Z9dlP~sjS`U79E8b5>uSaZ1InAgn2QRfduY*5Y8gCL{zIAYORa+<{KWF1~}!n(g}P)G-HasamjfS}>x
zA`M5DFgd6mo~t_eiH%E64ZV0F0BBox9yhqB4gzZha-k%9k@wJ`8a^nF$*7|~tXj3o
zW59^>;=e;U~l_)G^!p8!%xH90fh`8RS1oR(;n8cC9n9v&XhzIC8-
zsxFMnbBA?5nOA65QC8;by%8HL!aPK72~U
zYu1Omr7CTNdPY)`u)&EJhK!Adj!xPkD+KA-sHv&lzBckga=Z(cOrkDWY~5`MzuS@R
zp%A=`9HpGs@czczN-Q`dVesPx0Ow}3<3>>>ImA7beP?qk6cOv7sKWtgnhwTW$(D#0P^uNCKRQwQz^^=|Sz0m6Ovz1Lb#K6IFBe&#WBg
zh?q?zb$~$XEY_-#nHgh$y%P&)eRxPn$n;6Ld)u}PU<)Zkkah#tTvk!B7OctKZ_EuW
zi#Q#SMj)IHl+N^3L4QV1sHI^BQX>nWBnY3MUC`kg_U!(Q@0~M@7hQnhke4)v{X%DT&pXc6*}>OKOlW
zMn0d~C-Y4rR`&`#74%JVOhHxEvZPy=gbQXy%^RM>DS^-lKR=(I4>wSH5yvi^J9CMG
zn1Dlt$C?tmoXjskNpOdmL|_RHY-rMBiUb9z>7T#Mao>AHiYL4f
zLaiaq%DVO`bB7JPLW0z-nnU@!-!nEkI*C*UNlIamakQZd2%Dsi|xp
zLRTT`1ylS2CYwYa-u!q5Un>MVE`Rt6%pDRIfIZcGJ3@Fl9XfweCULMh82#8
znI!YqUrKbIFoG+uD&&icoP2Ox(7R@1HD2xLo=(Qx}PRFvg
z(Ouuw6xAQ8nhQW=B-x1QkIU0OE~p|vsF&Jpv%cpGu(Adsn$eGuflEZ9d6i*I{8Bya
z|Cd9y@2sEx0th5d3Lq=#ijcW5N!Ez35c{#KYcA~fxvrlak=g`fP_I~l_`zxlL8Gve
zD_4)X2;r)5bj){5RF@!BZW&v9O_17Uq417=a
z(|x_Gvi7g8&@Ge>&z0+bv;4wmwgsCVc+B?8>+59sl+As;U4HaPf@M>Bvj4>oA1wAJ
zs|sQcssgE2m~f_w3$rqi)C^(rFp2z+$(AGLNSZ4wJtc;(B(8(>1VBtt+4fRd-bkDy
z;2qYPTUfAz7bQLVW&q%N5~6I%lP8>;4nJ0h_h5d=0O>{d`RNnnFf3srfXlmL1juPr
z)q%b(5Vn!)7C?j^9STuJ(}HJ7C^EJ{U(oMl;mGpQZiVR&c%(2Zh$O;z@AD%i0=-sM
ziPN-JyXdU^eqHza+I5LvNmN+4zj%mUK;Vz&!i&Z^TpX-aPMiyFR<;C}$;Fa8amBCC
z9Th40{GDY^YjoWUU=DWuFN2tuYEv{qaQHZE*FuhI!LE{KoMnt^n>dtnH*FKzHR4pU
z2?>7>l0vw3{S^}QZ{o(~(`h}?SR&ePJ2TD$)(L05cFoCDrP(qNKSRDoXkNtLL)IOK
z+freKkozaGE$D$OJx7X>IM3=v-^n{Yb{NJ7QH_Y*r!bU+^B@?`PvBW_1nUsKBGE~l
z8~PdKLWNx&VQ^cKcrBQyTEqZ8zT)|YGi4|O$}{IChg;T-_m%QvNLx3TqtMMl3`9*_
zZ>O{2KUn|^O5p9={N#u!jD-!GHr;#pP`4zy8(ESx5oM6Zmsh7AKz$>^K0I1hA)(7q
z&r2s&Wjy^6snziE+KPUkhoj2y8&*(2rNW6O0JufM!S^O+A?M5njQ5_iFqm6@Io0v)
zsRUR4_mfY>?C<35`kgg?vU2iG-1xO!XGFWa--uPpir1Fe8!*_6=X1Io*!XQ`X-T$M
zDCFjzXQzIhQ(dd_sn7HKT;j}7;>5X+%};T#XtX0Ir|8k#*2p(|DxK`@r{Bj<0`wrC
zg`M;T+XCE-B>(*vzA4rC=O0*Vtls>*E@kv0xAbcu4M@?BU^LM}YHQ(8O+~DyM`=4V
z@j(x57mIudvYsvbZjD##5xv;z+<66&Qd>vIblpbRz8CdySjoXyP|Wqn>VWsco8r_v
zhyK*k(XmE@Q=5JBXDc{hB(03FD9%O&!ow@$z0JYNNzO|YgQwv=pytS#n3_iFr5bAk
z>7dv<;uJDSP@Z)Kby}IJEPkO2
zrU-;k5kYDbeYVJx8d5h)#VmBDq%L*T*paDf*;Z;|h@PxIxCe1za6_Wi*Z
z*>DeDeevLyGI@z2U#44`zV~Kuaq+pB>dsfMEPTUd7-39Ij2C>Ljymtf
z4u-qYVpGY>0YASoQxKd*yzSTR?^omEQfFtPdF4q{!WWq2hPBUfbIHLjnDh=#w;x~o
z4oF3La=hhJRKCNLw~o#~AJH-MeKv`X;@Y&!*?D};<&G@#nSy1<>Bn=E10kRC9$!<-
z%-i%3XTlztD^ypWDHjRq~OMTGMzA^CGA
z2SIoHj3$FNmQ;AMBQ7b)@DiWG@pliH1$TI*V7fSEEG=mEk-@p8*NS=Ga43Tl4@kHR
zCqVLOA5aWptJ}{`3Ilwkfnp>Vax!Kd+y_ue2-AIshIq8P?N-b#vu)(~oWLr`gCkq+%q?fN6}yqEXSUQ6X=3iz|Kpyu}UT8w_dRsk4SN
z4~(vNKB7ijg3eZl=*Lavb6c=KxTK$Lf7#iYfLi&pibV}Quo<0RXFu-v`o@EFK%@Zj
zftP=hGn0|fG(3QA$ze)dlTB4P(O}U!%$_>m9Mn%?6kWL{RB9)BCeDhbVi=&RW=tzB
zfs(=7P9~%?R+kv4NXp$cesmyJ^iU(ps=K$hgo?@?EdIqYZPQ{6$?Kw5>&xbR_wRq}
zD4&S%$41qDsv75B)V9TOzG94*(o9E+#tI$%pWyyH+uVdjPcetIL0
z4_Z~v0}otu*`hBRD}19Nw%-2c#G%y8OntEFr=t>?Uz+!h=X8%i^op#KrAC357>x_
zS?$vO~35?Cwg~6^GMj~y%{aD_T4-AW)8PN(Eh6l39pdKDW89V91*yigtYV}
z{CyA))A$oGa~!f11tmxa`w9sRm|Hh7xB*j^fK{Jywh?|T#+CD9W7V>ERCA-E=|(Fs
zD=H#Dp<=AtxG@s4F9H@-8HX|{`A^0V)J|ExI2#LKv{z!H>-zN{j*4fxwuk@8u&<9J
zvKsItip{C@}c+8_{_Jv)^Z_XT-Qdyo+_Orkm$5mnob
zkJ45Wj0c_=FCPRzmM=9p{Mr#73b-Ro6DSwHQ$C1;Lb$v=a#q2m
z`x+MC!1(x|uEAx6cV}UmVhwHj((F8*e}2{6AsFHAo}Pp7J&wU~xM;&tIdc=l!+YX`
z&8sUS2y7i4Y1qgZoT7|HP~FtD8VMt$Z;8kat?w&By5KR-rd+3MknM5*vBC_(IJNhj|8Z3
zMl>YK<4#Wg@+*_!UIMucqHZC>+WOFF9Y8+U$xsvyfk^F29I-q*GagovEiz*B?^z+VVu0rYilF1Q4^yt?&WcESS
zgObtIT9ScGhdLe?0Q}fj=9f9$9)#*!kk|0dZa*!aLjt)W5L-?jA~mW`dP1k+t1=W+
z4~jGIHJcccF;>SA&fnvD7fGdK-A_)SX^`g)LCHNNxfTd-#F61`Ya9qlW|*N(4TW84IxAYr>FfVVi>Y=-fT2^i`CMD^(P
z8;@iQfEm@awr=_O*vOKn^z2kH#`PzT_2Zz?D-cfOrG_}v9(QaSqjQ=;;M2t9B#+}k
z<2w;~*5<&Y1M886F5ZeM-6xn-g>qfMIn|o^^
zN!DYTPW}G<03sP_P&nfg8dZ2KoIQC+=3_b(A#Kgn)LobOYVg63Dg1-(2&z8QO*;Uq7!}6>BPS{xykGawW0P2%
zPbP^06i1|)9HAXtlQxfstWGFnHKz@V6{84PYE*!%VoaR6}3($laLmVX>
zDQ2-jYUM02&rNI{?4YK1`5(#yl98&EKiMuNY5Xt{lY;_K3uVUm+C>RtKo@dd#*a2e~-}?t?#Hi^x~{LIc2-FxBx7Xn(8^5$ky9(s0TV2udFo%4htswVxVsXVyln{
z&XR$}>eN=J(MZva6(ueV&g5h0v`~X@?l>)p?OBKHJL=}ezgPJXTEZK0od`cd&1fv}
zc;UAY4=ZvD-VFg%lV_&{!E3L{SXmm!UHpl~Regbt=qMoR+?y?QBiYw8Hcm0~ZV81-
z2L)eBawmNah9G{@*BUXaCoVnQcqN1&aq1s}2-p}tI*`UAXB#?vMv|e@5j(dI@B^6{
z8(gD+^4TkBH?2TFe-;;Y0zsEJ#mG@W$YY>HAvnwQMUNN$Y8NU$hI-wG4HvLI3m{D5
zDEc}{-I|)}YUFgOA-zQ+_6L`N91DIa=m>x|-1AHZv}Zl2x3TZ)qcf1rQX}+L0S!4j
zX0L5Lh|ms~M!mS8pdi3Z4g7wSf@7C3+YDB+TegsgG2l>70_Y>h+O>utbSLn56=-?v
zRP1_1`&rp1u|kJVQ2V2UkSV}oA)it%eEKf?v9R?-O!|WWk~U+((Ik?HK|qn0t1x{oOk|%m7q4tWgekQQvgf
zYn0f~p>?F20!1JIDj+DRr>o0~F~k!VM%%L*Am&$JdLqoQ{0Q_Pr{RJTO($I%3h{~V
zCy$QvuF)&Hl1zR8yh&~l1=hoF8T^f;Ixum_gS|#0`0;DJD=f@n^AHdV8A9|Q#17ou
z+%_27`RqJB8j46ayhz))r{6ibxpBxR+=O2txdoRD0|dJ5z~|4}K0YN3dlXS-$X3Sc
zhL#ycCif!t)ydKKo5c4eMFVQW>9@yT+lsdPwnjrVRu&YPbRyuT>c}ZUgfusr=G&Jg
ztCY*)xcM<$B96fYVzeUz?GNIa9k9Q^x0lji2jfJ|?noHz)G
z0#w0W!KPP~;slux(A5D#6^P@|Xc?Z5Ku!dpt5m&ta|snRGA;5)5coeHIG9&ZuueoI
zh?!pzdQpO=igSw7`;2IQr5My|4;*}PkF!6BC$ezP$2#K#=rt5oqk>}wY&7crKWdijsgvLBiNI+*x$JXWIdmI4#Rs$?gRVgWQb
ztgO8N03s!zF3V8Z<{vQ9-?`!sO{N9pn_b0<&8L~D9z{%kWUsHiM1
z&O_hHNfHu184o-~2H}sp(|)yeb@DJ%K+Br|0Ekk8t(UsR_ygb?v7oMs8>Av#Bb`8yH
za9}{Bsm1jjK4WJSpXlK@ArJbX|4cDj|yAyYP0OS%&S&W+O0aGBFVyJ;w
z6P0i%Mi!n!-TVFfHvut#y8F`&1&vlpqJzhfZi}Zd=p0!OlEaqZOhUd0a)Yt%j*fa0
zillPXR6p$%j2#~6@DiL18w6Kc1>RPKJqUY^yMmQQqz=XCA}|;N6Iwvaq>v+J@X!3U
z0(s2+>acS6@F3u8&qoyD1w!?q1524d-Rq^+OixHS5)j2WcS%lu!+gehOtysaC8{F_
z?I5;6V3x;aq79XGisJwTRwkR2YIghnEHeeR3)HW$m7sm#xjcKEoH7`6x!?bnCW)~j
zkmQvdh&Y}AO3p4_2jObPsWm*QPFc0@^$|tPHsp}(DbiCOF$n2kUXR;aiifT{2ZVm@eLau?yuX3
zO$*p_;Q1LB_f}4IM0mGt-;U7gYoH$;^?Y*30|z}Lr+x(sZ>S+?cVPO{_xCy*59GAWcnP%f@EAjvWFi8Y23gEc=%uGm2s)vDxL1p7b`V9|SbOc8OHTjFg
zuZDpAnZjU3DMaSrdQs7~{=vyBz$8?;D6N{kwBwj0Vyg`JC41nGocHp!yIAiS3FM3Lhtt9J;zp(JW}v2seP{
zL!PFKvurnT3=Wscn*X}D_YPnLLL`o9n|0!E-6E)n0f9->ZSd=;{-IdJ<{{P?`uNN%
zad;up45V@n@P%ktP#lnDitOKLiY*TE;S9^~m>by!rqc;G97
z0jY1-^v8f+h6vqLZUc~q5?#Y|%G$O9#AOM&?
z4s)B9cs;^g4kT|Wc@{5u=pv?5^0XBseRQTUWEL-cdHkaj1>Q8zw#M-?{9SjF==}Cg
z4%iF|9U${=sWmrtV+?EYt7Y2{tA*?p$Abam&@|TE>IC~bfI==Oqak8vZT%8%ThQf#
zK_*jZ`^rcKzDBM?Tb4#+{|=I|P*J&wiVIFkWVU^C^YhfO`X0AMb|Q3zGd3bn@mU+L
ze96Ouhtb4;zUOWE;;|id3X9+paA=hEm%F%*PYC2-P;ue}&*pUX7l94X1g24mVPVv9
zCLI#7!fkqtBd4iyv4S+;FRYWmNxl#e4I(Ah{9z9kW_T@ocG8-t3_xd0f8K6KdAoTA
zK0e}fEWXYwk8q>(f&t5I-3q*7M5D@rvPS5hv2mM{xd|={1)8Jc^_bdMC#S7WZ5M!A
zJ%-54_)xTqYZu)TOH!UNm@Y_8iVNSjXQ$78*b*aCRX
zUM!X$e&YxQtYl%4P*zR^iNXLDr=y`wJc%fsHL}W%q8Ih4lh*A+sL=qZgj}D56
zWMza@GD-_s*)x?$h$17qG(<+jsI;Ef&G|jgKfmWVM}5Da&wE_=bzS#$J3zRzeEi;<
z@d_ju6pX%MR2OLg%vvy3nF{wX?_YQ~PF|syVL|QeRcbywY$u2civT74sCd7Lkipz~
z1;EdG!&aN!JwL~Hw3$kEO?7Qn`H7fWnt?V=x@!baH0rg($$Qw*<}MGk997K~^k0QK
zE56L~H?@e4n>4Yjx<|L*@81G^Ek{ob$k4K!aLGnX*Y1?%#d*(;*~X}FuRK^5vE=&)
zt0QAW_Czn*f=*~#xNLb+nI|ziFWuH1r2)m_9r^NQEG^4;`63%ww)Wk$$tKdQsj#6>j`~Y=dXb;lu0=T_5k@u_Hl*&`YkPO?O5|EFu6c7UdBf)BTgO|P
z)UnNYbl7p#;ES&Mx^)8A-5+k6IJ}{&#XpXUdh6A^YmIiBUx_@Ow&BOWHZJ;4qiv4c
zo!mL{#qunX*Be+>T3dS
zbUOCy-`}>1=aw)$xqnwfp)NpAc>XX>M3yh>LN!Yzs8drrHx58MhF{AaSvgOC@ywaS
zInY)6_J-Q*Qo~OUOHSx~>7P6^m3T~Jw0uwt=W_5kJuL6JpN2A84Bvw}{+vPkS>4VH
z7d`}_9-EPX@Vzk!rU}s1SV|peOW6I%_9|=XF%dsI*VX-AT6;N#xe^*SXpsFodW_N9
zk~#`4moK-EI^v|*U!hRJ)b`k$xrr^an=Pq(CF*qA@{sMCAx)g?)>SuHJnqTthi(~B
zTO)=nrpOZ246?y*QEJ;z<(S51&)VJlo|~#?BSs5
z5#RY<`BpzaR-F$&nZlcfvTTZJ&TBj_<+Yt&s5Io>zlGVFA_JUgib~Nc0h56R@6dEt
z!fxN@ZThSum}d*O^TgF>PoZj@$P@9PrE!=i4
zv%eG=dRuXQqE`04Yl911^)uG}*4FO7_^mo+SoqW=q+V9we@H*7h6ExpIOXDz~qWP@CXf95H
z9XXAIX=6a}N(NW+sFgVNxOp7_}CrgGMu&~l)F`6Ey`XK8DN9*t~MaN?w9
z70s;zv?$c`Kkakkv|*hNT2rVRoEFwiOKaoKdGmf)j!$XSeTKs4in>c$7YCXbPo9{&
zX4PPot-drkOA6J@*|U%A>pbqRAx@k)G@D?(8EbfZx+lX>v1ajSm%@i(wjJz%G>A2q6rJ~OvqPngyRRI*6c1zgIJ#GO{P
z{TtR7DQ^1c!GiF1t=3O%ZKwG@Ne#?`
zMJbxNf|HInexEPcixN8Z{3Z~#7+q?~F_2K=4ntC*^
zs&mrXuE5$}{lo0pP1+gm;jGw_T{R}D122i>r1H;*s
z5oauD`_xjCPzy`T9Fzt*AE?Eb5e3Gwy9F{(5Y~vH2GJZvg2AXZofj-P7#px;9hQ})
z<;An4koj=ot`8;~b%R9@PEEVacWEV6Y9>{k%%Ldd$sv4n-m@e;8?B}u7+e;`*+r+%
z?J34iq(3uJfsqX6sP67x+hpBXokCZySMv@Ls(ps`+VTF_^Qd{pO`0842(pMiXfgii
z*^su8cXmcY)^9W(2+8B;D?w`bn0H~>#4gW~+7JiuQ%Yn`A8XcJ-+|Na8bm}!Hf+>L
zysz>tq5k;)Msx0>SO2xzm>~Az9I^g1>eqY=4=rF^7KPr=5<-imYDjDw*uW{Ypi@ydP63MrR-E5<=H0p>7MXC}O)m?XX>N8Cjh&6bjcU6l@i-1PY7CgYrN_nvir~|0Cpx=&Tj1bw3lT-WQ
zE5j!xD;4E?X;K3vOAsY=m}ckB>-bKnCosfB=j;F`ICJDd9dXL*2JZFl)v!M0tTB8!
zd3uyAoplE_E^7$yoXgH;+nNba>EI5r$Z2UtA0T%pCv>At+}%Rz!@)ye-?O`X@1CX+
zv}WICaooz5!{Wsea>=EkMyzAs(b~6KJZkYQwAbeKTMG+E&2{Uu>dhd?s>z+N+a|0o
z+sPV%w#HG@8Xggty_aJ7N&K4CXg`~97wzli*!=?dnrQHBdO2}U`>_PO1gD~*Hp2!c
zwmV*F{#?CXi?!Zkh{4mNMBRY+Im&hRp0V1IRNN#%?cy74Ti3v6*L-v=!H$sFsJD(t
zA~$8`NL5u8Y{TylFTNZco78Xi$SfA2+S_;Tzc<}XP2F~A$cjg1Ry%fl`t(%!S@m0$
z^j3XStgDF=vouqK);s5mE*gB3Au@c
zUE>Q}Sy|MP2mqR_eEF2Z+>mZLsY}E9kJUq`6PPJo{YKt34o%Lpb?mr%{j=6q?qD}k
z^53l`DAtOS!w?TJy3#F!%c(J%l$D3rv`=WWr?2PK({@S8r%#I?brLE
zs5+>HCnA^AgPZQPrajvMw>Rl%8@cE|yfXH0Z)n1ez4v(|6`V7mH=P!3EoR!Q?=T(E
zBTyr|M?ha|?OVw&Luhb0;bO?M$Nv4Nt@p(X8P)>qOaZb>3Me;X1ntyLHUmaXsDpi!
zW9Jra!f{uy7*D?#GC?Hv
zL(4H+5S6M*c}1NEf`0I0>imuwNLByI{^Gj}r$k|>ymv1LTQzzKuyf7cs#m<$t}
z%Fa0bt&DpEc65Z!XP?S1FK@j~9NsMIh|L8BtsBSpy3Dzwa`oUQ3yUj*4K)Jpl}0sz
z_ejQHf*_D_)qPH=k!KP|W6bpHm#&nX=JYt$ImXtK>rrgN-Nqdxd~r7-kVMf_(jI=g
zrk%RF{?eRLcpqrLR}0*2!BYi1>%)dl7uC)oGCX_)*Bcigs`ah}aRNgo>SM5Fg$K{C
z+_7`#2VCCLAZA$ejsqb#xWkYayFW!1_K6&Jb-!=Hhbj#zPhN9yzc@dqIa_Y@^yx$G
z%GN-YiG07`J}q>W9F)eZR5KmSe`(g1mmM8e>s2)^HrOlotVedY!-EcoZmeFjP3_Sp
z|MS7~6`!sDG<-qtXJu~deCt6kNBmfG&qW`P^>D6Y*b5ho28L@xb*kU|zW(|rck0=T
z7n2hc6=0{(7pjwAyub{r&${Ns_Gd~4r2A2H${9b5XebfI7t|3M8!P_`h4D_
zvk#mM(sNE$LP{1s;@Pw02M@NxT5+cTewMq_-@kyKj|jG{UxHQGijnCrdY^y$q;}zT
zZ{y&XZCvh-Z4$3FXwW{Y9qTCb7EPKqrFO2It@g2~SHV$9=4M;l#HcoJ?(gM3eJ>O%
zcfjhFCcQ{M
zrA%A+ou^L)7R>ov_sQ$m2UwKuK+&EF16lJFFc2pPb3gAc!rq%TaA6nC0h2yu^{jq<
zDxs>=>$|o2KTBhcp4TnU?Nk(esQTV|jdf)Pj&^ESnpk!)2%8l#DyhQ$;*=XpBW=oZ
z!P4Tp#}WxinI7i7t9INaQIKiiEqUqAGb*4^#q#)NTi%bbB1uAn1#9TFAo};^lKsoB
z8ch__JiH#GYDx}_QM(8_6n^r<#`5Qt_w!4=tuEDm&h3$VamMR!Vd?q*Hi^&Ye8$P&
zg0K5BH!MEuXnys(WwWblamr#_WdK1EfbGO0EzMucbvwW8@Fbg{Yc=X3^WhdCLsZ9Bmg|rqmTRek@EPNu7{Jtzmy?
zy50B3_=!d%18H{eZUF}PK!pYP?1&D@qBJDkoSi%)wD$5MF^EA2x4M^mwBg)4)L_Z%D&k=)i)iFoqSjIt9h
z{WHf1FNh(wKNB~%k8{$A7bOATg6G%oA`b1Nk&z=QozV{9B-i{3mH+4SXK44xDJyhy
z*R)hop)ad?w!^vKO9KPHeKhQ_Ve#YV&$SH=D-Oig&hbpRe2|93r^hFW9oh86h!H|_
zi3^3sts3{&Cs&NFFMZHJlMi?koLo1KT}32=wzjsYiWJ;isbLP=wk>PPhffRM(I-MH
zZkJ5{!JL7>qMO;xd4qpI|Dg;=)Hxq9XRKRPSo6a72_IfRdl+8r@+RqMZFSem%7Q1h
zngi7QUk}%vRQT#TMH5tOtNqfIQ$dptb@yGk9E-7INA(W&Uf%jQR^npn0%({NHVxs2
zodC(S>1`*R#%0w3hl+D%b_Od?-->#R@0c0UK7^$=2T801rOSi6#%}`tWwy%x?blu`_YvNUm-Sa
z6d92g)}_x5NqZoj9r$e&%fF`&OYry#yU@zuONeo8?V@K73kTmO?!9)wfOtgpOSyYr
zw(%Ns=X=(wHM3o!4`NxQo5X4H;@GE4pI)sFfzs@UfAqwKHK;Wr$@6qajUBuF%I>>S
zEK^CXOqsD}jhREun8~RxkB;vo4X(Tuj$Y;vbjg|_
zBMmfQrFn4Omk-04{ns#@yknI1%NH+p?b=oROfS3M2xZ^eZg+gumT+>&b|W*ky+zXb
zA=+;MMz{v41!DH*I`r{2zP5I4d&}|Zfme3cTd_93Wc#+cogNkSNN-*_M5pH8Y2`(I
zvuZCU#NJrj<;;g+JG&b6u9>mE#NNKWk=K;7nv=#6oGrdFOd$E6f~~Tu>sJ)D&f?nk
z?t3%yXzC0p+NsmLOMa8fbp|>wUFDqjw4C`EDDl4N{0}nVGhF-l)q!v{IqGbOzKXBC
zcT{vVT$X3o&Fgope=Gv$VDh&F*^iv<#FGVMoH$PbC@@C9pm<+m^~aC*#~Hi~w}I)h0uQX5Q!B}Hqx33#M7UmLV;PGJ=e3D^)?
zNOZ;&?8faSMi)Zi0RLjcCPV~EAa(M=b1EAa>wtVNmZwa20d7RQg7q_zmHomaNv*?(8L=SE|{XKmf~jpQ#W_O
zgh^IAOo5{J7hIJR0k}+is&TruM%C$IcV8Yfv;*mM1TG3$v+
zvzfDIjd*))d9tr_$dC88>RSf)eh3<9Gx*K}}@QGXg6Hb3o*63F8Fxv^OJ48tA7R`;{Yo_
zzNDvA`no`$vvm~HWm?}XYDr+q*oWs(8H<>`dk^ykpxqBla4C7&-t@E
zK=6v<5kCTy>rE>A@o6Hfd+wvdis(8tKs(dk9~3k>bnj8YS?c}G`+ho9oF#cdBs)PO
zDF5%1$m9(ko^*-K(|x4do-Awvo=)$LrtsR{!CUGL~d&eOI
z)9-5pT5_2SVM+O&gVPxWCWL-|Dg)blFHr3RJ1#y|nBTJWGSIdk3zD|^L2G{ZM|W_B
zVu4To;Q5kO4t%3aez>izHtDC2Xz#!Yq6fq6gE=!v%=r?hCzC?7?ze2+I_QLRIX2|E
zPvVCjin2-D_C&+(hSIB;6D`?5X^mSSyB$0nynwa18Q}asqURV`g@t%4Gc)*ygZ=Yy
z%D#H0qes(4I1Wl;?%Qhv=ztTKKcqzlTxMW-yE>2X8!;l`Wt`{vV;9C2GSvjyDHo_N
zrDFpS_Y0VAL$Uw}jhzspkKNU2=FD9uDi02H*FM#&*IAp1x9kqKzM5nB$ZYzdm0DxY
z&3ikT#y;EQUa;D96N!l@Hz&sgevNG$K|=@&Dp;Hlma4uE2-&x(18+ikV?gf7I205D
zEHwkEOE<%a-PgT~i(`u(Wu@0tZxZQ0-Cc|aVm;?bex
zb9;TMmvj@b=A{{qQ(W@A#%i~0)uVx-CJeKf;}$Fs3p-t|@8N^#-q}il)X%dt&nRSA
zmmaKr-?7@cm$TNkY_-236f_nW)9sDb>WwIWReh|%h?`59uRiYJp!Ep}36_C9^!6+%
zzp^6xcCyj)|IY=m2sk$NMC%@J-(2Yua2D42eva+Ez7~mIGh;@X9BtNN$&!CZO`O;-
zYi|Pl3awRskW(c?ay+TIkSJ=`!M`VDKn0
zGv5Bi#smQiklTb_KxP#Qt!*>6Q?X+Zu0Eul7s!Fi}Azf+!&T7
z>Xs{~9;D`eDp3iig;+;skH*^0QI~OP;WRo!doAj&^j_w>OSvh{htPYdFtT9A-&2hf9jQ$XLmSiK?<<1R!!0*x!1edcB
z^OYP!9ZbqgJ{T&=kg6
zS&8D$GG)4*=8PQ&)EWgNwGkI0N)gZCKQUHPWoS5s3>7~qb+I+wMy
zSznwtX=z54g!4!q2nU2Y(8l!7SBtmqBfKae^$9KPfjz^HpT6Y~>p1kyo3~e)3X<^k
zZR)zzV4Yrp`o_I>%o<){;x$gIPQtC)vnvZ`4fs&oDJATyhN;cg*5M20T6e8XoAQq{
zhLX)ww=ZnH{pS$p)F%DfIQMb3(l-h;aIH};E?X@*3&ob@m2N*Tl=p$YK
zB(;mtjyWb0I~!xm@xG4ipTP%dWuACr)pB-;V@^t{-yXpmm_^+&iE!K;ZuW*6T>+lW
zn*RvvuiJ|=8n7~0vFCk#csYg6YE|0&q;U|tZ=TEj&EP`FqXn-oS(#i>HZgCI*@Wet
zgWoz286bJpkZtZ|qv%J$P)dzV%+%hZ%L``hF&Qy&=uj{BWgivYMjp*+-?_<@6Okjg
zobDZ>+saCPpo^n_$K+rCj9*Fv{jp~KzZ1O2})|t&r>+V9XfjMZQ`17SJN?lQOq+G;|ra(qy
zZg%!?%m|-QoQb*s%m=dC%oS%8mpwpwrUk9xd!?s@%c8cXF*Q!^NGj-*eRNqa);;LC@`v>o@CT)NQl&wQK+0z4v~4qUnaTFwL0;
zXGYouAG&r;v*1IH{n2KfE-oI=`p+sn`78O5O-vp`80DZWY+VH7#=HYT-A&c|&!ou|
zN43!;uCvjSv4T^~GI|fR;rE;MJM9Vzx4PW0SIwS72YSH2**Tx_Wv}fOq
zL!~40&ob?R$)97B-B_1UOEiQwDgGUt(`I;=JgcsJq2A_^Znw>5x~+Dsdo|o5WK;K!
zgQ|*B6zkRkL**dTu@GJx`p1p8^Jg|R=H3ZYeAe7_iS2qZ_afH}zap&TI}1ORj9)Qu)9>Gd>`s_1
zG%)z>v5F!^x8=s6GH?D_z(j2NVnf7tOt2>2;Ej@tOIO(^Ju-@N7uJ(F%urQQ+^?lDINBzC@owvmqY37*^+6~llqWC
z8*Q`IB%FZWZN#9eHd2Dv9J)yM9Q`pk2AHa>F?48$jV2>FQX~(B&qhPX&jKzjvLU-I
zTAPRctqF!3sw)m=A3h5&GZOk!+Ne8RA?2t$350Jt_qZcfKkBv^Iv@jf`{YGJ2T{X1
z1nIUkY0&V!Rwk>1?v|tcaq`+}V*+)5KG^nnPj9`$FJHfYLc;XV%*Ai7nR3=)b!Xt4
z%7I;pCFLF}0ej5Xlo!Wj_I>tbq+0h=C&s+Xztr|{w)M74jm)ieJ8jg7-gD6VdZ&WE
zPcu`lnJD$_F)8WzGS~g~Qy(9-Z1r>BtgA0P_j7Qw_zEX~Bl%joUJzMuptD)~TkGk}
zcCslpZpxh|eH3ghM91;mAJV$U&E;{Taz0<2XUHqCVhN
z?Z;?NSN=nYXHTZUGH-E0?TqjVs^9!~$Jg|mKOO11AoP~!dSkso!%|_#L$AafwuzY#
zV@r-m>v@$(na`H@C)|!0A+fn$`**H%E&*CONF?%z%hj^OC6ICD?SzcT;qHW#GaxaM
z3qi|Qt&cCUEYj5{9Z^axxboGs#t>lx;)32eGHO9AA`>6!(f(Y#z&n__gxIau#K6CP
z`#{TNKFw2SqbGkmwSW7(Wtny8(1Bkcy=bXd+yJY%DFim+BExsm0tYm{ts1=U+C{^D{Q@{#)KED)c6p@LOzBZ_Ke)prQv%~>ZlmQdFVE~yrlIJ6HVzugDmsDzI;^Ed(y}weZ6>u
zEN`Ei;i*x@*}Say_WS@+U2#}E4X@<85XTHdCKJ*myPX3n{m{m^KB9X_>A${ieQ~!#
zoJMg*CU{t6aZY)IgEE+NNGe&PuPwo2
z1Dq1Lp>Kn)rV*cq
zHfV+c>zLTgp7kfhT|ki%jK2IMaiuaUtPEo&=z
zD)N-xc@~IIysL%-xYc2A6PPxDUq7eYTU0K-CC798sWm-rZ|knw2rT`qWelkhyGR)a
zE6N4*^;IjN2o6fe)O2so9Rkwcvjw|xzC#>l=jH|!+05KOlWO|GLBjyJZ<
z%X&RI{d$nJ&(BO$@VvFvPWo4|@4cj19JW4?(;E?$Ep4vxcL_@VDnDRSj`mMLqR6A?
z1|IV`O_z^+Fz0KKblWYX@g#O+NFufaTA$m&@Wd{coF-{|W&0A0Ui@x#(PKg};HwXp
zqzsR?5zCKcY0wQ&{^fkS<&W|rHJ$0#`gigwR>aOs_uKW`X-;~6=@sCwAuK^njW~r@
z(A?51;_t6_PGh7Xeuuc$RKe0m0N@P1vhebS3!|cKcJJLA)vDLlF5X;cw0qD}U`U8N
zB94Iq9Emj5OYZ=+BbP&&*(G^yWRc1cZGam;Pr{JqYd0eq(uuUZDnfs^cO1H(qSUR*
z7{qgkM?}}HZolYlcF*DYIF3gHX_$`M9IlXye?3v+0k#*81MVhg#a&n<$)w#DL)%!%jl;CdG_Z
zw&B2m*Qn4k2aC|)^*j|t2`8`L$?Nd)w|in8t|nph-+{M=8mJ$C%l=|5#}H85mlEwe
zRaU@H=!tarph5IHe;<7!zk5>Bo3&1d<1?Vx|(eLKSR6X1yW^$7&FC&NXHoq>?cpsZH47q
zf;qqt!VLQR`%nI7{5l>DV5Pk*b>@GqkB|RVql4Bt3xPp85W!2H&f9{Y<24qK32o}n
zYZmErhs75I3D#|@oB!5b9NSn3V6Fj&h%!bvX;PTRL~=)%Eh#QO3~iKrsqtcj6A1-1GHMT>Ax{A+;=(TvjvalT
zoCJIETgJojqGFi=Qs@Jp1kANS7uA$-v+Rym^)Jz2D4P9
zE1lIy9&tOM+3{y7zt|z5xsANoKEw~uMx5{KEj^!ZFVAVA!-R7x*=G$I9v6r4_bhU}
z&L~A$Z`_~4xFLA;`tK%_wZ;t^Np-2-A_)PIJJip&*vTZuR%SyAT?*L%7JoCb-@fbD
zuLmDHyF)4}3)4CKU$T?AQWC*@AAEhbM;
z3W?^F>w|@M7%4#-$Qo(Lr+Q}`ZF5hu5O8)&RFc`oB_SHsKW$=crSC${+p6-Va^8ZK
zDRr@=^<&-}u*WVU_Xu?d=!}F!0o~W7QA~U9h#mL6Ljgv;MU$
z68nX%nFPNy7MB=M(N$BPJPF_nw1+c+?;bGQyHz74R0kJ;l6+b$(=ajlqsr}2ro>7O
zAm+?*;-5@)Bovz-Obdz%!b5D7t=*^%T~W>)uCF*Xs;vL2mTVCVOUo|W+W#=lTW8Ck
zPq`Wzzzo4}z#_wF4h7!n12%}3Z$cfOC1_YOs%dX3pf=!fFq+VW6QAfjY28ydyyecX
zreVkB?^FO#T<>nTooVoUzy&~jUSNrVDAW>J?vZT
zB_#l1wq!mE1*GY_?Q%C7|8Qn-;zcd3JOXWw7PysG0QjtBXBun7UaX`+ojo8a{M2AY
zao=~YTCj5IjKNdm)mKWNGD
zX8#(8X1@AL3qGlIP?(SQN52Ew+-Aja8aQFMW0Z{JxLvhk+5{DJr6`d`g>z{?d(1-<
z<~Bpr_*15g7!@uFpWMrq8jZ@=zK&mWUG*<28%oV8m{gjNWjKWQi7}KR=
z-l7CwARc4M|If~rH`MGs3k>{Zy>@Z_t+#Iv!PSPsbA;Ub`&m43-n_?OA)avc>;7=SI9u3QpPJhhJyO!
zSnZ=wL;mBu1w|p3qbc#=D`hGlb%1=7QeMJL7nj>qT_0bbKdPFigq591K_B2i08M{U
zL-K)(WW)e^=DxXd_Rcrw5PjEUlTNh881eMJaww%88)XoZupnE^!VurtU%y&$HeWn_
zv2rS{t^2SY!Oh@+-3OU>U$1_{?8OI3bLD<_1+Zd;WcF1%9l!2yah7hHu}#1wib2-u|`FRIDV~9Bg>v+M@9AiQ|LNBk3xDfid-Xq%hXSE6su0bY=}p4NHW+?JHFiL
z>e!r^c2|0>fGv+Iw)!{ZHm}xt=r*V09UE_+pmUF8;)^pUCF#A;=0L%6!WqJzb
z$dv!ed7mwHzL%Er>{{Av@#18zPaGICw>Oz(7-4xp)@B5ONzK#@tz%wPy5+6~9`hKYRR``PrKR{gRzesRaU(k)G?weG(83
zI^|E8zVtrC`3YZsE@ZOs=yyg1+`$zF!$)>(<(be}+qYRarAhy|d
zjG=R_?0y(QwS!@n(bo!NH|@)%BK|hV|2lbD)boh8bGbjL767esD|agzbH%&Xm|L(L
z7D&3R!>4qt%n|Jn4xuHwIX1B9&Wo*?=tA?26AO$+<{6E;G`fv(2hbzwdjNs3RnjF+
z_nduo1tqzZtmxQ<(GQW7q^pn%%Ohtsb0cstIe`_#k5;$thHf3p$mQhHC`{)M$DsTX
zLoi{y&yfk;F{J&2Kt>2^6OX5ZvI`1E6K_7zTTP~+NSXns0Zq!iGVJd)`DD^yVV8&F
z7f+>%gS@)^@S)Y`UvEM9L@}$L$gU>)={+zF%=PlBdpDRA@yc*_O92nC&{~%e1?n-b
z{xOle3CI~hH>Ny0Tz(Pp%rgL}NWaKQEj+<-*kMiPh)lsTjNJ33?pHpK@Fg+w6
zgq&D#mlu-^7ysK(?ohd>SsX&XK%KzAV^)d)>M=&pUm7)HGUMqbm0nExL^#LP*YUrR
zCr(s8%?$1}?MO|$^e}Q%Q(cnsnJ*Fz&ne6B?8-n~!1s8@t~_5=V8Tp7hvl2J0ZZjHP)CLsdwR?NlbG;iL0g09Ip^;6AP
z67ec{#@c&BisEiTHT3>`SFI&K0i(obDgn#Bqt><)v;3msQO4aI8tcGK40vuju9Zqy&S7At>T{s<7&WYxk>&CjpNJUKN0
zO0@-@jKsRUb1Qzw6bZ}=1-xcl2hy-gJd6Y
zmcm97*|x0iu{JER|4}Q=jmw9xO5Re(^Ksq+m^81P?{Th{csQTVV4jkS0`$S!!b%yf
zG@O)>iT+^r@zu2{1wB2VXLbtv+tjQinR$rS2^WthYcKf>F(#fm#5=@!2V@6
zuDx&{C99wtNe*0p{`~pL-xA2^BXG|Py3nWW^;truFw06)VA}!n_SU=a1+84d$uNs%
zHDFw*dalm9cwr$`RHRAGCBn!$TiF;GG-rnIr-b;_K1A}EtQk7q@lo2thiksM*9>Xv
zU@a^$C20=*%5X;GFns3_A3>ik<%Zc66S}Kz?=Yk_z5mSlib@&YcT&8UC#5Lq1;z6l
zb}{)5IARLRcpICHF>Acv?U|0n)w}R4tbZNVPYUf^^N!l6w7=g`fv5|1Sf&*w}DLh?hdPG}0>8Fb;bXseL{dGZU4<5Xq57Fv(T6bJ^{&55KbY%U?9@?BY8xqs7M{zWofaoMUR#Vv`2eZ_EdKHfB2CiwkoD
zp06}be}aXT#ntGUl2weYP;lILyn+G_SS_q&3o>rlnZbwDRlatSJNs
z@(KbJ9ET%OEqHq^P=@lsA?cVm6j20+=153M)scCm{pCkbOWLu0yJV7sXo)~*{$E@R
znlX@OCf5*HtM#W}G*g9fKa1a!a&`Oq^@`%q!Lc|ZT|c>T-_1keGj^|g`9@3(us79E>oy{8;+&i5wWnkHlGq*e3c#fz%B
zGuVPIF#FvD?;q_bgHHt-0$hH4&h$q|c|y}Io)|Gh^Fo3ZYJz%^Dap+t1&U(aI&m_h
z$IzwIoJ_kwpS{Crka&YT_3noY%KJMcr5_qn_8?(tPA!8DC0!@vpjCtLk|?dJrr=y>
zc(?~)!)ExE?Bl_d@V2E*ZTYrrZlD+3sG_Lx`XQ5u?C%Vcb&ZI1!riCESw_={ufe-{
zHZ~B-63h(^nody{)mh8-KO#(~;6fVN-%4&g@`_2gkSr1d2;|2I3XWh!h2c&g0&a5LO03oS~{0xY;l}`Gi1_2XD|*-KRCcXRcU38g8Qm)<
zemIS?$WH`L%VYtrWgVNobZ`!oL`t*mZAS3IqB|>Ha7q>(2}p(e(4hBWL0TOD6-
ze{9k(?tYU`ucCONRMe#m9MReI2ShBV_UVE!E3(J(4kWwCj5R^RkRdQU=VfLhO}|)-
zc=bmwbZ_MhpN&OoGj@~-i@LW;qC-;Z{uy2+8+)1_nT63!+^-y7d!4ov94C;A3opi|Hj7zadD>PN-njdWwG~<{?{0L^fQ&>C;p?p@NLTJ#
zpv^e@#(ptUj1c3O^ZE0(*w|RrsCArC`*4J@6U2RBquiA8ahPlY5M6YeoDf+j++dWP
zyPe|JL0sZ~hJ&*5)9`&XMZfRsc0%v7`ODEEF3wAU;^IJI3ie)LB80Xy*E>2orXSpO
z90)|UvYE{Z7d+H(so!qFQ@rL382gaN*OfUvrC6XJ$m_@B(R>s*-(LI-iBmfTCc=d?c`2Y|mx;^o<7GFAQ5i!93YQp(E{*AK4t^gPS^
zhzAlQ@)Rev4d+yN00|*VZOfzkP|WP~Vg@ipN5@=4co1>?2pvbgt@L;%cS~1YT|ek)
z^S-wymim-Y9Y(d00la&I{zyb6g>;p}ef9>JKlxY3f%=uy{F
zJ=YQ;B62?;E~w?vHd0f_6uX`=KUE~a9o{))@bcqhZy!;jAQ{bR&b1JzI?@P*(-%3;
zor)hb5I8qV>yI=VrygiYe&;c>@lLHebxNm>+8h?vi6L$8DI#6D=IA-zrjM3Wco7*I
z?{rfXu)v0|-ksaNy@{a8MAoFM#VHfXPT;Pbi71KqY&qh>sD!oE1R(nMJgXTR!w$?)
zaa4e7{p%A*J|Sg3jugb!Nx8q!B7*qYR1f1(n`j5oz_$cAjT8H=lv^g`=KyZr&sm}Glw
z{8RVq=^-Ov5=GSNuj|FMpb$Bs$jFbc`8}WAYKpz{QKt9LiW(jklye@RBe>#_n|gEr1NR*fVaa|BMzbKbc6XQ43Yd5g-Yoxya0t2_kzjC?a6
z)gm6!COgfX7JJD^~IoU(8e*O9ipZ5IW>mAeB
zkQ}?cOQw^5|3so1dEAOK;Uy&3*5$?sX`F#cat0L9S~Z2Q
zObByhDn^_uh$pJKO469c{qa|tDp|%-rziL*
z5{uRf@yUy0h|L_uaist<{&q2a&nenz5w8fUD_}t{)4!qUIe&fGmzNW}7WDBrMD>LQ
zQ_L=81=}@>$lu?8u7t6m^z3$OFaX5E&EmT3QdRG9Ogg@NIT)ql(UjQ!9%Ya?82T()
z&6o*?CdNZhRO`xx3!z;pfeX|19OLeG3-e0idkNtLoy{P`${mijmvEp^GzC)3GX1k>
zx2N;}ElBGi1r4?kfw(@Z8Qg3HJm%9r5#ai!9&x?gA${FHj1|K8yKVyo5~rAk?3v$D
z<<*kvNqCV6A|Y%$>}i^)7TZa`VIHp+@d>P1z@H5?9{PT)gbEHxsAH!!q6^q6-%-he(thQ#E#`;@`u
zyu@<=L!|fo^(zX!<@=;XoI-duAIl3$(p3L-#fgs@(TzbHa1YzK-R^GvsV=QS(yPoh
zW2)VxlwnicSJohq~2u(d7dW+7YZ>Q=nQNRl#S}zT46LbTY
zIkh^=D?dkCwY)pVL90gXBD7XIn=6n9xU+Ov^Ng}GS+!$pHWyQ
z=lHNPT|#7gI+s;P8JW^b!LaEw#4ZERyDGJyUU4TVMQ4;T_7m!d^*sx;MbK*WjVl26
zUh7JW0ep!2psxmrEH!bb`2BaN4lfv9Es|@QJ6R?vQKz~
z1w|{DH~{5t(*%hzLuXSWLM&|#+{V#u^d`{Hpm5bw>(8%MTf{ge%FfAW-^9bCO5TJ(
zbCS~%TlR}o8;uQv9VJr+3~%~%mVOf9)B7xXGYG$FRQHWP0I|Ed;sA+6g*%?_V*o5%{Byo?fhzVd@WXM;ltFU?iw)gOE!Dx9orQ@}*2g;B?*y
zG7R{l6=SPQps*FDK^6s}lT-X`%wc(4*pd|Lo2VOL6yv23hH!xcy`efX_2tW#m`%Hm
ziTiL1*8pw}Y_)%(Q5SF0TQzlck5AcI-yi-ARR;F_JbqIZg9GQe7GW=d2ouI#2>!vV
zKTkB1Y&CGMJmCWeQiX-YszfirAKD7!D25F145XCKme%51##$;DAsl3eiX6y}Hlk8t
z@`3=spL=mt#VNR>W3{z|mm>(!cIRZ!(FN5W4zguq(t6`7WOgjwk0Nrv~MH%Xj!$FEe&;|H37SGSGwSexKg#Zef
zcx}qKE8Q*s!B#9G8tl`1ckk}zuHo_Xk|_kS!&cv3{t_R*w6aN^&jI4QdGiVH$%M9WK2k6CVA)&qX!t@1rL`?@
zeQmUQM>k*P#eSN>f3nEH+rn5da~+h3UTM_g1%d#x6Why2hWSC$*?6>kWs$UEntdsDb9;uPP6S13YhPJYOT^d#|`tpZih``6<_A_094O
zQ_BnfLW#AH5kG&uzsHI~jg7Z`<(*)oI7~;TC@$iMCtX(J*4YjCoyYqY`^3$*9)iye
zGT@$H;ii729fGINLB686mXzhYZ
zVujI$N{r(oJnm7<+*9unTM9pi8L|AOF=q&>*TJB%-|Kf=9(g|?GiNZQ
zCX}Qvu6X1)sr%%Cwrh8yqrc;$%$|h@lmQMhlh_O13z}+P)zWsrHH`Mb7CS@VgNL&5
z1h7#Xa{KGt9b=F9CD8^H#$6_I@`w>_!h~IKMm;rQf!FcNG}1Sf33)PGEBmq41LAN{
zb&E?&y9^#||F*Jgk)dbS_ZjkFU%zhW@$^LFnGhce;y8lxo`_rl-UP|X2rvaSpzP?=
zCBDizRvE!6MnwM~>SoMbU`h~I86qE`?JLZMNvO+LJHzX}XY3;*^bA(?C`oa5M
zP62uG$g?MOGQx$gL*L^AAA8)ZcB3@l$`#DdVpHt)iHpk1bCDi3j0_BM5Axa45YI-vnZO(@F8^RJ`RQI7UodsO@nVMxtOyF_
z+-!>F*vG;=PQ#za29=)NAAe64Y4-+m8`j!cKZxkqwr$1j4sQ(x^jnQdn^8ft(4>S9
zw9ou{1^EFh6u3Tr)Q89C{*`+f(gu)YKEiPX4g`n@`ZEXGG_lSjC|}0n0d}#lQa63d
z8Dpmg54Pj;&PD7y8d_iuB&QPGMDfX2W@RfuREkxGEjyCBNp_s0Vy!CRSGK^ZCtRF+I8@R;yk-Ze%nJn1GOKFvTqk?@K1GqY}8h)@2$p}BD)RXy+t0tpN%
zT7;NfVkU(9#Lb;C0`h@c&(7aMHD@_PPmiFEV<^~j;6M+h$6a`Cz{z=U*5)tQ%Obc=
zQXE&B)|W*;aNy({Z(e-A7BMW18wF*j?jr~h1s+lgCAqkwfC(;h=D6qHY06UjZELrj
z#j(?6tcM1c!$ZuS|4UgXPV~nM)^YXfN&h
zlbSlFdWvJ`VPNs~Bd>6BU(<@?Wy+*sq!qLroo>68BW~{rDo%QH_EF{>evTI`A0(vJ
z(8&}G_!Z}6bC{W3tKEb*Tfv)2U4w6LEHA`l_+i3&4e8
zE!Ogjmd&K%F|=!{f@>ddEyP99m9GdS4s7qURUq^lRhLmUB6%$9p}z
zbt`iCty;%|pZk$?
zH2%klKfKL6I*Mr9oi%4p*{ii?DC;OH`8Uem870-Cgeqkrpp&4qo(-s3*
zsGWR&CKi|9zLsLrFXX@hS_z{D77~+6`f)cjse4MaFc?q0yot7aac&QAFs94xaHy3<(K**I8h@4E59ROJ_F>R<3d4%PC~2|6!hLvN#sG
zrH;TooBqBA6^2UV#C%)`L<@Gpq$yCZHWGB>r
z&1B`plfchcSA10$BdU|z8EQ@iw>*8<*si{*jLAXZ4R1|86+3m?g+TilTm6Nv`imu(
zLxzdO2UUM-!PH4;5b{g^WEkN^JRZ)
zfCJB^JC4#jy(dn#Ee(46U*#Iw;JxYFR%>8wB_Dt!PZz?gwYj#ldyhQwTKaP-pgv&Zd9*4H%%+tCv-yk~)72#nZ{;FcPfY
z5U)dL9i4_A{de{-HjX#%=E}HmCsBkcllYt#@3$o){NQVJg#$|WiqpKJU|DpMAI-$i8^~&j3tG~)CB&KLZyO&UX&vF3E=;GL
zi`fi>jPCo>__*xMCBW
zEVvY_ugW7lKke}FEhI+w)Gaqxk>PXcdypjT1i*g_y;=ZR1
zf!lBTe*dE}C%TJ}frlCDA8rTt5M1N~_ABs7s!dHTt=*`->=!<9U_?V4ar1R)#s+mr
zpn(G7Gk$3XuSuiFy3wB}4jD8^Y-vb=B@Z$s4j8|59zV-$Ms01XH_WhnA*urxhr;=f
zj##x5cMP10yc2$w@S)tWio162wqBr}fVEi$QcG3Kp{&nR-rs8YlWS@zQJPAR35q~$w*hX6Z5WT@M90~OfNx`+VEr(?JqB2$`9
zY*t6o|pUl{y2|w&V5VY@8|P=U*mPXUa#wO;YA1V{5U?)VrptO-g($ZswVYr
z+h^{P$x$CO5CqK1JMyrJk3NjefQ8MP6yI;VCpIC$0ZJZYk0x;N}k
zzgsIme>DbUE8I29EA?qw^FN5EasE4ymg?=5>V~f;8tk$O(LM~^p_SF1u!Cl-D}-Mz
zvHC~HSa3TruhlS_TVV)sv*YTbqyhHZ`koiRyl*4=VH+6&&?YmQj`6|c{(-fDA&r{B}h9W_|xqk{WP
zP@zoCV3Pnqvcu)o0pydGYHHe(9tJv-4Vu7#?sX);{nZ;xU?LCBS|HwL{PXYM59OT;
zWt2fzpx#+KK)1M{`$3=HW^JLoW>^}4j8W8IS~@WKuM}}V*3G6+4Em+V)(7;u^!t4`
z)(XI^yycbns{cM=FNNZRb@UH+HSqDamT2W&j?T#Rua|6HVBpOKQ_<{UET8BoHfhZ5
zLLOBZL+bdQxU_!Az#-2PmL`ElI!p;r4Gr+LXrQ9;AT5>+Tm<$XZqsUyG4;cP|L66R
z2?w1j=*wCn1*Le?%`7|408bJvM8JzXgCMbC-4QY3nMLB*+cJ!gTFzkwRJkn53^8hI
zP!$Z1y2i7-m3C8S;<{{WxbOAefvY)!PRZI+N^4#qOGfZe;WzP
zIU%107qfAZ>(+hkx8VJzB$uNL*}lkQ7j8JEx$^vmMq`esk`-nBh{@8Ct_k_O&nZ2i
zO#kliB(7r@E(aggy2k@wD@Kpnefy5PuAy>h4Cp9}n2bJb@Kj?UrGS>t*0eAaj|T$x
z!LbP-NF2buB%S{~o*TLYaTeqYvX!+i#p|Tap<uL~A?%2(?DOt1lty_sJvO4fD
zA;)YQwUvTw%P8u){{_?-^}i%IbzH0--)Jm5Dj+Y5SCm`gb$;0!JA@tT^WPDS{^vK)
z+1PjsLqD)a*?I-Ti=Cm7>POeK)X)fkz}9c^8(n7QFyhckFyK8@`v#pp*t&m7>k69X
zO4Qu;rA3*-RHi$~1-eD@ev{Y%3#}0tqCV1EiGKY3KEK1yvV`u{t02GHy=S+-vz&Eo
z?DTn4*QWmG>rwF3yX(y0({vSL1%NjslnzZsD8Dxkdi8n|$|Me&UFU#@5Lggvw;_?s
zqQ``U1f^@Xar!&T-F})b1HVK4B6S#O+e*kybZ%Hrjd9oflV(=Qfb+Z8&++5O(_8VS
z`bnk2V2JA!V&ZPe$-fmnDC4-Hj)bc&EkAR(7cr_5BOREo;8ymd{6~zGV~>R|BH(5m
z9PhXH5;J!1+9k3vur6ezfa3ct{Zs}MT_t-%9tPZD=*S+i&O8f1a2~BSFfm80=vtct
zG`PIn3%$yA-n?L+Nx75`bGb&749#hrHh
z_;Ra-LeEU|yL~?T_wBs&-ZSgYOZA?34qoD-{?IX6zsSGw%dRhvja^)tS+Du$7q>Tu
zk6onTGhu>?hM|AhW`{~g0|T^bsO4AB{?+|+(P6Pzy~%yO^4
zC%$C#EQ?ubh@1GMd+*h@8F1I{&80k7mW79M1_j}f_@QY5rYj3`Ay~8TAV8fBh{*T)
zt&s>-H;y0MJ7D=XW+MwRtOiX8q|L;+f6Vc5A$+9K@a#CB8^XeJhx>g&8dc$bUYT`6
zi}99}ogv^_1UO-`!wQ=+f4*q5#9ah`8p&>>6wBT*q=3Z39CiM}*YE$Guhi(PZd`KM
z@WlAOPFtE|)|vt_v^EBWwd+x6|tip|J^aIzW2Khf37Is&ImTq0>BH_eY*`<3CbY)*?V%Yfi|h{r9RtT1i?z=WpFbKjsdY|{=h`_GPeN)&&Y
z9QNe>lK#L*XBm5mbRA&rErE!E{~msVHETop<}{1#as6`X)RMSRn?zg?-zJ1Z`Hn{f
z;emj-Ju*_0-{KhAJFV`+)v0{q|2jksin4vRwY)g6q^mqup|Ejh5WRLpGC1dPQuo(m
z1`W}BiG7lg;5Wa$zqlB+`t)h0qq`a%a&gu+3mN4<;AO)nZ5}*5zQ8TeM;*WubaTt_KzWQ%1OX6gh?kkk)>umX-yW
z+VDDK8M}c1BBeB`XJ&P@=(Eah0TbDik{=({jJG2D^0eS->~>zDg<
z9BruayLhdT1&#V;Ec+C7sTMb)`AiSDK)xXI%IYtn#1uIULDiz%h`FkyRGDi5^$TJL
z@GDO5@HHxPlsay}pI(ijKgx%C@)odJn979$XH!I8MV;=*c$0%3)P7&NP;g;cj%q*X
z-q8b{W1|1LG9)oRe(Y_n#GP(~`z5ws8kO{7{8~GE@5!y-?W(@4z#7Gm*P3jbgs)^b
z*Kum9{`N;>eG8Ju{P=d2@@id!CLSMjF90)DURE1V{T&O4G=kYRAsbU+;~{2Q<|9Tn
z-5S@tHC$ApHIfV_KQGw@!Egb#R}1d5dDYJYPKJi#lXV%r{$;O_xK8^0S+@GmCkyOR
z8tXb{en418OBc7+x^akKSOw*m(0GW{*LdXA*lJv&ScuU!&$3;yItO;lCVwk8YbMMq
zsbwheWxybWdYYNI{Li9l|Hx8{*Be_bF8#f9L@Sl|Mj9g?%{tN~-1I=}&s)YCmOiT2
z*(}8N<&m+se{9w)SXHV*(!6Tq*zwq-W5cLPaVnjAX7}0}QZY0&*K;wOjjy3G0hJJ%
z1(O8b&NkO~v}>NT3hN8(W~KqP09%SI80IUkd6!ub`R=w1nTQ0kYlwg@E7FCHC_@rl
zZy764VQqy++Wxdv(8rK~L94z6Avie2;~{!!(Jt^wZhN~{(>S*2Ggk|^?ln-QNbX=j
zAq^c&?E=T;-lN+KSrWPsoz0K(r`LX=vpC^4a!;&oyyDWO{bN(U?!tSgB>>Z51}ZjC
zv31(=fReJQcZP^GK{@4uqjvQW1~`rF@ZmR%{ienOUx+}D+brx~I({DFLC8|VdHE~5
zbkn|iV_!zIr8kHi<4;bh*P;KCm(XQ|rVj*-f-kJ}{B!o*@8P1TwQoOyVrv(xp4qk~
z&$nRyvhS45m9MV`pPpS18kpkY{JC=GGQ}35p}U%_Sg-rC+_v5dg+u;5?d&EtTxD&&
zaLFe(!l6}N;l%=Q9J}$S4I~NU%2ske+P!(hkT{`q(|`*fNPND*0`g&54CnQ|J~_?hH1UC5#d+W4t(3%Zp5$RMo-h`k0iMQ^p1aD6Q6W=@1%1@
zn?DYXb9s1hXd{^cz*khV_C3kl5Pukm88~~j|7TS=kU}n~=-hu6pXE;Tq_IukOY4I3
zjTPg~T`Z4);SpYcc;l(h|8DN1
zsiIWV{Um+^6m`iX)Fd``Be6326apF$7aYgxTN<{W!Ek?G2kD
z+h-G*BR}^4ATeb`wtD#|PUN@Gh5M8Ng}M#g+pT|x52QLHgBGO?bWq!Wrh3bZrhdq
z5r1$h+t7f`#)+Is>3qG;sztRuC-BD3Ok|*Ik
znAMAaAIYh38`)~E;(NbtzV^fHuj;DZm;lqHGg_8zt@mdD8;>b%mh##7JQ
zRe|tb<+S`Y7W(}yBxi*qy?Zqy7|i|ZQ~Yxnrfd93d^l^@0MF(vuW!F$(PP_fJG&~|
zMQu-KWp>=UA+CRiKRA>k%!0t6^{;w!TZgkZEBQsYcK0+8D&>@*w5Mkmb&>HdA}#8M
zY4dx0*y3x9^#)PpE*v|75h>z#9%!uH)I`}m#iJPk;w6-zE6<)yuzyt_b`~tzq4b9!
z24ca~@L~o4mG1!Gz61Ex^-Wz5Um3gUp`oGw{7j>*4VT(B_Vf8>8uhu1F1_T=?$KSN
zbZ+-FV`akQNb2f#`GMvki&odwEPulcluGXx-&U`4bmhRKPNvhYHij*qNdZ@YN?x8(
zU%M)xk@08dwh;#qXu&Qj(V2|AXJNTZl_5I}8m4cjTJ{~V#Bh6$$t7oPhI$$GIW$J8
zs;5zKlw-8{l8Pxw6Vsg`Ui-$`9Wa1hw*~L
z;2b+T>Ros|6I5Qp8&X%*wtiz!%$F9n0F$E}ett=TNRHvC9H1irD0vsDKx5w~xxKB;
zJcI7=n+38?Ntw^y*8x$QXb(CoUww>XH7jDS#$s
zm{n1*FgDwwM4^&ySH;s6aUN%>?95hfw<+Wr{F^W^$nFz3+rvZF-0=WgfXSL4@!K$_B)D*Eck|2ydg94wkY<
zSF4R|Gb_0rxgDyGt!aSj?#O_R8#c(O7cRt}B!?+@Ep@cr|BF+gwr^A;|51l?>-PW4
zBuYf*lq<7M{~*IM`tk9_?K4-cH{J9c<$eG&qdh=
zRd#M4u}=Y8i*0U+_m&0rM_!#Y`QjYalsKBFo{MvIFXfgkjJ!YL(eW7y^@4-v7M)s8
z1Jky|(sE^`(u4`S6`f|7Fpd%o^ieYl#j={3(%;5$*Qw#7zqz#BBGDO{yOb|
zn^n345nRGq!IYtB$upbx<38p+n$H{{r|P4{*8EWyO#tQCwDYp0L})U=#bOI^VqO>i
zu{aCCdYekm4TXaDzF=a@%1Q2DN5h}WdiLyDXOB_}KZ{^9vn_W5IyS!HKWcEjOr?{J
z+ScXo%%qA;x?GM;xYGoSf4e8#%xGr*Dytm^o
zpPF_Z&s&zRIWhU>xQOIV{S1De@2Y6aN8DN{e(
z*|9sPciU!SUjYe(^%wpG!4#tJY0o&^0;c`C
z^o&92kb_P;x(siA+AXoRmMrz*gBuOtBvzF5mFuC-zkrmG;hcpw*Ft>6X|!&iw1n^+l&}{U;$hF(n!6_EH#x%z8o*ItV9j_zHtAg-{1
zR#fP?HD9vpT-;B1Wjl6wptfS^>9-@Nv^`nBZ}T?iswbEv?(E8F6V}wi4~rQ4;f+x;9si#T0I{#~&0QHlgk=~k?Jp|~{3FLM
zVxXcAgpQJ{16A9=(LT38KV$#&sinvPMjFZI%3V4DlNF7wV}g+15IZq*2s3F1KtG{K
zA=&eOM_gT7WKz4!rdV3tvuDp+L?v`CXEa?x8s_N;)Ms@$j?;hW&KAMiJ
zz+3W>w>xTE|JXUTpn#0Q%--tUsBPcg?<&vBD4N;63Y}E}%eQEJz|P7t!gQ|$S9;@J7YKx!u`DNX6n+5W0q^zy4k2M42Dt9`TkJd0nsZ%wXt
z>9Hq0I-B3EQOlekI^pYp*yleN+be!noNKpoyNUnk!OdPSczL7gw&i*Ub_FXVVd{O>
z7wJc9rc?B1^2%TgOvNVXy{PUgPqqAf*vTi@dhFoWQ8O8yDs+t05Gef&t9a
z{Pz7U50e_>(aTq_gxge@KB8xeWy4$Dp2&V@VnKFfsH)ENunO=P=aW;0tZ#OeA&cx)
z;r2^XVGOT$VaYYi?_J+h%3&B|0OCfpcfl}ZHRWZCRd>JPw-SJjEiZX@Yw^LGmgwT#
zKkUq)rn_2%E8SUJJ@|c$@q%{g7gUclGD&@Yc=7;ug>Sm)3FnNQ_La2${y4kCtij$t
zD||EWg#W2a$om>Uv9GPtYP-dCCPqJ3)?exJc|z2iHQg1?hwEOP7WH=iy?|Y=4)(8(
z)k_LKmRn}Z5N3gINOv*$LCA!}?AVfL*J*=gFamP|bKT7lpA~L(4?oJXl@>DBKY|z&
zP7K$P%FRz^FA9D+)j7+5ot)ZtzyLOR+aSS-)Dg|@^*_1Gyb90E+f(}!#BZ9X=SE^M
z9Hf#+_s|R@?(g2x+k-IU;7EAhJmV5p!3_D`ZLB$`i;xO1XAuEYhG99=U827}{k5Xl
z3bxD79H*E#hp_vrW!VUeo>#mzzkBERxMTb+AKn_PTLm?B$ijc3Y0~C7ZYP%IG8I;Q
z$+i0Z$L0LR)L9Q-g>gAR7)yxzlO|Z!IsgQ(49^}hzR4fW1KPKypXn6(Tzx<4l8%LA
zTAkfrnR4xXM~4%WRqcNL>JeAiyVUdPvC3nvwUIw`)*0MNe0wZ>PiTo*g@X2O)#V2J
zwyiP$vdd==C-xxklrqSHrzm9*nJ4yHKe7BZso`YLukkd==L&>oqz(MX-DfrYugR-M
zxB;mCE?Qi2^UH~!hxJz|ka-`~6=FcifbYL~%38$+lO3L>90RehBwc7fv!9ngYF;V3
zbxlbrzyGc~kM^x|b*S1C`Ze#?AndgHs^t?U-Z+%LduKQuliW7Z0qxFx+coTks)vVT
z`Ao;A`71)thqX{ls&{{M?dmU`?x$31l-8=vZp;2XC`t#rFHQmmdm@E%*oVWPg&AJm7web^dd=5D6
z=;qk*mq)9t%-DI&?jP@Q3euFqUNFSex*8^ZCjYz{Z9AagF^AAS%HWOPn=N*ZN4uXY
z>o@rsahx@#O?@0^-v}%(NZAN}Ic3hAO#ttN8lma`9$S&Nbl93bc^x&W*?qlMdMw+0
z>3en6%OUZPZUv|o^ip2FaN$eNq)mt8qxOX!)`I%kiS|+qgo{DS_!R31L2dIc;dap}
zajXNydCz&B7}nVxx4BX};>of(ZTkk7jDHv%_PqDS+Z)4uvhT0fHR=7M#jWn|4g9|u
zeos0Yah*5!$2P@e$MgP|)Xxt0j{Kt(dj7tNc6!1pn*j&Mjy^WApL~0Tl|DY~go>X1
zVFRqpPX07c3sFupTMK3pFNBHTX6N9FRLywiFktF?B9~ak@dC(~0^mRpuDqV~Ww-V}
zkxq9aO^O^Qea)RP=Vv7#^-Emu?_YNK
zQA~!J=x8hLSnCk}?-3_)6@ufjFR3Rq8_1i<2$qOQj+8mgR=NND^lan4efutbx9myI
zbvkCNrnun6H2a1N&KumClrwt4(Wmu46sA45n_THN=$6CQxU@}!$9G=3D#d$npliq6
zW__F7`IXtENm;S(LOQQ6&Dv#KB&aqtTpIsPDp_G-zI?gk;Ijqzg^JH1fkwn#sde}-
z<}wv_T=kIo=`N7cl!e7saN=hzoXLc^5mpVu&doc-;2zTmm8~{m+KZ>ho|)eX0Z;Ub
z!y~im)pTuJ{!G0PZmaATqBTUfw)vs&4`wf2=Nq5-eM^E~k
zQMtn@*^TNMm>tWe>Fk?`jEuaz)VT;c4|C>wdA+mh!Rg&ws;{p*qhi?&h1a(GWA7*#
zhI$umYGJfIG_wDmhG#>br@m0B(5X3hWckY5<+JbVue|%UkFH~_hvK*$&;PyHNj3bk
znfviEE(a&3H{GLP+xSZ@mIZrZ;uFv$e?tO&{r>&2@|chB-`fkc#RuYFMK9Td8b|Q}
z<$vFP{rh+=yT!|v#j|t$cU=UV
zi0U#xMlbYAFFYTG`7KDS6zdJg$1kYF%(4OyX^DHnq{dSRnjhDDj$J-8Ng7b{?oMm
z;W)+X@y{oBTfh85dA_ypitJB`Gym+b`;&H~?(rFye8=>eXL1KCn5MXz{2JRT%Rg>j
zv!CC;*S!0D3Xa*T6Lm}(amPiNUfw9<$DhI$0FW=2KlSj?D=sS-YKjO&^qv?guH@6?b=Zrfi&D02-q}Nt
z8(+SNRkobd+afk7Io$fdMf+jl`#M~#sn+dmrByH0Z6q}_^d1YrHvP2}7BBv~#Lth7
z6|kDEAvDC*My#ctdRKUH00hm0)VF_O3Eg4^Sr7rO5{{U|w?U_<7f~IFRbtbaRs$2)
z4&Cr_%(-4$M~paYGAu&xoqv+O-VI~pU47@dof~DW8&GvxTr?7k8+KkA3`00&Nzqtz
z&ukZbi(gzwrst00D;vL7eVJZeqLZqslM=R&p$;^(%K&NP@SzOPG>CaPOLm_zpUXEX
zo(lXhVprdpDm#1h$a6~Q*KL@`^H#%OE{+Q=8Zn}3(bHbS8X2ON6%>1BL;2IxZbKjM
zXni93e9x}RX;iIn4~0ohN$yaQTYx{m0Cyn2*HOGG>IB?;uG>EJyn&cY1M@FLu4okK
zI0XiiXjJ+7?dEQd#n~my^G;{?UzK@dj*c_A<`F($;otl@wWm`{*Q9t{?DaKKHERXZ
zjR01**?X^0aJO}@S>$r
zrD;lG3}O`AB9`y}Uz+i(uKL0(_v(S4k0F%b%8mar{==iwjeZ+ySUZ@tZP1~jr;~G>
zRoVOOb#V@Q^)$1BfYwkoirhc0_rNd+4g)87+N=r7OJ4~$ZP&n+-W%?z?=QU{*1yA`
z>WcaT=A-(;Dsap?odDD0y6vYx$APT53re!-np$6@tv@x`FvNm^>y(Ouleg+R#&p~I
zQAe+WiC^E=?uLz*mwiJ1+=y4bW~KT6?%K*P-2TF|(^DTkPfyOHDAWSky?t+gM1h-Sp0s{0
zaVkXq>xe37*gh9U&Yl&h+Au2MFvHpocwmZ2-P!Z!zkM+x;HcUL*q`KYZ
z0vj>7et4-9Yoji8=oczkJ7wY2u7(=EPymgG?K?VcZ6|-7uB|p7Pb@2bdm}4)O+5S)
z_4eAyyI_chLa`76?&X_P`2JU@*K*IeGppwX1-K=~6VQ2n;1_
zfM6+ZOF&Hk!h%IoMY~l$acsBC>Q-8l)tkS>xt0TZbKd|9Y+J<4lse5;IZh3h1ubPXbLT7n{Krx
zOZs<^!6)T7jPlJQ2J!p7L|6&9NrTp6RlFMarRJ@Dw2}Wk_%kKW+rN1fBewL_(BN7Xcpm^WC_gA
zwMqWB_&FqBR8&rkiCGbMnHRnroSBwo{GS8xw=fS$(KMdIw7w)^Lgb}6y_Ay(tJ`oX4}LRuGne7@&(kJ4$W1NOiv_A`c(}$z}qg
zUn#VQrdp={2y#U$e>--C(5uEVQj_KL;6tJ&gjw92WW_HeJCEAgU6ol4KPPU64A#Z)
zn4PFWsNkskBR@5N05<*g#S!PZg~@@nAm8b8$6JmzTsS77EWxQ_K*b2l8|Btb_9|U!
z^gM4)cD=PLzq&9!;HTdhPqZc9#Xd_{Ef_a;x~BI5EVSBNI!g-EQr
z@$_kwc+tZolg$nU*~od>;zS|VE|{ALtBXB>cM0>~CfzUENSjeuSSW)qU~DIL0?X6`
zav%@zRMX2EI(y8jo-W%zaHV)9a9PE^&Dia95XOWd*gi0R#-H6c*?M^az>U~iiNud`
zR#pVjcQVCgJJFs`o`>-G2mWk?)nCU!tHpk%>g!Flos-kowXJY5oE87jsfA$!!;+L{
zgZnPjc!GP{&^VoeM#`Rz8XbFlbbWNBCJ^b)&dS>Ubqktd$7)U^eErrfyLF-W(9jx=
zIkMyOXLrb;Z3Zke#;ya>U?3HbIi@zUlZN{slPPoGeP6d2hJj_uPDoypprt}wXGDY_
z=IYRS?9|yr&Ha$PAv#2|omfV|@_ma$c}I&4usYjOQh7rxca;>zAPvN!5yE9XW)?Uf
zQPX?LhHC!M7X2zZptrSo-DUSn?Br?pRNR9m<1OHaGaQRwJDyHD^)(}nH(9&=h~ov7c}
zOl@?;rpnUnhi5!Sa;tl^^3uLKAs<6rs`_LYg<}7sY}-)R7bc%YRgD-NP*#7#@_`8*
zGtJ=y?uT=@C%Aduj}3`wThTw5_prbj^`8=d
zRCnIA#qjj~fgBk2&qOv!Htsjn;EqfU85GcGh?yJ}C#qZpw7%2$MQFHmZjZd)4VJUb
z9mHlhfjo${Ye{?EkBzMyDVed)WG*$h@@%}+-t1P}xUmt{N*u<0Wl>ixeSXk@&KDgx
zQ`SR8C%{t>cL-OEHt8A@TS8PuuS0M4bhYLm#GzUeS7Nf%DWvnX8^ie|C5=bP>OP2j
z)V0C}QIN`+5O=F|QK`zaBzFUI{e}uYsclJ1>cPPpjW#JZSgSbL(md_^#6|-ZJ1Q>s
zZ)cP?C1TT*xX?d#cC(iBHy$~1QTK^jk6q&4{d?LF%;GS{72e6Mt$a_OPAtalAk-fY
zn+@QhbV5ADL-}*WnU|TVaOSAW_91*`g^(d}hr@*$!5QiiS!i8Y*ywZ7hdFbk*q5AS
z5mpi>Wr*Ly{n7{~Gq9DzySa+O&FMda;m)$RqsY3vxg&s?b5h%aFJWplRAgoFfZcH|
zyYozkcd!AL?_R#QxHv07)DW==qhd*S|7nIA>bfs79(S^{&>an({HeY}4IDc%lDS1q
z!`T7~g{Qd9>R@Je_9mErH6P~mY6~UF9>%7yC$TgGo`(o|jhKq1kYn>Cx2?F)^Y=-d
z;9X&mdbGtk1su@>;gayDJRz?8Vp}Rnj>+-GpCz3Of~jUmab&~cYl=X^uQ08n#jFPe
z@*024_US&64{mI?8`1NR*W}cwfhr9e^ln=@EA`3R+MQiWJ@19zIOc!4euKHJ#rvMW
zzy24aj2ns%7n~hts1a~9|9Sn626`d8JhqPAxVPV;u1*R!tO94fDhXhl0vGV!j!j!s
zb$pz=MhIs~vIYo*gV&1wt13&BSx}L!eq%(ix}I!yCX5VC$Y)&A0VdmZzKOVONkv6<
zt02htfx-1yzl2y_RtmvH^_v7V*wCLfM)#ACUQ8F*eNp`tzrPs?GkA6NE|5;R&%(LI
z076z5{??lhBfu_1XBeX9I9y{Xz!WKs*3477HJ1+~^CLMPG{6tz95gAAuZSxO(*jf0
z0JYd5;lCp#vR~TwF*JTsAwa=?zd-P)HPv4lA@>Us;0yp
z2op_~%)ppwiEZS-|7h3KUPHV7vE@lsRg1z#JzKUs-re}*nbTLZpSJDRuHDCvpNw|(
zG-&m;L>rQg1Xa-b5Q_zq%SQDGxaq_>c7)g3=p240wNxkS0
zHnK_i!q>$vC*CtUymrV
zYJk&X&i(fDr%u&w5Opu7qk;FAp^ZZ#V93b|Y}vU(F|F4*G+@mR-ia){p+GZoJN<5r
z>xle0HR=D34Olq|=wB8=31N&*FqWe$dP|wWfB4XCXkLJ+dZk^BHFZliDY)=_yZpmg
z&ceHaYcm$6;)M#`c8C9JF@O6
z0k}Oh@q}zLc9W*bJ*+!B|DzL#e&Pd+A4Kx^sSC1MUz5o~l=yO(=3V
zXzW+D3Q9XR`fzg?$DxlZ91Xjr1>DC4lqXZn$!oB0cZ6?TD7Bv0hFz-pF+m)uV6Kbr
z&wuYLRd61PD+XI?>t759g_D({oJA`d_I`f9dnw4+o$;-CLOj3hao4Y$hBwfVp)=bK
z)>U4Uz=2=L56iHz=3Jr9;_>X|LC{Kyv4Pw?8*^RSAM?o)e^R!Aqf0%ya8gduRkWFK
ztktx%8fQX=!4^v!(ENM15lR2g1t83f@ff3N)%c(*R~E@dZ10Jo6=|zr^wIUkbBz9a
zEEZ%E^piQ%BCBKh8HN&VyEgvHL63xMbmUL|!a7qe&95kvl$elkvE7+6ftn;yfWNw{EM-z>xI}
z09Nc=A&YQC-oxvO8JiI2zx57f%n;@l;wAzE!xMDUWt-Le>FRWaQ--dLCx>}uz=^@6)FPD2O
zvI~-0j`E(fqz4KYcH=U|4Ci<*vV{N;$50U!Bb4H}jynW#?jlI|{vhpH}Y8OSD
z@V^_H9}A^}Mr(%Bzs6Vtiy
z4n}jr$=o+!BbK_{ndy*X2m1UN@i(?zKg7CI>1dcYdLI25N8VtNAjS*)0D{6DM0+PZ
z{P83b2gK9cnrK_MX4T48@&TtVVv4W7aMK{wczr9MNi)oh$82_vuF(@?>6&n?6I?P
z{8BX~*Q2qZ@X`-~pBdLpzIgFs0i#qFJ5`zf+)A|=4DzM%0;&Ly`2;T*`H^EVF9-z6+2MnK7VO7qHn
z7aBzfiv?U{F$!r3+_dC@tnZO7l^ckU&miv|Job=d&}hU&WOq(cEHRk
zCV{@N3Zf&$*HA^t)En4b*3!ZvZEiu^+OBkfv?O9aM$sz(83|I%?a|p~l40V`pHsEc
z3gK1fa($@>){iF3z9v)uXnGP$E
zJn)<~U*QK3>)#Y~l3wEoo4ez@kO>k)#px-~&+@K@j?@=8&M!-succ
zvkF~!1+AnE^v1IvWy#||nhV}1;kRYPJ2pM$cWLq4
z=5#_+$aWe%CXQ2az90>NC}U60dnR@|I4hfImZ02xZi+4KBA^R=^$jntXf*qL!bbA^
zax9!or$8gk0xhM#2ni3@t+#SBjkA!~mvk=%G0mbn;(NO!q=nO?gO9d|K*Xq$fsRi{NH1l0=?`xR;i>VJJZ5fglPqvhyeWg
zBe?qJEW>4{5vzJF<2?^Y7?NBsS*wLhsq!ZI%X@Iwe$%CY|1G;L#cIF;?dy%ayt$rSLt<^xMi+`*Z~SL$OFeS$o~p}dB_$w;kHA_
z42gB^+7kj)FOmxxI+MAJuGRw8jva;k3b@^8Q_Vg`YxG3SLniTiFOn;w=?@=pC4hh#
z;zvPUx6A6e5l$~v`*fbMh?gmwd6GMLs$X!S7TAR{QC$3J2;&K|GG0Lpqa3bk6`R8U
zmn}K>drnzsvMDIl(lBR7kJMN1}KD6cbBeU_4ndNwb8nyPKk3fNGiS%@Avl6DQJi
ziy;&OljfBz;lvfQ4-a{V#@;GJzV8eQHe&EL4yIJ?%A0oRPMmR^dzQnzefNRx7hBAa
z3nkts=3vNJ{W0`>LGP`n1#!oDs0DTG`obH8j(4Q|gs)$}
zhJ6jKDnadiwAgRRdTFS2u3q0eg9mvv9hm!IwWp|0-r_WJFWhFG`J
zQCSz`zp3-i_+9Iw`eg?<8F}p=yG16ZKeLBz(-?g_P-C~xflk?LyR{l}Ewe$7F-=F#
zi2nNLkjY!k*?Nk}lYc(jI$_zp;)#z}?%%vP>GAi;V*Df><@Et$f&XBnDU&BHEyX$(
z6v3LVpYZKLwM12L>c}MH+h58vYD@AtzT%V4>cXhEK6^gR1N~;QFcrHlZY^R1oqzt>
zN9`sHzQEvFkg&hKzg4(8uBhVfOWSX^`L#R#!%*EP#r|9bOjkvYwz-UpJ#r^HV*Y89jLi&`LvmU08~+2r(|xyLX?wYc}yqYjR}
zy)HXYkChOdwFhAQ1=v
zw=Z?a?}wQ4W~C1nWEXsYx%6CCaUxN(pl
z?%m^$-g`x(|KDn#d#sptD*0j3t=nc=QMQ*|r8B5qd?A%jU)@)eGVcvMAlZ`A%j?ky
zE48>+)uS-D*Ws~JE{ZN5-$r#?wH)lH*++!0T1PlKl0NvIg7eD0x}7~H*%R|l{aW|O
z;>3W5E87(q^a5GHy$0?kmZtqi?Zk+uws>VZ;jl>Txk8wrV
z(D1rRFdOO8h)=?N;G<4nTkh~fImAA0)u$R9i4;zJyiJFvGQ($$w5ST8q3&%?L`F|<
z5U6@>Gqywwb*1ksFG)#0qFIdX`!Mg|WHttxlI+=!upLtf9@}!82^02E$BU{O(fEBq
zQ#7=Poi!W>ZiZ9pMrw`QnnTnG8WVYmnjo3v=KLbN3lTW6n
z;hcPHM{U+}WPiO6K!80N;M_DWKK|I6-(?0%X|eNxQ$~BSVuZDq`GixKL6BsN&d4&W
zHIt%dGG;k(F@w-s*T3GqXY6DwsmXw)HbX=
zwEk3F>|F8#J>r;VM<~X)<}sX;uHWf0^M3#S{T^MWedm~~`-^TFII0zY{+#sr$*Dt1
z&UfN)1RZK+l?NqyOuM#ikFn<`AL6nT9(rSIC+lAAYgIz2x2ufq$vVwXKbsQvRuVJ0qr!lR;&ok>e`UHgX1&Nl1CR;`MroOHnw-StY&i`+OFFK@K^=SVM%9yvtAUZw|evm
zuSu2an+%;}8=L4bT@_LR&kPV#Q{7e>o8Z9!rLxC*1uX~FyN#oxRG#gbEWydcLOagm
zS6iG&Lkv2&xrrnXV#^|0DVOh0sbkn=*~4X66bC6q&pTz^9>MO_RnVYve&22$0FpPT^Dcn2I;s
zUI-iExb}8-O<5SJRG>M2v;F49@M$sao5Gv`9HyV?u#hn%N$I|9Y#PFAGBR<$6#n=3
zu4&V*UC)It1(P1);SsOoqJp-3C%I7EfvI=Ivky)*sF_BiMvX?M-K7JyA7>S$KAp({
zuI#&DE2rnRDjs%FubQVOp7HWlVVzBEZtEP237R~6#)?ewYQe#i%p}fF0@7k+n2AN{SaSQU{9~;MgWY?d_(g>XDV8Q}zXL+X
z?)E`3PsCt>12l~jDvhFg&<>r6B_|*pcoVhQqR@3}V?4XDxVNOpFu>f0e6pcMr+bvh
zzgTw;5GZ^paRkAPS9wHZ=eeVf;MzS}d;n*U>)spFN(Psj?=4Yq^K;u6Dh}PyG-SP#
z#sF^Jbjlr2%DuQR*#7;pYp}G~&Lo+aw$t=#H&_+7iTvsKm0QezK84~1qk{+R7b6%8
zR})bUK96=;>Q)-F<@++ngb1x(=SFcd1jT~N(oSOZ&xak4$jHsjTg7nGVG^utv?FNf
z^o(y}lr^jtvki4RDU60(vQ+DifVMNd7RL`V!e13F#Y-8(1BxoyoFvE$>MC1KmClV@
zw}J%{DKB@A+4zVr$$0B1UH8FHLur>OtEORKD=k!WOXs<>y}V=t$LZ6jXT306bOXcFI@Qd4kw5=s{h(H9w9iltq`Gm_rZZs6a5^UZ_uB>SgTgxQl5cu|c45?i
z1_bgrDaIF<<>-mI2mCyHvbJf@o>#CT!F7EnJVj2{mwR`43`ix3IIcSG?kM^ttY&4s
z+P7c;6(LzgdPKg9UZXvR&ir>EzM&*MeQoU_e}Daj-2D7&|1q5);PqG=5hQBkyVOMQ*(AAu`01>)fri?N05tQag2
zxtw<)J|VJUe#Q*j&M}$zzB6aI#UGTy2b(UA8jO{70-<$t^?=*U{6fZ_Jl>EKMEEw?
z=?Y@Uh0%MTdhO~}4SR$F6r!GC3b!r(MnnkB5#M!~8p
zdErAjzSPESoymBTu^)5pWadu;2M%O3?`QDrPImSy6scP6+T~KX_d8-8{O0xRq4fRr
z&SI}5VN$#Ra6widWoI&@=;5zPLmq^romj!{&uiocj@K79=Q(tad4Q<>0c2j;B!EjN
z)sF`axwzH;p(!%OyLnmHzSr{QKV2^s
zVd*U_2>>yY^Ej)nw*u#y%~y^zE53GXd(Gyxif>$$n3o9jAv#cfY^
zn8}(T!ik=$1Y;;dvI~yxJ3xCA?c+b9Jm;3fDibaO9Zu!T%k^<{+0H5kl)T!K5wWaS
zY-v?p7uK?ScOho;tfAvUUZu6K-PfuZl=3fE3hXy9=YwrfObj2HN?i$4r}TFwv5hrX
z9wH?7U)q(7B7;*!wQHN8(Qs+n0(m%w~qxafxt+SS&Rz=6vk-Ms%1<#(^+Q5yLbqi&mxAP|HCJO
zPuHsI3@O)k)vEq5V8hlAZ=8t{GM$@jIR1;t{>}3nEk-zOChn!c_QLr^SJhc()aiD;
zW@Sn=Mg%K{kA{XpuxDl9D@P9L0m5=|ej!ua&jR9LFmvH&p^x-wuljEP$j4@>Oy>3L
z@q^P^f^U)exNg^|SDLrit`!-U&}I2hU9gP)n-9_+vj_rK8kU&OS|?BeJM1SQTO%)608?cyh|b$nC-O6CiVN&5fV~i>&Z|NF*Rf0>WV8ZNJZ@_
z&3O`6!C@Faol(-aRz~|K&R>M)h75YI%_*Wy+{&?}^Js8pE3!>*M%}oAZ?-zPqrOd0
z20808*B*Y|B5W}J2JOQAq({X2GGES8jk94OE)R(8AzR1@kF&=%5tRdsT86Ap|E|Ci
zSoy0mE`Z-}iaumodJ&PP$AhGI2SJ9T0Y+uXiET^Ru
zF<3Hspd&6h@7}+E_32Z0!gd|@m&qJ<8N6Yiq${_1g%9U<2;C2f&R#1fPf|auLwNv*
z;<+UEaT(w@)tkaAjZ!X@GYb^VcQeCte^9H`r{r1~8Z_0ig
zK9V<~T0!vm;hp>fxX}Ik*VSnl`W*kogAjrWz56ULuUx=x3Zh{w0b{9M=Ya#;WAZI9
z1!Fw-HtYRhF0hu(0iqJ%#|*fr6#w*#W~Qb$)0!e&8qY#UkeOH>pSM%vFB>jZS32O&
zE;D^RveknOd$F+V&R;Brc{^{sdNbF+Y-*O$U8MyOrA2qa{FRb>9WXy9`wnb6Zq1I}
zyNwPiJ1bpW@ns!P=qrOMep@i^MX(Ejr%DYoiGcTMlF0=P`D`q%r7>zI`&`fkO#WkG
z$%+iBQ(I~`X3B4Yp7p77=wB*j=t56IrFRci>XQ`D?QeeO&FL-qnT8jI2YuJ^m0~_9
z15Qf2OEXp5x4%OendzILnl3)G3@XQ+oU(RAjy`dE!F>W5MdnFF3ob4$oU%gy{rfxe
z!e&4c5PL#uQ|R3GoJ|?mN`@ep$5P-h15)kKp*!C+pmz25Z{MnzK_Z5JczRkLH@x4b
zJZ4B#bh1^HR7qJ@pX*bfwQD_Cw#YQ2E_?AHzu)1g0Jag*Wc8|oNH=!weAF$iQ{RPy
zzxqICHegpLzC!{W@b0y6uil|=14tCZQFR#tp
zw{H*7mM^)0L?sK|h`P!{gVIn4+Y>p*vmFR>BZB)4)LBsS*7H$828s8@pFZ8?OxoJo
z?syPjsLOF+V>oUlql9UIy@3L6Zo?ZhklB3SL1$(?RP^)nV;9hDh!9AF>ebFYSbLsV
zip9n-mP)*1yrMv_@B8V~nn~)5s4SxV@Cv??ndy)84&$C(2s#vy)jM3ePzx!9iW03;
zD5Xal3J-A~p&O2{dun|!#H?5^)?d~$2s;Ddig;mfXjQV77MrXmT_^VZvfF^W#+|%|
z%_=Jw$#}yly_g3a;IPB(8nv$Fs}543$Vh%dT63mZ%q@Fg^FH%2SaT09joIbr)6+{=
z+;Oj2y4z&>^w(=H)||L_Klywrc)*5rQVeL6K)^ZzA1$q
zMr0HlZHSFDgF7<}zT9IVp=<_(APNR*cfF?Cl_6)&^r4@f#u7NLMFHM;#dUGQco1;}+~Q7Ze-5b@y^}ag1GkwLD%fo&0$?jolcZlvYHn(mOJfJx*Ci^VN|qbO|~CRX}^wP
zosu6b7Ga>p)Dh`zy*0}^v}n>Kx4PsSH+BWg1`n&T+SY|2?&cnMv}iqtO`8^@q$B6l
zlQM>in3UO^bGrO0q&}!Or(nmS3{kA^my`!(8Z=W#Se7hW5Xu*um1sMjc}+mgo-Q+N
zkQIa%SGxU7om-&p&!mM_DmFCJJQ|ozTWGLnuO5^^@KW}d9*x`#Q`&!!=d^9xw|jCn
z`<*-Xb}Y@GZ(FLqx*BvLwplEBIv(I)7hT
zTje?ff;;@DX(rTEfjGn|i+YhkenYBz`e&Jb6ez=OO?
zZtC6gP))?<%}qe^?xy+8hBHOMN?#|M#k-#-om+D2>*9p`IXuw~WL!o%J1r&$`O+a?
zrJfVVG_=Pz9zJxo5`5tK9z-t?3uSn5=W)xXaQKeG4xs~yBk}-m0F2!t#mUD7d?r#%
zpEhkB4$AR{*sCWi8C-2AoI(0b$ai`-K7oJx3LBh*=es&SflDnc>d4Scv
zfA$0!n*Gn^FQfK%HSJTNfJO$iBJAx{od_6S}?OCNsttr+RiX~a=;SN;B-y{G%t
zBIJ&5Z~ye6@~1G6*)Qv0ZBKfyc6}MEcR`6S*Ot0n5HjwgeP=5TXE!%Dsi@w)3xa6M
z9Bd+6<7t|_**nc!vT<{33wFvVs11@YPQl63r^71(ss@2-{*MkrW&hqPqlyz@!|5aA
z-v7F|LBobIE=MaT-f5y;dwy3r+yKZ92l)OuAEZV12F+Z>BIyVWUvjXBoZU<_W|vdf
z2pC(e1Zg@Lm=$i2>~Wk3*Oj6vG%9Kn7-4+Gb4*MT;@GZ;8PkrWB5eT%QodEi8w1ao
zPyj+Rg1~fl;6L{{sv@xl^va^@E}WsQ@`QOc>s?-*FTZpM9ngR1;(mtQr#CclR@_v{
z_i%=e{`t21){*b>I?()5R_cHkkf3#5Bew_1X
zOl|lXR6T9@ngA5+0JuyDsCu7pLJr2&t4cinkq+sMLIMozFJLkw-j_hq6OT;q55CCe
z_j&t!D9MXu6vfga!=fUq3qzS?&VWZFxX_!as$8kG*Kgm-lJO_0UXjLLI@Wg83XdNi
zBv?sqCW?n4tvh5W&VcmUpaL&He%!(b<84zw-&|qg{Ax_1TLd@i6k9Iwjoe+H68$#5
zF|*d|6Y6T&@{EaAEaiH#??Kh_UBqllOUpZZEVk3UyMUo957TJeuSvJdl(|F6u(*J#zhJWY`xn=pe
z8mcXp@iO!ANhr#I&q|EZe1t|Ks(?$PKvvAA8N*>yfcwzmuE-
z4X**kVk3y-+%^yq1y$f)el!@f1;8VEO=ZTlsA$lF#BhDEi*QCerR=29gjSU3xe>70
z)h#T4PH)*z*r(Wq0vcPi#GM8d=u>E@d@C+DLL+pmosqS6TO>X3l4j7Q2qpAy2l^al
z!XYLFzI8P}*$X#GQO#-=HI)=YAXxQ~^eQu-60HcwN)mZ2h_|1lV-JQsR*-F_um>A-=FxZW-d!F#<&spvv@JK-tFS78?ts}B
z$j}0jX26tM(gwWE`q8+Nv_A7Wc+E9c4
zCvv~LejGk%oOkPxUEG&8lF7^6Yl_;|4|
zi@7W;9UkXWdi#Mf9n1ew^&U_?|L^~QGArJZ$WAE=Wo1i5sE}+$wrr`4NW)AaQ7DwM
zM=~RlqLf)e8L6Z)id2%6`rWR4zW;On-*Z0ad+?^$>-l_)>$sxhkb>YH2S8fCZ
z2S4MycO23g=$}PGOs!~2=_yZun3rr&#C~v_r)Vu}v8js;l+9N(k
zJ}YH^p4HZ-^=Y-$_z6Q9!KlEoB9v-5X~_AukiL`!ZP8JCc2eV6C!a7Cm%xu)dfEfzeim&-Ng+J|{bBdnF+%K#
z5t?#v3qy7#FarN#7**h|HhsNGgp)CX(oic|MfU;?gmh*l|4G)G288nP;0lV6>iHUu
z%q9ZQcFI@_^XmT1F?iN7O&qjfUypERHrxBYO$=?2Yh__mQ=5C1tvNIpVpKowk@l&UHkU4zua?U>(Eq1e7R@G^gnS$(+;0S8pY6@VA5+qItYC(Ti#sxU2Nr&A%I`IkQ=56R-o6cJuY!waaxydRbvq
zGm62joE%RAI*zpNb_gDx_6<=0(7GmFfRdTqH>dH&QzKqk@K8-gi}h)#qUC)Ajwt8r
zS1QZ%ryRU}d_Hm}Q7LZ&9FfC_J;6|!K9O?wiH>bipkbbnCe$R%98?2YPJV84G$0@W
z=}Ad=qng117TZYk>DK(NuC8t|SkJT$5#iyxC!UQ5>_ZXw_;8nz&GP8b#nm9GCB_>g
z|LxWOOfp1wG@Y1s5D~v&Tbr5%Axh`M5G^Sctb+Ua-zo?b?B!`8Bl+5#iN_r{GKOKnMyQROo68Yvrz-3{)@?1>V
zfs#9j=2%M{ZsLlmrQOO!rU
z8A?#%B8_~^vivvskKaBI^TJvDxhFFiIjN;mGHQPOeU#Gml`yv-3;!{EGLMdPL;{AF(~PS3MhiVuGF+DLD#*
z*0>SHRC=;e1kK@(Pg@;mGhM#T2lU2AU>ZL4czF0z@HwOVU3nPfP4$q%hl+;rjaL7*
zs!VwUXSt6e9g24q24$=gc{_sN!tly$tG9V5qyrZ}Zsj4F$qIGQ?${G$0`C{uT>&
zCRb2iMU#RR5R#j_05EclxjEyHk8*71|6a#!MN`QiZ>EoerLW-2W-ira;H|!GWOpum
z5toDJ=HC3971Qmi&s@^65O?LrAe^Ek6!@lVS7i@s%Dm|UO||C;B*NyD?$8@quej1q
zqB9KI^eR1l^x4T1BbU!$Z!t+T0U&i!EZQ-a)-Ws+J&lZ>6i;fir@xkRDl~;k5$DcJ!8Pc-4
zU1CM~^&8+GXE+#wOK(E^phrd?pQowm^z*ZIMn;BF<^JH{@nF&lRCiq{fT>WPCT!zn
z@jm+2cmMS<#Ke94(xvA*zu+B>Bdpz|%g~|0@-o4YfRnpv)V&hGAS5v0ah@w#8Tzj|
zTYN8H?!~7#ErW)7%(g4-lyJMLJJEzgxlrX&owyY_OTs&tIjNQb7Tpl8BcV17#f+IH
zy(JQJ$p0Vq{eRuOIRG42?4lHTRE>OF5>bF|A?Aa2Zo-r7X1LvqRWM)fMj
zDd^TONAE@=7De-KipB*+SU%_&rsu6EPUrx)kC(;Ng+-HC^tGTs@F9pPg7+kZ0I4yL
zlnJ-biucdTURa}^-?*&8+N18OYuBc89L$dTz4!h5XueA`9ZE_IA4)$z*{oI1}Z0I*SPBQg)MVv
zxw$f13uu8eyfpk(FEqu#yCpw9jDXvY=g&n+Bavd}Q$3vbNWQB|a|P;g%kW}_R%qzfR8+PimeEMg?1yQjm$
z$5Gq#>eJ^qkBAA+OO)5}PmvWQF}VlUSYKH4CAB_fEG2LPiip~7XTA8<6A>R!8a}6h
zR~p$!%keCd3-E{ji5K2M-pN-51uUf!OkbUzeD`;u;IL#Q!W>9Vo2Vut?eYh94v#IK_SH5g=2);VG@O4RMT!ACL7&Zg3-_C
zU-i9Dp|u)0o$zpwm68#GfLU-bQ2wX!E?jhy8##Bb4Ea1WGpaZLe2^wYz>;wqX1Bdk
zpU(ve68}AYam%I29cWDT9^kdWrazCq-}TRz)2oI&-;3$deGyBJyPBRJjHgRt_jeJ@
zlwQyK0j3Irc;I53zz4)aF$OG++V}$sOS0b&vO(yUoYdcNkxK>>ip60sCWidoZ`^r?
zSJhN3cu!aVS|xLRDC+TOM&^H_Bf#3bof~tRw=>WIht)1#I#VZjHp{A)0S|ckXwT)r
ziS$w^L3|4T$k=DJz&Ok#J`Ih;816$aS4WDbn(nbRHePj^BaO3m510)XTWm;ywG83~
z@W)2*FAbq3rn4_?485bN@&NtK80z!mZ6)qZql5|+HGR9OgG>(Vvs`9o1Z@`6E$Gq&
zItJ4~tKt4D**KzD1;jGDNskn5O#Fn1(LhviyxedOm(O!->Ul=ctac>UB>mBQs<
zxlrcvmMF9qXWZ^~h4GYlOG>^(Dj=t|v%wR-@w6vJx3b28!sIQC*T}DVZJQs8AQ=
zvQHvnvT;_f`icUq#mw!@>?~u>cK$E*Q)r0vq(sfSqA3SJ~T(wFQqp#B!^`g+`XAYAo?6qX~U~yK{Xpsg16x&9&
zcy$8j(7=CbJ)N)>Q{aU%$9ZGXozZX%OHxlL(L)^AZ0ogdB+V8N-Wr&Jisiqmf!%e`
zG|-VqE)++l4BlZwvIp8rU?y4&pz=lS*J)*rM6wi!bOmQB;$O?!%I*f&W+J;lfO?Qf
zfNof~M#b*HR5B|X)J*Cq^ovp_AdHpPoKJ#sc1qgtmSK&!y#9gXX97oxQ=649;y954
zgBMwxRHHn@;$e);c55v(TOB~Qm_&vlCaApt%`(L-KshD`EQ*6po{h9Du5-8U(KLAm
zGkg>70;pA{KT&Z=;0{N@0!2M7??9o^DA>h)kP+qNLdFYv$Pc_)0q7(mm$OwK01XW$
zsNNWOWjM2NE?hH?A4W;r5{&2h;XYj^GCH7ZAl^=k+t(rhZjlIG%f*ZJ$a!H<+K-1_
zLIkPd$?IBbD_JZ0lyG^7_W_bx3ONpAD&A5Q6mo#FY>^O8Q~%}qe_DVRtZcPIDM{7i
z8HF=XBxj=CjW(vl5@w(BTV#zh|F$soSEi1noHLeiEY^EUn3;;GpF|K(`BM*r12-IH
z>I3X2NDO5F3Y6~aTWh+Z6OAQ!f&1h+)L+-hXF7TYI5OKz^bdd-=FJSN_fMXjV`6}B
zl>$ZNJW3-PS#ck&BDVYILGBr3jOe?q(u<>PZgZDXWn2
z78c$hd3x47KUVmB(mN&Yl$+d%SbvywnMubza`M(^gPq$Bsh4O+$T|`#mSNA2sW7F*
z;>MTWnIqhC*vZK(DbU9G;_$7NOugqkboo`-n&LG8NJe)QB2o&?0oG4?<40|x|YK)N6N?`c^^1y`%V0ZQ>msGd{%9gaD*oHJ8TU72RE;*rz0jHu?@Og(*kEMqz}H2W2AV<`cdB6(tSCgE@m{!~6*
zy@w#6q`zTaP#RIyDX{&l=w9@<5?%ut)@q<*ZnxQIOdDCh0v54>
zID~^g#uTO_sI6JIww-!?YG=vd1&Bkf(HIHQg0txpMOPxZ3bi~-&UJiyxe|UOEZz-4heSNqzfsvB1w#wg&0$E3;L#X{?EzWL
zyw*VUjJA1Kh~<${>O53Tpp8-RNF~#{bznhxb%^&GY9tZw!0PA!!mKF~?jYAxgl_hU
z!>vWD4tq)kj8R*c$PTNaTP>}dRli$xm!Nfn+5@24HB`V*u(#&z+I8oo_ukD(4}K+c
zE|6^QsaLWcwU8w@4NdCP7o`BKiZCf7IU@9#Cj>zamd9_Lh#Ip`Bg?JQ0Ig*h)iK?Q
zD18)Q{FPlzYPy~7|NaK$;$?JZlmRI(UL2LabY={IKIh3k+p6MPEi=w}+`_2yZn0EVCf5@?{%*w*=
z`S2*uGY_T%SrZgZ9j
zC;||{FW}U+IWEu5zf<#16ew;Mt^r5%Uq~NhRx*dC`)L*mkfi}6i{q}x8oIWe9$b9w
zAPx@p6j{7GZ&ECJ6iglGAK^iolJjW!Q_bfM!7r1~rDv{MytvViNA|z|XJI^OT#p!*
zl7P#vD+X>ij;ORdRN1jcJuDLAZvO;?SCFey^TEk@Vtc*l;vr!EeYzUyKPXFQT){ijT@D61
zAj6qAC*W}Sb?6;E+nq?==V$>G05Yj7Mdgh@3m`3GSY-J+LVOgCfuD%1D?TxCKUF22
z8Y-8z{0B1jX)7xjhXr3E?1vxV^_Qo%sjdGb#WH`+10s2;potmsn7G(CjO>
zkNz)eApd5#UWbN76TvGvN$LB|hwiczMwF5zA#}tJoyNt=+ivODtyC|10@GPycP^}qkz7FOta
zAYm)PKajtE`LC>}S*spD{bg5P$?`x-S5;8(JL1K@bmOAC-wF!8S4M0~e(F{|ZG6<5
z{*(`>gvuU$-0-VV2_DmW3_Q2s!e{WPCJ#vH+J+-Qm4kltq
zXOlf!a-=pT{#$y=oc#RGn(-A$@8ADD>U#HXI5G-Jxur4Rb>P6YO)b+)Z*T56dP~xN
zu7c;7PDC)0yJ1h$urqDJUO9Kk+lq-Fw^!zu;d&Cr2|PR^Ba4<>b~`5szZUh}r%#Do
ztypIxt`eV01~*o(6BtHd0#V7aB>&?T%eyhtK7a466=rdQ2+VOL5_KpQIls{lv&!jD
zeNoMl1kHQ-5BhEbEuaWi=Br(=DoLMO>Ap0sM_1OOLv@vlRJ@cEClz{ZL*f3EHla2M
ztFd3H;^u3V2@IDELrYFsV7{n*OS94~<8AIG%w~1!M^dwZF+|PTiUhP9^tmjvtuA-$p
z%G6X5-6IY4C5RAy{A(yy)ahz_Vu`Ne6VgcO|GTY7uRcXlQxTK96OWz_eA?4`3ju}b
zSQ^s7Bi^)}IrB(8ovrJ)*@WlAy0g($32K3=`AnccWhq;1XzJgYi@~{?0f^tbe;<7y
zyu3TzID$w){7?ac{J@paSW}1bmlF9LXj}LV8KTB$0R#S(fW2Gw@@TQed;s_*OCL(K
zH^dNCmN~5*^54CC8CS|QL4{q4(
z*&*0aKn3DN6DYQ_Uxk77bVuq#Qa^?{lweIr6^XQ_RblZyp+u*J3cu#+Q`7w(RiqD>
za?g4SgVa2eR`H1z(FsT;PgNqXW1czvDtjjN(%ccCb>bG_E|-`(sA@^{1LGZw0*4`2
zo?~NlMky7!BjRB}+K($BF!oI)w;si*-}9yuDa2YGksJ2q%MKSo}eEZ7D9q
zpNX1qf$NQ={;V9hLg}(0kh*$GUlo^X>q`X
zb5SBwUCYckPUDlHGqM8+v8khM?SW`qupBaRrNxEdY1*>ocHD(9u;NyvK?EqHMfZyx
z=_zD1VLMmriQc*XV`xW&nwvqKN}&zruUsi$xy3KdDI-S6K~ho@#;c3nva!)@-Zm|6
zDY-&Dsa8n(1)Hp$426Y{?r|a#HKdK(0d?!?$6I>&_;f;|k8Z~SU7xi62n7FS{PJ03
z0vl!S1A-kjr=uX_XQL^=^JmTq1Pl$MMwQ_UB3fo0j@9xY{Fh563jRbQR{2VR
z!_8Z?s6XWMAC4JIPCePn5-mv$oLV|EdMYauM$d^a0kYc3lQVK;+-e7hIk-?|DlodK
zY-|`jw4Xs7ofzl=67z?ie(?oju`?x`FrOF&#cGB4l;jVtHVQ*a;6cZOI})Ox!qqjx
z;veFyV`~av29t!PS`2l3M!FzP1;0CL@D(h!0MI_*T{zXM2T#3PW%Z&D!v
zdk*6}EZp3;*oO7t2mUC25B+<1Zo;w&T{z0c#JteivZpkB0H!q275vj#f?%%xUJ6x6
zJ4e+|x1P=ECDCv!xV+5s8LwZr;=}^D9|?z{H#?@PlMmC|>vi{T2X=@21~_uVf+&m%
z%#8y=Yjbq`4%Q&g5r=w9@B!%#U|(pQiCY|$T)q}nIsfzmdwVB%PI*As=i@lq7#^J$
zej_uCI29nsX`RQaZyN>IMk)wjx=Za%@?qvop1eOtr`WvEz6#0|Us~nY&nTXt+8e)r
z-=GhYuK^JGKSUem$?m!jw&W&5@DcvLCv`iD!IV1CAbjDzaq4
z?jV9VHg+(ykjp}z3a^R$;k@AEpec}Z4R{%nb3}2>U&blO*;<{t?Fy>_Md)ct3Gs=^
zG608>Z%MHTU-3!4;XeRAX}~P&YzegEh}W&XX4)Y(n&W_I|eQz5O;
z|3}&0@uxc(8t$RcK*WeXy%X*RZiWo)!N8?}wmh|im77=|I7lJI^=I8O!ty|51g_y-
z6aWZIYkGTk+q7X;`<>=L~-veYNoUV*`>9#*|r=m43MjFD-
z!{wCyAgX^PaCw}x^H|xk$JLvjoESJ*D`UUdaEdaG1xrS|Md^DZiMX~l@FZ|Ad;R>1
zlggRrl-qI=NF*p2QpCNs7v8;n%hZM4yLXTK&i^+E-(`l4NEwRfA9Hr7@;7?8s86w+
zJNC+tKz$AjnEDjp(#)SyP2pB7MMCjb-w=WQ7)
zr%fYv!W^+3P*9tct%2EyO0cmrdqEWC*8|0^DjU(F;PL8`AQjyW|uE@)1!0b>G2N7pelw7=gX6oIhbnDZ(hlhsbw`5S_OYiWkg8%e?n3mRZeb3B_tXUPMZ`TD_
zl)WzP8sH80oITv~n#D=Jc<%U12jQ%dPAGs-MQM6e-RNR!hxU>mG2O-{|9b(ow74LF
ziBnU1muIOSbg!}fZdG&F>FXBNp{TcpX0x_L|IXqP%Oq=P9MvYplG(~$y74~i2P6)k
zKK+pT2T*$Jn{eK~$lrzOzy~A(q_C|n(_h~?$FU`&UjQw}@SG$!16&D?HuTj}C)@W^
zx=va7JovzY?c-062%q|?6vs`q1U&3m(H
z#@WgFZWCnUVN{gM_Y1#&%$Zqze+4JO$j}LM7A%kn(bRl`XhL9Sq#L_Nb5x2(31EwD
zfPVHlj8tIRU`7&7U;m(ncpSeyC#rRA50;RTH3FbSOIgD7
zk=BA2?W?K0TO$+q)V!f5N0VoPbbJiu8Cdg2;)|Z#D376(VR_o4eqt}F7aYNd_M6CP
z34-O0nMVku^!xIWI7%{Fx)y&;H>3f9{gF6U0#0gEy{Yo${H4Kxlp7YY-Q4S7>4Ubb
z7;D^h_~(JGUu)a>a5l?G)%O2o>MKOX_7u8hzLX{Y=aU58jgGllVeH<{>
zBA6*omtGrxEf>0m1Rpe~`}WpT;9AilD{wO~pjWzd%5cYh_!zXfT0dU`-iSvIVo=Om
z>oQ$V$EIB;p@3QyiH(7E>ohdKR0Sr*k#mWCWlj*gaXQncDT-YSW3se04KD;($XUeO
zNn-g3Y(uhA4r)gq-9_Kcy=@
zG<4G-@Pqn8uR22yqVfSmKaeyQz1djE?chVl5zTTUN-syZJWxrcV4m&ST#zMTlz}2U
zmkNp{IFtd*LgNA{>gpJm{D|C|syj=*`E5i#z}c;hH(UeaY^r;Rk!C2_z@dZQ|1HByQ&X-1o)!Um#
z615^ASZ~TXbv|dRnxWavrV%0BF9Lhhx)ORkQq%xydZ7u)
z>FZB+nAO<@P%-phfwX60)E;Q=(~(vT4RbtaniR~GO&?|F-N
zD@ZF6`AOr`w9ld26Wc50l?)sNIUt)yg}5odm0xv_H9GkHvwZgxuxw;&i52RfS{TIK
z%O-pglpj*l&FoT1kU|zXyPzNEv=eB&goIB^6g7v-&{Zo}J`>$Kc{6%?jgF4*(q(0{
zMQjvA_#&O4nVd)C$W7%08lwN;WFrlJsGmDcm5d;gyaB`nwRLb^3VTWR<_q;gAssqS;-tX{Xyp6*flVJbTF0SRXs9SrR#k5h-dYf)n%+p8eO
z4Y|fvz0GLg0huL@_!#sZH*Gz#P_(b_W<#YaSJxBzNE`w@sNUALy(Zz}lEa#_h6IyE
zsD~t-YVbGrZbkp61t5%SK!Qico@w&UE^xsLQ7C_hA6vV6O^G?vM6vdm5={(RX!;(!
zetk1(g*_En=I(~>&1AS3odh;q^W>}ZHiR3o{sn5LucfpQ#FT1L>9ki@<@?UEQ-_r1
zWJ?Ff$&4*j6er#uT;QohY5;_Z6Pt}^xbGe0Dox~Czhy^zNSzwU8^-(FduVj4^C)4s
zfx}=v#YD&6DP=1>ZrII??xq<>`M(ok7Vs1=S1KD0EcrI;cT*jU=mGMYB1ld|tbN{v
zzaAtc_#HiJ@G&kVj>C`_e1!~LK|}Y>dxV1=%;;RGTPBm_aem)#Zs2@sh35G8>GZm!1g1#Ry5N+z%4DHNKey)qi=f&qhxe0nF+`EqBtElyCLkT-VIM
z7)EL+Wwy-iK%^2eefdUO1UeV-1A4mtC!r%q@4;v+TIqUFSmM^>$?U*DC3h%y`U#&6
z&Bzea(-N@e`RNII5*`DV@;_6Am#IX~!6Hdf#>*9n{ouj3=Ux3Pq?uw>eM;yTWE073
zIx0xry=r0b9LQh1`G1$WEJ34j%m>V|`&cwZ&Z~q`+-E9I5t;m-^Gk~gyu9!mvgygIY*f#dT
z7Kv%QET2id!Vvt1<9VYC6^
z|*isJy1QEvg|Au-4(aL8>Gg#jU
zMd{grBi|Rx0G0#-ABgX2s;wyn0oC0;#GVv`j8fTjIqNhL1xu9N&AGI@fKIT#c;)_h
zs;F(|Q9ue>223Hqiqa{Mlc!H>+zvIC<4r;G*AeB1QfbirVmoeBybpQ4WHWJriqe4h
zh^WZoNlXh}+E_6(*Z=jWYBG8bskv#vW_MpT)toa4g0L@u!~ZfQt#dc{
ztFHq$6_If5TKeTXGp2FTewkY7
zO+3#?8=IgfabcrH4MR~XK4}OLZE6KJrc!53ABQDM>8rM@dFm-i0l=#!i_4zZ%!R9D
z)<~(#WC+=rNKH*XN}7APPjH@nB=wrHS~~sU+mbQmS=u`F8(e
zG6`q%&`Ahj$x+Sdc~uW42vMs_!-9wY*o!Y@VJTw)4Fmf3jXubqa!(eLn~co?@xgDB
zZz{xwHS^Q`v>&L^98lCq%{aIyx-@B-YP$iVM_^W%8prmdO>Tf~5|j%Bz5nP@fs3Tm
z;_W8)U2>E)%5(n669~gH;1ZLc_5NPXJF}_}ku~dP2SuymtnAK+?8IURNWl_F}^3Y3V1KQ-fM6ubyu6#yia
z{?GYEa)C+^6q;ZVVcsBwN%0fKlDW;IQL`n+{DNgt$Zc6N=;$6wLBR5R{8gB#5w)Y(
zB)>w&R?|l(2h8WiG*VId$o;*|t#Rq2%7wZ19sa;cZ^Nbg1FKJObc!tvXHsPyn8Ps?
zG0f(k*-q)prT)YVD)sc4so_BanGJ*97tBO93Z(`oX2H`Z(Et?ivkWUNjn>_Z%nhbH
z_T4*?nImDA@FMEW`2Y+qZW9<`*z;nBaWqS`6JV$a`dBA&8Pmr7ebNrK1We03)+(B?vda;g=!ni5ypPb1h@QP@
zQ$^s8%odFan1u_a)D{9=MnrI2;Ga@^W;tg3_){=FYFj((Kj2^;CeC&Ck;H@s4ufO%
zKmaCHAeDR|qo8@cDd%lJ3MOAGxm_MSyNG5KoTCp1=7;y|vJx;QaYxh{a=C3KXFnYF
z_?5#O)hyfAf%XldjC@u0Jv0tWc0-}DoN~-_-$~WNiy-Yu(}Nn-WR1HF{wgjW4oC3Y
zoy*-_RnJCf_JyORFh@=_9H|Kk3jeJoYM7=)4A}AKA=xix47H``0N#j$-*oD60zf6}
zv-6;sn(B?j9GbKJTN!U~Ub*^|6@`fccz*1|hhO|6zJcg+x;YP;6`WOhV#X_ix+-2f
zk?+tTn{L_d;=NPJ#`2Y9HC{Cxyxk@5Z|{=9s+;l%6eswWHlrWtgvc%d(}pAm
zch+9TW1F}YCUoROnRO!qF2DlGZ$ts$$395uM#m`LA^6wY$n7;>Puyre6`68Qc6Kc9
zvh?-HWxRyZOcg}uLX*FPOa)4eGq1j87`}HQQD9Tvw+#WAjaJ<$YYD_D9(x!L1vH$S
zIaY|@^Ho0>J1NonZqjQzUUufD@>yWKdn3$ljymr0HNCV`(dMKPA%Q_b5=|omuF(f!
zdr}q`D0c39E)Ae32t0u@P}@$u?BAW0vnbZ14nF^*^!C|!>Vz~i84S3%1Br061V)*
zg|zD#ON`n`yF;puoG`#J{)$6J)Af}cc~v>LyowtF*l;W;Gq?OgV51(7apAdYPHfxFu}>3*oQn
z>4o;izkV^USwDH>c9SRKW^swsn|5xpk_Uf&fMS7Ge*pciW!~Q*|GuMeg)^|gA~--h
z0qLmYe;VSpF5skb7lRLB@xA-?TLdvHMgR(=7D;A!O4RG)M1s}Z#?vLsU;Dm8HOFQf
zfnUv;KL)KpvUPH;>6
zQ}QQU`)+AdRmyi+OHyS3o3tX+m8gKdbAo?V{+YYT)E)3u?>W|#Cv5A-z_SBMrU%f|
zTTNhExMIDzO4^X}ys5qIRIwej$lc-Di~CE7$}Q
zd()+~v#_YS72bJU#Q|dZWsRW?diIGXPg*aydhW-4eanv-9OG(AG
zUlxxYi_|CAyQ42Qbzvcj+l2J-oh)fa<9WCqjyzfPBp0gJ}*e1#uTe&Y5|+l)PfE1?6M{!@C!qe%r&15-
zSu&i0Qbu?U95~}%QC7c@+P6Ym_&xv(unqUFN
zhJ;13S0o^axoKlo-u%mu0>qGi{800J!iL3A9-1-QfbFg842$|?#ISlx0IX;iG9VcNZGDEH(Hz8
zhYOn7jlj0SkLdbjz8mZ;ClhX2Kkb>`4*K_W6+^AqZK#Dh(NS{*$TP}akrg8Y2Px#G
zZ{{jcdt`YfGV&D610zbUhk88~J3DZ}5HNBLh%75OR14hXrKaxZCz_7M6#X&goj
zL1WNdqGqv?Qhe>-yZ6|-)*RwtFTS8bh(01sOQ^dD-w~0u9^lXsLIMCSY>ecuQ0&O7
zV@*p!`J5ZSpZDjo6Y)J@;5LK~O1UlInC8%d@!z26oXPH#cL888S-r?b?!2v$rntIL
z-g=59=))<_H|N6F)~jxtDQS7P44?X##VS9UYJ34t1|!F$%@6HETuaPXobemi+PL*6#Ngu%(7#|Frr3#1I$Wijh$R3V6JL20;IQdc((M#
zlK6&1AJ9q+DLkf}>!L#?C{8?2wDA(p40oz?NO3ZH9^o~JIKeq`aF|_aBVRACy|z|6
zH5T!>IL)6gHrn7+d4E73>zG~pbEJe2
z^xD0q+?{ij=zK3l4NNyO0Rx2uD35D{!b0Vw6_`slH0YaKLR
z2c=q~8xUp@6lwAGw`x%#^gA4`LeT+;x!aO=Kn)|7O1_JHr8#J4MgpEOx8@iJysjB9Jm
z*1(>5!(`L{j^GJN{1Ww)Df1Y(T-Pte?{SLG5zy!o;~UUaYKN-#q5~TnJS32JsAg<<
zz2oFF6Y8*=C|^6x^_rH({U@=%uv8W=x4LuVhoO6M^pGVmxZG9fsVo%wmEk@q!3u+RVwVQK_ZPIso3B7;e*d;7xf)!4I&S@zI@r
zWDf_t*7F2-@quomXqDCv?GE>_8P}7}p?S2sJA@4bTIM%(J%{X|q8ej)Gie&4Bq`3Y
zpY&Xn89nb#4@xV#a`Sc0f>tFTN4WX0yVtRa+>
zM6HiSMair~;HsaK7c9t~bi2#!n+~2}KWH+aak2>eYH_kE(BPR!s0k2H!h(pBZ(MN1
z&OLh$tSk_Z=gp1f7mpZeXHf^HFfU#%9A3cXj2+YyL?G|x-fTCa_RB4!ffU}TxAozs
z8!V1LmqkcP@&Gtavm$8fc^Pvp;p0F@EZ^ar)wc_!0Y}|xr33|OSdDarn$NP+IO1m+KuH<&Ksu8%3t4%dn4+7GnoKl
z)Mhu0S_eT+NuLJf5;g&umI5`IFn}`Y384VFM3*`GV=WG$W^CT7Rg!4nNX>98+Hr@_
zN19t0L)o;{?*%JH!QCmq%&L!#+tBp=9DRW{r%(FteO=mW(Y5di$#xpHbetw
z6a)=MKE)|$i{xaG`9y4h%dhh}-T7mX%e{5=Wc@AVOYdQSNLDbKV=d1{)|!7GWNh`Qh6=WgP>XvH1(c`3fanF?S4
z?b5ZM*-e~g5FCy_Y8iTOd`v%{kJfz7CYo1jexYejf+$O+G1=)f43?I63?#Sz(oYc_
z$bGx?`k>^9Wkzj4b5KA;DWWz
z)_OFxqfw#N@phi?HU9KRt}F2HN5h+Q9r;Pn5E_r!vfNw0*rqpE?1D3AX7uslhdc$x
zkc3;EYifpwA3#x!(~CB6VzX&)Vs5uvxM2SL^_QjL0Ke<@w=BJj-AklHWEnekc`NrRINIK9i$C@EPkadP
zy`HniDRFI^Hf@HMZ6wl`5Vh39yAWC#ynk*MT0QpOmN$tEDw(8nqnJb#A`~WZlnk7c
z(+t?TX~ZeA;6Rn@px*=eM@DjEtM<=*qmC2E?gs)a?xL|{Mb!a(48z9#-jPt5UAr2m
zGF~EOsnLnkr}ATVOg*n_VBCIwr%nV@IfnG_)i*)!Qiiw@b^NXP}z5C3v}=JN|s
zV4MZ$3fr`8dw6u_M;Czqo+L{8TjX^L&ql<;mXru*hS$+uV1-IDjw?f$DBg6@?11c3
zHOl-$@J{XCz12_~f>3>TIsSaH)GR_D;JwF^9HE!vsrNJgyF2*%?cYB_Q*V#38_wSpfiH8map$Ge!
zJ7o9Ro_%|5s&bj%{K+oOv1=F4o$HCBBPFHhI>nKqHB`L^>F?;f8=WiPII5<9pFWQ_
zEnm9y&&w6f5^U*p$nfhe{UH>g+0u8?T?4m=RCpexuxeDSCdDOpjTq$#j{P?#PTd~?<
zb?7c16%n!GE(4-Pnxq*uAcXO*E7!faN|`*)
z*>()nqBj9~#w;={e5xp`Bzhw0+sXYc%SM`r+8_&p>TS8ig2edru_dTvO&1${L;Uk)<
zUG&ScANAg*K1MOPUr?tON$2+u8}uXFsZL`x$4FyaPwl?;3HlywQat+Q#VI>~8FcN9
zmFA$Z33@#Z7J6Ua*$Zf5`hhF)@vrMyI$2&YxN!N>rPzO&f3T1NmA|umm@vVAR8n_B
zJe_0^ZLFx0ur5zO5&4bXBn_?O-dd50KO46>QPC_)!C*7`Pgl2WvwJ)Ep~NFozTJOX
zkiIi|^%k!)dRH75ZRt0)>+1eJk8Pf6xng$%jo3O44ca-iw2!qpUHAH}IsGP|-*l;J
zbY}-uuLkoxH@Us5O`#f@iV_zS_LRA-$
z^?2+a>p`80f9@=6HRu1d06$&7)>~nUWOE{6EOgeSnsp$QvkPq?vNJM*TS=#OezHz4
zeZQan&b=q~DRtGXt5VlL&Eis>qr3kL-`#SBi^25bIlUT=b@HBj^+;mmz?%M#^@GRN
z@i~&-)?<6l;MosVR&8{A*2DAVk$J{_KGmE(_Bhqg&wu!+4Gk8UE}z$Z!I0k_%=Rg6
zscfKEZ9PnT@&0Rz_ZjU?tlGG}^Y-b}XUti+V%yQ{?SJ+A1?l$5P=eB4zaCPOaew8D
zusNtTr6Wf2C(!`?u(Dy@hoYJ0tL$WNKjZ+g|I>@$L@dN-{YD
z65&P9YZy9o(pC%@wI$SaxXXwU=av|`hP|5D`iz2q!!b}T<4kVKQ@snKmK_q)g4ns
zP)G=^h{yShc2&mna^`#a`gAeq)nvC#DIie9)}UiOCmgd(Uw!;%lh6P8Khr<3!clqg
z;UfucjJ5~On`Lx&t*Xfd2iME1UWASq*Fm#Y=Q=wFT0Wbc}3L&R#SsxX)(j3>{#+I;WmZ_|!$Cr0ud!0rHhzk-2p}DR{!8e)ebohG@?^IO)jjI)(`k
z9CK2%+=e=CP3YR9frV}hQo@#!pF+n=xy|TVgDz8hk#k&@l;TYzq_(Ml(Cy-NJAHgR
z=^K9W`<~h0Km)y50iS0&pXqk)!-`{#=Y(7x@VDslgUQ~QdIs6+mn+tfZt1nr+1$sx
zlR@X*X9ipA9rtfFv*$34<0Cc|&CWWl&~0$H!oqJ3UB8WNomkW4u**#mOZ0N_26LQn
zd+iFs6=k-hUF9$IboHpVs0DJfYjLa}ci*I?NqR1R$sdg410ZVf6+U;+)3H#noXGJej9@>owNt2
zwMcl`e(;1Z&g*a9p7c1e%MHDQ2TH%-2KF*mdV2();!3|U1OJD)o{q6&pCz;ZN0YV4t}UJ
zx{Fh{uPwLNvUjU4RW44mUG*?o9Oqv5az1>BV=pbHDnv7OpmNR
zarfrILt_w@*_?UR`1zN`4F-I9Zm_?^vh}eQ&oj2;!rk6Er%^+vr0k{G
zC|JmJU`EVED
zJPriVm@#@`vDL2;iIygQf8TDf>mh%pL4a;g4nJj_t#7t2rcYg)-0qVmcl}=X?%B!9
zO;&GqN%Za0{LHL7gT`fTItXC#Dbvn8IvSJ!0b&9Lcq3KSE>$58eBUv9tQrhkaxOl*
zN!^!rOPzN+t3H@Axk0AF^#JuZVXbDLH7N|6TtbT1_=uTDdMBqG4(nJG{d#
ztA=B{k9An>6nFn=U6w7l33U7DorReaNG$m!V!O5R`9aH0+o1C?uD1914|UC4^fNM*
zVA`~F?dQf8{_W7li_Xf#t>);V-|I8`87&C188>Wu|mV}
z(&rqrmOZ8q+rKU|bm94_u)ia=pB}Pf5)~+A$A^m>ThL2sJH+%lsXi(z|O(wjY
zaO~HVE$-t(XSKfe-R7D__|`5zeR^dFUah(@%(tPbR>9Ji{qL_VNzQX0aPwowbMI7#
z9=g8zTIj@W7LzYd4(y+~UcMUHNIrj;e}`7cIl3-WOGl?(j~)@}SDL`x$1w*-a+n5o
zmEL9z;-HRLDj6AIbi?OX(+F*Rb_Q9S63U){UsvBNgwtYSwf7iq_}Tio#4i!Z;Y
zp8i-_xIC?@qRWg^c1p1m(?t&+HbYk~FiV;+I!v3io?BP)S$ZX%o4%b$;4d`P!
z$#zTHY?W7@Hpk9({CH*Cdc%{R4F;;zwDEP*9H6@MliEP1>c$$+W(~VDZnvTBj?00)
zf~O2QJ1e{&4#~K8?{2@oeyIPF)UABEEi8qY$nV+WQcF+QpoAT$TKkp^Ie2v`YN^Mi
z+oY{F4E55WR~1Rpn=5ncYJK=%)VF8$w4-|tEbUv;CFb*$1{><$dA0PSV(Gkjzt?6C
zNxL|6XI7Vx6mT)D8D(Afa~~Pbj(I}+Ghkg_yYvwe$Dd7|KRK<@nofOh{dcS3i?sB5
z0RfFqHr390qd3l~<*;qalVe*X4GnpEviZSAe+R3Vb?!T8_kUg1by+$k+^la=Z5S*b
z!WLiB90^<@ff_DA#&AaDrL7XzE#JWLqLOs}@7KUfd{YDLYdgSD?@pFR}k9ZuK->YzN
zjIYVGnw@`Q+F06Jx_}5tzfI)v<
zSy=e{OP4=Ci>z3`7V$$RErtrxsqkkn!{EDKqr&4ZUDP^f*mb=IF`Q?focP=_Y)m(^
z?~VP*@UDB&^lRMY|Gp1Rird!k`H5wEMKd~Ny^WFR8dO&rJ5|*g)+FhcuSpsR)%buc
zfG*@xeAUp}o4gR)`=`ti8Hg4D10UX;P~3`+l;u7Bj-&1?e~bSV#$E6IVX}F2OcTWo
z%dWe&Z~OT7>W_=F3m$t}ANS5{=!|jW6U6(1WKZMx(TVeO=WV?e(B}Pw_R%U`2kYP3
zrkAEZpkmy
z-Y%QJ4zGjtLNh$a*%-Ljxk0D|LOo#WQ%E5}kQfr*KE%CvAmm2&j{PQ0v$}P7w&m*{
z<@d_%ofQjn2dG%*w5mJKLrEIclIMwO)b8WoaANn2u^Myz*0jQ@cK6j$Jw|{+Xc=Ue1xEa@j2C40|yV4+X@paepFD}n}
z(|c<8%oz(J4QT6;Y@~ds8Ps$@-@Ox(C%jhqI;QWF`d=RxZ``GAuYV?8*OM-H
zcS|f?<4}M5=0Dfkm}*rTi@JxU&{NJp?`;~JRSbsuglN;hLfV(SG90RX9lH*zlszrm!=fRBdKCg@d?95WroKbumdacW+{xcsyi
zSFOKw?$w2~r6+@%MGdO+?mC=8<`ZDS*^m2)oWI7dy8Em<{x64~U0nS<)_K_XOVfKg
z545y?*I`z(jD7=y6MXek>b{)za&h|NHyc;?2wC%Y7f@-7*N2MH-vX>j4t8i^4lnjZ
zw_8$M<*?9WL`Nl);D9zPI{tCobm~Q~PCfT3fBDk=<(CtlqlcX8tGa!9a${xfe*GdR
zBtL~?>$U##oUrKQDR!y%jE-Gv*IX;UTkm?eJpP;ux-fS}ce|KFA7-oe*jt=K{enxg
z<)hr?;+KxubeNjpQLmO#=Fb%Ct;7t01J>G`MyggJpqM#m-JYT;UG36$f9>$=``9`$
z*>Q(*Cl1?~o#t-0@ps&s<_qf_KZ3F;+qWiqe3K^)zUD_CY?iI)T<_PN*kyLkl~+7l
z4>O1zVt=;2Yuz)!m$J@cAD|mWgxnMm3WH9hdcL$;ktE8aR!=c`A0#kW9-ayFSGYm@&UYM*fng
zlhP_CgdcHV_4MO)Tc?<*1t!Vv2d}p8=b@BcXP3f0<$(pC{%W~r&Mz5aa;S%+Ye;VA
z6vI!4-YOV7J*#7v(@x(!@oL9Ib@mV2IP&qf-zxPQD|hVNxpUUEQgq5KAGPlyj!hJ~
zXJ*gtoYXYAbGed6SR+AX5uUr2JnVuja?&lMfH?q5YaSoyHTHO@YqQtE$)2-o%8Eif
zU&OjpuUmI^R82)NKA>l1Ki5(Qmu~WSRpYY4=*RqLVH=;NM<@MSm^X8o=AY#9qscSY
z*4gNI@o`Ofse9%Py@`sYlh4)I7scEU8uYH=9`_OV>^5tADX&a0b~x72d!z5IhB@&z
zw$qlCt$AhqC#}pW?c5rzw`o&a+!(NW!U~_XJ1NRuDhto-%`G3TeKIY5^Y#|slh@=a
zZ~fu$Y*Vn)fOWxUd0^Zw1uZ#blV9GwvHs1x|8}$=b=9hF+3v!4%T>D7H!SuwzUF4>
zxNc<4`m2iNb<<8(R9~s?+xA%L;K&=QJtu~Et#SEiAMsda{U+<@A$r4CMHg+a-hbwO
zta+?q-QshNXDS}?JaB94PSY<{?|%m87CDV=?Be=j!`6$>vMfvAT4ZMp3O!M7y6=5r
zpVK8<{rl+0_w!vCf3VZ^>&pX%Cr&=7KdG~&%{^p_TYY`4A}{VXbW~lUq@le-1exMa
zVmpx*Jyvnv2@dPnqn{W5$=kaor-$O{vWJEG$8J^+bzU1(el4ZTRjcQH9nI?al$CqG
zcoDtoc(qICrcKV2l}DNsG|&4P)1}k9{USz;d%=2hJ`z)#JJol64UO%^MEB{jK(6*W?`c^ExVJN;eHIDXu?LQ(^8L
zerDC6Q&tUTf38{d$tbVN*73CBug_L{(>t81n0(egXhE+=dZ%jqzv|!muJvJ<@n3VJ
zPFsq-Qxd;e{dG&Lmpb_DnhjPZKRfLIdE`)$lk1D1<99Flk9CSudXe~HR_4^vjyc6o
z9=6k3=&pSE)aC9oV&adPA*(ndH_+F7z-JOLyBBOxF~7GhDICoL`bi=Gt}TAP8MFD6
z&GWVr1*3m{V&0*O@B_ze=RRtDCcLIYb>`Kz{?%oUpYzl8>$&dBxBJ^y<8Ns*XQ%by
zqtZLRvT+Xy%q(B9pwg~?ON08iLYK|CR_9hhQQV7|F5wG3LSizGT%6(9`%~|VEm1{j
zn|92*loMj${XqLp!0)WC4{sT@SgCMpWzr%Kjax0#`-j(WbG%@D;pUa*{ZrPwSbuQy
zpZMObU;F2mjr)^sbIkT|>+IV`Cd+oLEZnX?q0`f%mFD(G-hMTCwV}?C;ptP}P6^p%
zWntg?i}Ay(PlsBS{62O4qk+beK}!q=EI76&_r%9Q+t(vXt}IylEn{KIj2AvFx{bno
z!Jxn=*T&r?&l83zb&K^FHN%lBPfhMxfeTm2QA9%!$7Y-7Ef63z%HO};RpV9pGb7p#
zUv)A{b$OW8hU3rs_C6V~w)o(b&)Z|Z4^RIyL_sh7?p@ctp7K;Kehsm$@pV
zQ9suRl;y`7-wF=?Toh3=?b!zv
z>H*KM(bZ*tTf16qxDm0dx?;s{=fP#k3;UGB?XDi!<+_qdy$esD_W8E`enLY1Nh*`x
zpP2hV?|8uPS$5M-T{!eAx$*m($|=_m#jNc6EiUV}$+Q%c(La8b-FdMm#4V)k?(7~@
ziuT2+YG$t4XaDn1bV0mPe6c}w4~-*7HVoNqu6CyA3{J5A8TL9;Z&{hc@*dDXe1Upk
z!_PC$X36s+d_NU@?8XbbcMP7r5#gpIcdB7vGS{x|fGP$QG1>av5n-q*SO)t2bZ2Rb3#NCMK9T)3-Qo5BHej>_#Z1Tq5
zS2h{{OxCPxqM-CAvGbgywngT?>6N#4A9?Zmr^gS4*ZEfy
z9zT9O$L#I9cXb(Rdr1GV>z|)XNX+hPzP{x~;|E!3|F5w(f#$kxzemwTDv}UNGLIRO
zF(ee3LMmj+5GgVg5)vXwA~J+RNTeiVQHsnJk}?mOG7~ABeb@6l|KB=it+UR%*0WYm
z?^C|t&*#3cVPE^&`?Jw)XHM1cJiUY+frs&F*K*=it-Xc^hWx=tv#JGVx-KP;o?Z0X
z@gROEXuD75$}ftg8Ai94yxFt!Sx#*0J++cj7}Z+ozf<4v;JT>s_Zd`t+}P(GkkdLk
z8q!ugJh8+^Q6WJe9=$jiKI%NUpp~%
z%Eso7{k~olh6f-r0nz=0c4wRWAkkA`3duvmtYUp`*ER+Q6_l<-Ttn+~G$k@RQLvJA
zh+oen91Nwtx<<~?ket>*hD}42I|>f8u9}%+?b$Ln->Eh`e}!lENw(+V6PC)pZu{hq
zE>0_atgnC0y=tQR{md
z8uS|^<|Y{V_54Ck@mWph(_D{Of0Fj$)AxsXmZcvKMZZ~;(+Jxxo~VB
z1A>K%Lf!{36a6tbN2~nc_*1K*bHCGgOoL4;zkZFweivjX1nf8KreiJ41?*e)f|H2(
zvm9c6%Yt`~Mpac7j9AsF72bp
zc$6V#a=LP8x4NXu+xJ@WClxq3DX|59HKqhY0kECDv959Lr|l=HBhH3+;71WjEs({d
zlS>bPs10fS^AB@2FTN_iFslCR(NX&x{qwq;?s#xfa-6n$vm0cxcnO}`m{$Tv9;_CM
z$$ndwr}VH-`^X)GVP4L|5vRBU7AP!}9diPB^=dPj&K%3GpM_($fIzIvMtsTLot>RU
z7hH1Ne_(=!h;U2E*9Qd+{fYQspo@A4eVF@jZ}wdWf(jyHQ-`*K5Ef`FAtZ6`oJW>n
zdsGF_2PwIEF>%YhOo2$dhE$8!{%y~;WBv*RH`iy(EAHs&pz57l4AYK^F?Xa|LnVF(
ztA#xOQj{KZjryGdtZ7(QHgC%u^{|Vsi&HbnYIz3dLo5{vR!OsgR0+C?KNxp8?c`&<
ze+MQZ3De*cVNXNs;Dw?F8UTj)5}13+96XnF@7{LTm5G(y-3CEkf?E$hn{*nSO|uPi
z7ihm2LTMa#H@E7I|4KuWF0`(mU5${|qzs*vY6LQy5kM=^#R4#5Si3#xLw8VT8GrD0ioa>)>=)ra-4A&)~p^
z0UF&ax?9&9Q?;<}wZ?Ib53U-qj#1$DzFOf@oi~8hHX5J9U=eff{rC0W
zPc^Oum5Ncscu(uOGR;l*=_?8F4Ag|4o}Nh6e}StvmN=rzhl+}edtqt`!5^@LWxY(k
zK~fW|1hJB*)+Qc%!a$+bPE9VUrc4rpGOItH8WF5MxRViAw-QD5yYo{O%
zYzt%<%5v3%B!P;L601WMV?L(r;2l}Lzs0G~j6=~_FVcf2ZmCN4unW#@2M?D9j7Mw0
znjnszq)KUp8$1zy#a^zYO(w$u7i_OX#T1K2!CMbUPc}8Bd*3&jp}D2I$tEYA@&(0~
zqk=hS^4|}sTNQFQTAZioyo3j(
z`xOR#=79H78W-=5-#8G
z5{V~*Ok}^(D|8)Pbj;*>MU7s+ve>P25K=euf3}KllK#mdB7NuFdj+N
zldZ*0&HI?$9AsA%Z7=?fx3TIiB%{H4m%tDlYd1*)qoTg=FRWX=no$3UBF^VE>+-}z
zZ=N$2S@qo5n_^o;bY2I=XNs;zh6-!q_Bu6ON{w3jA1*-Dct@DTFCR9D3Ifo?9WTd{
z6&Cq$?}&~kuF%bp
zjgW!CnxC4VAH+)}D;-d9v+Il4ZLUakfw4fVjM1}8-COZ(5V4H=S}%<`DE2wNPqbGb
zo0@7i5FEaBTjf%6q2Y?~JUcbz3*?Q%_q%!Yo}4>r3$`7)TYLN`~qx_mehjMOhu=%i0;X|0Dbfm|1g&(5DKwwJ-s>K5k|6QA6{L)^!N0dfcS_N8sf}LoY=8+SXVbaesTl>y#c!TRrn`nw5qlQ
z+iFn|e^puvoS5!n}l0jQ@)XnPZJ?j)G(-PLl_VPMYe0gZ-qL;NM(0`=m
zk@)*oW+=E|34{C4M5f+-6-GhBVBD>=SJBvz2Z&1eUkKOj#U7#WUq&e0@Cr6cQM<37
z>~ieBXu5je{VSh_Y;AV0E#x}Ypf;Yqzkq4}YixN}
zsN~3lLp0Z42w#UdBPhyZgld7IRA
z>;O8W)-nnTnr^we?u+1^1*)(X2Ue;(x6b!Z2(6ddz;4Jdlc9f}Pj;L9;lmD}Gb_F<
z&JpoaBO{4m**s8Z$R+{tLn~;-ibxT5hDOMQJ-)-gM2bmC
z729OTp)rBY<5P&$>iJFxoIpC``KOaGrZp~%VlyjV!K@n}XRtA_aIT*9GH#%PF8
z=hW2T4p6@&KVNk5`wqGXJ0^E6zcbhH+P`%@^O})24tsJ~`8O1fa0JfrU6-5|i1tuA
zU!Jtc{d?J*?p@M)L*GZ;J5m)EgsP3=1gj@*a~!ifwqBmOGVMMsNb@*@)L5RF#>8cC
z(91Ihs9=E+UDb*tv~U(Go-xTc$~+h67N1RNL*^bdV>IDf%8*)eea9wGBY++%P%
zt^j5+)pMaab7r(zs;=uh#7sakL}EAqaS?Ppy*(Az-u_{&xO$b&gZ=ly6V>f{cLnui
zGn1(^?lqrI*rTVMlpufqcQJ*Y-pb1`_Ejl!>pe1j-%>oae-*w)?FN<2V+OV7RC7rV
zdo<$8&b_HWD#PfUtbB#4u>T^bZ^R#&qpoj0e%#E=9MjBgqS$>&iMWYju!YUaRtp=D
zVz%SSoQM-<1OLB~|Fc~Tw`s6K#-WH^L>2?2>LX=7mx<6hpWUxn=s-a>3`P#U0|SAP
zkp_uhoG>$k^p<@`u@nV_61q{45(Y9Bfb+%{-M@F0RMqxdN3CBoF1U4|*+KL4o|PdG
zA(+SJX_rUZJZI)w*RN?kDkz%SQu^Vv*b#pIAN>KBs&5+#9lbt2P(wVw@y2&EZ{o5A
z#|kw+NRz+1<_6!LcsX(U&#>!lbMBR~i`sSC
zdQ5u$TTZV(ttX$s$x(c5-^w4kvAkLShPcoQ1GnsE`=l=Kw^ypSY>Cp1qNj~p$J*T3
z;;ZyobBiOpV{W6Ju40L|%z*2g&gvc4BsRg{j~?p~SetsEq*Ug{+-MCM5jcad#Y4Or
zmHzQGx)#f~!J8S{9r2{LV8*@B=)25C;?>1x`S~{3BPN3(bg{WudcfMj-N9Zl=(ytu
zVwrs(+#z%m6Ha4+sF&lEBXsvL_aml}I7Ix(05Nqm#D9s02S{@lK
z(>r;1=H|z;vjq%ucQ&Y9?>_RJU*!7M6tu^X=s0ywRPtg-K@uOzG@aJ3<9M?lcm~5c
zw%`;kdcd3sEj^dw-o*5R#A)H4ZIIpPR?yukph)BXY%wV?*6))}NJuu#@gW`V?0Zbv
zX-xUq#uxQZq{>zc&+^4e6;F*_K!z|J?m7^cBrwazzt~l)(P5@P%O^Ed8CXj7e9?Nr
zz4tN4h%S5axUlZ0(!`&m=@!#(R;&3_MW*u>w{B7XZ5qmJcJ+JO_w0BJd%KYQY$@13
ze=+`MWq0=4y|C?j)f{u5ScF3ugZR9D`E;CqY_ecu987-;?AbsYTU&5#@%e!#
zTQ)g4O_X)9rKdvoWEBzWMS#VXR=B(v1q{TMYj1QRpRUJjKTYH9+sk&pKxB9#fBb9Q
z-PDHmWjVe>tIZyIu1IDxg(*&jx1K)q(RYd0)xco-TU*avC8eLgX8LX^DJ9;#*^gbe
z{7dPO#L`OOmDp!lkemB{Qn(p!W+x}-6Z8<3Tc+KSA$rdDht786ufPy*#zwh;m5+cf);_m!LgPA
zUj<7Wo4|kMbKgru*+s449TxFWZuUWHSK@f=lfjRfhMBil%fw?8zucfUON8~ah{
z%Za@-06h2&9MWp$S&!w3)8>3F_OFbxTCk^OX>0qlKn`wovBg1WzsB34L;w<;`~3M^
zP?6Eg$tv$^<(mj%Uxwu)#9KIdcmlbX)}4ZWFNpH^0yyFnrQW}8g($bYM(v*xG`5+N=kcQ
z7aWDHi}wkAVAc9JU
zWLK{qNNign3{#jw5MesBEEy+DUKn1dOEnby=SgVf&#AgsjI-~dj
z)+j}=QoUe)sYioBG~oRmNMIkrLK{FNvcQQxVlehHOa+PH6+R;5YiYsp{rdxW^Kx^O
zac!}rdj|pr8p?0iL8+J>g|Ou5&|$)!Y%hGO)gG&j_fOCQ?8xnnyL_7)bbmT*0kF$o
z*Z{H@(XfJo&`jN{_}JK|V9bNNWe@vA81h7|hk{b+shLBSwk1}t8X
z-FrBg2yexsl8C@n!KWkM3MiQHMJ{2R6@GI~So67h#}iGNS*gc<(1>`-+pMwQ
zonf>$f_xPv0?9#H5#JY_cn(Mz*Gfts#BNgNibp&a8&7fv((j7QcWfhym{_yrdQCrJ
z10E*o4eY}?a
zstg+5M92m=0Ba)7AZ}%D0Jx`tttX(XHs~&Z-f1eU23?--xQL$e3e}qREPH62;46WG
ztPXT>yh>6;kSp#Pe(l;i;#Gqp4cwLQvC-Ed`*t37eE|G2jy>5y;Lw_#coV6>svx&9
z1uEEKF?5b^-xBR1sM%QLN|22&bcCq;8M+}maQ0gesJwVIkrZ}U5mQgu*&PJSF6>;Fp^FE)P;B%QbsO;3(tKBzO-nr@
zK#Bok80J+5Wym%_Z$!5a7YR$+e;U&y-~y2L0Fv>SJ~`x*)(S*v7B_zp%TkoF#1$U9
zp0eu$-&hQcgDwX)8S8Vch~-no(3QYZ4Aivo7RXY`YH=74qvhf8bQG?LQ?2k4B!616PJ2k-O6y8PQ}ZaS)DjpV$ZLafp@Cs!F_P89Hw_{3r|)Qi9J&wSa64OYK#GOk{|I{d9*H{OE{
zATDGmUGh9xZozgO8>T>tG{xS^AH`}NKKK;KFKmzNuw+M!=And7jB|I|abisj7%kbx
z+`^}`r{;2$INl(gey+uS3}-5^j!1T@A(POH4qv^j3lXFbA0k2i;!TvAaAp;~v!9xC
z!)XK~plF9~J5TAr5SaHLNwqn702^%)_#)U`*+_JFNZE+C@WurWOaQ1YKllcK%7$$e
ziAFcwaA>!W(d0r?4SaO&t>mvw#VRey;}Bq_3Gxs^1i`Lr)K`%XdaRoeRXKuDqdjHl
z7mkAFIW|U#jwbfG)k~5fC>tPl7xUCy&>;e4
z7}%DFZ18GU5!*G~kMC&Zqb-7cW&U}D
z!65}_M-~$wGcz-WA>lm`T!v-jdr%o~_?Z74#XV|0^)Q@W_6<}r>VZ_wy!DF5_Ok85
zWDgMID55=x9cmJkrV{@wE#W$Cz)G-Kjnq}O%3q6%y+3}`0pc>UwVliA_TD%Rv@RY3p*;$NpB1chEnY;kEEhrDcgQ3#2I=+S_(Nh*y@
zysxPpMC%1oAAo!Btv*1c^@y7zG*}!yxNsoTi|lt3byQnhUdrke
zR6cmT4#;<9H(YxDo}gCjDdX32udiXg2WXBIUghch-=wqFXJH4
z_iAxO=xTI4oXwyogGlg%yZZ!g@?o;fi`9pL%)-Oudzz3P58&0P!78Ci1FErk;ukp3
z70f_{gLuW0V1-O+^TM1n;7~d$6$W3
ziGI@m-WqTw2-Ov>p(Rd|r$g63gD!Q12<-mIP7(pTvCl|Sv!_*^8*CQxY6e0O-VGUK1Ijl$lpPE=6iy?b6O&;mN?eUbxQ)flLhy-Ptw5`NSL_PDJzj0h
z;Lnp}j{XAHO1+pjTHynb6?a?}R}9=!x}0Dw1*^V#gz0J|CCn@!-w^>-4~;bZ$cy~P
zb9WMm2B8iQOb4^$hTz7N%mxrU&!4Ee*78A
zQ@eKU;@3~5MouH@%8>C->6$>G$^|rno~IDnSV&ugk-CbcFVPnWpzrl%k#bySc
zjXvf?>EJ)HI3(yBtL16O=b)~o1D8YW3Gm;kQP{Q)R5g`;7AT8i10f3NC$T2NstJSx
z_n_WaFPngz6fVPgFj&-Yu)M~OCr111c-(3!9}f-SiSI!?4#K)2Y~&*>Sc;G;BrsmY
zZ`w|-HpSLHtmcR$5RC^DV)=TA4S^iY*~r^X=4dXdWI~MFO~m+6aS&B(L}X8^uCtDg
zcVWXyx-Jl(hzuR3sCcJbTZwW!>cegF7lettHY5zFtlnZkeiO>ufUu-I+Hei~fW}BL
zDTES5K;pTCtpx}-M5mu36r){sW(itaMC!i?)R|C3+B#{kuh{`#z5ykLNnFAs
z�QF7N_nV86ULYY7n{(i)GurfI^lUIMiw77XeUkx5~QfMW}hc?D!9-5m3EF>#V*n1>P_uPFd!3y>N6esc^yMe_V9*D-E86ktf4gW;Kuf`}rpM(OZW%ug_
zYaobZcMi2J5q6`YmJpAJqc=J*OHjN?Aj82X!!wBMp=PO;6ep6Bv9M4hy6)&3wQw!*
zhjCt#GS_oXe_sj2(d_B#BMPcKJJq?NNP=>QXbS_{-eCL_kC}`d5SWRx8^aH+S+oYt
z+dd%G_!kuwtyUv9`y3<6OJs~#X{Q~iY4E&YdVEVkD$)bj5>)F!;y_VRaTOp`6Z$5N
zO4PUwI8Xm=%HR`dDBmc@wIhz`c)qEKi)(k6AtnQ}a>433;aAXk>uVRlx|Xkl>V-{SNi#(O7i6+R$jl@6`h
ztimFKtyGjU&gakfkaEheUmwna#IuuBxlo^AEBymv3r67L1XRa0<>`*KhORg{YZ%*p
zK*LQFOd5*PZZcjnf0ckpH3L)+u<<*bLb%v^Cx-0D4@QKsfp7y^ekF}LluO_iC*0Q+
zK(9__@|bTXpeoa>H4#Nv-S@~STN91ziBgZJuWRX0L862qv<3rP6J|Mt36hzh2L2J}
z>A#=<9r2rhAW+~8!g)n^%|@IBKr^^)$8ZEoCwN9gI|^82GHwFV6afd$%F>eajw*Eo
z53yf|NTTG@L_kmw7PGmlYxYAClE`#*{o0Xl9ERl|&;)zX@2y=~f$;J{sQcKX{K1F0
zef74+UndQmVPXveMG2mrBpgmlU}TFWMe=hIpvf6Tj{ghFhfCL1BPZR(ok8D$ews-B
zqNomtkB>KG;8cCM9_#@$^O=XY+C*am1rP)wYB)PP-zvAu!*((H6J?;GoxffYIbcYA
z#39&19=Qbu=4hXOp(&hce)jN$J%TJ&F`_Xm(xehav{d=ifnSrw10sinX(EC5VQ3?x
z>wfY^NC=4}L@f~ga8s$LE4~X^nbJ(t7Yw1_)pUFE!-o%AD<(v^Ur{j;1F1vZc_&aR
zEKAWSVa5VAV{!^13V;r#+tS284Z6uh_7}mBoHH``P6@w4
z|7HQ2Mr(oBLG&k3Jte}gUk~*I?vIs&Lk()ajb_U@nZuALB)gwT5Dd<%44`JSO(KAt
zS~Mg?5ryVDX&)kd@Ikx*7ZCdvbc6&{05n6UfJE8_?@Mfv1u(QEvI7BNi84D}FHkY2
zdrb9`>xr?Th8hH`hz4CBpurQLmQI!?q4a$UQV}3B7<5Q`{37)S0CbcA>PBnCbFrd_
z!F40(TVGyXmd6c`N(OH^fPMo0WQmYHbcW;(+7xwkGQ>0-(X$+Sk!S%yv^di&j_sB-
z#5j~N*YJdiBs!XNbaexzINC)egMNu-GMO5F8GFftjg^+j=t>E-sjUtJdDV~R_}+fa-hI;P91
zdUGpyHK1|hG2XH0-${6J_$4`5o4M3kO8>x;AP~o#7_Dg~g(*Gp+!w9`88~oV9^l|G
zFcK)|%_N3s|3q{Wpud2ZfV{K~Eeb{?w-8Nj%dbB}5E6q4lbQ
zd&O^7Duc$N8{LEci2!=p>%dtcyJCSV8s=Vym0kQGj)GSfUQ%-D;f+<{H*XRpz~^Vb
zMI#R%D&IJSrU5$rbiis5;%=6dl>ANF0q?zmMMJ^!4pERq3xgxOX2`bJ{lbNZ05gHL
zLCVkKxwtbtX-QEGxf@b;qZ1?gy`XDk!2B1XuNO)E@8|Nz`X%C{CF|%Z>W>KCf@*dG_0^44(#a^UKOxFs5C0vaF^O*f&fAGbsBHm>
z$b~mEfDjU`z|#Ybt{V=M75xMds-=bR?X3z
z0te-q=GfB)pp1M>S%({)isMA|6nWjYs91;MyMl%M9!DhaEF0qY%&gWJ2KSaw&-8};
zIQA{Ox(6E3G!khrD1Zpi_?Wp3Hf%@XRt1SPBCh-JMi&N?aLU>bn2F98HdlyCJJ4VQ
zx8OG|6odcKeax>fO(tH-Q0Gf|QVIj;TDzOzqC__mdAf(xCDML4hCRc>G|DL0kQEph
zR~cYQAUgn2%kteJDkUyE9R68(UWLUbqED+cWznUOW)g
zv-fdBNV3BBAVQ6NdtX_Q;Sg8+Jl~jc7=r
zk|y}`tEHbNfW&#kVxfc%D473D^0lqlp9B5gq+1~eleLRDes14$zf
zPC;;r+`UzedTKw!l7=9=MoMy&4;+S{aSJGkF$%N_`7A|ROY4ztdDiXVB60so7LFSrQNWa7R_8Tr8G#B0>h{yuGMICH~
z)J`rO()S1A$+~?z0lX0Mdhp+-{PDA8Y{iH?Mamklo5NJo|OGsDk&XTKP$|^4L$M@(6ERBEZ`8!56>o`
z`Hdz4lO_mRu$h>cWWJx^gxVVp2XR~>YI~?QRFNKd#7J`xaA-FwPcm-9@;GrBK4
zFB50xCcwu;JfxStdZiAxfeZ)z)OFjclAB4g*20J~dfngq5RaJFO!lnjnAR_zEE!O6#
z#5GF3-}&n|Zg68U9j^qojx|7l0fi@nE`zDoCjhoM&}a7?z6G&NqN#~9{{?d(NZ+M6
z_q-vSgSewj=r1vYPPL+r*!b+yy-8^(xPy19i;D8@9((;D-xb_bfJrMHFP+f1!^q-5
zrlAW+2F9KcH?;|-E25Kg`~+^a<=
zo90g)5#!{itwq8E(Evf!Qc*SmL0mMcICh&rM+bEPWGq^uKJDk`9rLT
z+%PZu&E$}u?J9EMgjc*AOIgpcY?eGDA{FzTLml_QVtU{FA((53=*APBrD#X
zJ1Ni%643+PLEI$Y91i*20SgyV)}bMSmwP38Xvsgn@n!q)a!{3{bS@{^4m~Xe;B58n
zCD@e}gWV<>t>Ln7SmgYUK{=qD&7GZ{sDchKOl}}g5L=U$_*e1};FWliVKTvMDIyw8
zD!o%AP9XxbEOfHKnBhZQ0VqI-y+40GMD0#yF0gt3J`&;r1`FXrBBc+a(15iL4!{*`
zGCK%)qDC{YlVDP@Hq~*Y@jVOAJFK~gL5Oz4Qs?OTPA5T3eW5T~+1%XBA8vzok~C9f
zSb=R11|H2)N%d{^G;{zaFfk?%3&q~R*K>;}#t^HBj5De=!g@diIza5C{)M^sW8j8h
zxsUCBz>!zpo)B%cv&#aUhM{}XB0Sn*m=FSvb5r5@
zQ%Dc|0MHGoId5>&Kx(q<*N?nLBBhO2raM1@`R8>~?t|_SdSY>OmlE}VRsFyOdmi_-M%35q6Xpt
z;fNeLL_G}D?#i9N1+E)!59xFNgHhhUUgQA2h@za(1z^R%b&}(FDjyBhZi%xYfR}>t$xNLIfHhRTzeBC@Ij7C(p9~t%_95QjM|**BsDcKjd8l0t3m;Ern*Za0vFh
zz|A5ZJAw{o^f6gKccZ#RGIR!~0!20rEiIRTKnQ$K9`g2+FxJv?6!YpLews2M>u6jF
z_Kxd)xKnQ*nDD}4i
zYezRrsFw^0rs|5Kn^mYS#SmH`9Yw%bVsuJ@99)M7Ou!f-c!Nj6BPh5YWKiP!ot|zt
zl7iGP>m}j@vu%8jgXq!7NSV;#@qO7L`iQKQcdkRu?3K3$W)!DCBvT@|!DpCU}aQ$ZD1OEa|syirJR7
zqFK%i3?a_%(xppmsqH9io4|a>p_hTDKwaJxF@YdlF);a2VQ8WUBjk+ozgUin^$mas
zpv|P9y8u~VT^)MQiE($)Psie{-q+8BbicQQz8PXJ+%p06d*1xE8#{Wqf%;C}coDJ~
zY#=ES_A9AW0Nm+6KYfJkcHvzRy;QW}5;PI?ZEzK^A_fuoLYS$B>Sso+_I^!;TSENj
zH%9xd1B8o+N^*ssp5D#zrKo$*nLS3#AyLW`(;-|2`~}DV717j|GN8JF@uJ4cGxtN8Dw_yFoFfjp7k*t8Z#+g^oRWC%Rnr^7p3jM-B5b%Zax?Y>8kx#Q8Fk
ziX}B;GmZZtdf_vc7w@S3uo20)c&znVNsXCX#6ilr2h6KCZ$7@6jcR<203|-{ZM{!4
zoSYU6RyEeP;~W|dyo_oN42n+%Wg9u|Y#ZtlT-669yUZg#|FuR#0sy**Bj<-+7WD-&
zGA75FOp9m7JAU7$?j{2{Sey)wWIa(Fy&N4CVlJGvkm9SHe1DsqbgY71*LVDq
z4KI}os)~w6R;yyQ4WM~DMiR5q@M})NS
ztA9Pb4~L4izP>9=Q|}%QD_;m#n5L#68IgP(JSO;~+(p=LyBm5L>=FXNBAX)q%?)Vv
zD=aKc_%=#+d6S-5KveWQ%HZJS5i?(8xy6Y~CM$|50yV0tsUqMC+5x=+-;JdfRa5S_F@k3R8sv&P
zsHXN2e#J~}(oa)U1JD2y12S-+-(tR<@%V8+wp*kh^@nMHk%ybqmfa%yhm#&W&V2Sv
zBJt9=EcAuMW3`%FS|}~@jXlA8`}Sjx@uq9U0h_G8n0fRBx_(-{Pf>9PT4rn{9|toA
zC%NU?{hs4bjz9HpHMPhR*s=9XGV2
z(RN5;2kJ7#R&sp4KE#j1%|8gApuDlRwl;NsT})3qOFgY^Yy6KuOr>UxH|EFdd3ezUr
zhg-L9O?~)~_2>)=vFX?AIGi!J6K0}C-Uj~=L0$&~StJHXC3cUDc)d3F1>J|BrLYB)
zer1TF?jG9STyQ?g_#VfeTG_(FB2RAz8{4IiK6>=_0vH?Uj`|=JWlt~NiHVtMke{HK
ztPy%L2|ljX-uTQ%k9vy+6S-gpgIYDoogbL7EXWQJ$G(lsxi@~SS3fE)t_GZl<8bak
zEN}N+SyEW&zKSqDbl_r$L6S3QI8FV|GNf=qnS-x$J36`&z#b6J-8fOWaTrO3>5(^t
zZ-dD$Gf#rFJH`@RQ4k!sgD;?GYU-#pzZWl^>+ZFnAhT&#?%JXj*2MsH=rMeW9{zy5
zhXWoCh2haaquaqfDZHT5D-_k}se*6f@%8Ngic=9RcvjYFO7@OR^aZ)3=K{&-J>?g7agPYm*7L3=>y)9SlH_3z6v
z&A=2Dj-42^v%OXj5eWmvwyj5jpFiV-7&fHKhI$nE_^#nLa@{3wq^3p_(_bwFoG{x7
z;_`}V^<}v0zV}1Yhao{;F&!#I$ooVn92%LQIU>9jT(VIY@zCVcVTzAC59Vl}4bKR?@
zX=Xo(GbW6+;!l_@qSU`wH70sT6rni35Jt|o6O;}a0R8$4BR;lz<$S-AYd1+7~utzmWq)v)k5^_b%Y(9
zo!+NaI|%EPZ)5M8HEXg30~%u`2LtF~8c1R@7`}=&6ubHPu?RLA^_vjK(eryxy<;Zm
zWz~*LKW-=7Cq|;={s7l>X|stD2f1<_-FK*BMmj%&T(IeMPpSCO-k=$xCg0Sgp{>mW
zK{|}s?t*b^i~ld`Ag&LrB%GZmJ;%zF9nWK
z*7kFD)@_GJH|6r6@A9?s^71V^V!$24UVx^y_C{8ejszkWLrzXP4AL=FOY!IUUZN(a_c9+qbVHBVmGs?m^qUZKx}Owh$XFZtl0o
z!A99VH0sD&2NrK7s;;MPj;OG}kJHf7+5j^-vWSI$0iD;~f460}ed44#;*7#vvw?c;
z){K?=4#1Er>sE`zHVyY*S+~Oi!~FCgedqSZqd6ad3AeH%
zd?F8!EI2SQa9o%PaV>2AGs_1lmE7mn1la_|L1Dh#72(IK;9F)^78m4Rk3zaA2ZgMQo-{6oca&4xPvg774Q&TN3FS*#*STah*vBZ}|5lCxLqo$oCaoBa=pn(7u)nP4XOFvdJ@+ga>OuTIhV!#LD#bQ8J^XJdcFizOvHN**~R+nv+x6Fkf
zEI8YJsFJJ@14x;NWFAWjEi58!R1sL;1Nz7G43052kNFf7{sequ%^E~43peMPfDu_g
zH0wy*;PNnF#)W7yHQ%;&7BPc?gF_c6{L^~dyr)>J)i*E*`~HL66wp~IfP`({vgMRv
z-6<<8tUfe_M9(}bElogE8W|sd7=sDYz2b@tM4#`I@%r(FW`4r@DYO&CVo$fy8?h;g
zC`mm&lc4h`LC4{qj*p9s&Xh`aWQ
zjQqoy6ckKz8mjvZkyB7^ek|>D={#+*7swIivcbMCz+*TpXI}IV4zih##IK$0i$SnT
zYHD@sE~iYf>O8)$tH6n4)3sHYbfcP9S7&W1U20c8;&T3Z(3Lha8a6g&#S6CbwdHL)
zaUv=@rghz^tUIrGx4d$9@0Y*#mMX>XT=~Gp!_wt*#}w^0I;U3aukWXnRZgr{z1gC9
zvq{6gqTIQoqkWQ=x?D--E;1cnVBw$X+W0Y7jE4zlPx*4BYe|O=a&;^K2{iwT_aB(R
z>K(KYB4;1BX8vKFtt3WDhyWAEj*gi0mOH{ijuiU{B!sSn&-n)2)w`$}(1Fa*R2lre
zIjW-9hUr82c6oJtJ2Hy~ob%5hg%zA1%6}>@7VetzJPHQ7Pkw|a*6N+uHxY-ysM>ZlKu9pqbgN|vQl!@RQOryPG(K<
zTkC&Nee$zrX4W4W$(6n5@pEXus@tBWb(`K|M+1gabbbh*#K{tv$fcTJud=eT2pJUh
zG2gu{OA9-F8u!gle(PF((Guuf($N4$b7R@^+-^b!F*DnN2CNSuz4xa4czIRTK@R;R
z)ClPjpV>adi+Xx`s&ARE^z-qSnV*@F=W7}D_opH#6Hb0yHrqijBuE9$glmC;J>%m!
zksdSw&go4W`0^M+_Fmh__9Zg%0CQ*{&Tx<%`v%IjRM|GAqk`S*EVy!CJARjs3^}zi
zDIb%J^Po5Z$#AXV?Z+EnHuJaw3QD5O`aFYswO~0sU!XTLFsMK_J@LYE8`vtzg@tzk
z>JYXS(6@h@a24G-|+)UX84a$PsK^uV_uPh
ztR`1<_Aj)Ih<2uIIqm`1LF$X=IX~L=t+lw@(dkgO;UYzj!*OSs3<@TSwsK|+@T*d}
zy)KScP98}q94{WlT+-?10T|KNx{jPAT5gO>vv1zn44TRbi+{ZK7YO^#U&9g(fBC@AFqGod1}o|V6$?bWuF3b
z0+6ip&_Ww|FADsgo~C|oT$Gh_Q{L||Pmm2os9&I{=#PRn>XC0T=L=BL
zu)5457K`Xzd3qeA1URA96m~BC`P1NGJxP0nX19pQ0BB}*;Nm_u&R>si5-F%cTe{;d
z03qErM)QBytX~EYoT9xSHxDXvag9M$N
z!nffoOx-6VL)P$a5yj?RR?gay-(d%l#kMI4_LrsBO&53
zi8U88^TQ7xJ;3)jiR>D@U2ls5Wg@`2ljSShpo(;Sy@#hKL{Gw>JbCgIq$Z+Of$>~BekUt*GUPi4V#Ll3fwuxYMejfBJ?Qu9u2TV|BG92;=?z{dygGG|!I<
zdw6*z(l@ZJi~)xLYH6F-#&QS?OXuqovobQU;kE(L2WSFx_%>vPGa-Tf#|3%RxcJ8S
z7V$?L52&c9Ss}{a|VGY1-&SRt{9ixD=HTfGMa==
zQoHxJv#4FB66!*6W`6;9X&hO{oZeK8kQaBfLF5)uWeIvhMtDLZW#9^6Hr&p4RE@CvN|&IHAKk5Uf)3hijP{5&zD-sHNV1=Hm9
zcPi<%y9cMzlXyW8xiKFT6H^7~8Xi)TFYcY=`Gd9zjDbq(P6)D~!zN4gSTj>oQStvi
z$rOlIdvnB#N2#fAsfV;HykJ*{;9L3JjyEy{=qDTHOJs1YGs0}~?4FBCZGZZKh{)ad
zdW947EH5u?#1mH{Os~_;LE7BBtBFVypn2}*G5nPw1b?=c>;|(Ti?{y)DV^)YR-X@iC3q9l-;ej
zCG5?O((UEgN%OgIU25~c&wcoiz5TGjDlap$#E*R2N3uG!5B!?q?0_5ybxVGJKI+vc
zCzPEMzGfE|vSe_Gyux=;=XY|9=3rzDC@b>~{o(7AqvC<@ynA_A*N{W*?aJ4&8=ulV
zbS~3qJu|lN%Hw-d`jZuaYuUnG6r6;%fstKYpi@?h=E;-yHopP3j9Ks1&BbK-5qC;Z
z@Hz@Rq%}EG*F*2Yeb>L3qUv)GjJgOMigV{gfBpJJ3K(BzM#fu7NxG)hxyNO93)6pR3WQ02nIMonZ4}g
zEW@fF7Zjg<`*?hIu!^$#LTKO^6YcKOKN82xJO?x;j+vP?ppp0ma8fM3p07^2v+yk9
z2w2R-5(_B{7vBbqYBGu+%WrkycKw@?H%H7g4!h|a8d?WTSFZPJDq|Dsq5viSw2|yQ
zyv#Arz+uMm2rDW1EpWhV|9;9a^LezBx~v~SqZ@87n(_2}aduZtW}N6
za4V*#r)9uYK|Ew+WYjx;{E&~2BFZHq#)8g?OE|XP7IQUH5s;B(Q&SU*U)GBk(buo9
zC#5TjCPa{^n3z>_21QbPBb*XkU)oy$&zjdTPwF^V_3qt3K<;OjN(4LbA{anGlN;^>
zS<$n*^OEPI7tt}s70jbS`7^urEqgCWilv;C4>G+hg$JDdvGuoG2imk!RCSBF*Z<-n
z!KSQ^d)aXKa8zAMb$#2%`}mX0Oz|Ckvj~P|XU4-70JlV)4Ru2>z0E8Oj?1_Bo7Qo;eK84WQGO6g%OPs)_GxoHLK^y3aVjqj4H{xq
zj+z|+1QuCRukt^jCplx+pmEdp38I4su=o46iBZ{Zp7H^OSDnV)N#(mNbSqw1bc=RR
z^(B@(E%I11$(XyEf7Qs%l9$6Vqf6x+SMvSV6t1~TS$MDi)x)GV+f`<3zfmD
z#*u!<{N4n$Mt#gZTfIi1AoA?q&yUSI9RE;k{_n{#S)nH?V=2xy5&{qD56a>}I39On
z6qJ6it&VCW>*Zk628-<pa(RkH52hNl2yX^MYf5h-YwZHWEOD`$Rb+DJt;0GCw
zq>bGaO3_gt(gVZU+NX9Y*d5MHm^&{gAn$#gxgc1DR9V5#$z~$ohQgnR<=n>W!kLHW
z)<{1tQ>Z>G#P*rf?x}BA&|Fa83#ZfX24|~+T)F?}x~+AaXbGhX9q?B;*7x=VonBX2
z@iWU48x%a+FE^;3lAK?6g(HKb<KCU;Tml=xg(`Gj|v5
zNcYLj)9>D!>{{U>@5~syZijs6l5Uz0eg4bxt$t@Jik}vDeqC$lLLq+z)W?$<|MLNu
zOx;_KvcLG&=0Ta5wSBJ1=jB>e%CRe(Hm)SD^=_h1Xf)IvNmph&a9PPkx_d`=ebBd(
z7tdVtcF0&2NMid5f05n(loYfm!G}UWg5vz(&70#15wCM;#k5bzcgF?|#K<&u|J=FU
zR95td3q-oN@}vptrrtXNpsY9l{??m9gqX~93I-LZQW
zufEco1qusI{2ATj*{0$a6j=lm?XQCZ-ml4&Z;5ha4mdrVJaIoZd{?9A;yzkFY
zim7{tyQ$OOJoRgR{=e3;oZY)c$%UIPb6fF$=gV}Lxlz6CL&bUa)-cw0{E;Ejg_8!Z
zCp8nArDMuIT3hvpNl!O7XWp&9z5$)ecY@w-j$o>{mYC{q((UUTt!(u`{vZ;w+#Tg`d25obCrC~hL84#=0j@SLK3wi`0h=?1$poTrn~Q5B)gxOU%G7$`JcanbW(7YP6Sz&NVY_an$_Lyw=R8bG@g;w
zr?YdBL7rFMFWgVfNNC;2NZ~HIxMNNwVHkO9%k%!1gj{|CnL@S5L;gXPioPL8&HVB^
zC0N_my$dvP+*?U;?88sN{tFhgb4Mq1PnsSoXq%jxw18FwI8fcg!||9>a!ec)DBR!#6@=oBU?P?nJ0L=iUWX+0N-J5o|^1;*QcIE%gQ
z*~t`4QEL3<())E6R#lg)=f!H{R5
zY*UU4r7dCKR5NygqSR}4r-EEuNJIoNgGbddH^uZX_ieWSg(pi%Nr_IEpeK|lB2Xz3
zyaZkDtx{hlb7m%eL*wh&oYW@Tyv4@CZc?#@>_fH%O&vYSCo7nLRJOhNCjB^z?ekXU
zlNDWQwvuT~v
zY-D#915;eS`WN!-n@Cl8
Date: Fri, 4 Mar 2011 12:45:09 -0400
Subject: [PATCH 0960/2835] format

---
 doc/use_case/Alice.mdwn | 4 ++--
 doc/use_case/Bob.mdwn   | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn
index 836d935726..cd5955b3f4 100644
--- a/doc/use_case/Alice.mdwn
+++ b/doc/use_case/Alice.mdwn
@@ -11,10 +11,10 @@ server for later. At a coffee shop, she has git-annex download them to her
 USB drive. High in the sky or in a remote cabin, she catches up on
 podcasts, videos, and games, first letting git-annex copy them from
 her USB drive to the netbook (this saves battery power). 
-([[more about transferring data|transferring_data]])
+[[more about transferring data|transferring_data]]
 
 When she's done, she tells git-annex which to keep and which to remove.
 They're all removed from her netbook to save space, and Alice knows
 that next time she syncs up to the net, her changes will be synced back 
 to her server. 
-([more about distributed version control|distributed_version_control])
+[[more about distributed version control|distributed_version_control]]
diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn
index 53c09bc18a..5982b5fcb2 100644
--- a/doc/use_case/Bob.mdwn
+++ b/doc/use_case/Bob.mdwn
@@ -11,15 +11,15 @@ without worry about accidentally deleting anything.
 When Bob needs access to some files, git-annex can tell him which drive(s)
 they're on, and easily make them available. Indeed, every drive knows what
 is on every other drive.
-([[more about location tracking|location_tracking]])
+[[more about location tracking|location_tracking]]
 
 Bob thinks long-term, and so he's glad that git-annex uses a simple
 repository format. He knows his files will be accessible in the future
 even if the world has forgotten about git-annex and git.
-([[more about future-proofing|future_proofing]])
+[[more about future-proofing|future_proofing]]
 
 Run in a cron job, git-annex adds new files to archival drives at night. It
 also helps Bob keep track of intentional, and unintentional copies of
 files, and logs information he can use to decide when it's time to duplicate
 the content of old drives.
-([[more about backup copies|copies]])
+[[more about backup copies|copies]]

From 88898d427d57f27d26d50d5bf10d8d22e50c5e63 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Mar 2011 12:48:21 -0400
Subject: [PATCH 0961/2835] format

---
 doc/index.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 00a3315cbb..b2b1bb257b 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -36,8 +36,8 @@ alt="Flattr this" title="Flattr this" />
- - + +
[[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]][[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]]
From fbb588b486c6fac122e3b1dfcdec63d40c0eaf83 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:04:49 -0400 Subject: [PATCH 0962/2835] format --- doc/index.mdwn | 4 ++-- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index b2b1bb257b..a5a2043273 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -36,8 +36,8 @@ alt="Flattr this" title="Flattr this" /> - - + +
[[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]][[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]]
diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index cd5955b3f4..57862c83c9 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -10,11 +10,11 @@ When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on podcasts, videos, and games, first letting git-annex copy them from -her USB drive to the netbook (this saves battery power). +her USB drive to the netbook (this saves battery power). [[more about transferring data|transferring_data]] When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back -to her server. +to her server. [[more about distributed version control|distributed_version_control]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 5982b5fcb2..5e6d93461b 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -10,16 +10,16 @@ without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what -is on every other drive. +is on every other drive. [[more about location tracking|location_tracking]] Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future -even if the world has forgotten about git-annex and git. +even if the world has forgotten about git-annex and git. [[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate -the content of old drives. +the content of old drives. [[more about backup copies|copies]] From dceea4d5f3c462a8c6806c2290936ce99acb64ee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:07:48 -0400 Subject: [PATCH 0963/2835] format --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 57862c83c9..f47dc7d016 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -11,10 +11,10 @@ server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on podcasts, videos, and games, first letting git-annex copy them from her USB drive to the netbook (this saves battery power). -[[more about transferring data|transferring_data]] +[[more about transferring data|transferring_data]] When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back to her server. -[[more about distributed version control|distributed_version_control]] +[[more about distributed version control|distributed_version_control]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 5e6d93461b..8fb8e8aaea 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -11,15 +11,15 @@ without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. -[[more about location tracking|location_tracking]] +[[more about location tracking|location_tracking]] Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future even if the world has forgotten about git-annex and git. -[[more about future-proofing|future_proofing]] +[[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. -[[more about backup copies|copies]] +[[more about backup copies|copies]] From 400ef405ca5aeaebcf7c22411ae3a29388986715 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:09:48 -0400 Subject: [PATCH 0964/2835] format --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index f47dc7d016..c7b70468d5 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -11,10 +11,10 @@ server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on podcasts, videos, and games, first letting git-annex copy them from her USB drive to the netbook (this saves battery power). -[[more about transferring data|transferring_data]] +[[more about transferring data|transferring_data]] When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back to her server. -[[more about distributed version control|distributed_version_control]] +[[more about distributed version control|distributed_version_control]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 8fb8e8aaea..21cce21dec 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -11,15 +11,15 @@ without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. -[[more about location tracking|location_tracking]] +[[more about location tracking|location_tracking]] Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future even if the world has forgotten about git-annex and git. -[[more about future-proofing|future_proofing]] +[[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. -[[more about backup copies|copies]] +[[more about backup copies|copies]] From 6072399dc225e4ca333fe3d424b587d151b24065 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:12:48 -0400 Subject: [PATCH 0965/2835] format --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index c7b70468d5..4e07e22813 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -11,10 +11,10 @@ server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on podcasts, videos, and games, first letting git-annex copy them from her USB drive to the netbook (this saves battery power). -[[more about transferring data|transferring_data]] +[[more about transferring data|transferring_data]] When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back to her server. -[[more about distributed version control|distributed_version_control]] +[[more about distributed version control|distributed_version_control]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 21cce21dec..915fa1e0b1 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -11,15 +11,15 @@ without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. -[[more about location tracking|location_tracking]] +[[more about location tracking|location_tracking]] Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future even if the world has forgotten about git-annex and git. -[[more about future-proofing|future_proofing]] +[[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. -[[more about backup copies|copies]] +[[more about backup copies|copies]] From 1cd4393c7fef81a28e4f57adb8aac49458d18182 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:15:04 -0400 Subject: [PATCH 0966/2835] format --- doc/use_case/Bob.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 915fa1e0b1..847f5d16e5 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -15,7 +15,7 @@ is on every other drive. Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future -even if the world has forgotten about git-annex and git. +even if the world has forgotten about git-annex and git. [[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It From d73a6e2e7294d6778ba43d20d643521952c96646 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:16:32 -0400 Subject: [PATCH 0967/2835] remove links llinked to above --- doc/index.mdwn | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index a5a2043273..e70c5cbb13 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -50,9 +50,6 @@ files with git. * [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage -* [[location_tracking]] reminds you where git-annex has seen files -* git-annex prevents accidental data loss by [[tracking copies|copies]] - of your files * [[internals]] * [[bare_repositories]] * [[what git annex is not|not]] From 354402e6b2be7317da3ad681873666f62cc8167b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:19:26 -0400 Subject: [PATCH 0968/2835] headings --- doc/index.mdwn | 2 -- doc/use_case/Alice.mdwn | 2 +- doc/use_case/Bob.mdwn | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index e70c5cbb13..0838505e84 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -32,8 +32,6 @@ To get a feel for it, see the [[walkthrough]]. alt="Flattr this" title="Flattr this" /> """]] -## sample use cases - diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 4e07e22813..ccc0727bce 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -1,4 +1,4 @@ -### The Nomad +### use case: The Nomad Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 847f5d16e5..18cdd4d8f4 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -1,4 +1,4 @@ -### The Archivist +### use case: The Archivist Bob has many drives to archive his data, most of them kept offline, in a safe place. From 38f1771866cd539b6f9c089eaacd53d1ce8f2b5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 15:15:55 -0400 Subject: [PATCH 0969/2835] scaled --- doc/repomap.png | Bin 129065 -> 67316 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/repomap.png b/doc/repomap.png index 4d334aec947e33900920351ccbd094b22c4aed3a..dcd777e1254698aed16dd2f6a2804b69b5e9533e 100644 GIT binary patch literal 67316 zcmb@uby!sG*EYOo=k(-lsB%|t^BDD_M}B8X7fe&3y$?vGACk`l^pEx*coH7Z0` z!L@un9Wj?2{-1}^Gi8xEH0J+2posr|#{B=s)4-(Mz$@j~$6Xi&;6vbBZqS_px;_yR z5dw zp}T&WyL`)j)4)I0xA52!AP<~Ba0BGoVO|{t{WW@C3D**KoaW{oVi5Bv984D~L2tah zzNm1Tt1r`k@mkpC*s$Ht`+1|o^Z)B~R3M!7@XvPcOng|!<$BbW@CW!We0s^!l`o0? zj;f0Qea#8|&?EBCX29jVW6f@~hV+x>nVA;n3v&)&zyQ00RiW1(A zOAGJIWAk=nvEqO3Ah>}d3e2IaIvP?%e!q9}W$}tc7M>DF6|VUINVokyp8xcDwejcQ z{?79a6G>-bcr?0=4pYA!+e9sT!c}__zqWulL|W z2;s)9FRSiw$81Nl%48NwwI19qv8trlt#TPQl)gNwHw}o*NHY55^1MQ?N*f%+gAmmo3*5_4?cPrqXF`Aa;Shj`1=$MW?YS!KK!&&ZBnWF6@5Os(YY%G z&0d+dPadZd5}~dFUh_~86~_IVn+EZPSSb-RcdH#7hW#oE^k@6U@}mmV-N~XeKMOKc z$~uMmXmRho()mV572Ldk#ts??Xd*0tQX2!(dlebnZ~+lD72@v;+PDk_t z%HeXmM3ofd&$Z!1m}=XL{3maJ```i4Ae%Xx*Yv)tkh9Gs4TlP=cldO_9a(~ERUt44 z3|vIuDuTc!H~_*Zw?qUN22x)~KlJ9uQk>2~$_WX}MHpAx|>~-cnl%No5xX)nMR{=7B z6uc6+0Z8#sfOJH}mQVyaN8G|E*BW+f<1cd`ZD+`3PU%;e)W_Qt7EL<>{xCz4Cma2_ zXA3Y5X%ez5XmBy+0SedD|L zj7bRzuWBWO-sfxypK>e`HoIde=AY4y*Vn(dC#3w(?JCx~-{PG4e_Hmd2!p{$9)Ib6 z*BE;NUrrOS^ulEtdzHPJSZCS4x6-#~yN;*KZ!Tp(E`Wg38?dLe+D$M!^Uw-fDT!Y+ zmi}v=rKSI04uCxH{Qv%d|2GH!+Q@&O|Bu1^KOTVZ1x?9ePv-6Rz=s2QToZJwB6YFoxd+0EK405$h~I8e>h~eO&Pvl)TV%KMAIGFC=wEFT zc^|c3l!LZh*627>e!r5y>8{QYune2dbZJqJVURIh^4*BBuUQH>9dQ6n%uv6^v`WNz zzVPMG#j>&M&f7!Rs|x}UtAgTIUZ3<*oQc;;|Gm5D6#k0I^1q-r%&ZXw$TI8Shb{hG zYMttfp_u}g{!|DW&ZFA1!v+0D*A>(IMPII*#8w`7dsPmiZn@vtUb(|mdAZFw(}c4aJLqy1STr7yuXL{)*lA(uBJha0om5AiNP#GpF(x6}z+^iZs5yetCvOAt)`$ z4K#GsXXL=XcNF*D1>6;r9z&kMhzQ;Gt3H({JWzBaZHRbPqO=tPNul&o~`4@!K-LpS8r(SpA*tvpnHqW#+Tk>|U-@ zqV9KH$6U`>>$D!$w?42v>xyKG-_++P{SP((q-O@qeU@=j#l@=W1xp{{$VhUYa&%nE zJj>JCG>Dq|8FZ%?8~=^7#Pm2g0~OT54m-Ec%XK;LQNPi$AAhAK`rj`sh3${P!6IT& zB~-*XHlJ(RE3f#TA?)~jGq!a)&W(akY4Z<_Yg;~vW?w8_h2bZcs;>BEISVGx0P-%x zuBV5lW&c6PdvMf>Gq^+$8aIIh)I-A|*M0x+`Q!k#%+2u|glQJ?xU3{+k}!=+pBHZq z5UCN1i%Lak0-E>S1KfQ_Sye21UuE=Z?_kzrYp^W;>AQ0F+yHHM80PO@3e@cDWERrc zUJ!@8MLa-f<7Ggzj;{73;$KLy(@lD0t*WFBh>S`cNoMb65M_n4EQ?=W?blS`(}|Y~ zS`U`exHQYYC@Bta%;X9A4={q>nwPsxHJ$ib3MrSKY7CW-Li^dYQ432X{A!W&MZn41 zuv5=5DgD$j0qd8G*unqnv=A9Kt9tJ;OGT#5i)H^Y&+Ui8S2@1x5#d!15h;ocuOAib z$#Mrb@9qY!Pjg%RJ4>lZ2E7L@2Jy2edlfZ)(}RjqvV&!z%snGy9GZ-u-@uz@aVF^KU!O#NhAcO;?t*3QOUV`V?g10 zD_Eo@j*{sBrK(<)VG+2nizLc0|7(Fog8O36Yu*~(QET{Kl`R3(>rn)r9wR8 zaX8;Nl_ln>8%56Z=<;Bt^F?P62)X6plHB)nT5fx_7lOk(juxLiz7_h`|L14+r*7h2 zJL3ib{G*MUuz5GCu>B;~daDRW3+TU#D#;1b zW7r6SYjchaJQv6&VOQVTz9B$GUY+fieAej%X~RdxQ6yB8gcn%f6Mo`kywu`-X1Trr zO+_iRVC{K^L27yUWl|I*Vhvk$D_MwMN3SFPkY0j3`5g>gTnnm4hx>2ebw)DX@AkXu2yWiR5F%c$W zbOo;%^h2YBv%WFpE;hFyIyW_pdGAij+ip#^qbRXVuq6QDH+R=Z;N5v5E@Ake75mSk zP4$PEu5+qhB1SV^JIZoTWgY;y=n9g{_0}r$p6ff^5SVR2-+O)2G6}oKJW1;2L60|3 zW6TDIaDg~_sf_bQEW}Oces$~Q@&{Fw0?6_2-EY*uoauj73dpy#gslfjg`H+=!D(#P ziV%{6aEnI)(pmcX!0P5e(wS$6QPDH9!{Qh;fL_c)t}{+0t+|Iz;$^9Z%&qR$P^w1% zpA&zzj7JkEuciEcgWly&d)uqiJrMMX&4~}8f!obw>6HD!pwk!j$-w4l zx&5BDnoKdk|HWNa!e*yAL1!#Uw^f+&C=<6mTYd=c+*=%q!M9`>0;`Hk^4!^=aekZY zblF2^1~|Zo72p6k_Lzm`-(V2X(vQuDdQ_ILQ=xa=C=WwIK27)E+ot*V1Y2^PxS5LGLW75}^ycG4e zWAxqwtf_XXhdiGa3uO7f&x$7W0J#a zdPmSqBOTD(L@U6Vls^xKFiX(GJ{s5WXwd@}MKXNyF}vLfWbxrW3^!j3=_Ot|&AaEv z2{K|Df2O(v=lg!r!8N^+JZ{craF_I576UVuiQflgLOa9{Su8Yj@e!1QR^hxg-98+3 zAvCkUV}yrLX*v6FX%o9}NVO+Je{qb8`!kBktTJ05Po(k`O|dqO`kUD zL;kd}6RG5!4iHduJ}AabYZ?^DVVNwywIXVXFiU@^T^KH{U#mM|zp)V~Go9X<4``GR z)^y&LSNW4}u#senM|d%C8aLPSYMazq$ls>a#p zz7?wpKek-^V6)O-|LwQ*Iy^D{i}cpt`@lUvE@b{d#VkI>8_D!G=MVmN1jYKjS^SrE z^w!(Hp|5_vSQjUCp4L>qw)cS+2Z3bK7^;n>lW^E4IPSI55X4xCu<7<`Nidj3A&|@q zCZ-TP2}WX>JqcO>3A0vf#>XqE(UaSkUt7Q~kqh_W!cg2>SKCaUcBj2B?nK^#t#NhmI2ZhyY9qNz5)EZCong7@SFe03*1ii~4=ze{ddoba2d$EFgVHT*rC z0uenEoCaqte*EH#-`8I;T>hW|&Bx#}*3**;vMCqAf}!PxjShat)@evGupC!4n%tmvm+i|CzQwJ_~OLFmBH0lFS;|r#ZRE=EjvsygiUG@ucsiVhIMzahA%>Gr8$T^)rlnNoV$)G|uJn~lq6Y;{lc_do_XT^A2us7+hX>Gg9= zbg#9d-Z$d7PP`D)zocKl{{73S$327Qt!7G>fDIKq{FCcS^_Xx|>YRYYJN~`B@6O82xSnaP&q*Bq(V*rT2+hrkER+adp1!$96P*$~z=xv@W(9-6E( zg~z*nF&AJavN%7O6K}$CBqT5JeSgHi+U;6v+N{Dkr46R*oF}^dnp~E|Zw;iAhOgX5 zuR=g-I>R5*_)ynTpSe<(rHzs+Z?`t#>#b!h>q#Cwibm9>a%lj5z`T33Z(OW=od4pu zCxYZTNG=+#(+_)N@ba_*!{Fm!Fk5yXZ_h}OKjqHK0>i#-?zoLG4@oV{3E?dV)x>zM z|8f2rWN?w`N8G-u;mwe`a&zS_zqkG|_%tV|?PmMEmW;ovTYpXpyii@Qq)>iP91y+W z(6=Vo`xeKG@b8b}gXhy|6_op777-|EiQaeaH`CQ76_yVmI-{L%gI!!os1y2T>FL)I z9M-ph+2SvF?a19%Fsgj0Qykq(i301Y6iW@C#}_5c$G;tDa;&Z^N?{lqo$VO+<(J@$hI zJv3ITZIpz)Rya!8L3e-V6Tesu2?)vOETK;#1*mEPR7Tm#j561qVr^FBM+iMiaC3Q5 zV@{t%tJ~U>*Og)v%7l?y9iG!!k6?D(0nOp=Dk%Fbya`LgpD}LCv#qjKMe4PPwH}Z5 z*bu*j-LTD#CJQjfCQib@5OBIKt$DF7=aklN(+330e|#y4o)Z;X<`#1*m_bF1sNTX3_#;s^s%g)5_vi0ch&H&j0 zF{RHB@}lHO5*y9I=J=7!O7E9!Bk`Q-i<={&E0tpdR&dQ2Zz|Yk#;Oe#>pm41^S%Nc zb(#77Ew(sjldH3tl^IGPeF=Mgdp5HN5{qe8pKh`` zmPn8@f&mkH<1YdGwT^Bnc^qM5XSmVJqvfil=cDwWBQg&DNKNBug1O+-rmXSxm|x75 zsUImlS=`Mdw3~u`<<7a#;vp1l-l7ydHn|v3#%GnZton)V%d|ezD4)2j?P@cq92a#|T21*_Md z+4$6tl-@i5240~Rl2lwyC^Fbz)FTW2D^G4_xMt0!v1kD2P@3m3YO0LQ{csSD<)Xoo z_t>&74Rd!6%-g`*T^e3l7O+2fuPmL8MZ3naQ18B>kZm71lr`g)PlEGF@6T-QuY&-S$cujNamAxI5Jo!GLpm; zc%2*GNfEk6%fp-dFjkX%SLT%*xbb7i~HOBt!m+rSW2E6 z7B%;4(li^1bW~*kDQ-EK*8JKZKrVjvbi81W^KI7yqT@DoTpZ9?QvULfF|Eps5;PK?8Zd8 z>f_SxPm_NmJ>IbLz~iRmd@#D4(P$x*s-{+y?4C;&yQZRf&V$9r)Z|3sUcUC3aMzZP zZpLn97N1cq+O_M$Z{u)CXPH%&c(!#bv1Kv3anYvhz7pwB^36tgr3b_ji*m$AJ}15f zs{{>alN{3npH>2mj@Vs{v9$7&@wbW*a)~xOMQXtwmhX#E$>wHyo+t97+ z8}XT;@HjEdkdJQ)r$tzq1b?mRxOxVbDE#Lw)DHMpmC!{55)h32bocPOwEqL??S%*K>-kl0FhXu8RvTKc-Jn?zZPh zm$!Sz`R(a2E4kXa4EdNDS_c|6IFV?;gZe=?{b$)Ci;?i})tz^KbX;HLQgO%K$i;rh zm=K0vbP+4p83A7#`hN*tijmC_!&J;9^@?F!TqHM;Ah#w_dfkGA|91`An{C;#ZmAjd0a6e zpZA|xjYS?&hwn-&$|&Wgp90V_k-Oot0aQygAqHQfty2m*8~RR%nvz%}S$hsyo$3ci z*;1k{-$n6Fh_XaLVWHeFMr8wa5X^yQY43p;T7L_NGV^D?_}hR$rmstB)qt3C;w-o- zh1)u>_A2(ks7GDjkNqQb)x!x45CYHT1&J05NcVPHbwRb&w5+2mw|Hx&7a-2Eho!ilC zw5=lX&%ubgp6sOOW0tFwGX&eQx#t-Dxw9P4ynS!eJZ{op>ALQ7M4jp3nJIFHKsF(w zH^a$476etL2B0VUfbwR4Jn6nyiGVDpk+FQ*8W9 zBxd|gBW&RRD>>;x{AnnoZOby{Q1U!IFv#7QCT-zt0YT3c`jcM<_VDVAC54h=dnCb>1YB_O zLgj#17Cdu%(kSp7DTP4dBjQ>`)}`ET*(v~j7gO{wFx|PgccXgT2i?_o%+;y85>Q%S z@0+9=_2CpBKKX)o(Cf^8ndL#@Wr2Avy$vZ6c&r~`u>SG8Z}imBx6v5udisbabdLkK zQym(7j_CeZIoyC3b_&d5SWcS)^D|YRF5dLhbvdeyUDl;q^QYN6@6_IV{{$T`;!OED!=D@VR|7f zJixtnps`AYlNT{n2&<}d5QwL)bt4}rE{v{g5hGE*3|RH2B`{UYzIFKnW`!@ZhM*#X zS9GUR9|L>vAZGBMOG#?@O2R3MXd($-63jV8GQIbtWqR!0Y)K{eV4xi1WGB5%=%qJ6yBo$|y=F9lS2;g3m97C#S zYrIgp-j13G{}fGH{jwxnt1Q9w>{|0#f14=C%}m%rE|fU(Ls9$Dd>$3OwENIcCvI({ z8HholQD@QHpiQ_)tC;w5XLqPx2|dp!Bv`%!qXEiV$*|pG|IwOs+%rdrg>{WyCg8F* zFn3m|(uOVW=V zer|8j##JYqf7uUlkbF`q4e5>dbz4X7f>E-D(_<`!-}WLrf1zXSVx9MZj9`Oas3e$P=3OnSXfk8JQ zR+Ku+SND)?108roBKS6DPMTXvP&rD@f``SFum)YU)QZiMZbQC1N&Yd0GA9(Q0YMRv z<9B3X;#R7wCy}Dq+_Sh=8BZovuaLp%i#h}AwoPRF756Ji|eK%$qiaWy)2S*=J%r834Wforx zm3EC4D$)S>xFee3f9SV^#W2)x8NJzs=GD!}%_KoNz#Vt+7Ml~qQ6xpqcp%FXQRu!= z-%%-ZMuY`p?<3|%BOSS0sWty%px1Bp>G})Z@XNg?)S@m;1YHf~=Z zD|ldS(45kWeq6L0Z~X)21?#_{0y+dhOfMD=L2sx#MIX%_fQrI)V*?gazxjRhYfn~u zWKa#yo@My`F!s3z&h@L=T61b3n;ceOHezc>ChvnDn0wLeSe(m&v<3*d80*5bd{~l? z#plY}FfO&i(2yTG1*=U_5J2|8WS_Q@Erj@0?XcKVCX2h-E0KMS05rDXT4}070&a_D zrx!uD-nVmL9=rrXk|J&D50-g<2Z_@>z95)>GAjE7&+XlYxDDN_>`Z2Mpk=EVHC4o2 z6FwaCevtsMV{_*Itoof@BZm|xMrn8ptO-q>684^)c)5L1yTrD{-Od85_BD`kdof1o zXq)v>vp>G@Pibau`-#I{tFdI^5zD_}YVyaHxrFTfaJn=zfTlmA;}# zZ^9v#>1U>*hfqAPb@U$s3(n^;Mx%6#-)P_2jnaHF0Pv2@srtlum=iH^m}c-T38!r> z-f(92rpYd}4tKCK13St%j43E=LE^SO-S3m<}9k;Xby^UHuiU z{|g{|`pt))2IyIeSa@%RnS){u@ASn*wb(-8#*G%>w_vT*ElX5iUiIdV!cBn>s@`O| zuYjJ;CwzCkwXfcGS!m@8T9n{gCMAp=fLd|CTd(`!6Ky@gp(n+7100MrmhIY_lpzi^!o=)voypv_TxcTEaTBGRsS%V#jSX`ptwC<$BOSa zqlJGX1PZxPBe<A{*+TbKtzp7b&Fak(I_&krlM$NnjgZtL7Gk^;ZO z(c&-p*6UOaVtwByny>deJheqn1a1}!c)kOHYzCF~@LM3k*T6Kp6X(GEGxQD@NIY|3 z`-?R!=gX42If`RLpe}Hb0)BNPip`uOkpCw)IQ}L=t$&APrd1Yc0@rYTz8}vF? zpp!G%$zi2eB^yW!f34c(b`@D%@15?6dm$l}o6SUSiUuo*?}|Bliv_PlFuT;_YMX{# zy&v{B)+lozBL%!Zqq*H)or$l~oq9{I-+VPK`a!*W7MxA3O?3xYqGLH6tPyMU1$J3S zIVKO};7sMTaTz7wmYxoWi>&t7a~5?mW|teZcg^o=iKAU`MTl6X zu~mG)dP5?RoBt$uCOH$I*a2~ox+z__!Ul3&zsNa{SjZcYym&f}=UlBPJmRx~sc*xM&+l?!vO z=gFs~F??_@uIV@^QbmYli)+`#S}8oU!1QK~=o~q?IgLac$E;oj3m;l+P@i8l2}w95Daue zA*|XgII07cMTL-)+`u<34%bAMyL~J%R)-`o2F$`N4V&JAb|}K8n`I}*4ZRK;=_}t% zK0seQ?Pw-O1%V}`LGvKOd!0magf4n!HtFrE^$kM>q2bCu36HNUqH*HYB1pdtJxWm) zsjG()XFlO-lIwx65x=aJlW2Vxe#a-0+AP*mY2{VrHv$beGqDXI!6ZFNd>%h3keq1I z)(v<2F-*0R^#vRa1cv_0r1j-)NPAuFP5M#Nu5M@G>m_;>i>gT@!<}4J17Mv=?bn+H zV94shH>r?}z{*byjPbW>DMEU@pQ*%*KI?(>T$1>V1mz-u0z#-y?fobSlmw*{%1NRH zV>)xT`W#ia+gUYVL%;2qi))Na@ z5DS%oUDGra)Mr$jSex%GA;9{N-e4m#Q1s_K z6APAh?<$z*0uw0{20+AVx4a5_s8W((RyFhR)S`Wo)I(+ZU7-Moh4Dep_uke#&q|Xn z)tm;AQ@P=O<-6u7j+z7m;8l4rL%b{of$`d-^8Sn;!u%rrzM$lY`nWv{ z`K5z>F<&u`J`o`9R-=<7`ZFay>(gmtP2EcGw_0ptxmIjF1#i zWd?{Bp5}cD?zCIXK-Jd~35$3hDONI8-lv-@>S}5dHLRl78NCkKeHBI4TQO~dpj9~l z>-K2BpP&W_UbtsxxS{c(Fbdj^(hB(b56^w^F_M z$o5DfQh7V|VBAe*(TY?@`JCm1%K!Y}#DunRjy&?cg|hke<2#nyc>dLMCs!c;{7vy} zliiMc5?F;&t;pW-1`5COdQZ#A-Jphmg~e}g$2zZnO3WCcfW`$Nz-s0=PX|8jW5T*E z0vw8;mX$$e$vfsvB;l9`Cb+0;4=q=~5$hhkkU64RSKQrsVr&N1#dMpT<{Db)DwSRq z993BX?-(RrjY(bX;L4q6r$XYDPr&;7xcg@-iKy^P`4w1jH!Ya(=A29k(;8np2-mYL^w$V}l zd9f;0b}f-IGc-5o&sOTxR_cpt5*ls1J5}dLEf#>d*HkG|LCK`I7&j4-j3M-%{N8Kv?rXT=X#DT-r6Z}Dtp*bBD}k}TlOH+=4RUTY zzW3Ny#schT+i7jD)Din&HKqK}XQ@cUX})l}(x^nQ(zGh2#<+}}50q0H)IE_gb#4#b z55sd9YI5HDDdq3A>Fk9lnApYsi0;mersm`kvq6%{^ z(cmvK&mxzl&n%cZc$#cE5V-B@fvf2$smi?tX0xi}$n>5k;?H)HKIp4bwWoWyLef`j zGeTX(vb^Xq#N!7QZLu04ZnT>y@Ty6=32h~C6Iih}psDmk^^xxEa{qC$V{!|sJa;A) z8un3s8Td?3Rq_qKtd(l(KNlNAkwIZwP|kHV?gG0O%J}+JMx#J^rV@8vINw`=DsZB! z>!H{H8{+Q1%IDu~^BtQvd4RrR01RhH1na1FEDMUF2k~@@ck*5j##)V~YFH0t>XQxR zzL=_S_<&|MV>MQ-U?ko(nNIGj+Q4vn^JV0Jkd$7NO0bjAP!JhEII34Qje8K(+P06%~2` zC_T5Ep%En)tK0|`zpxM7Pn#%+4{T5!waWhXG##!t=o268lAIAr!ltUE^NeXYLs$ir zB5^M5f3N%E(Cf1U{~|2Q`Ji&6k^uImHT3Kyz2`cuflOYCggy7#y=YXvPC1hCj^ryU zv)GFp{C7?`CP9Fc3-{6w!$_+v;});7j$J~c6D}OWnpk!XZUmu;yc9Z7U4TeQWKYYFWc;>mY2I3 zM2)rv32lcuw)+>nOu&qGEe?xjADtKrD1Zwpy)^5N?X@C>OT-I#4ksy+ePVRX zLDOeYzd&H)!Nax5O5F{>%ZYV9@1> zs{m_cyrp?d1**hYqW4cK0QF`$;8gATP>cCBI4M#=w4XbUnm6#%<>UgmqMt6-Mxl-O z{{rR8djnhGHWnMGvomW9f38e`AQoLy!l5rW^-9>YQVZ}gyfP(sOR=N8nV$4lvZ`%x z%70RB-5`7SJ%*@LOJ+O}TfF^8&>0dQ70frxWnD%b!tv?S+1z zW^*uwBfkkwc-Ht1IaX%NeOpqLx;0qjAmUJH(gpv%2n3J%KCAS%n(#OPBn6Rm4w`K= zzP%2i!WxLNg(t?$RXKq4x|eb^WRT-#zD$Qb`z+c!UNauf&WW~^Z@-mQtLNQ9FYhx%7~IvZ4`3NBgE;{zG76v>a7@Hw@S<{q}jn`@gcUlCmi zPGWHLm~$#~1(T;*S*<_=dN$1Im?e_1oHoBE$Vv!fZvHW9Ze9hUAeqOZ%+P!dWiiM@BN-B&=B!34g(ay)kJb&zj zfWUzdsDqb?IB=DpFsmjJdr#W7QolDCk35Oau!3Oj8j&FWV>^!@|3pm{NQ z5)pOAk_;2|P$b>*l7pd0HT@#2i!DSGGEVw}t%6T`h&<^@?sEGSnRH0UFw8BcXzHQf zcWc>oEy8&##CqR@jg+1k&>r@%(tbT`4nRAXd-Ap$l$JYS_82Pq&|tstba_j%M>Gum zdPGYYJWIAqnvN5(v8bZBIw4?xfF;)ej9vOX%WT9DbrB zTKmU3wJd?0^ma|#EUI6S0G{#5tSimunZQuD;Zq{(XLY|1=5#K6yqeAg$juTiyLAdz z@T|p7qyvNZPwxqIcRx$ACQbD6XntRV%_dZVgHG-pS9$s>sElUO1z7B8XO7fFmq>7 z;CE@j*_3{{{kP)uKsH^#%B+7-F3gdIP-h|o85r5Ue%_8}KhZtQK*jxTCboZ~`*9>z zz*=|OceNfgg3E-7-KB&l!!Utx-D zoOQB2Bw;w*JmOyCy-6Q}O9eyuOP=0yp^LAN zeehM`MRs)}uFGu=)=Z?b#XVS!^KKJs*dHIjk7q!R@6+mrKfb5w(k6HF5ERotGbHkX zAUQA`f7X{BrMwSyfE1=sBlt&45ze%DFz{j)$p(3 zZ8ejJgL6BUmhCPbA3e9#Z|Rr#FL_zjJ+?=yEaz0?Z05^OBml3xaq|b`&%S!Dizu6# zN!13PypFWW1iLut`&mSfTO~1KQQn3^rr&MVj#e2qKEpsr{y>kqdR=K-``Q8^#QfmH zpKU;#t!W+@#SLeBXI6(yhvfYU1m)A;%;qYf$VsgMDpxM%b@sKe+)N_~i~xJCC4bTS z9TWj$aEHOIyBp!=y;RgiXoniDvmOf73zdM|i})j}P3Yab+e?|)iyf2zQ(WV&8Uw$P zoe84P!D8s}x3yNRLwjIrvpBqGSPlKw0Ya7M1#0E4gjA0axUYlpR>#HTM;~M*p;Dp< zwy9$fiw`owQ}F<8*O)QJ;zB$oynueHZ;@Nox@Fq-fYJ<>VF^{&vy}HZkA;B30Q|UY zJv9rl5!>2usxD?Li^VKxGbG+g^ohot8K4xS?wp+?@WA;6B@R3Ab^pGJt9m1?l3ON6F^{b27l2(4Yl91{tuK$nK!_y&9ys!j0pZQjmm=0KDid2JY$(fZ{)6V&Hw-t ze2OBAHALpk7ED{OP`3pokjc%H;)7u5w-#dFNF0Jk;JpsCULZ4iRV-uj+WQ7Y1d0c+>Dq$F7S)iv78232Bw&(5zG*n@hrSv=I@QCM%*VS3{i%~(r_Ah zk8Ae`pXeDBK@Wu)!lpne`R0)o8UlAm)gw6wq%uA~m#b|0sDMr{$)j`#a7V=qEIZ0b z+XelVzdqR)m<_ApK3x20+XG<21?t!w>mdL{HuH(0Kkocm@)PoF)))~B>S7aioA1!<^_+M0cjI~U#8ZK**{8+7Vk7C)fsCs z5bUQyL~YM`3Hw_xu(41YmV~ZH0B1&kI$ues+7{avWdt4IJhfb>O2P|<`^=py0fbB+ z(k@kNR-8~S)T2Izhv&jWGQB`+##8FCYCq|t5kP?%$2G6It;r`g93$^;_Z(VpR2q~? z=MD;Eg64nfn%3Ctvc8O7mQ*fl4O4WybU;T}nzMI1@4|HFPr4 z%oO}9Ok!rgcP5lGZy+#6=k=Zl@pLpth7|MAX5mpOi&iBEq_zzZZ&ld(vqhz||LS8+ zFcV_}Aq1#-`VN0S#14qx0)4o8=U6?!hm1bh1~G1beV#kX8jnhl)fB{lXO3mHG&f^!r^Zeep+X&uqFvU!xyJ?e>72=xd>!heDB|+x9Q|>zGlR z<_IAVh|l5t^BSRj2qO^=c(GfE*~zO!H{#dyX)6VrgehjJGZ>f$rnpppQ@(`m6Q7pe&b0YaN>8uZ#m&Xmz}0iCq+_ocj2p zUr>KJ(7CPX_5hV&*`Em$faZTP%oqziYGd>U?6*X^!jw&ARua5v)rgg97C%oN(ugW# zL54RYw|ZHmSp2#nK%Es4L4RP^WOUJBmOmR26zGw+Tg>T6IF?$roWyVeU@qK%RCcLl z_9r%+v@J8?{l(880|_{~q3_1W$j3Ll^THv6JuQjZhM>UOYajO!)U7WjS4&NGdIHFX zJjo-t5HWcUyfFRn4Li>U8MEcC>Z@8n+DV~`P$ZTG#L=gb;s}=>6=rTVntgqk|~ODNJxwgWSpj0r+Lm*h%T*wQH>#P`Mov~Fw;h7W|L)` zaa$rXfH3}i#wH)K`(VWQ(DQ7HdYO2xYhw)X9meK;YT)vSd{~2GIZuI4mnEN)RAJn3&3doJcNg9_80VphFnRETOz-pQ6R_AfRjJ*D-zf*0`e6B z%RZ}FdR|=|#{HPHxI*%ps5+~tfBH<>M_m_@ahvsNhr%MS*P99uE-cE7ecrG3>3mAbwHiWB1#fshH7Krh zFkR^2$q%4+fNh7~QjaaB!+D}UJrC#Q?_ugQBIh&BN+IG-guCZRfT^QK_1}4bq$uqo zuFVvNq%pAB2DaK4*{%&#iq_>P_Y+~>H;x0UrG&mN&^1eE$Yw0V*YH9VB{RW+Fcv8``(qOVnJUZu+xQTJ~sNviYJm4Rwz;Tk~KE3V-ev-=Il+-?jxtymyzqFc%}TW zRv)JGzp5`+86eDkvYH#~T%bm0LbDDD3?|n*=SeSRLFdo6c+OOZSRY_!HV9_eCZL*6 zHTH;5cUUVkknVs^7j;!{($}jEyJL61rj>^u8&lWUCQ5|#3bf^Yi(eMKGE|0SM-|ks=g($0l z{{rA%1`~=&wo{fJHpFI14zoaTyri!Dp9^#Ih|j#~$CBt6+BXETb#%1p6I?+OW*$&2$~{ViKufwcxr_Qh?MYQiqXP3fZq=g*WkA|hC^PhD3h6hp+^$U92x+<=op7t~_Yg^JOh zbV}vg$Y9B(5<;z3=SR1&)dwxF~agdIAEna&fNhZE8$Fu~Rb(BvYILw^#s;xbuZ&& zYPa{&bEWWG)}ld!Kd3X)8T)}7V_a0?r@sFn?+1GVG7~-JOZcfhw^I4-TnLeSs!`rb zpKKk1P=KS>v(4{p999uyTTbUjGzRFw z%99GL6J~ivUL3akdu`CGOz5QP=ig2@*}c}oI>jejQ}3Fw=_~K^ggw1j`sn9n%v@=m zq_!!4zr!`@5vQj5U2+DNJ_T%2n8L#t1*@Ik&Gmem5rY_C`j%|XNb_*U)<*NpILOC> zaJYFY_U%cO_{~q)dV{S(bw{1V8OD*&ZHtiTxk34f1*Z>-?8zzr9&Z1TcBe_y-6hKQ zV%4-#F!So>!DCJO&5jy8H*(RvUvX@2-=Vt5rTaLGUUCXS&_`+1JUC&;8_+xP-bvNu zZM)L9Z#Q?^I{p|n&rDROK6@Ranjy4RSfZ}`J_uv_Evwz}-T zDp}KeED=wJc1ImXp9(k2j6EdOXBxfD-?g`&#Y%hdJW5VyIrBVOj=dejzV;HWdkg?$ z6R8J@zr~JlZcuO)kp#ZX;uCt?^s`e4D2Df*{NcX1L0uGshv=0tYjQ8n6t;__l?`m& zy+0(=>wv?Fmav99r(4s{MQB-{HE&l=>fJq&bT(?pSlRxi;}K(-`1-poWc_IZt7mX| z^!dZ@PygGk8gZQ)W6leEXrXHA@1!w`H0k!RUdaQ&@dTB-ekV`S=pEfZh0%`-F3+yr z^=(Hhuc<~}slEbMZr!@NxZ+p573NP8vn6$J$XIcwmJx94YUfCK&&-T~Ld+Hy>Fl*v z*yTHZBLOdBr&m$Jb5oOue1y`|)`-xW3}x06Y%&nEY)klN-=(e!Wocl|0F!NxGz_EQ zdOWr(Vo3@sJ{*-bGoq}W?-gU|h8w(ACaGX;H194)R$FG)xV-i1{XOJ}FyYcw{y2n?@5CS5w&51oz?m`m z?;@n*Q<9)K^f0XT022pJK%|HJ;oi+;qAE)sq#C{2-NwGEK4P-^)ykHshMoCohAi>2 zikTOTdkv->e~BC^2EY3ut5L?~Ce@p>bS?x;W0={0wr4e3@Tx1dE;B+Hc)Fg_&3Cm~ z_yP6RcFJEwYCL7|)blQPoAQcCOwvQ6@>MiMDJYkfd?tJ~sK&Hhcs@3?8~M2^$*ugW z-f|ipygZEFy9mjW)BI$T{o~W+p);x7UNm+E=o`6RfNFD{0+>Kh*hAXArD|V!U!A-b1sXD0`2$H-s4h=>IeOoy;13k{emUeo9 z=r#VYsgF;}suv>JZ$QJ;O7^J3rr8jQuSnL-NVGTRHBlInGJ+#&Z8@C+(iW}+7QeaQ zx5$hyUs(;c*}{u)y*Ea3>gj4hOZHrlPuqA3*@kKxTBz@N_>hOD;vLetny`^1ofl%UV+o^f9GMI?q?k60xU&yp!6=|*EU*?&LM zu(IvkVW&B|9-J}NhNjp)YqO9U|4`v#rGO|>F6MNU*C))>9};4;h2F({20#CPdh&-?G4M~ zODg-%npH%rR`vILQdBtj#CC!MP3!CdOsY(H08+5;VKE_r(=;cY{IK{5iwa%IraMm( zr7Mw@dQljy;>&(xFRMtB+RAK~DK?Rd_6ka|Bx~2T=fxlM@}LCxDsrqf6ZdyFmD#4F z=|9wm02echTYnPU6S;I{$hxy6o!5lHI(U%!1VwF}L2g>|j6sEHDyxWC2rE^+;|H>C zL+BgMNHv!>$4yOU+hM0Q-|YM--B`;$%sw)^jT5Uu!O!@8qPLuvyWoh86M%q_mUbqa zfB4<~zlT4f{6!z=C($1Mh}VQA1ZA(Z=SX=asR$@XQW%zKrDv_YxBd#8x_Z**n?-gM zm2t;=R?-;foXcwZH~Fo_lVjv-!c}bD{M?*I0uUj5Y|eYfzr>-}R6QR zEE35)gr$vxD{SCE^HA*ec7g+0{^Z1i7@#|!e4{a2F+r{oAGhzArG`Xl_R_p{GpGp|4wF|Zknumd1w0IqGhjbGUMvM zm2YtqA(H0;w4if`)&@`z(Rt2!h08k+zlcA+p!qvpY0^?R2-aUQXM}#ysZd4dY59}; zsc*e7gj8smt_c78T86N|g9K*WR9X#`w=r6V;Q zb=%qeUwNJI{bt96Lf(WAu(I~$KPvKL_xyAuZgZ#VccS^{XW#X@)$ywK(}iw${!o6< znprDNYhzNC4@Qo|WTHQZ&J(jceXZ^tpT}Yd+&^QC3+jS{UF!v)66^%NIeAC5h89WS z8Rq%_jvNd5S8e#npC!c0oO4!s#}_~dvZY^NYrS~o5nW|^GLyEknPH8`Ra*&-_ih@R z(|H&7|58uw9-!IQ2UgW-fVeQ>9_0;9y5FB&9LjL<>iDE)ZS=XCg!o%^JHzS79x7-! zT`8N_DqlbJHAhs-|8E{o-StBSVoy#eVg=qUzh0teX#isy>rx1zZ~&YAlYTOC1fzs& zejI=2j${4|QNCqOWi$nMiIHWZ{|be&<>JqxXbREXjIn;WQH{1z3^g-lJEj3dGQ#1! z(8?9!A_8i8d-;FW_)*I@;5lZqUM8S`XvS`P1rJW`kP)kJTe~E+22s4yb8Kg-wnyPr z>WbGt@aP%VnhUKFT7Z1|z3>r&24r_sKFWUTiJqt#jh|oFgZy{uXQzPe`R<+6WZX_7 z5`*FY)`vkS)5Bj8lN3o$Ae)WDjaEo+nuIJr5rqhNFH1FRvAX{Ry^ZO)F7 zl(>ONWSMSR$?UhT^XD+3p>v&;wbsLt$=;55S8LAAi55^9LwNMHP?z4Kd?) ztg?KXmL`&bi=7W_WXOM)(XnF~RDcB`gwzF~^+qv}DDabm_AN>ppV#U(owRS=Zk&Al z3Q$yRi#?Y!D>tlQ8}RM_+a+pJrpLSU`#4EA^0>D&YAxPf68Tpf=2|5-ITClO25URT zKy;>mu6GLgA^x{WV`IPK}@#`sCJb$i4c4eB2V<0UyUlI7f(N?1ujI*|nrjrnaB&>P} zO`<39P-t(xo7bny-_hrQoHf}Lem3g3E-zKxip;=+Hn-50vBF;pSiq z@ML-_FN>S!`izkIEP6I}LW|YOvO3qQUTjY=br=XTcFw>(oRk?PdIXSnfEPoc*$DO7 zy~Ef%ZtMB?j&^JK%Y3kVNx}RoA1*I0vH~BKSHa}w3~$kiJslebD8!09;U&#&l7cX{JwcgU zXjzMCIFO5kf7}J!poeeumC&5cm%jl_hvV{*pfS`X-@B}uRbpSE+}HwR-m)gyT&y&? zBdw|9nfB-7=E|_WAI!Gl)edl&z1e8*K=Q+o#s_Lu?+9fV(#>CWTCzgavXF>)XAFs1 zz_BhZw1mdT<#lvomoukm3Z+W1+z%rhpK-uI8Am^gMid(o8@gn!5odNt^w#vrKWI8w zTNg=k%0J;GzWv|f^6_TfJ$CzBwcVXqa@eEaKZkFpXDhEP<@#>jMg!ZVIX#rrEo_a2 zTJ0XzpFWWnEp||TY$0#)!cWxM?OR*G+?FDH`Bdk{n;X;l#$*DV#MEX$qAu>7Km?KM z*!A=SoC^=pf-?0_;45w3o0zBb5pGvZZp|E}46`EP?@Z!QCuw11jXE`?gm0dBj7EtY&s~N>2jdWIw;fh^CJjz#%D-9p5$SZ>nd#|8m*z8Bc=QH-~PJ)lnR!S0;odD zr!GoxR5`bxZwI@y)dBp*pI={#aHJnL>^~2#?cc=0y}CitvonaLz>HiTjh=TDs_ysg z-r5cox)b?$Io=vmw*I3B9V8^9Jrn?>m`}z4(WS2Lk6Q(;>;28c9X0sR=`ww(xe@v9rxjp`kW2lst=pRkb(3==x{0z_ed zTVxT|{>z&(V;3~cwxg^eh!nfC^(>bOX`J;_`s>kb$>ftIpr6RvmHg(B8ZS5~Xk*T) zpiYW2*eSnx=`7Ldjpv=1>iX}=wCh)-GD*=J9RN~n?gW_@Ian%R&7OU-;vL}?ld2dA zVIQ4-(Y|LGFB)-fL)*H8OucdCPvOfjKUnrM{$(O@@GGwAKU%F&0w$5r=*O+^_rCC> z01g)2NzP25zIWmxWQe7Rk5YTqzEnSJT2?FX=Ali+`>NcZrs-3^ChO+HHt-$+N=A!+ z`cfC%uY+P+%ti(@y?7|wC142oogv^Yef}()qxuJs)TI9ZOJ64s^0la?>8gZlC{VIATbH<)L-X;$^HeIBQTLl2WcWCaMQwMq9AgwPa&SxSm z^P3K^_Aht=eFt}abnPuv`f2}_q;dfq%4+x-)%{6B(AbmXf-4HIUsX@F26tzST&7*+ z%JXIoGrGA|DG4~eI#IIkn*-ES#WPaDtedi(!0nR-$(wdpkz+wDNSMm`VT9swDKWcM zP4a(>z)K|+gq) zS8@(lq3^r=-9L`i8uFm7`T16L!Q}PUNSAuU*ja?-^?&cabJ^o*d$*MG7DEn*v{#9) z0gZPB9^z8)3~V!g%&50!VJDuYa{l}nQ?5OYm72f7o1o4zsY#vRLCy@tSMxdQ*eBbf=z8aJ6$?a5H z{IPwS(ghu6*;fxkt8$S3Z+Gee0%O!@2zpVj?`cOeXy(c1S^0!N9Nzd9u3V4l0Qrs+ zsbLG`0%Q(o49kgHcHw$~rXK4wTqT-~Yk}<30~!LII*F+fZ2z!+J}~Rg<@KR|5_`9t z1jrnZJiQ4$vY+Vos-qeMi}X9C0AMUFsbA{=Th3VocAFnx{CrC2XzTo6lfB1P0BX*r z*dc=jU|&*)psA0l0RjdVO%a~Ial#@P=pSYZS zc`9vpt-5mV{-duz?CF>MZv<-a$hu=1IaMv=Pmg>7E+{m2d%A4nd7Ak4h@>XK#F#OC zP~a9!Ia1%9smMY?Ry?f6yoPn0uSatw+&XowWWG~OX%tJBWNonXV>Bq^!C_wy}hU-{E_w`ht`n)i0m7!VRK$}vRNktsjW6C}mEG$hW9%-LDZ zKJ=Zbb+elM`SRD=W8PJ=_W{wO#{k!M0Ex^i z5;9)}xpANOXIeLe-um?G{(GtmQIx!#^dcVS+)5ES@A{mj83EMuKsmd}U7j0^znCj=@(-w0ETb^s}Qixyl58ZG3q2;Q7`p1P^EmuwBn|^8C%7ekKw*%@6eL%mP-8ECY=wMBSB1n@}o))DETfMaDUuo`m zwHwZ35J%+9`16qNy5AFYdFQY)xV}!K*B*fH8QS|2LBl>P>$&5=sQ?U;dLlyB=mJCf zErXON=N9z`K9lCRG{UxBY-{my7}>JP$sgyL_N#FyK2@7 zHr&Rlni|lxyosaAE|{M(86kF{BqG9%5Vv@@f*mIg-2#1HJpc)7-hdJggPNkQSOCKa z+H{Ak1?H=*!nq7;URSqaEOyD+xjJ5S&~~3KyjCGI$$M1(B2s6B1;5yX@hl*Z0}i3NTi|{ELj~$0UC6(O zF-tj2g@$X@t-iJ$&8@H?N~2+5db_GXB(W>={Kf7@JcR4XU*=ZR<6;+W;oaer@V2d&oL~lUIJdno2m>!0e=eNh&}h zz8@Ucj((6iq*0B3u`!MZY~h4?r2y95l=P z?qewh3ZyOY$|E`tEQF0g*ia*&YHD=DGdo_*C6PNF`@XqZtF9J?6xJy=$(v94ts0Va zx_F$63;pQv1<*K^50E=qp0wXHlVhkI<77x;)Qz1yY=b>4yBF*{%5;afpWHXm+qf&2 z@eVXJj7Fb4jzZ{sJCpe?eGsEa`R^uh45NoLW}e8P&DU-~D_ILA_>=;@@pcEw8n(1i z3t_S$BABhN8o9f)?fI;uf`tSu)U71}IWub0(k8=ZFjcoq|NY2VRJuJRG7xAv--vO7 zuOR3K>L>NjElidTY8`e-!R#8jzX(tEom2G?uV%YDtwSR=v{hl zLWhH(=u2CdCPn|A7!N`O6)jb^;NmCXjj)3u7Wrek3DQ>sghC^OT7D#{j&P-@3A@Tz z&1BqB!Y0H{bgg*I6Q6_&fa?QO6={T!waFJH9zP(n$&Hl6nDPMnR^~DO{E+U~%89pp z=P~~#y=mN&xY1(+W@=65Dym{oO+}1T%*I_i9x?H=5+)IG6aNtiA@~D{i`Ppn>HfpO zl)b1oj$sn|^6ABOT{YQ#jJ;|e0NR4dQ9=2W?sibZD9KY7Y zBrHc~YqRi$$hR1ql+cgUL`|Dkg_vICZ+n<>A^v;eM~9LasG~Wyzc;DG4(|blzgLXJ zd(bvgPs3qZiXX*5Vc^_QyQA@MF9k-u##?Y$0>KokpDT`%`Cby6s%I-M9uqrXm$578 z0;rstYj!;BcE6vYkc5XS=XWMGHgnz;9}EEnL#Ietg&DhDxjIk;#Hr3S*`Kz{xjozR zIIrh3FXTZjyk(<LR$!dSUOP7zMU*6Tgq>&?7>=IQnFJ_dVJ<}RX9)b)ihbqf((QVfzXm9a_t?#*Mr&HUJiXz#?jsV>&lQJrS+W<-roS#RI2$dEKX#OIQ3$S z6Ko{r8Gm|;LlfG?UM!D;|9(21RRmK%|681TA+Wxe((#}YIaT~ptc5J(v~dl=iM|)= zGokO61#ojpN^|N#CT9%ex4>i@%NfV_6|y&G0}Ux1P{DdDGFu*Aq*!qR-dou!84K@? zN59N$Q*Fk7Fff&$f+B$M!uAog&(D0e3n+HEm~uTHLScX-1FZWHbo=u?I4)Ke%b857 zY?P*wltCj56c6(IVz2g+iHv)WAN?VRS6x|OeJ)kVz-#VQ9u)WHvS6DE$&zp@gv>GB zK?Bbr?Hv_tjFq5y{G1#8(xbLwdlsMD`F-IaLnUaXFoisVU2K>a>N~e`D{@p3ioT!q zt+-UCS|PBuQ=od;`OS;^_B_8=+ezE=8f+G%V9YPZ!95cVDgWYtl(!}RJ=l4$d^fku zdpXT!U6&!k#NWkAL{$=6X>hO2vTPCv@aD zFDTnis*Aaax0^;A3(z|Q#mf{2_>fdDqDStoJoNNI-ajnm5ho;sTVtx$i82GuID{OoYT??BS7n1frJ4G^s|0f=fodp*QQA-g?w=M^ z2*~e(1k2z=mKo^+qAN<}L-A<`K+Q!qn@)|2cjs)(jr?OZR;=`fDf&PNH5sO9dEHuI zyxIlVW>YDU#7T4N(;Q@OezQ|e@z5T%Hy2el} zC-wCpu7B4;_VTpAP=|&HnK6Ie z;SD9nkF_yxoQD$rcBgO(&O4H;DwkKFE*Y!k>qZFb`n0pgV}rho6E00r73mhBUVSj` z2#Mw~Yg;(1MuA)0BO-6ziis-M0Fd#~Ofpv+RZ{e^UIQANO7k%NA%_$D5kvS<&__Aq zPI*P0ePnl66CPCk%=!nm;3A3TRhEd+=%)By^ERJZ*FXm0HV`nh$Z(q;J;j=9+Ff<~ zdqD!x(EXg>VS_L$&yRj4J70EStdu+r__s|_cl%55ds>b`f_9DwY{6zxYfKLEM{%Kw z?dp)dc^T-O+d1)j_cq#ENm2{td+jYqzo)F|)@N|uP;VI%cbfC096HD_Usei7Ew*^~ z)-9!H8Mm(s-dhNoca9S3-TDwwW?ATJXc^gLjX2RM#1to+5@CHUmp_m-P>%#1Wulj+ z()?stD1fM|Y_m8bh1}I(Jn|+(KAr$u*iwq9`?n@zYA&dUn)S;t-S3+= zpGm<}&J=iB(3WwSiABsNlU<#Au$f_(nO7b-SDcVeY zN^-sal!8iC&W#0jkW;YypEUCRa!+;tlK|6o+hb2u55iSdTGJ%SsUB@cV5TH zO{e8_1;aY~07|L2HQF~n`r$4 z|H~xS9xMSsfy_CTQ_HNSLz(fQMpAtq1erCY!F+d`y&OPET>8A|L8)gDoJ%&tEj9H+ zT}A!-UgY(LLG$3GVMP;7&`T{r_^W4`LcsdtZMGkEh-lwCSy@6VA&CI9nFgk%+XsiU zW-rufA@_1>acWuDc{4yl@b&L)R+=yarT5YBHJI^wb<(Ud$^jy+ptQcgiGUg8oq~gL zUZRngD>iFRkA|v_JY$sdd@IH%kcwj%Zj~bfp2Z|>$Q}_I1TQwZxobdMkD+9}?iS?! z{Vy=+34WDJ?e6X|zo`uR9h$waT3qWul{4*~R6U@njtylV+q&L4-70H#-6onWc~>&C z=+b0Z|F|fLO;^?tGIRKOl`3QR^CI=H`A-^EUHT&p)lUsa)YamtyVXbRsx3wWrY-VV zy6>D#rkm_E>h1eUrK_xBBWZWaRcMJ}ha2oRff|hC|T#e*E5Wm19+ek@c?n5Lt-d zm`w6P|IuY!cRl~VpWUYzi-pC_xUVQL55_?;+y+maR3%jc1H<IZWc-3IG6yspwe8*#%A6xCL+JL?Eoiy4nJhNPgOII6Pij6@ruor^5Sxa zHe=Y>TXL(I%6+?|C?>^shGZktS&+`lH3#`sPNDb|+CYh~3(h4IR*&oY@1xf_25Lca z+&fK(xy!~ChK97Nd^O*|8nJpN^VPCWmBnMaD?>quri%_T;K25e`@ALUIK>uAw=LyI z=Jes09bZtJD{5a~_4@wZ!r}n;ADZ3e9!nL=w zpqO9-n7T~r&~!v%w}Xd=oe#3#s`nW2qa`c?b+Oo5ilLY1R)It zT7EzkTk(T_z?suS-C2y+{Vp(zC+Kc&?S zWgE6%qA~GbeDG{4rZP1A^vtP#?%5eDU32;ZIzVjuLDa>p8L{$^K6-Ly><>rTVW80k zR@kVXtC42ml5w4^EldDec3-?CRs%AEJEGQo#2{o$Qrga(jgSfxO(G;fm7LI1)&ueS zW=j(Y!1-KH)!>m6;Zrob)}lDQ2gDXK@AZ)T1&B zkFBr(^+b+%>k9&0nLCik%iGn)o7#s?zad{l>A6K3X_89!$k;f#e`% zx1efB?vT!!lZyAT@?yGl%Q;Kv_q|cn&`LBfwX;J`bQckudnel+2FTM$2{!&^+Na#? zQ(N2fV|gBP5LW>z$yi#5W*!%>(4XMzwQctC*4^*M-hXv$qo1y6I^P0ilRt2XC|jRu zr)13en|`Xw*&fX?0Zp7*PaGuzRF9X8-wcngw2SHSD0ygG`NHQiUS9pS*0!$^x_Eiz zp5QgsY#qUhI-R9ZJ9z?TsM3jRJED-abbr204w@riHUxfW6u|ZW^wC~{3)7|j_lfN{ zziI)QY2+#V9B4p1JG54?BJm<^QOhkOjY;;Qw2!TWY@c~k5fCgNJ(-+3n_>7p$_M*o zCS(vVyYnX*7VJVoXYVbn5;E_cinV5a8y`B?hfycBb1E|pyH`1PU-%!Eg^Uuta6qIi{j?Rvdw5Y$f_B~D*}33`P1FA(-LF&Htu z$e{Uf7Q(LaK|*f#culM1M2TF_*o|tx_EUE(UGxQ-v5~;J$$3k22UeW#P$usH%)*g6 z3d;Xv1-p$i3H}wKdXI#AK=j~?RXiRfwT?5lOVaS&ThN2a#!{Ksf)7MN9Moea%pk4D zNyg~-E;B}T?2}V1Xg=mm|H0_46TtU=o`Kgf2$Z^g#PC8`dw97G)lioH73G_i!Hkc8 z>?omAEy}!WmZ;-MTe|ByqZ!d!_Fq2DJLylK$SUSN;h&Boi!J{_D(+jC7V`s12eBIT zS?}LT+u`x0{}ph!aF%=6q!K6u;VO4I_=BhuK$fVD0LPK}^>aKud&#LY=7|l1on)V1 z{HguM%F&W1>Q!`j;L?GPPj^o2F+XVm3+_uo6G;#aSG0(UK2z)-M~?+qfF7E!<}M&! zjqsJ|X47bbLdj0lYwPi1n;nStYYu86gU;>}b;73Y`F+`O7UDYiwv=JSj`y#b?@y@; zVzL{y5!KWs7uWK5HK%MFore~S8_jhdv zmm1ZEY`0d~;faH|HU8?l+F}fz08#9Dhqn3QiO^`u?qU2s)-cuZJH9Zv>-+x{#5^7k zX0m`6tWs$z>T_n_@(ub7F1!zm&$6)D=ggBN%$KY#5hE#kzhzdj;?Y2PbZi)irMmVU zESQjgZ~F15vDYPC+vot2Gb~mFW7Q21P+8`8@PI4ix9h4CA|*-WXy*!qRZO%>4^fqv zFzg-7fl<}l4KfyVq5}~8(l4Q3l;FL7%?~(X$jGd zDMOq!_YXrJGm^9g$lJ841PF``MM?KAk==Nw3cDvPs)O_U>;CWm$!0A+J*zI3+_@Ir z`7>mfWn|d@_r|Ephi?~C-{FBhf&K$py&6XHzczGvc`crp3^aY&V~8GafY%=L%0<7y zO32Okp_4F-Q)bW)9d%Lpn3U_3?^8#k(l1M(9XT3yy$cCUNTD7J=l6NM7GGMw zQ+K17k6S)RZw7SnogW?!Rq>hIY>c0{?>(}IE`!(Nt{5vQ5}NQ< zA5=wQftPX{B**s`^!XnI1bN(Z@BPslkptoOWpu<6%p(8TQ;vjQpjCextVv@Hy%1tM zJi)lryr02a1nq1Muww%85qJ=sd?BMD-pzmE!E#!E*F+7h+Sga!{qBpdz#YYK)jtHi zcKT8>cREc?5fB&MzSw;HUPE2}l}|yj%~MX1ypLbQhLKJ#WR6KO9FX=c%zY|IEfy&! zG*msN<$srcum*smJd+C^{U#<>2&T89V6chtiV=(-^_X;k5@I}nMk{>7Gw*w?e|&WU zkD!fO91Z2Q*HlA7s52X^fs?~cWjQ3T}Eci?Zu z-cR59H6*w{aB9fs4;=&UhPV&*6+m z{&Tk0ZZvS_SRvctF0^0vz6~__^~;e3eicgG>m_;Bc{?TI)weZ=U+@%;9DA`>Xnth= ze9(*CPBTOOR6c4*sEA?(s+P;lT}LmGI;a6eK<+}D^j{(lgsC_KSexjciQO0_gf5P^ zsiu8}B5qO(AWA6Z3^l!?nF*(B3I>tAe+2UM+hk6N%MoX*Ke0PpG7O)coNd*s4lv~w9yIQOUYSz$>56EC2woTHMGryC<3&oM zPtAvgszNSE^E(uuLi-?)YW$n%uBDXve)dBG2(fAj)TqaP$g)lK!E|Luq4G!7_&nS`-|gm?Pvs<2ozC2 z?a%0<%>yM^-FsSy{8x98x3UJ_GJyI?vKrAEADMz5o_hkg^rj z)bg>fgcol);frDMzM3%?()_;oZ6QOyWx* zXxJTKL4{qC0)p(FSoF9kT<>sWd&0PKj?q;t)Ku*)E1pz?c1%3)-5GTZ-SQgL`x;nT zkx99vFeIWv7qGK3F#YQ{7ckr{sX_ZUF6wn*ZoaYxQZo~UCx7OV=khTt*X$NHb?j|E zwlD&yq-4dAHCa5Jf`cC70qCbVjvr@38N#|?Fz8tWTcr>mXU}HmhJM_M`J5eol;`xN zsvr!}qfDxx#^YJuHKjAm)5KaPKgPP#ZTYe4w3-jOmp#)K&DXqH-qeMM@~6B>a=|co z`+}JJ*2UsacpMmVUdazVuM)fuX?BT2$~hb!BoC5tExQax+Rsd@GBpZj@P?jB7T`N$ z?|!i|!KHQm8t`Ya)ORFXuq-}p-0ww*)|8u`S%*&jAdYw-h?%{ zYT1d|+UjE4b_8S1&Cv{h$-o9aP~2(2pPMyN$q8>kCE9 zOzk}o^iD`2Z9k`qe}o{!WvRGxStid}&Jz%HeXcRwQ984D_n2Wdhk^@AZ+=Kf!>3>_ zKoam~aS9KumYIRlUlntgJwl|I^_QpS=dI(1bWvG$zkG#4-}_fsW1bky6@LNpuI;X} z$NJFtx%!wwb`nVWz@{dJ1xv2>=2!1VGnMvz@K+5PKW^$^D}#&w2kJw0VXDaEddCJppM;befOS>tWkVOZu>43T=Fd@(-DJ;j z8C8?P9=Hys->xPPSj+B78bTr$iTA93{$49>EUKz$|7JE9zsok$OX=&4+iGDApB{K}udIGe9}WmQ-&EFs;AI<$uKV)ygipb)Wyp+=T}+T}ifV6sr2;Q)wshI-CihL{@j z_4wSBJa7NJcl*r2#i$BtrmQ}yF#^(DGT&V=h^qF{15HH-$TRrnuKC=&mNxgq4UwZ@C8-4Fp!8gR}E(Ci$#J>pYxD zxLn{MxoI30l<7tkk6*F#wsikw-o$is49_X?onwjfg!FY-gY0?d3FbvDrPl2&dDs4}_gOE=KL3C|j$^&cF~yTF_(DZZ96?vCKudNvPuKuJ zj&8L925DBv9TvF+cs6*{)b;D74XBzAy_+112JTXAVcfi%(6IC!t^~?4SF_*OAaKe7 z%mhjm5^t40jiYZ)+hbzHytm;S=-|y;pzK+}r8@=k&n6gNS~o9%JwAs`{VhSG=g*1P zdC8HGTq`q}TF2jpuqSzq8aCyxkeJmsnn^I9>^zu%gm0vC5`+9y-os&PX9~IA<{PGO zKjlUDDG1E-`{z-JomWW4ooM@~xg_b`s;(-@#pw2dbhY>Pj5;_WDfhZ8fH>kO8G+8Q z_L!56L;lu$F{h>Suy~rHEm5b^Or(Is&~Ep|Gs0-T-ic;Ad^K5@pS*QlHFa8`xuLi* zvK0pM)I#pN%?~}~1b!j*L*w@I`#!3-gvOGIP(U&}9q9^Q-N}f~Kh;$r>rmHdgj&_D zT3d>p#6zK~2R@|1>OJO=;#D}0-<_il+tYmIgmDQ{wqBcj_Dr#4JPPjC0r_5F2^-d^ z1%C!SVqLKP>7uhtZ>nY%Pj(WK-3d*za=VA&S0`8)fDS}H4(-jMUc=?wTj80CBxnQ> zD|GPjoH0}7@$E#2dh?cTo>1S|hdTYerBc_#TU>s*cS}QyRe#O6$bJp*cu^b8c}=6V zdB^IH7<wV-A;Y`P&Buy-dV7wq&oy!HkVII>vnAMY~GPNVo!Q z8Ng5Kr@4W%vHgIskzEDzk8tSQvd8ZVqq&6*dfek&N666XY*Vq-ad5`<{ANMES>!~& zO&%-W`E@+BDg1c^9iAm(xFKX>5%_J#k|AmzX4e8YL06@Z0K&KUCM_L*H-_3A#?BLV zgv(x(E|bONzaO+aeLAgg_6`k;ou$77C#NIDE+GTTVtZFn48KONpKJXL^a+uUiLmH9 z5^rAhKX;q3XK+Mp%Li?1u8tEGDnm@xkzYA z$AnC3L!s=O;^W_II_l+D-%3=eMBO_v>$@_Q8w_4HFJ1GuZTjwmQ~dCz;i_@|!mA*- zEF6Il^4>KLSk2~y2p1mo2$FtB-J4khys^h=CeJc*nbMP+kyeA?Ra3u4ya%(wXR}SO zO{c>ZUs=yP3!bn}sqg6a2O*nfK(+Dnc_;D@(iTkh>g;gKfN=TZXfF%7kBrq}PeJ*3 zqz;P-YFHSMN9RNNdoY38367F)Y~IG?=O9?Lyp7-oyg2DDjXwDttExEq=q3B!Lgfcv zB0*#eSJET<`LH`wUkLu(62s%+$!oa>O{x7o*u*4tvH4bW^9vQf!A06VR{(%^Xu}F; zDc{+isxRYv?qLDepih%~DN3}d;A})%-!Yez?dbZF9%K7BI5o8rsWn|{VkBBS63ZBH z{??6*!@&_|kVLO^!i3sE}b5~(ip8&z9+A7))wa2t7UEU~=TFBSQ z45#z!LFXry0!Y$k6G}3u8gw!LW<4(i!jAJsm@hRwT8+fp6r(jnk@3novKYGuN+txgEz!D!gzndhR(5|dd^KBw`)Id^nk|W11@GJf#k#;@#JAe zN_q@6v=}TKae*j;bq+)6U;ldTTGi$~ziIOVguqt|WnMY8efCJTYD@bCk{;y7eYhz( z*qxk9Ndqc5fS)f6tLF&^287{adwU=@5#bsqV3kPIg3_NFA{!Z%=q*>syYmS0gfF2J zXE%tGEO1pw!JRT6fSK1}up9)WPg=QI&=D*2U!o0i=&bt@dNBb9>=*ArP&^29f6Al1 z@RYPLY4h-w7WCz#pZ76H{BD9n!$>s~`MdW`+d7802&gxEk`<{ZL6QnP;$Koksdh?o z<0n_)dim<~M0z0*&$**bE$(dG9AkmjevVIR2cO+O-_|EE2iB~bOEn-9JF?Mu#?a#f zOe3x6#Bp?}^jv+c)@SSc1eoZwbKAo7%O~bHJiD#TZTAbX$tmwS1f$7&Em3a5|HYuk zQT#iqkcn)BF~NulQj7org^VTKyLWrzQKJ{6Vpt%B^<5e5GC+Z)3YkB6_;hZ$lPJG`cSS04#*A$&ci) z<&A8QxMw87)Yx!y&~cHjzE+szCi4R{$Y~HJnwYA~$K{aXm;)y3N^X^`TiIQ-lrE6< z!c8pbj@K6XY<21wone%_$6mMx_TT{k#WIcy7F^QQ<3*Bm9kK$6f!s_2 z?>&`6j-9*-_Q-Sf-OUGa^zT#^5{e%*!2sNzRj-fxLPO*uCD&tRqvzIU5!gBoeOjqe zZX8;lBH3^Ukw9vdWfnyX5D-#)^N|!uwy>%2BLLXkR+~?_9KvyO?>2WYnPm- z|KQn&NFV*iu%B`rL~rs~vlT*p`atOM>w#3>vPXdS#$iAyf73>eCO!v<6dq#a2XQ%^ zXjM%U$s5YL33*a?T)vpr)qjW9zX9&PW*5h*`f5)s+Rx`-tzNg#<&ijaCHdm7|KglZ zPwyBZQCHDQMZ1H;Qt}l$lM=j^?@4z*NsEw(s1m&lm#??L9*f3LhcexcD#;8O*>V8mr~-U=fQTh7gz8Jzao zGQg{}_$KMjE^vyd{rg@;WCCLfzcOcSfP|*`ZJ^Kz%fo z?%=o?)o8R?Kydf@nc)r?T6V6!`Dz9{KH6aDoMIhDJ$(D}jUJ25%eVJ9J-`hB|7U7F zkWkJhbI3Yvq@&{k-;_Mc;SxX(jNByy&g(|{j(&>{#M-rP6JldfKC~lY;w7EG!naQ- zeqOmf)>@d^ecaz-D$#M^pTtOFBIT9&cg)Ea>a$)pmQ@_(bCH0DSC^9 zB~hL<_I5GLs-gcnNpnP_sGv3kE_kJPoyOgR5dMbH27fSNoC=x%$=@?>pi}(p6M>2h zxP8vqPu(n5o_(g>tlv&NF8=hql|cQ@i+CT=xsT%;5?UABTC3snhy=_$kSwf$-qL_z z0QHW!QJC}Tfa2aU*;3Qr@S99CJE1_-^}Z#~l}g<)-}*B`(Kym3MlW6Nr`R&NZ^1+$ zmlhy%kVdLyO9&3<{45p>-{#etP5a7k)bLab@UGP7wvSq4XHProv-=J{$9mw1bBa97 zkB)wB0v5&CEQfTcA&g8mcN4|Lw}J+i{)wOmPCQ@GNK3*FPWcQyEP4fS(s`oDNi`F& zxOc$A^l4M*u_=a}ye+5dE6$E{Zy+*oL<$S}>@TO6W4KKQSz%RNXj4Er8@Cwr-X>TK zT%ot%b3-)Mbdc`Uq{^Mophh9B$bRZ=;#`aO`*YBioitfKX#HYv0?MljoPBh7y4>|ZL*)7X4=|YL065CIuPqN$@(JYA z)%M)XJ(TK19)euZH*~V-EfI)d-fUM01)4*yaP#HU4k1hFz(xkluh*>KsM8wJ?+Jd5 zXU0IL_Y(Du-@(If7;3iojLl%IE=kYG0hgmO&%v*g1`~0+#pj0$T>?TsHCt0ysZ8yz zFw#B?!Ff?r&F+CE03j*3$Or-eltZ%^En&&i$#+SgZ@qgn(PXuH8w~$yb`y8W<&p6B zU~u`uWarx_+0E0S4GfTf9_L(Z01(1gQWtkzB}eZb4jE9&t-Q_2V{VX|wg$4y=D+1Y z0rcJm=wP2*vFWG-5c+F?!+FC6X5?XnG=(_#?flhvI5>S7J#hBOx3sY6+SMZ&<)TkC zMzqDj3{?AnBwcqrm3$V|quWsmGlbc{sy-jT9}BYR|Ig@o*g zNLKdDe1G@z{`I`i=kvTA_ql(+>$<+zclhr9Rz3oTl7OPFO)F?(WU53|z6}VRYg{0( zfJ$N}>T%fXy~$hSv^PnC&%`y(R?p%4^u^%j+%KB|PNfyFQgpNC6`LRL%JB{`BVFrB z>bq7Rz0=+cyUjfK_#_MJJ&mHD;k{l)vPg_dvR~%2fIG>3wMg*61z#%k!wVZ5f<+%9 zI?YEil!dSS-vd7ID#2IuUwWfD()Yh`IPAv}#+7D4wYipfn3( zUt-gSI0daMNWeQA=23X>sg=R)h_f1v`o*nD>^nA8guWjj9D5F!h@ps}g+9-~oF!{~ z%uJ9pJ|sqm`pCiqC2VVbFc9~g^_*B$#=U`nz4pLuP)>!yD{Z0=HE>0bEb<}mew;2V z3qhdVXMxipsPK0jTA3lSepeh0PaWk^VwSs34NECWStOKya3M4@2nX|t1{KAU=y63*j00d3a6pkqC1T+_0Q5&Y1d z5bl^blq*g>R?Nh63}(BOCjiP|PIA=JgH?d)Hb%|(L`eZPzZNJL&%ze_K(c@*b^1pd z(){N7@j74!+(RM502KM-s!kkgf-@-S{ch z?rnf>?t03A+2+t!Z`L1_x9$CMKu)g1%5l0_eAV5=}KR0{ixG;Q3!@oJK)L{zENt@;y3LL z@&a-r)$a6Ah{Q$Uisk&;0iJ75E}q|54a43wk-!CXEaj^xbFc_3XOEf{qT~ST?nLxr z_^edmnU6t3Xxld%u}YiT9MPMPudaDt^EHJux#B0Nu^7uD=?{;cntx*7WP7C8YaHgJZv-_hm=HvhlE zrp~S4`YN*og|q|!9_4S880lTH-S?atSGhrofJhKAMf7PB`8(|x_<*ZBnGb0RPfF+t zh0lO)VcId$$^wx38^w*yC>~c(?1L{G+w4_uvXE^_^)*zH+p(-dKjjs3YA9(Lc%Obs z+bH+Fm8`3d0ye$TnpxsNTs19zotl-XN-*ceg|9eBB-;?-i2Mt}ns@Pu0L<5^MO0Y7 z@cNdKubQT|FVLmCGTy3>;ax#4sQn^%@w3_YQ++WYRg&ahX!KoPkCnM*V;&V0N7@P+ zqP0c?%ijt54>P$1ao}t|F)aOfH?M=Xtdg9b%4d7?Slx)%Q%`M~ zRw@Pu^M*cwQT5g}skass!DziNpoNl4k-t)|HQlwJz26ZCeqBV(kaHc_0hGHJQ{~uJ z*;|OSX?E~|)xBR?vheti;L|sGe}bOMf$#QB3p3ByIoO3JX2(Omz3##3}f1mfB8#H|NJ(d&4TyfWHFlas@9z z_`2r@-uO=9`-Ob`m>L9B*i0(^3FwExKo*}w*}+wmyh`^}p-|is$>V9DKC(4ow?n@p zho{w)%!D1B0E$yte+A%>H0kNRSX{Lg4@aL)WL^6ng-5;c(no!Yad|mWcoN_}ohtSJ zIOPV(5?5?;0t!xlb(?D~pL&dOj6ZI^b-_}{M`qC|83Ftg4Qxdf9XP4HAx>75BpaGw z6|aMpr{)U|KXF8wB)Ak8Zg|Z63hN=8_;2p}f$Cn9{t47UfEoL|qx?yH=NrN5f0oa! z%Tg?m(8_9Jx|hVhoQFg5g*Lu?C__7t7?r^K3OX&ZXN5WLc_<+RC@JVMm*`vB1rT%a zX8(XgE)_YCCffW>B2bSaFaP$k*00m8Uy=t}@qMds9>;ml@?kv-5te{jPx0{-kixIa zufQRkt}h3->WgVvtXv@U40m3hdj64stSuOAlBzx1cJ&I?S!W4=`LRGLP>}oi4hyJ^ zW?$K-5@}=bUrN?1Nxr4mr+5Qe2`JY6kfE*~*Ti@f!XXe&Z{i~lhj08BRL=tU4WJCt zJRvbcLM|ZW)P5*e-X8R$d3mcz!K*Z3f6h1zZ!@lAno^i)kQ>9uR|kc&%p6~$GgwS7 zSHTOsPe!sGOu4SiQv}MK_8&C_%DEp@#8H5Sb-mw^=k2_r)Y48O*sLj$l>^{j0>!o1 z`?#T4s0}6y!Zb_iMI-Ie;>+WnyQ6cL7v2GbTISao*V{Zt^6rxu|CRufaus6``B4Mh z{acLPo}U+n{fZ2xJT;R8U?&QgNo;OQ>?I)eV_u*8Al{`P$#C0I+ z{(fLaHl~IIc$qWw`);#x`YL5IC|B>*%h+OT(mL!aX3 zs{zfhrN*1Ulhtsf$p}Hb2Z+M8G8gnkc+R^sD!K+m1rBiuyD^4n{P@gU z(x;MdHO0iCp{Uf(kLn=c@Q*rp3WhNZp|~p1;m6;lm5$=OBI(yTe9R~$WS(ft0Gg02U;0_~{Z^-v+M1#l$bs2*!cR7&Bt7 zCvM|q`ZhInefGd`mxhcGYFKT%sV1`v3$RJ3wEfvvD8xtkp2)tf*DsQ)%Y zRzN;}lx`5w=gt8ptPx^`QI-MFrLBPxj{GO+g10Juc%86pzkI^jsY{DV1*(I=k z`NyJb&p}Q?`VD(+S088J_hLZwTM;M-t39LV77ZTb?IlWIxTvKST+hwVukVC-s%kgq z6`dl6dI6X>3ZR~QyK(>JMpNT3N&*2WKMxi%QxaGNBZX;GWO_4btI7?zTEH{%qt3bQ zXfUm4H{&bvHk<&r4QPG;6IK75K(AxO%Z5g;a5-d840r?I+S0if|0Occv}752bPDUr zCKfwDaS`b~?fc;CB`KcY`zkS4_Vh_rAkd&Q(M!dyP6DK6Mc?j6?FTuq@WqEdGHpj? z$psh$Y_}6Oj&rygSa$!4S)XmHt0iEQ@zIp|JZf`Ne0#fP)Vdf4DJtT9@s_HsHE)*qAw%`zdWbt%IK+bk=hHm@{r_=7_#Qd zVlQ}~0h$V)C7|6MA*<}V0t9O1_*1D2g zO;DL{S^AB`{?r#(a3cgkE2=oc!Wa}A?#-*~B|4?mw_=FdZ)pN{t(ch8bhUERnG5k1 znv+%rF|6O9I>!v}C~!#*N%xID;;SXuPoH>fBDxb~d5RM>SpTFfba3t+;2aTGV=U6M zOc~`_E7&DnP_IN>)cba*A<`~PFdSVK>-^tUG$UX+O$|V&KF?D zuyY`xV-gXtqwTib$QrfM+LfSmb4R5z8GXVI*5+>sEm=y=RT)qOX6`wITDO{@sR-UF z>S^bNrChh*fgErMX^h&EUQ#`&h0Kr$AyHn-Jgc@F4=5py`J5BSdTm}+;LBm_*~98> ztK?!pgim^oA4K!MFom729?kXx-T-i%YnuOK^IPGZWqh@b52xbZ!$*ngQA?v--P7-a z7KzVhFl)HxXFUX39pE6FF3(xn6WZcDCzq`qOcFco0${oEe5r6iM`oGKP~mpmx+(k? zNIE7s5m=cN^d6X16YOk*Tm{{~2jOdqOyEPnL>=e3_5)tiPt7sK^i_c<9xDB=Fq6C`Jq!d1DaY@|4Jl37kADiym=MCzvzajzUCcCv-w z#6IQfN!-%3{!{zIsh@H!iiGKwI}k-3G;lA+KxSZ63~empVuQOYb<%CmwNbu}-}FSl z9heTXw&cXy#mKJpL^Fy$ut}zVanFwG&7wLU1D0B(A@hqVx7SfqFcybe(;4YS!`8FF zye1NU&;;uFm_@8?K~NP8NG|+~s)@DA*b8=NpAgKC`prTWO#o4Kzq`CRY8AnPU;3n; zq@X_p0MogdkPriD)lZ1i)sW`jjX#lIo=H?vAWosC#s$!>8}k_$^9cwn^SleGap7I^ ztWzA)H6bLNjO^gFGhvrC<{UT_RrL1?AzYo(=v?4-doxhXs*gr?>d9V(X=`9YkN^`c zl+lpDtYMyNbpd9%GD;Gq+XdiKU`7a)c2aZu=f9<@%);`Z9T{clfIfjxF`EoPAg5^_Dt_(r)IoE*0+;>M!*t_H~4 zQG+!wiirdT;U|sig863{kpZYe;g49y*+!jv z*CDEPEv~$`t3c^n(;sYdFfkE^GyNSf%p9&%D!_hp zh-j04PSelc)gBVWERh$)ILF90Zd%qH)X#xzOQHfGq~6fnTN4gC1t~Th%rKOjes{Ip zbDzs>ZnJ~JD8EKMYEl>?eoUgnVjx@Kio_=V6&?$OL761<&OhPlHuJ~AY$PLpi=(c z0tf-n3Sav<2$Ta~R}LgpiWW0!uMZ89qgJWBV$RBC9i=d)XY84ramTyd>j0u1`5_7Z zF;%B7dHr50*Beb0LU5xPuYS^~V|0I3NlDN#=L-yn0dN;zPMci>nTZLQVYWa9)Htyk zyefg8WD6g>Xx;qI2ky-);OeZQY-`hc8Oh+F^G3$kBi??RcQ9S_q0QUfz{*hSpva-4 z1Mvx?MDmfzieIC#Vsr1d#5C>J&SwExU^T_?tpdK|zhsN+On|~(1!HMH?EMH}B*yQ- z0I`a{fPFbQfd$tf0$dv8BQ){w>1LBg>dm+NxK=hRcB+g_P<%T@B+ZVEDl(p?#OuQP zCg#cyqUx_{!q>@0)0FNh4+`f#lwnVlyq}xB`LwSk%?ftpHm{+Lz6D+LA=4To0|O5( z(Q`mBsmaF$Xe%C|LWw|OSHfg^JXO8;gl#P|78JN~)8nozgoI0o(Uq2!h0+L^02IzZ zB=iSLDCt1>ena-!ncR;DoHj_ef-!bo9kXck%{Plz$hE1JqIPF!Fx#fr%B6Z zzHK*pUeDc&9=lz75aq=lUcd&4ke31Rkx?5s3$%}T07c%!D-_;GrQI)i{AlReRMmZc zqk2u7w+Ws`U`%l2@0?X$h$uE?C_{@+u+1y*-#Zk~QKoUhbE3dFAT!Fg+CvD(z>1|{ z?z4`H5!ck1>T}q;>?#C3Y+x1f;EQdSBFx3gjyp6^~g$i-;f7QKuu?$ghapt5|@$ufq@)4e8{#+>TGVDI5{*f3GH54Z7lp-EidP;qhBF<{LT&fVBe5QHo{_%kBuh^!xy5 zca6A&W|4PT=HCsF>yKi-gDJ<8Y7+pJN|)S2``ygWMZm{UDd2F*W(~A|dN4e!zdQjl z)#%6n^|~p0u;Cr42 zLHV{BN9P|7ntdI$cn+>m=7cLV?FFI%6MAAX2fouupr`U3;51F zQ{2JqEG`Mtyt9Gt+sU!kRwdw}*aPXJJy0Xr13rMAuC9d|jY^&bSQWM;X zl-zG#rAx1%5$jG?Jy)MeY2z4g1hHqgEpKHTO>AQBO}j9yU!!{u5R7AMKr=i#piRjP-riY6T2t}fRmFb#N_$&1$L=te>+vK~ss=)5|ioE59Gqs(+ z@p0G_zM}J&`dRN3x61viMMczn6+(M?p!Rrlbh z@=mi%n9F*`{hMs#0}fZe>Wn5Z{EaPiMS40mnYrG>3A`4MCRLv2Y|2tj=VCAGwG{sb zEZ*ik;#@k=aeXKT4Ty^-q(M{%tbE zDUu>&;}wHBJxPO5Yy2aJiO;MSh@OlofFB~`{wGQxTk%ixe??pt6F07Y_!iVU$n4Mj zE9!3#-&)##d3+~Encjgk+pC!&vTnb_2e}K$ zZ-|@m8L&G2az(U#1i(n={slNypK>Jdp#Ud*Be278It8O2FXVIk?3c;)@~wqP!EqBd`3W=;#O=O@^jE|F@c~k^fAaOWv==t1zNp1f@KKEZx4Xee9fyCtjm<;E|vh zW&Qo<<*q1;#NB(qe{ATlp1r%)>0_|99QsbFwB&F8u;}p(UGUciwYJX3GChMG7xA-J zBQkAQ+=J=H3Cd#8vqg&$&ESMfBf9viQ?5TCMi>Pp!Pe_NA%GF;>}R+VOVXRMgone? z`l>XuW8@V%YoALCgW!P3fb;cznmHKuse^eB&BrR=FoFZ3D$Mh7Qwrj1x{>NQ}V7#ynREc8)zZXY^CkgdK_Sq3c%%l|_Qj1%N{t zhxYqI#h38wF>m_Xsu?81Hu9rIUcPAx632@NUK{J%(H^)32W*MWzB3@kq8g}I2JgtH zF5;o=DRs-x!m~CjgeNbGW$bxY6=wIpa~8h*da{%D zo_X0Dk`Gl~Z<~ATWWq~_I)O5`(&jwMdoT#xkqyy{t9PQCK)oPbcD zm0XqvzA5~xbn=HJ{C_bh5NW$ehWV>fPAgKG==ub}ZPa@~LEYQM<9{B&JLfbUTD)Oq z2ZO4Y3<4ri*$VvifPf&d+%!9^Ee>!rSZWk&9iOWMIZGm#&*Wrnc+$K{?m z`PO4m!CO#g4f%>i@x7IO%6el|<(`oc6t3Buxrj-Q3(zn%Gic$Ync*S+OeD?$XvxJN zLcg$WVSkR)O`>_gq&63u<^QE>4PF~OM->5R_Ae!(ElHPK@$fUC#r(-=@-I{Ebh5(3 zShM0HhB^;3js%;x6&$5JUIPUd`7aI-5<8J{i;X$b7__!CdQckQZH-rnC`fcx6X!+b zf$&yliud4#Xt^=oE&hJgB>kn#}|<^PCkBs4~fBLZ5_5`NvxzU@IP<^4IgX0$fwmf9aRW zJeT^CJ+;Vog3$zEOT6`()V$m8^#fd$1)LbtT&hNFoYLxqB&Z8CRPH#7_HcT`fA5UI zWrth~xO~1gbJ7d9ppV4^ZObbbISmF30*5({`Y z6ncP#%;-0SH8K)-S&M*StNNVijKE2cLrHQSs(OY>p9sGct@)*z2Pxlb3!i#m&XBGD z`3uuBh@~ky%Bjb^yIM9F8Xb-g?p%6>8M4iEc*OMz0}PuGj&j+Oi_E-l*5(@Bp{&OS z*6Z=Z-Xi||BaZJx!~U7fUk;=&_nq9io2XA1qj_{+a=m@MJy0|&;5S`qoV#r0g^Aac zmGl6jk{*h3YSrrQ?L?dm1DGx?SQll)izYbp)B10BZI0jBC+;$jA`2c3asSaNaE+S$ zjOrb_(ZkNMX4l=JUeN$@T~2&|gANpeIZ_5Rf;gt2s1y)vZiaB^Qd27wH9SNZo47Y_ z;}P7SL3WB$e^$=o;l$1Q7RV1}lR}E&m+$8r;Wy1W^yEP&YraD3J-C{el?9!(F2C{j z_#hK}{)Hs~%u@Jx_l@i}-eDxMaUL(yr9Df^mu%ZK{1835y+eg#Z{VL%L688Oq>)Ob z&jU*Kr5<;ft__WKK^cf*dE{StsqEEjt3XE;LI0Q2^5zq~%=cHX%QbMkYR`>S`4KLiNU3%T2!g=S4l*K1N7l7xqtnU z{PmL4iCQ;t}NlQ(&;5LXK72eXllfr-{X<0fRSd zw#)qnDjTWfHV4gesY`>MtO|E>v-^2}DVK+V<%fr(;$S8W_z&`p7KkmUw7v01V)om~ zNfAIV0`0%lM2$~*!C^vT9ybWqU(%Zl(J@WuQDt`JPJb|zaJ=yh&~NO4aOcf@L{nGZ zXL0(K2ZL9xDK;bgiDs`$O00w{>;yPnS^3)EfXbFLI3G@q2)*7xa?E0OZ&qqvqwSk9 zr1aQ*dJP68Pqd0&-3E-}bPTMSGP|Le-n)9uY~BSB&Fn!mT?tTCm%xJ}^}8m#@q4eo zjs)PO$+#|rI^{f&Vv>F_6e(8fpjAF~+@3{+^g-QmJcI+1?*z?DapIMi*~N5MW4@AFx=sJLPu%8`g?pcp~lN@`h4tJ`{!M_>JILCfFc{4c$RI zwPFfK5gO>>f%jo}vN1Y+Km9CyAJGBy(+0BJZMcnvTX04c@)x{rv2cT7{>8Rl#RqmR z<6wmNZ~JY+eSII;UpDqQAvt-H5fq5+m4^d6oWF1?nMEwa?LlZ*%qiX|MkeMVV9iwP zq}I#67QCvV#=?%WwO4g3Yw!xuKNxTaCU`7QkZJ)Z`^u-R$Zl=|9h8tS$h{H+o)s`J zc4ZEc@P%kS6iovjVj^w?0YN2`$S^Cw_&Y4c7_?kgD-!*O3FQPB!^t3!6hTq9N=0;0YG`_ITQz1sE_aI)s z)ei2gO=mctn63d!1$g?S@q9W$+}8l%ZRco-qp9tz9=e(>Bw?}uE4&EEE{bSx>B6R% zE0WbS#N(&$^6DlpK;HC#3g+v>ADIE(o-WB)?Li-=)ajYR) za6k_}F@*~`*_7#`v8o-2aU=@ITLejMfkyy(oAZ5sHeMKE9YgV|wz)$r@T&s{@HIWY{*fE7SXUl(9rQdL5gm_k6tdf7sXv_mgIeU^@dm{%_; zh!nwAy_7#4ed1Dh${!fxhzP_so0D|~N8U)bSoJ<~se)X9c-)T#ka?heS-gbyFe?1SJ%`P;mG$b2N$l+82CEZU+(6%klr+&{-9La$-upddw3zi+)F9XoSC+iBkW)&<}Xg*`X%#+n>f7l4U3f9O4V)4sLZ|gNTDoJ$fH&b zew3<=w>VreH1hEOraAD+>-qCF)3XX%Mx<2PA~5a=cz7c2v(+Dx+`xufmFX z>`!~z9xsaOMj*JND^T3x1LJd@O_PYuP8H={?w+R2?F`mmOf=*GR@v>7rFP#7O^;>MvNR zKcwATDU_5)=5^jr=nT=JjLe;qaQb-{p3Scjd*LtY?ifO4;HTyA_%zQI~0Nx6oi-kJr*wPc4_A}UJwaY6f3cv@d_ zsA1w)-e&e_`VeU8OL;s_Cc57V2?=#M&j3*T&GcOj*@L9bx2i!awuxqEx>rZgcwg2E z1i118X3*Nw>5vjb(0^a+BQW0qIKR;HAvi0ImApd2fz`wyIn;gTQQ)>A7e$;hshOXn zX*`u`F@w$RvX#rS0(2yn={rS6jAl}-Yhsr+yb8IdT!jOpnbIMu@bNGfm;(%#65u}# zUyzF0lTs}C(x~`r_dV{2RBQ+yjsZ5`fh9Q?37&Ns7sb**`jaDe?!-y5)0^oKuay~) za88%3p2Zs?CXvjj zZb*<`kH7?LUPW>Klaq_C{AmXX<^p)Z>oY`mjcZ#a3pNcU~z=Bb@ZE05lZ{Lq#5vSUf%F0qaNb; zq0B5X&$}&)d{dqCd6!1T`@|~MxZszxi$Cv+s=#wU zy}}LqBPu#Ti%=sj^mDlYzQ*nwaS$Ab%L6a9`1)Y*uZ#0}zqv`g1;+^`XqK0bK+p8{ zbRJK-&d@jPg;NaES?#~@XapxF#^?3%EX&)_s9E^6#a!A#OafH}m|w_R{m}EynEoq>#De=TbAk<=O@m zZ)bP)J+_p)$T%?KjGFZwMcugK{To)C_3+3sBNrz+aTe^oH{EglY3_C3h#nh+*wTm6 z5gbca^-li)FR^C?Gl?X$6$d{E9CdAY%f1x}x2aAyFfpP|LnsHWEGaV@ueQ-;NU&R@B`X(-ndNTiDMg%k?nEsH{{U}Y9=5)7f`<`~6n;{?(}tP&z( zzRdus45JhMDO(-Z^(wQ==j>-OwEcuG*zt5E+WZ^`L0Kf)GlX0Qy}!mOKM!Qy@;hK= z#i-CaK>rB%IQyK;Jm6#029L@EKfi?l(|U1I>VX6L*Drn4L}jDp#)j~wK993g_16M! zfl^5u+3L#}ntB~R)8HDA(U+FM&Gj4-Q5^wZr)_L>VkWqKO20XrqAq_y975o$B~J!! z{H!4ZTW%k=rP^)+>ajiWFP+v3PmbXtU7#187MLfm6PqCE3G##v2`JdgjQ6;tAg4C| zRze0GPwsIcO6aSbKKc4cp+-*i(zcWhwGtfjIKUIZ`}dyFTqn6Se}u_qqZ=GXD)Qi& zp;$_rC1M_C2F51CU5FB0xtV~M2t7V5`w7u_t!il*N`1>_zFTFZ7os&PXyf4M7I&UW z;X=(5LkW`tB8FrfQamq^m~P|e00Fd}Bre0XEwiz_tU?~rdoE{96>_KJafs{kKv8Ly z2~-1qPx(Epw+KonUfuBp4v?#Fmch2hkrANfBPkk2nKBh&W^}vAnuf}f{(1RL7v0M)=8q> zzsSk)>k|>}EVd)M^xO7j_> zCI5Y<2m%~>Bs$ml0eH0nb;=&l9Ou8up6F{*eD_RHHcIUuO@%YXc?{o~Lt}6__n$uz zBT7J}6c8q%_60(T9D%(sunGlVKN+6+`R5u1Tps|!+^uoPeZI1fIwHA3U8scF zC=KSim9a)^4_A(`Vla$;Qr#!wVWZR&avYGfX(0{r2sAU~?L{v8E&Mu#y=;MpyG;M< zH>41k%0JN70@VP}4ZOU?Kq3!#yzpr2=a<(Zt;qQs=#$ol2t@VsU;mIu8{rJa+4eu$ zJlOx#MvV{QTzC7#^QrqcxTy;rtf@6JpOJ7#c)=IrRuotoUQ7RQ&*NN_*1lqy9eHW{?ciVv6CZ@ESF1aYZa%B$ji08NMG$XGhe@ z@cbAtpHg+aTZQOP;(N#Zc@c+ho8;3a?|KmaE2f}WnO!AmNv_fnl89^Q%gPHSZWlLa z?zM@Eip0o-r-+%4HnDI7jo>mJ81>$hP%3GKv#b3DoDQ9`?@8{$_p^%C#yyzx<>MWQuB9svr%LFW@d9opEv@m*(Lu-VI}KEyR0ByGwc1zNR^ z79X-k^PV_--zYXBr7p9!UoBku2@;CX7uU2C8MAKbm47_s zKTnutvUsT01Hd?R9X4rkC8l1>iAmCVX|_wv zBzQFM04Y7*jf3jul3C?@ydM>4J>@H|*Nv_oDU(T>BbFo^adC<%dVO(8yZr+tUFI>H zO4ne>KORpX0k^BO>LjHYC3`?vJ4PU5?l_?de7Ma-98LZHSvf}g{!7;LkRqZv%JLmp zZ`Qy_<}CX^&snN$E^CXPbO1_UNnIKeA- z!_pW!{5{}arNBPfYP!*Q>I9;LKCF%3+0=BW6VQIw9jg~`a-?V6sJ(rOIU)P zFs-2DH!N_PtAFfRgZjNG>pvPoA^RTpbgF`YE?m$}fw+Ax2}%bXyTF_;M&{6U?+=-8 z8dXbRqWaI)PPg=PwKEV!d*B=(Gawa=AGR0D(0oquSMPLlws7;OXWcY5;MKI>vTlv< zqIJzqZM6MHUyc1vbC-R0wCL!8MTOpVf>=%5Q!?#yzk?@w9!(0K!0V;Pe&Uh1!`9onMGQA94{7-1liIC$SdPbkHq( z>^#a*+LPU zEdx{nptTw;Cu^my;t#6r#~x+N_>R==#R9>V1Amaw3Z7Xc{*XUpE3IJmug#C}8NVr3 z!PpU@|20xT3%@_L^`Lje8+m?LrA}Y^+X&V8R`o!RX>xU7>YSkEhljEx45ucZ z$zha@8>5~OiKG5ceM6F$(YW+ zyd|A<94qZF%{O#*Sh{p}aJqD`ue-G0s*n`Rus|r`6+UHZhGsHNHWuF-xuU9c}>e#@Y+hSVDPJZ z`Ay^8>YIA+vl=URUzR8@a+het%Gv*2!P;+TH8eU4MBHxUOi?}*m*5Z5b*aedDIg_X zN?)O8`9|l+z^R@i$N%3YVc83|^aWcTT2oh6tRXS{1pT|*OeQ~S{Ob(d`uW!~fJe|? zf!bJCgL|nTosqHK>NS;|s{N}fI@~8VbXz@-=05Jrt5nof*X64r9W8^N_TP3KEmBS# z-PI`_1z>D}KJgP-0-?0hivoZPr1m+uP{7G&u>LDtrpQ!t#_Pn9y*#eWYOOV!Abju3Qg2X0N4E<1}S(us*J#*6fv+@nY|J z%|Z&T{r51nwKuXyD`|Q$@2=9QIX<>!TvW%9C znx}S4&hGD(%A_v;>zA&Z#))s$2bMr{h9~dx?;4(z=C>N2JiPMRsQKce@@S*t@YBVY zpGR%`bNnr`nSFr)Udg3T1U%IlPt%qLDsF0*50UwPa;#`fy)@y?x9X$AgfNLoqY0!+mD6N8mG+s%gX&$9GCL*e*3u>)=lrd z4W;Kss8VVN3wFi6zAKR^DxVg!e@Ae{ROZxt@qo2L$p>#)#_72Qt;X+$Ej-kp;2>~o z-`T9c{%27DzaKVx{tvW~K$!vl+bCe+r?^q3sY0YIlYC)Dmmq#XfF!iQFl5_X4%4EP z8^`eZt=F!?#9;Oq$nZ>aGHswDz@7Mlc71GA2b@lH)iT9(r1qAyr`#-`NHObQ+Dz*< zZ{e?L(a~YKFZxGrOlik>Nom*4Olcn)P3dg-)p#~3mMsjr^L=y|7zfldA*Bg6gV){m zCdaV+N~Ut|K0q);DV2QFfP9i;$sUiJ1d5JKy>sSzmAi;Upjnfw3bVWeG3?$swjv0= zi}n+ww6#lC5!Cs(JUEs@RcA%w&Jx`fruu!yqPvt_O}fZn%%K>N-%6joyKtkhzGK$( ztm`$;i}dq=r#+lbH6981*;U0Pf{mIrB|jcTGl(dBFl{oC1wPA6Tm}ToE$|V+bi7ZZr(m>uLVyEugb0knaScAX<=;#aEo|(ISys6} zPonb$u$$!HBw4GHlRNhc#?zX|?ikY9T5utS8tq}h14!kf zy*1XZM{e)nAynrIl9P{P`)dtq?1gdLsIEWg{Ll?%E`Tf~Q7XlDgVezlBuKF7CWo+0 z@&CNa@rAw-@88e&HZ&L=IRNzkSkq=?5chRoK$Nv#`(0dOJC=9T(9>aIc7ck1Ge^H{ zF=vXJe(TRcXUtvWTF0Pu`HjyH=qOYA-##E^ed{1^&PTyhDwl>&`BrEj`1nyJ3fBRI zws9Z|{s`eJoN*U<)tj$vIESVFzMvFP*buPFng=*O)qR%Xh z!(Kfb_`%;+=4YalTwx-Yyb*Y(^t;205F~5Dm(yha08a8&!*(w{CgWwdNh7CKEB)sM z^@K4;sV}Tzm;a_cNfy*KX^QxI5I+{J^#H23{>mf{be%;#KVZTzq>2x^;TB~StFzGS+&p8uC8WChv`z*aA@s=J_^%-NK z!vB5Cx4PW)8Czbh`)je7v+;!)XA1Blg0|z^*i#bL%bV1fcysPq`ClIh?u&4{cft6_ zTLiaF^94`B+}9N4ad%09iKy}DM!)6GA%!$r=z=lt)~utz!re>z{M44mUY@?C{hk^t zQBFxBI4HbW${oz`03R6Fjsni_0yNoFp2O7vM9r!lWk6=*z@_5FL8i)nuS}- zaM(>YQ|sblX34IR=EL7Ff_c^~H@3s$F^>bS+0 zOq_A&w-)Db$c|^93IAkAUUq=ISoJHyDXl>P1SQXAH^$ArycwW3H0Z^C>y{iR={FGm zQI+!5OYM7KS<~X48ViJ;7y)2XEdgfws@Gj(w!3RS&Q7(hRhj?%w*(!urbS%VKV`hc zcL~X7-k|eH-}t76_+gJpHlMt~T5(j@__l}Kp zR8FO#ZMD+_{=-YQn7QB1`4+92j`kaq%iknZqJbfAF;}+6;9ORQns`f@ok4<5(_8g$AH$r^XJ|>-GKk7?DVr)|DIl%ajWXhfBdr6-XpzR zE)`xM_xYO+rtkj!HX^+letsNyD*Nfj_z3=Op-<+m7lJB1cLEE8$s!0A#?dEf=RPm< zTc`Vf|69Wx-1%**qt`Jae7Gm->LP4vl#7{e@b~VlISW7PAMd$e-W=pru*`U;KkeMF z^_)#8b(Ww|G-d)*le#hP@{R>c`#zF=8%17OZ3Y#wRf^%Ge<#yQO{1qM-Ep0+I&AN9MYPJ4ET8wE%7XJPUN(Okz7*NqGE6T(Cfln zurPqzHK6Q;or=cfthAdHwC(7hEj>AZ_z;3JdSaylbC$R6picZwo7naJf?DQg&1jpY zG8_Hc#^h()0luE%Y7@}rvIIcr5}R*t{@bXUgrXKEHSZd?0T;>5D*uDkcbif7W^xe1 zduMX5bT+0r#%p=_&pSr0rKD6;~T zAH3K=@;@`Aef49=qr2Q+kHd^Ts=8HFG@RrZV;njB@83*1YMH8M@2NbrJ;8B0d1kzf zQ875usYaaMW*$V0+8qWm(JcCO@qP&Mb^9}X|MD}U4J`lcZu!0kY)pZG#AG0y26-Dq ze*ud>L(09txwh2qd%?68Jw5dN!gmaCFKYzVA)msSS$#+*r&3W_9oQ!Q6>Hq_$6o{QS>07=F`npSvM0Kl_ZkU@ zIKI10|9R@t02$;nN)VQf*v9@uU}}P)uc@5fkTo9W&p(yhL&;dv)9%2QT(a;7AAhW; ze*M+)b)YXbJ#x2l!%e{Ea#Qfodt9n4%8F%No(IaX`u%b4ayal)XV^;A`119hJ*+$= zH1u+9CM@>vvZ=no_{i|#dMV54=i;du3*$jk;nqg~quHkj4t_@9{X(}RIaQ#>%xaN& z)3^Y()rEz>;GTLR&#{BK{xD|$<&C%$MGDz{O(TeszP}d)IrT{*%*a$;)b3L-cMH`0 ziaC<2yIa?bz%+ZcVWS=&`qw{?l{Trc@4?HMTQ$FX(OJu3zS=Y!xV3fY@BG8_UpWbc zT9(eSF(kLh#xqY5Wu)J4^rX^<`2HzWC2oFQCxD)h5riH6U1t7YPiGw#RrkI90g+ZI z38hOyKw7#>x*MdsyBq10E7&CEG7d!N16Uh7_; zTLdClYBp?r_13Ss6G5>0_u5+<1Cik;msW)g46nMPz4Q?o|M1wTS5_I6#u8PhjxAeC z`_+>?{xMD%;;P?4h4O9bgQO$NZY z0)dGiNNV8$rV8(bTcs^a<(*Ea9Nx^(Q;Yn;2e4w5jfb|yx$@!Fo>!f={miP zA`&s@*B-@ueWQj8F@K<*`=N?9s~#EDaOBiXN^!jf#R>x_2@*B~I#x)RC#T!tVi-D7 zR1h%0lP~5q5* zPg#;3M^M|Db=GB8D1mu_*TTac)X4fi!)`|vaK(&1=k1(Gm$BJA^&FwXxQ@{M8XJBq zCLzAJ0C!?#$|paMTdi*WJ-I6J%@VrCN)_y~H$ zulgs`X?(HDuLl|8M)k$riKBTWJEJ+WCH$h&QuV+wybgm6(g-vcs9SOpi)j?hM-7_LNuw#QPRwOZUAL=l5v_3Xt($reh7M3Clgt3`MK zsI(H07|*UlNPerSbPtXJ4se3XhU?`-JSxS6T+Ty!rR~<*sDPL#@HYJE+uvqFt~6!k z!nFtqI8}Ga8+7s~ynfvaSy&jK1ztQvKMt1p+~@C2O(12DoWAO(LySwdj~~JYXy~(= zLK>_zW~0y;22epgX;q*&2ARN9$}T;ihTH|vnHYC_)B?PBX+7W7%ao{qa+^vsFCa%C z;TLo!nlGeYTdUI9;sP|~Na#SgzdsC9p2}pu*PJR=*kZz1#Ej#j6QuF8t-ACvd2@I^ z`mIE*q$KqDtCDQ*T`3>pPmWHX2_Q;68~l6X^g6kVGF1b)l**FHynv4aV(dqTfId|K zUk81>YmGd7L+7CVEjr`ruVVDjtt6%&h?+tBIkZc!-z#0;PK+qEl}ce>>{pNWbe*Q8 zv14qw!Dvre_u0*Ywn<-IpC{Sm_O&!6V-RGtOlCc_a@_fMN~Q?UzZRL zmxa3pvZDx?BRve}+;wALwpCZx(X##K$S84gAaYTy|2{b`CVg?y?t-}^t7r<5#P$bA zFw#=ygiE{Ncu{P_HC0;R9{v}!c7ljo2@QUSNCs~GqndF22bd%!>CF#w7q`4TE6JR5Ea7(9EAnvK+Bq8y z7U){_L`L*Cy)%^&04OhwGq}P0S>KX>;te3p$*x%jB_Sy~gmRF=QT?H(i}$e3UN1uW z&66}PKiPhUT?gX)e0s5L+3Bd&GS00RF;TUI{2`cF2Acae;}_tE6@7d^3^kd)&pnob?(qxf0$7=VCsly7*X{2gEKY(sv5;EHa!=?g=X7fCj`E+F7FUp{<1^86O1Z0;wTg^6|B)%hF)!dIl zwL^^icZo;|tcGxY%L_iA>}qO00y@rRH{N%kJl(xdu#VKv@5OOr-jnCW)y4UHaac&F zUQ`;fPLqe4!cgj{nw?wl4QCiXTR#L{%kFpx|uT9?UqVN z364@oHJ+M@^k-FF9yisqt>KyOL@u|6ofVxAT{pmn7Utf)3$jf!oP8@%#a)kCTu#&* ztB*ZV1YbGqmH@^mVLrJXdVgS70tz)}4r z^Y2dK6sSM^sI!^-HeIGwXp8R_w7XQ>T(VTuG@v#GA9TjhIyvq;_W<98bkH%NshNcE zJkjbWtKVd-_xN}cV0F8(H6H&E9!h$pqC~{wtgtPzY(NO4-?t~sl>OYD2njRzJ~_y7PaQ>6OvS#prn>FlGRdYrax zBUR}|)JurzI(ZMO{en$>G2`#%@2k_g*TB~EdwC_|&Vg%vY^?QO+Ja7l-2L<+hcG85 zmcFNUg?gO~foju-ms5U!1VL@atvKjMTl(he1^0Q#cbRKzp5DJxK|gj63ghcRbE2|#p7*f-Mo*{ zo9h-kw~P1Y^*vwNm)EW7%%GVi_7fc?&Nt&Fi(DHjNeQYd_uLn%IoUpwHMA3xM6VBj zl_BcajyRAG2BS3`@D3rvu*LbrKIdZwu5TmoEQ3vuo$%RnvFN^51$*akF0hkYkK z1_sVBr{5+*{{^zRyaIv8JEsl?|6D=%aJpEHC(I5(ktqzyd|u}Ycwohi&`?kv8MjSl ztT@Mfjh?7@bVMY5RO=jCV8g&*WZ@vqRHmh>`x^?aJ63E+OXbg!TtPOJXFnb+xkJZQ zF#95q+}TUwrNejrM`X7RfCl-RuZnnVn?;73ERtVwyBAr`6lXORorslaTo}?VG`hZ6 zN=VK&v+$Aj(&io;r%JZ1Ja=fmpObKHZWPt*OK2Tp<@iNZcTU?Zf5MQw^s(1c&(uOp zTH56HJNw5<>k@@2;S$fF-4c`LWJ|$EI9LwI@;?3IFaADAg!45#3i-$hTudrkOnZM` z!^3CYopxfvha~h5&JvX?X#R}4DyAHAiPw+@3#|9E00a#6TGEd%yA7z`)@hX;;GWDUc@)j=@?9OhoHR+*xuSNCBkz(O)GxLmgF)AJM>bhc;!#(cXMM2n)#tax zj72({N#&Z#Q-CL;?Y5gAs^?XvRbw391e)ik=%=~IOQJ6>g0yFBDvERmNBIg9pS<^* zTh_>!UDmLzs3_NL6(7!K)|PFuO?b8F2h!mMpDevJwOur`233O4UKeq1O({-EtEsEf zsVF9WS5w>A{j9vj!K?n#T2o=;AXVMuC{5R-c_1)1H#z2E@U%cWey9?7Kz@Jw+ZbTiBA_UeG1WVWo5Z{p&sJD>|r8bt~h{Tt7B$y9tz~ zd0>JomTtD>3*A_LFrAl!rfQ{@m}-IHOVu(Jb{ECQNR@lKizyR4-EUf)mSyGJ&LOYt zry%T?^Ocr8lLVGGRU7m0x4`9RXKx?qLzR*Axr+3BcQ$GocrP25M>Yf@#&N7XWzOxM^lhNUACqj$UB_ zff_|ARu?|sLl-u8n=Pn;TVABITOPJ}Sgz(vahrQIxXcW2ym<9q;o@*g`WZ2sB>e@{ z{V|kQdGhWl+T^^;-XvI%cPUBWr!*M{1wdU`&b80l2Vp+JPtW*VGlpmjt@osT0%?5Y zwN475QOpH)tUxY99IUW6;=Bk7l>CphLDiNUbhY+BgC>sXOR92P~WB2M7p-|0B~nY0EbvgA6TAX&WMmkcS|SHC8;s|j&_TXO=laMe#)s0 z5fi(&?(TvIlFnS`vSCqdBRFGgMvaOuj2l>o4L9KS3o23V!SDeMmSJdJ zd~4LmY~9~2EF|Q6L6fjagD((w<()*kP-WDBhNdb3EOh@!JSH{Jorz|(PicDGfyOjx zWL7e41`>&G-Mlb+qO{0RPpL`N{GdSXr;<i7@6642xK<0sB(&LZk#8pe;uQEYNah0ZxN|$A$={{{WTRXj3O> z<#`$Hr~a}%@Fd`^PWVNn0K^vn&31Rc|9C!_*S!UE$b#9xe9jmI?Xd;)L=6T@pq=C- zUj~qFGRAhu_Nh}UFsE$*D#8qoRA|?jC?X=FM}oFr*djiQ$8F#tED=5cLYkEUG}GVL zm^uTX;2`uJlttQ;jNEvm5&V~dDc*j%3+yu&#f%Zr2RLv5rx@JskW2E1s$>Fursn;s znyc;ccm{0_pr2<{R2+M60mK9sE`gR==yfjEB@IW36e5|(csJO6A_r$4q%AWTuPk|0Qc!k7Pl61@_t-YegLY{ zaup@)<(L@cASAO0K8bTbRC$Fa^*OhwAp@31;0zjrb*cF?h<*GGuVqTqc6TcMo5y{! zNOAXkX97NuoE!zC%x12f`F;2g`k)hd=Y`r<|I&nM&!4%AEi0E%Dj z@n+OdY2f`BNNE6Usy+~-gmv8-c0OCIx6_}3Wr*BX>LL#TOOG2H=3Z`A)*KclY2Eu| zwLI)4ZuoD0U6|BS*7dz3;3Dxc7LKP%5MBd7oKB-H>_ADB5BjqCvrr^^(maM`gPy}p z8Z-0UnxP_ttGK@X;)TJ!wf;9E3&S#qohfsisSmgSJNKumqy^EqDYtsM%#;^`aN!4& zpiUW#W8CU_p#&YWJe ztIh0k=WCIhR29hwds80exW%$H(B3NfK_^UO0U6nC8r@mh`w#G8$Th-q1FZls$51YN za&l&zzax6SHyc$Hd7n9#+(o7UEiUi%(reoWq2VkZPLs_{AbTzX=+GulUD!EDoTgU^ zpiFgHdZM2o0;|5hBt~8Cvby99OY!h9cNbyta6aj$S4Ow{qy-L+f^4q>`xs z=H6_+6gJRXLH<129qF{&yq)0BHmb9pN5v!;jbFz|IJ*&{VtJV!UL`FhL;GzgpOJ2g z)QFtR*pQ8SJXL{~-HAZ|`Htj-1)U{b{Aj)@_2|`Iwr1>J*3`V$kJ_vGz3WHa8J~mN zD?O{$Bkq>bTfd{PecXK_7NAk|^5U@=(A@0@JC^v3QbNS0J>dV?`Q2umAy$2HX#bYp zZm18O!TBaYLLV$+uiv0{>IY$vbKrN^f|iC>Oy+Vi&|_gAj{;TeIh_TEoiwo4rUA{o z+4=m2{%fu+Af1wljC4g2j88&K|J3FUJeJ;pV!XZ2UnK{Uel$vN7Vm5OYVgLeD@X%>>&?uUMB?PnEkPcbxGP=?cEc%>Gt>;P~4hk`fM}VG8 zR{|o=vYSflJxr#b;LV{`k7K4=28@#DfCXU>bQETYX?2)mD+xixr2g_=2w2=NO_piT zT-c*-w&o|`=LfJov&~?$Fa*W>3w<){tl40i>)BkQ@E6j%8no9wPuGV(GE4 z7oSW|y}ITH-Q`ayCk~n^;b=sIgI?kPk$phTS~MF6-z+d)HP%uo7O6|Ke80B?~0YtRw@i*g8<_64HfF-`&(S?I2d8{Wn+dmtTg zkqxZ!OEhVN%IGlI(<$1ZG6ysx*Sye?UvE7DFr9!Qb!R*-BV%tKt$-fOUs%+`MaD@U zD-VcCw-S7UWKER&Df*_aK%&(E0H6*M>Vbr``171R+3HU~vNez4Ieeeu4mBUkGbHF` z(QB6=@u`^|jUVaBkGQ+LBPpfGZ;7L51d0CCX>>m*c*iU#tWtfQdHuKglsG;#apZ?G zyh4~^SgFAMjy$7$-0{0N7;=)u4LQM|cBmvTj==^w3_dgeg;GA>`H3iGd?*XyZWX^9 z1?Xd2_aN;KN{d@k>R#aVI}F0Z+aQ;qcpHJ+xzUJ=M$#~l)!7eM%`6Tra z47VbDfEbPC&Uit2G21=D30?lG@AzYs4+Wcu{7(QmCR@D^?!$%aV?OQ`co_T4pgkQ< z<~wQ%hYWhKojm+1zeU;Wi!K)zX75gS$>^Sim%fwj(LXM{ou>uEp83T*n1cn!D2j+@ zI!j^OHwRUizTp5Q0@6XhR--HDIx1rCFiJcFJVs|CEOFwr5`ssJ9WqA;p*06gkNHaC zua%B~v1n#&;!;tP7HKw*WP%FDq>G?D7}z&FV|CtE*wV!Z>d>1#{l|1H;dc!$RD{Q` z)4j!UA7I!mlI%V@xUey4>?0qGu($J)1tmr3c>Xv(*DiF^El{wQ{)7Q=3FseENq}Z` zt68{N#xojR;PZGM95^*QsK=-fRryyatOiJjECv2NU}?B36u2*@w}*8?c-)=z>Qa5d zNMnRlX4p3MtG?`I#fyuizjOd{#X}EFNt~F1JP4(HYF^xGsf5a{l47P*2hA*Y9Z(wD zhB{M|gI@?fWEvD3twF2st{l2Mq7PbtN(uS5`LwpceG0>4uovD7$;F)_iczBow)iK} zsj>iYVEEwK(^lU_(@@UaW)P@`T}dTFx~fCANZ<$u9S)A_fett^F>#GL#JD3JLuU)N zr6;DOY@Ld-zC;luOMs{+*O=wn>4Rp0%qs)HcH9J+eklU7YY;Iu=G&K07Q0i+M^J7D z9x{nGp$ak%9s7@w#4e15u^x(0oq(ud7}OhvfZl$XB9_+>cyE$#op=Goq%6?C4}mho z5ZFH#A1-$$deMn_XLU7Bu8hM4E`2~NHRJtP&~3*^j=5JUBN4zl^1gHbcQfZ&n?7jm zb;`o-+Z$PTkpr_I2SfYqwdkfgL)FS@T~?xwqu?S1JDw7$kF?ZOwA7psH#nCnYWbE zca=x6e3z*Ug{!4V`raoYXLp!OBl$+z7yE#W#Icp-mi-zMxAW|r3KAkoMuxn;rYOq^ z)^(?8O|vvui0HCk_|u7QpW^leP^%ifdjOxTH*3-SVHZ1}3PG$1MbOS|(c#mX<;v5y zyKUCZAY?QB(;mPW997YwOw0&rBq=6d)T^~jI=eIIM~7owt_Mof@)w4sz4TWmx*daQ zQhP@^k9TG>ofu5Eb);a9M4$=F;YQ_seZ)!R!#9RHZONqoC?w&iXds^dC8rA3)zR|u z1}^4+>GL9t9dZb!F*0XQCmDQ16LfTsLTCF?oJ_R=vhMm0>D^qTAMo>t+MDV1cA8i1 zuA&B6-@aCX^$*`}L$a6Mlyvb=!c*Ur#LMDUfQKL-1(`1RrFsda&4gWC%=z2ZunJ?5 zhV;X08N#3ReW?tmNEYFV*70dEQz3T;c77Q( z@ECRd?wr%;jN~(=nrA`czr+HzUcjZ%;?1!+i(kVuzU4mZb!GCfYp!&>RY#+8>>7|&D#i_NTLFv!ozL-jcv29<_4^gF98h5eXnwCeRMy@JcZU2#Ab_6_Uyw``MCHz= z)e^b8(aYWO9Qu3z&5~g3)0@w=EZQ?<;BjFQ#ZPE&0Y#0JE;)7IgyEBp-@t0obf#Pv z<)!XQ2-({Pcce7tfjsH=w;+G$*1s;C9|1WRA7RL5;WZ8U9h;r4a$L=nZ$?^#5wM?f z^d#(B#G5R;ME%F%!)Bxi*2ay9b)CgTPi^+8U!*t$KAU+7O;4U$hnK?lCJJNa++uh$ zQUP`wGRTOzpGeS%qKiNw;9y>15y&LV`vAh7PmI5<=A${U(e|+<1|xNE{I3PTN>2KG zkBQ~K(&!I76NK!7#xV`p(I{PJDkuX=Yf^%-@1Wt!E_pBm&H?bbm)fYl?Nc@^LB z4Eoy~fz)Va`$Gi?iE0Fh1+M=*dE;;4dgOFpMFj4nN)&&5+W&*D9Q^vTZ1>f0#!ul7 zzA_KT>Yd>@`o7TjV+sM`g}ET9Rz8frsv2gDEa(fol!sIis-pwD((MX5`g&PR1{#hS zql1(U#y{a@<%f`(cDAO4pa{lD_jH_idWW$Q&G6dkSf6SkAMLFw!7CtRHgyz|v9F@N z5CUj%nXQqm-@*Q{DCqm9B$`zV-tKWC&l`Plvq}N{sifdK6sI>@`4RhA{z6vMrD=c) zV;S%$p%6DA@j~7alBl_k$@q{u9-otQ6q6&_7!{B3MIgIRzw7K166^1n>OXB*s)Ake z>1_QlXnqqLiM<0Q4^Oa6K@!s#XF0@vpy9D&q}*ehzVI*lgi zm-(x%7;HS{=JZ6l>@T={pCF>!6DyQGfn8APV{Z%*V?82VXi8ih<{OO7r`X)zo<$v% zy_8qPeQ>Yq+~$RnxTDf~lQEMpg_KB z`$RqU<1KKaV;n|AO(I~2QfSV`XP z=s$YR1+AHA1oc+kY!Yl)XQ4N6G{KSv%*gJj(7(I%oZ~U?UPb2uh}NNSeefLUxv~N| z029!oLNs;O-Som8dG-Osdh4giJ80rgdasmOrwl-X-(jv?(hn|7vg20b!xLp!^R)Gh z_v+<3b3T3vn%gVUb#gkMneUA9|jO?Lrp_7%M6v8O; zi&4LAkq@bN8bk(4r_F?mBzaGU@CxH$vMMFR!$$Cw34dNif!Gldcm9Ar%*$#wobd!D z6#D~?5?F`#Yqy8>FhqXS2X98T2FY;h&%<}xdT4k&V=;l!xHxvY0MI*fHn6vO^>O7> z90W($xB(*yVM&;pQQ>~j-m8n$XyZEk<5v%+?B~3pZE)67_0qG~SJ21SMu0k4l;nf* z)jfXKzaKyulUx|@r}KIGW%P##LMWQxx2d?!^r8wTTI{HFv_p{EBZ`U6I!hD?%ocp) zOfOnHF(Q7~=-2X1<)we{qSRBed_W%BHyU1kxtRp+B8)ub)l29(N-416;!)h(*l0Xv zG~k(^`Ljadhis?sPO;OG%`?7?K?Z0W4rEq!-5gSVW^!*AekS(2I-h3m|Bt2>qHi+v$Vhse=*!0w3p_$`!HS-N|8I*nPAKu045-c8q{1 z@rrANRo@YXga$Ylkcoy129GfY?nCSHj;>GP5UKm;4;M{lEttRRSXI|{wYv4_L! zrw~ewSU(5-7tI)CNfGX-bI;4n@f$yJhEBg83zLrbhQqf;m8BW~6a=W<>k+hAl{Vtl zPa_oqlR=EyVjApnzbvWy#{v8b&Uyl-J{06ZT!H-<{wvETV7hTJQTHO>lJH(S^ZvV~ z;!JN|oFvj&b5)TSwcyy78u5}o9k4@ZLT)gk!PIHGxm|v=eNN1~T|$lKS>ECJSyB%Q zQUCXlBN{Hh$f?jAe@<}@%Ft6p;pj-@7HB@r zF}AkdW1GR6xO#y!%}jlAINMjwaSmF^2$ir7wh)tya!l>44pfOi36=3358ZbJRd5y-A*UC1Fc8l^pC2fKZ1mUhcs( z3gIeL`#I1r3M|!8{fhauPY>)O%R=y!5G^4^|g7`cq@^B5~vlXba&uu(T zkbX!EnrW=&I@qt4E&C=bt9sj*r2}m%R<18EwkvZunE#f0UvA&5*6K+;`tZYZt!Hzx zdW0oftCoeY^t#b<2%mKr^#iZ_dEMICrGW|4JVumG96~n<-W#3w$G?UCj6E{xkS2uw zIeQYBj-WK(I}J$BnqGfyKS-6i8B!ym z3DsLp(WTh+`J5bz@i3*K8l^9=cm%o<0biLatAL-*FXzYYl_WAxGuvKhvE9Cf!_WHk zBK}jE$1ZKP`;HQ;$F9mmp_%*y8|Tbq)q=H&O>nJM8K1FDeQnUR436vMydLkT>13@8 zqYk+z1I2h~gMS7Sfu>(LYQMZ_R){r$yiqYK)*_gezuJ37Zk z;A1%Rgi!F~2ZeapoVYMCLzb&v&bjCVh0qZ|UYdcH)f zrw9Tk=kdSf-yZ`%GBMjTfEZxT@wrUPaQhGzZ|pli_U@_%=(Nk(pD&O_5$)WtoQP__ zg2%99TA627!^s-2$SsafF%wT=uu@w%Xt6s$vyvc3ZtC#mWJc~3nZDHY>YRTv?YX(0 z3c9fxXSe|WcKA#U9kDeiOn~ke1&V-ONOURvGq}F3z^(Z6PynytsT^?Qye9g5oCsSn zO0mLKc*xOrDBMAcyjcw|s=#VIKb&4v2;WiI6bqGDQ0~(GZ(wefeIG%WbeFf5@Hks# zkCi0VhP>}7w)%yP+^dCX759YlH|q(gf=m(au@(nTdqi%jmk0A{xR+dSG!`^v$kdh4 z@Z!QolBN+gt+dKB31pm38=Z6yNr`LiM`7Gi9Z&ci0U!7irCx0aB9uud^O#s^QIr`8 zj{-;FX)pbUV=1tr1S1Dsz6&0dj451USB+Caal$IFd3NKUx!w;4i`52mR=ebp<9fxh z(p2~;(78~Q3IA-pQx+RIbKafoUj#r`IebT`b3P^fudRy}*@NlUX4;fQqr+z&e;!^^ z_ry?3A%{l({7QDTW;N}(Fj}c~vHMNu=Jb%Y0FwcO|=TVA^G?G-6596sY}!*)BG ztQZ>}3-6odl|ODu9rCbeRg_t-N(lVITM73T8A78%Si2{$uuLhCDgmm?)|j}Pj;%oa zTCoP>@InM90q?T?$%}?#vKx~JE*f3}PTNfER2;k(0L&Qa#NKQBRsunf5#AGUui#qX z_thipsPD%J-YebE<|!0IbGprPYeEamql*aQdseKF&1R86z~gGIlEX9_o08^?c&d0) ze!`E)L`r#03HE7WN+FsAzXpdjako5iQYFQGkzkwzQY@~WiTC&pTN%o?TWfA|=V2lp zV>q?Pv#_HsaVRen`R5Mo$A*q?te?M;hKWJ`V$j@FB$d9wjZb5g4dWy_%cCdfCFYjE`|RMV&Dm}D6jwe ztMRrc@|bI*MIXiY$uO=3UJeB(VtOG@^>FBKtyI3;2P%Ho3UK1yK#;DjvN!1M~zgHx_7)+h-TNfSr}XBgOxzrH{sMc`_b$}p8_(rYB!2jvoUYqf5RJ0KIm1Re!!%epT%yvH{Isf*YsMNkDPvBk|VG14MH5D_~7t7FEz(qym*`HZEOQB zY_B5|9bg5ih$4hP>olznPv?+oMp3**a&sSvknj25o>z;I*4*lDyRNmbF{0mlbDl#7 za=F?U+j_JLDsJ6SQ2G)=1ADb8Ua%;|qdz5a*ApTZgzY*Q z3yETdP_9fc`L=!xAkeF`)}iy9LGXQa+qOIFMEEm?Ox#W|9rNPuet{g}~W5J>Zv7K!gQyCg&aKP;5oGyR9{wmwn#%TNT77f$C z?Zuy0-w&K)EC6%YQ{Af#glD`mfvNTXIvaQ1;6I>MrG{D)c#u6_q9 zrSChiMJ&wX1uWpWey+UK#EE_6kGY3iO<1+d zM!5c#%3VKMV{g8>-oxQ+Gwdk#e_zu9-X2)Ar;VYcLtpo&3Omy3y|MA15$MxOr_$fYZfl@*PyhUb3zxI<5OjS#9ed z_-c@?4`a{=YumpctrX-|?C_1&nw)oy<>-`qtfhs0^`!WGB%ajAYGjz}OU!*p8UolM z-M)I7A36WK6L$D_=GqUZcxT&rV&9}6A2tbZ30ceTS!Fv~X1`A}R=f6uze#`nMve_I z!rkrX!q-NdRdQt4lxF10vp zIg_hqKc6>x2@6pFjh7l~`xkpd$3BvWV6~ z$k*Vb|E#r`psoffgR|??jm_-aBTn@wd;{)mpE|BG;ipb{zaNUH`zZG94lJ#d%YNI~ z)YaZhZ))9|NY4Jo8u-0uScW;AUUougrWG*vL!F7$wgZ9HZ$csN-F$gL1L; zFl56*n}6dR2Z>_QiNfy`X#gdHYJla!&>2<81wfM$XhBmc42kT+$%v+6$c3>C_`ieb(N2X4v<#IzKTUYjf zV@(E^Dx6I76Kq5OwnIsXX(I^#{!L#j|NjOiI1cjpzsETk;miNtm_Q4;+MhE5#z18B zp|D<2Y5@U#eNg;>t(AP&>xuF*phPDN3L!GKw&kU%94@;mK57515_bLv)CxGkc{?HL zR}3Ln9#bN0wjz^(D}hA^e89i818j`2c>(_X`$@nH{(U%j!M|+`ne5+(ffxMu<}l93 Z7b6D)2y7pzKSRJjDKRQ zQOXqdarM5x{e5eHYyY>`w%&K8Jf7#iui-q8>AEBL8}4RiJg{4nlHDXQg1^w&>hIRZpZJ>7dLDp3R=Vk$dQvD%9OQp0%B@5m z{3E@Wo`DX1&k7n=?v=EcXkSq%!W2Di4db)jtF{^vK^|Ic3C zrdeM=GrdJqOKV_gs7{lW%_uYPc#G&+AD?_yZEbvVR`X!pW%tjo*v_6mU%;cS-S&x= zp5AVHxJ8PFjZH4};Mmh+Z}_-_E?h8E_CLN?TU(mBE|i1q@0Z;F_=Zhu+dn`0@89UF z|JQA{eST3|YTg#7)4U@&{!9${!QZ=o6DmZ_ieLSoZ`l$R>$vj5@KFi&NH<(14IQ1R zii#k`+R5o+&>v;cuV7TQT!>U!Cmz2kkrI=sI zcrKYgb=FWH_NxbmzoZxm3nwFN9{YQ?}#jV%xpO{!p zDLV6w@2N{qc+WG}LEKWpHh0mb*+G^MFV1mDN=lmKT2sAv@dEE39U4mW=SPD~t6fQP zvCW4f_uOaCBJ%UMT)cSE4IkwEu}qMjWv%Z3>sk(u5NY$LDJMI!H%Lo|B_|67hlEVk z2?gkC%{C+kSX&EIteXAoo80PyxKl7Zzq3^tRphx3XkTk zOYG~}r^h<>m6Qf7FI~|Un;p%}aT{q}zc+E4QMY6%e!5Cv+y0WA(o&VO`O!@y&8n)Z z+kB=swC7k|{`ASV?frYb@8@SH2bhw39FzURvyVQfrg(XI<+=@M($Led+rEA7dXv@e z-SnoWrbV7Zs~H&?txud-DIp1bSG zhKAXC&2Q@KSC%jS=Ku8RliT+`73H&&>U^)1T>GMP%9n1j2;;qsgmU zFh}=V{5o>1Tyys=cI4*f4z0~8EA*ZCP%JAi9}*l)P3dpCv(t6)&sa$5y?g7#4HGZO zl+99~|2m6LP&aI+eIO-@d(S=n%r9xJY^w>equ^83sqN}Uk}Y5!SyrQAX4fmp-+uVUBz zyT|SA?MZQ=T0v)x;=;;6v(HlY%a=U&!Cil_wu{^bs9RfGg*R-(5)7@ zkY`u7=kmzd7?X~U4yJI$+}xZqvxKOqsFUWFxFLf9{pnRpOG^n`92m{3F)D{9fB&u> zsEUv~_rsn$1O=HUW+izwzHAx(&u>J--TwK}fBz;vkW4SWP$K^M`rqG>Isg0hKfn3^ z^6H1*uAt0c7(R6U#*KT~*{dBK99Sc@A7*Fo($S%HI(ag!2W#;D0=Cq>2M-=p$c>JU z?l(1EN!gF8l9*@_vcSd3ney=AhDB2>I_e+_D)p5cHzF=xWaQ-HO6&Rj`EyD}#%uZr z+#|WehYsvfH5)c;xYEbS#AIY_9NOA?;M(=;dgbi_R%T`wRW}Mr@&)BIJTh zOc?S^q!tAa9Xd2TI;yF!&$MI5j<+o>8g(&(ceAp3`yV`dv@%GYA}uYAn@~S=hzlQy zm4%Nu)jlSP7N;wwX==**x}m}7;K7j8RM89CqMU2i=;w@$n4{brICzkJ#@+oFa4U2b zrAsBFIlXV+vd+)X%WmCz&0j{Qd3gs}5Lh3>cf{EvCugVZMx@uRTd{rn_E+`w8kUx8 zF#{9|8V6a;Qc_Z+-5xx6aCmI2>eVZT@bGY1S=qq6Jon7urlUeXe*OAoHOKSi%a>P; zjoOBWtMdyAVy|Bh8y|Nbd5Y$E|G|SjRV9gu0@Ksed~F$i3%^O>DlXnmLrs+&{Vzyp zJKWaRCcJ$+FN=_>Giymfe!ehwln5&WspBV4o;-W*Tt#K2rq4-SkJstbrq$^WGcqc> zyZPGN+XJ3k;fitBvA1qrympQA{{8#?E%$aaFfmo#)rt^ij}(@WpkKFc-3ehOC8db# z*SVymrSGPsytE~C0&``gf7sG8=JRKJF)=aWjT=`+Mn>xD>1iB2D!6s)R%;iROPi0D zSbu&Mn_E~Ih6aV-?&9lNg_}u9Pmfru;mNUUZ0+Dc%g@j6HQpI}wGJOmI#Nr|wVO91J3CFPYimVSRo8CZxDo9U z)1-|zlRMEW=JmZ#(9qB@_sJ9c3Nr?*3KXW!t}ca=@3=+M>#$JW7Jj=301{yOu3W#a zv+(;{OPc^-s? z@Bv3lyn+e}DT7v-BL0vYu2nO zDk`#``c#9#C4<(4Zs9r9G!P|I7Kqp3a^7MP0Fv)vD@zIdJ6u%iC^wp#nrJzp!^6i> z9c&sC3U81EZsiKt0^i|Lo_7-y}C}(o$Ea z7$zx40!^;s+sc5kz6#u`Xr_T%zW zL~_6quU7b~iqG{|x1T{HdPr+qt{|FvV?VZ7>G>Jk(Y8k@OO+1{ZeNLw4RYyuDi`?9 z+|JH!g_@cgY0iuDzsRuQ=iw+6cT-b?Y5#43PDe4vhIe*^Z9Mp(e|qGd$@4SPYn9Ii z(lT+Y8yH-oP30zAwydm7ch8>6(f66|KR)a2O;YAfH@p+9EouXj^S3#emHqkS``9Fp z+TY(lLFK$6mRD$+{tb`D8P8X*Ub%G@KC@s#eUdWe%OY@x{Nk=j657GQ* z{@Ogk!omhZSX2qVtE{Zd1%kl($dg_5BP;mnQ$Uv$qIS`&Y86S{mmZ& zDn6W>nD65Zrm@Nf9Lpd?j|>ZcX&DTeoh__J^?SH#3`Woap)0gFc++^UEnAA%Wb}ogJqu zvEn`qcLsFO{&6UYD4!@+UVQ$meCDy;+8tgDeSLjzTU&$Cd{EhPedj!|^3{I2-oT=` zcUe{Q`I&FRGBV4rPct$z*E~7#4xoz>VC>AdZkx9$8o*hanP>eM=I3p3CzC&aR^g`R zSdst{n6Jj>HVc>11Ox=^y}5<9qM|~Iu24@wK_NlmBscJ3#J8Sj&mMYmFoq3x7jDAs z@;M91+0qy$t1?rVOM*Hg+E2WDFwkESaw%qJ;d{9qM)}h3fc5~U0Pl}c;|p7cet6rI zmt`)rJvun>{rhsG-}tw01j8Obethe;K5Qs=tWdf-pF2BL#!5iT`ch8MIgI{(AZ_-9 zL1JH0bzkXubfxa`@)NGEt^%sQJXa<5ePu$$C}eO3VDow1Rp2PX9hK+5IM3~UieJ%L z{qSLK3=!+vwK1ckPN5MI%XTNf0;#%BebPbk71^{&pKrtBpFe%G3NKz2D+`*8Y0bPF zY()8I!WfvCnDkdia)7y#DHC0{ZeU;_2t>>G=NqLVW>w@y6++K)iqm?`+$P7Om>SS%}y&wHP7k#S9p59cUg zV_OUWj3&()M8NFxT-yEn6pFHn3cvI6SaZOaiHW{`M-C1Siu#EYBmCvzxE6F)ih7P^ zDId*;bn}L>lfRy*`pv)iUA*|a$2DQ=iB?ao(|}=_K20;SMdjSX^W@d(%NQIk7onnpF5jDE^ z@t~2>xo@84C`~9<+#p0C(<%A+?Cg;=Si@OAifxV^Yws`K_ph4Zy4O#4VQjox8r6iB zA}l5KxZ-#D(w_`qA&T-y_~H|=@6xH0S>J1=xuf`d9bH{n`t_n_t?HtePyac4P=D`U znn`=*!b_K!uU)%FNqN7zWuv4dH3it{2Q8rk1mV5eqNo`D;DI=R0)~uPNL7F$8?fZ7 zy=#{yU<<$i6SvgyxWSz}c2HAPmuBq5OnDi}A;DMZ)FH!< zB_u1b&5d&6#0hXZozthu6HjL#gAOAGjbvtqufifHKh74e7Tm!}kv;Wc8$NNXiG=Uh#+#GBe$_xC!j-Q|xP4no zn3eSKr%#_I$UCmiwkq$F8u~R5Aw+$)Rk{?76x_Z7dw$vd#X0@cll>axit_V)cY@** zrBWMxFqjoAm$$=6>3sxEe`~!3->-lxB?@X^Zh2rA!tz*X$N_wvAM^ykc z4Pn-ojg6}^kL1*%WZ+s)L0N$=q=kl?TpV5qJq55bbZW4!s=As6oaXiG*N|fB{>=Sk zTe}vQS%_Nv*tCESgb3IleR;*<9Lr09m(NOu=D(LO$3W?^`&1qICjyIUn$A6Pi_b=j z)aX9i-7x|cXxeOie2nz;5!?Y~YRiA*kG;7?i-KiQ=6jPxc<~;#1w=wyfIA+gQx><& z4sSLBFp*|92YXr~vYVTm+e;Ktwl!-yQkz^L7?KtSx^HA*5nUC*GB|iyjV@48=)uDi z)z#J3DCxuPS^Tvzf(^m{nc<^bwrrXG^RpQXo({!=hKA-zg6Gb2Kg5)kl~eBDrvbYG zi{=75b~|&XHgUV>Y=6tg9iCrT%}%&!i3;Pp_^V@ba`NTm zKotRc#)l7={rb7%{BJ4nZ3e(Gtb!D~x)_tE&b-0FP;^sX(#Iq{N=>EyIn-Q7u+qp# zEbvu6QO zm?6p7P*8RWp?4C5l#qAs=P`0q1d^6`jRap-ooB$ll3HE3fhdCGT?H+LCy#&{eCly6 zuL1hLo1R|x>E979+-NdM28zw8RU-F}2Pt8f@UCrV!Q! z)(Fl{&@cI{Cr^Cu9(DouBB+jO&E{&15vzbeG zl!B2YJC_j7%*=)EfFt-cf{xLr_nVkp=ud6j1Lo%ZglnL@?}s=-q;oI;lYIM?zJ7i! zceUtOtyShpN=hP><>pOZp|{?hSXZbg7vH|!pSaDP8n|LVga?qm!;ejZuz=LHv_iuE zh3P)>U?6EC3(&Gq&8xAPFkZCh&YgSW`f)AT`sCD95D)<%QJ>TCjp!8G+BEo8R?tKU zGs7bzHk17oZeCv6XU-@R8IZ65v?G+k;dkjQC?5S^zJ!4WXc!vCu=w&DA2|~7-23|{ z0DeOFF|k*Yl2)UtcujmTofkQP#fP0h7&3?$I?)P{Y0ztXHV7bprO=rD8=T+f-??@z z6lex2>BC2lDqg>41hT5Vz_yNXc76Q@I*znbX38}wEtHy8DkzYr6%3$61>VWEwy>t(iey9k9>R8)lAwfoSa>(k!2h1j^bVDuHyty?(( zEBN^M2oi?6|F-nJcQzGlr?!i)h)>m^F7q1u=HP?98c=zoGz=bKVX&-KKJJ$3x^nB5 zEt(Ol1#y4tOQC!fCU`c#dUfTg*GCAM0AaFFN^oNzK78o^{CT?dYM!1LyR!cu9>CI* zve~7NAFqPOKK!9r9u=VF(LssM?(S6VQ?l4^-1ljKU`%)wfGW6QKN=wJs}?s8y=nz& zgV$)=O03FI;1*{W7n@V3R`c@ST(jdhx|{o0yR=2QzaX{~B*NAOpr(QT{y@CNz4J2@ z23yKC%m==GyM%gY_v3TD6y4v7qqVK{{KcrK*eh287iPZeukeN*M98GAtthhuntyJ7 zzLWcy%aOx}MR)Ar12DzzC891UYzqdhsJOV|{rf`G5=MP7cKn{OoE#ewfea0I`S|Pr z$M{EzdW)^m+0&zc($U6-iiwFS;pWY}UoTdMoPl?A;J|?^n3ntZ#Xgh=2zh_+tHBkD zKzc&M0Pq16scLOy$MP>a_fs5lJw0G5%nvqTYX7V{Z7J?`_)KUBnkFVZ>>M0MN00Uj zU+d}VxreuMb9dJOpW>wr|wC*=sr7Y2`w zvhP@>l7d3mqemMd1*Iv$yCA4+hd)NgfAQB53or;!P98OXOVux!oUyVGmb#IgR!^`ANObgRiR2M3YO zn^$9lqPQ-WFR!wHH}R0+Hu z2rOhJhxpbxv0yac3(?Wq$B%EA=zSgmv`*K7%cEssVSzW6YY^c^fSP3VKp@&aFW&>j27%aof9WE?%A`4FZ8z$ z{@4#M6EhCr1n2KAY60{T@Mm>l)~#mEW|##d=`poy8#ix0HEppl_7wdP0yPKN`sm?9b(At<&;cL7SA?mQ zl9?GrJSi|6EL*;(@7vl^?%t(xaQMPK*9k}rO29D-y^xG86upYNy7Vsd^cq_^VmCr_ z(1tB7FMn`eBMS4fckkY}sNTdz&(6-yI%7%oUyG~-Eu{6`JO0oK6bm#)l)H+)zO_&W zKn?=a(i}3K1Ywfb{u=8b0t(*u9yAA3TQ_g-i099DqQhGre=yqq6(Fj!^t=+q8kXhR z<;9tvw$ppW*r&hstf9z)OiCRsd2-0o3do44TLfHUjQ%3QeMujjr3ahiAnQZ1z#NFI zTX(Ob=*PU=BoUqQ84z#G$LaYA2n1hSHPDv&arPq$FzM6TU$Axwz{$#z2Dqk%uaD|o z-Xi;!6Whnu&dz)4EnVH>ar?r=bScHUx|k_ToDii{-k_rTa#6b#8Sj)v2{upA*C00_}SAjvz` zTcX(Y+G-y>a_m5q3Ic2KS-I8~!H3UDV-b?VOmtxosDTcvfEo~fS@;jpf;*pf)n%v| zU~R6TUtJBXn%3jt>6uolhYnAJsy99D1;#VD+-LEsK?~~|W)|@;z;ki`w6H69059g} zxc))R!^P+ttjHEXQ2$WuNe|PMSb0E%C1<`}tiLL8Pyfbd43JA+)LZUU2|wG`55U zp6a@~b<)zzxKQGa?b*YKOKI;XN-KCdN?}|~R|l$9%wUeyr>wGHR*(xy3{b@=_sgw> z=Et7bHsIGVdZ;VS7Lqa0I9=?#y?i!~$MogD&*1`qk}mx+G(>#%)vH$%EQf!xIe8Ms zam``BEC3+@TP?H&;DWm#&TvV=CsRP`q%(Go-}XR^V*Eq#!5wnl0ZTuHpkREzQ}i3c z`T*$TrTC+_e5+)+kHj?e=*7#kb+JM@)d0cv1jm6VhaTd=H5 z72ept`~x(ts1`Bpbq;8<9d+jj1} zcO=Jha(dc%N}n(^t!+qJ$!~%m9XWEp6%_w@2oYKjMXI*_h)wSGTM9)h}o&W7m=>zi`*(c(`VD2+>t_X?#kCG znfPTTZIVmj5Z`8;j%V$?0+VjlKs9R<>k{1|v?L_6h#Lr42JS))9{>oLs0>6j+HrccZPdJE z?0S6sC(>f!FXF~lLXAMdhDb#?8_1vUf&O}vWvIpLfY*=?dieNp^-X!l+%wBHJNO3V2H#gyKN3ty{U}m7F zg#hr`wWR7m+d6()|DHlRmi`|I)Scx4YAANO=Vwkqt^Gu*@)|4ODMLCpicFhBXLB

R0`k<2u)YQVZ7(r_m8Xf2&VJ* zFMD!*am3t@={&f09Kvj1Fu;?j8$-nbK+_z+>o9qoCJUDq=6ec$4-|R*^>=+%0K2ep zafM;yL4v`qVZm^Lrd0g+(lGGpQ|s5@Y+%xY=r`{)zrE&8ZvZ5Ds5hY@AzHUf{J}g{LGi^V4l6HL z^PeA)NK`q`4aW}(;#ak#fcdwOn|z@x zO)nsv0GCA1wRu#Lg@R>&N%OqlrUk2K9RXHCe+sexv$K~BrK)tb+ioGRRLXf~x9bjk zers1L%t{0h32gd@H*$fVyJVZ&flm^p;n(OPLIKGR_x9cmof-~p_XD$;+Xv7aR|{+l zwDO;$CF%mk3fgEmi8<9=brb-ggO73{*FpieOqaTVIJ}ei(Awd}rMX44S-QDxxEBp!@|VC9RE@v5H|) z`!gA4_N1YqfuH8r{P(ePZfh;WL<;s-|%*;Aqr&uS5b5VJF6WV+io7ly}H0#Net&o;^AyT!T{5?7OS2`RY zZ&W(BEDRosn3pz+%dzI9OF*O=FvQShh#-%}FxBOlTzJtT<5tf@qkE`5Qj5i9n|?zd zlSL2;Vr03e0!Z&m6pYxbSBcpSS1|3p%)ZW6&DnQCzRrCwc*#abCS>Z%4J%tLrZ@98 zPA)E4TNB~>-$TFrhrzhWZeAL3A)z#3tL2~X^s7}2fZ6$4lUEPzqo1GWo^uSFoj*0> z>xY2W1=v2fVojuKu|0|Gg|0%1Hj)(Bp@BI$GJq=G;u2H!uU}VwX}H0LcmRn1@slU- z?MqfAzAUP4UzdA^%stQvwZ&-;0Na(g8gj98baaS1@GZ{`B@@9GBI6MMhvghqi`PI2 z*d|0B1oY}39i7KM@2@{Q1r2%6b3>~Vua)}oQbBM>l4T7V;Tb@zuEHA1LxQ(4NrfMF zHjLDe^mNCcFCwqr*iE`6kV?mn7+ohdrN9gToBq1B-m;wxD3qqoiPEn)AJw zqZ_RWi-@qpUL{o-S|TEHFX?7nyYhiGX=g{$lRK~WZ^ZBcS=68z%i2~mJbd_280rHO zy??!&t`Y_sYRJqf0n1CBd3N+fk*7jP59=FL@GD0;Z&C!-@QS6y?*Vr8aqEYBpDU^U znbdl0RbEz;y?rw)!~K(4<);#ITy1i`?;$g6HTL{n`u+P`yNAvl9S2ED;%yt#nlG4m zhuZxQqu)dZ8Svrs^dh~epO>G>3@BOfGcsa>Jiy-z!K;0^ z2-U4seaUNE!2H5cZ!=45Bs8Rlw0ipbRgi>8wgt72*aY`>UDEtm=GQas(~kDYuySPu zS=@l%xIKU_>(={vsPQ%pp=yvOUb}MY0W<#F!k6{ z9Ba20swNveYB+Max)kjAmMn9nKQM+m%lvjO{rS1i_eZ@1i8s05s(=W8StEqbW~4QZ zNB-EVV~q(^YHH=)OImk_%H*!!2>5pR9H;=+CZ;4MH+O%@2aCm-eXi%HhmArOAh+-d z?n6q=$54jj(;;Ky*+BLdd9-Dpjfn}&See?ox(v9&2w-}A{WYyRay~N2xW41d*^Y+1 zM#fSbpcJwOxyom*Rw!_#o*XI)>wz2zpawkQL3&1Yes~=mR@I*dUl50mMgRlV_NuhJ z%)o8eG7Tt(s17x@e{?)NwvZSIO2gjUip;3sCpZXxZBon83C-!Z?Dou3>V?wqO((?AW~2xQB^K3{~1yvEb8Ni z7vcy8$U8RmW7s3&J?06aOhXodbl4_=h~)7dg6b-#)$%W93R)wgltf&L{ufsVb1* z;Y)Ax_{@MdWP@lP8XGHAW7mSCD}|5+BJpEcSyr%KqD_Box)X){n73fhQ4jSCnIdvj z0H%)}l1u?Tw}}l2CQ$>Ml}FS{185vdN)Tc@C`7r!0WkZbpv}GP?G+sB$f*Ncl>-Zh zBYblc@75In!~}gnuTbEYu_R$109F|NC1MHYP;U zrv{d;Xr`p>fHV)_#t^Kmf~|wFd7$^16X{<;Tp|FpZLzIo6%=aF4=jhb7kYoUB>WF_ zS?bfK@Vyv!#8Tu#mzlA7X6NR}k%-Rc-ZBKg!B6pcd{$mX0w%I2DFVpTklKrmR^QR% zeFg9Y;e4yJU#?+-aX&=wCQKQ0?Z1Mc)z#WWRmWH$Hrx8~AhayZY4oK_HZZ9GJkaK= z+S)j4Yio(@j96*Zor+{qq?q;h?W+UD3CFw=47qN1Of@d9=d;8s)Fje~$h~7E2{l0t zCDALg0Xam?@mW|{kVSzoH8~|Dyi7t0fDU9! zAx1``h@T3YRYQOqquy2TI5U0$OP|CHNIacvd6efo+3D_4iIA}m#Xmnzpv8m({gL@a zfEeaAnR-b6&{SiQY#kpTUuFVL0Z6?@_Y*vjfF6KP0_wxUYUCw&)|132Wnamig@uJd zXqO-yl`yMikH1Y(oLTuD&Z9dlP~sjS`U79E8b5>uSaZ1InAgn2QRfduY*5Y8gCL{zIAYORa+<{KWF1~}!n(g}P)G-HasamjfS}>x zA`M5DFgd6mo~t_eiH%E64ZV0F0BBox9yhqB4gzZha-k%9k@wJ`8a^nF$*7|~tXj3o zW59^>;=e;U~l_)G^!p8!%xH90fh`8RS1oR(;n8cC9n9v&XhzIC8- zsxFMnbBA?5nOA65QC8;by%8HL!aPK72~U zYu1Omr7CTNdPY)`u)&EJhK!Adj!xPkD+KA-sHv&lzBckga=Z(cOrkDWY~5`MzuS@R zp%A=`9HpGs@czczN-Q`dVesPx0Ow}3<3>>>ImA7beP?qk6cOv7sKWtgnhwTW$(D#0P^uNCKRQwQz^^=|Sz0m6Ovz1Lb#K6IFBe&#WBg zh?q?zb$~$XEY_-#nHgh$y%P&)eRxPn$n;6Ld)u}PU<)Zkkah#tTvk!B7OctKZ_EuW zi#Q#SMj)IHl+N^3L4QV1sHI^BQX>nWBnY3MUC`kg_U!(Q@0~M@7hQnhke4)v{X%DT&pXc6*}>OKOlW zMn0d~C-Y4rR`&`#74%JVOhHxEvZPy=gbQXy%^RM>DS^-lKR=(I4>wSH5yvi^J9CMG zn1Dlt$C?tmoXjskNpOdmL|_RHY-rMBiUb9z>7T#Mao>AHiYL4f zLaiaq%DVO`bB7JPLW0z-nnU@!-!nEkI*C*UNlIamakQZd2%Dsi|xp zLRTT`1ylS2CYwYa-u!q5Un>MVE`Rt6%pDRIfIZcGJ3@Fl9XfweCULMh82#8 znI!YqUrKbIFoG+uD&&icoP2Ox(7R@1HD2xLo=(Qx}PRFvg z(Ouuw6xAQ8nhQW=B-x1QkIU0OE~p|vsF&Jpv%cpGu(Adsn$eGuflEZ9d6i*I{8Bya z|Cd9y@2sEx0th5d3Lq=#ijcW5N!Ez35c{#KYcA~fxvrlak=g`fP_I~l_`zxlL8Gve zD_4)X2;r)5bj){5RF@!BZW&v9O_17Uq417=a z(|x_Gvi7g8&@Ge>&z0+bv;4wmwgsCVc+B?8>+59sl+As;U4HaPf@M>Bvj4>oA1wAJ zs|sQcssgE2m~f_w3$rqi)C^(rFp2z+$(AGLNSZ4wJtc;(B(8(>1VBtt+4fRd-bkDy z;2qYPTUfAz7bQLVW&q%N5~6I%lP8>;4nJ0h_h5d=0O>{d`RNnnFf3srfXlmL1juPr z)q%b(5Vn!)7C?j^9STuJ(}HJ7C^EJ{U(oMl;mGpQZiVR&c%(2Zh$O;z@AD%i0=-sM ziPN-JyXdU^eqHza+I5LvNmN+4zj%mUK;Vz&!i&Z^TpX-aPMiyFR<;C}$;Fa8amBCC z9Th40{GDY^YjoWUU=DWuFN2tuYEv{qaQHZE*FuhI!LE{KoMnt^n>dtnH*FKzHR4pU z2?>7>l0vw3{S^}QZ{o(~(`h}?SR&ePJ2TD$)(L05cFoCDrP(qNKSRDoXkNtLL)IOK z+freKkozaGE$D$OJx7X>IM3=v-^n{Yb{NJ7QH_Y*r!bU+^B@?`PvBW_1nUsKBGE~l z8~PdKLWNx&VQ^cKcrBQyTEqZ8zT)|YGi4|O$}{IChg;T-_m%QvNLx3TqtMMl3`9*_ zZ>O{2KUn|^O5p9={N#u!jD-!GHr;#pP`4zy8(ESx5oM6Zmsh7AKz$>^K0I1hA)(7q z&r2s&Wjy^6snziE+KPUkhoj2y8&*(2rNW6O0JufM!S^O+A?M5njQ5_iFqm6@Io0v) zsRUR4_mfY>?C<35`kgg?vU2iG-1xO!XGFWa--uPpir1Fe8!*_6=X1Io*!XQ`X-T$M zDCFjzXQzIhQ(dd_sn7HKT;j}7;>5X+%};T#XtX0Ir|8k#*2p(|DxK`@r{Bj<0`wrC zg`M;T+XCE-B>(*vzA4rC=O0*Vtls>*E@kv0xAbcu4M@?BU^LM}YHQ(8O+~DyM`=4V z@j(x57mIudvYsvbZjD##5xv;z+<66&Qd>vIblpbRz8CdySjoXyP|Wqn>VWsco8r_v zhyK*k(XmE@Q=5JBXDc{hB(03FD9%O&!ow@$z0JYNNzO|YgQwv=pytS#n3_iFr5bAk z>7dv<;uJDSP@Z)Kby}IJEPkO2 zrU-;k5kYDbeYVJx8d5h)#VmBDq%L*T*paDf*;Z;|h@PxIxCe1za6_Wi*Z z*>DeDeevLyGI@z2U#44`zV~Kuaq+pB>dsfMEPTUd7-39Ij2C>Ljymtf z4u-qYVpGY>0YASoQxKd*yzSTR?^omEQfFtPdF4q{!WWq2hPBUfbIHLjnDh=#w;x~o z4oF3La=hhJRKCNLw~o#~AJH-MeKv`X;@Y&!*?D};<&G@#nSy1<>Bn=E10kRC9$!<- z%-i%3XTlztD^ypWDHjRq~OMTGMzA^CGA z2SIoHj3$FNmQ;AMBQ7b)@DiWG@pliH1$TI*V7fSEEG=mEk-@p8*NS=Ga43Tl4@kHR zCqVLOA5aWptJ}{`3Ilwkfnp>Vax!Kd+y_ue2-AIshIq8P?N-b#vu)(~oWLr`gCkq+%q?fN6}yqEXSUQ6X=3iz|Kpyu}UT8w_dRsk4SN z4~(vNKB7ijg3eZl=*Lavb6c=KxTK$Lf7#iYfLi&pibV}Quo<0RXFu-v`o@EFK%@Zj zftP=hGn0|fG(3QA$ze)dlTB4P(O}U!%$_>m9Mn%?6kWL{RB9)BCeDhbVi=&RW=tzB zfs(=7P9~%?R+kv4NXp$cesmyJ^iU(ps=K$hgo?@?EdIqYZPQ{6$?Kw5>&xbR_wRq} zD4&S%$41qDsv75B)V9TOzG94*(o9E+#tI$%pWyyH+uVdjPcetIL0 z4_Z~v0}otu*`hBRD}19Nw%-2c#G%y8OntEFr=t>?Uz+!h=X8%i^op#KrAC357>x_ zS?$vO~35?Cwg~6^GMj~y%{aD_T4-AW)8PN(Eh6l39pdKDW89V91*yigtYV} z{CyA))A$oGa~!f11tmxa`w9sRm|Hh7xB*j^fK{Jywh?|T#+CD9W7V>ERCA-E=|(Fs zD=H#Dp<=AtxG@s4F9H@-8HX|{`A^0V)J|ExI2#LKv{z!H>-zN{j*4fxwuk@8u&<9J zvKsItip{C@}c+8_{_Jv)^Z_XT-Qdyo+_Orkm$5mnob zkJ45Wj0c_=FCPRzmM=9p{Mr#73b-Ro6DSwHQ$C1;Lb$v=a#q2m z`x+MC!1(x|uEAx6cV}UmVhwHj((F8*e}2{6AsFHAo}Pp7J&wU~xM;&tIdc=l!+YX` z&8sUS2y7i4Y1qgZoT7|HP~FtD8VMt$Z;8kat?w&By5KR-rd+3MknM5*vBC_(IJNhj|8Z3 zMl>YK<4#Wg@+*_!UIMucqHZC>+WOFF9Y8+U$xsvyfk^F29I-q*GagovEiz*B?^z+VVu0rYilF1Q4^yt?&WcESS zgObtIT9ScGhdLe?0Q}fj=9f9$9)#*!kk|0dZa*!aLjt)W5L-?jA~mW`dP1k+t1=W+ z4~jGIHJcccF;>SA&fnvD7fGdK-A_)SX^`g)LCHNNxfTd-#F61`Ya9qlW|*N(4TW84IxAYr>FfVVi>Y=-fT2^i`CMD^(P z8;@iQfEm@awr=_O*vOKn^z2kH#`PzT_2Zz?D-cfOrG_}v9(QaSqjQ=;;M2t9B#+}k z<2w;~*5<&Y1M886F5ZeM-6xn-g>qfMIn|o^^ zN!DYTPW}G<03sP_P&nfg8dZ2KoIQC+=3_b(A#Kgn)LobOYVg63Dg1-(2&z8QO*;Uq7!}6>BPS{xykGawW0P2% zPbP^06i1|)9HAXtlQxfstWGFnHKz@V6{84PYE*!%VoaR6}3($laLmVX> zDQ2-jYUM02&rNI{?4YK1`5(#yl98&EKiMuNY5Xt{lY;_K3uVUm+C>RtKo@dd#*a2e~-}?t?#Hi^x~{LIc2-FxBx7Xn(8^5$ky9(s0TV2udFo%4htswVxVsXVyln{ z&XR$}>eN=J(MZva6(ueV&g5h0v`~X@?l>)p?OBKHJL=}ezgPJXTEZK0od`cd&1fv} zc;UAY4=ZvD-VFg%lV_&{!E3L{SXmm!UHpl~Regbt=qMoR+?y?QBiYw8Hcm0~ZV81- z2L)eBawmNah9G{@*BUXaCoVnQcqN1&aq1s}2-p}tI*`UAXB#?vMv|e@5j(dI@B^6{ z8(gD+^4TkBH?2TFe-;;Y0zsEJ#mG@W$YY>HAvnwQMUNN$Y8NU$hI-wG4HvLI3m{D5 zDEc}{-I|)}YUFgOA-zQ+_6L`N91DIa=m>x|-1AHZv}Zl2x3TZ)qcf1rQX}+L0S!4j zX0L5Lh|ms~M!mS8pdi3Z4g7wSf@7C3+YDB+TegsgG2l>70_Y>h+O>utbSLn56=-?v zRP1_1`&rp1u|kJVQ2V2UkSV}oA)it%eEKf?v9R?-O!|WWk~U+((Ik?HK|qn0t1x{oOk|%m7q4tWgekQQvgf zYn0f~p>?F20!1JIDj+DRr>o0~F~k!VM%%L*Am&$JdLqoQ{0Q_Pr{RJTO($I%3h{~V zCy$QvuF)&Hl1zR8yh&~l1=hoF8T^f;Ixum_gS|#0`0;DJD=f@n^AHdV8A9|Q#17ou z+%_27`RqJB8j46ayhz))r{6ibxpBxR+=O2txdoRD0|dJ5z~|4}K0YN3dlXS-$X3Sc zhL#ycCif!t)ydKKo5c4eMFVQW>9@yT+lsdPwnjrVRu&YPbRyuT>c}ZUgfusr=G&Jg ztCY*)xcM<$B96fYVzeUz?GNIa9k9Q^x0lji2jfJ|?noHz)G z0#w0W!KPP~;slux(A5D#6^P@|Xc?Z5Ku!dpt5m&ta|snRGA;5)5coeHIG9&ZuueoI zh?!pzdQpO=igSw7`;2IQr5My|4;*}PkF!6BC$ezP$2#K#=rt5oqk>}wY&7crKWdijsgvLBiNI+*x$JXWIdmI4#Rs$?gRVgWQb ztgO8N03s!zF3V8Z<{vQ9-?`!sO{N9pn_b0<&8L~D9z{%kWUsHiM1 z&O_hHNfHu184o-~2H}sp(|)yeb@DJ%K+Br|0Ekk8t(UsR_ygb?v7oMs8>Av#Bb`8yH za9}{Bsm1jjK4WJSpXlK@ArJbX|4cDj|yAyYP0OS%&S&W+O0aGBFVyJ;w z6P0i%Mi!n!-TVFfHvut#y8F`&1&vlpqJzhfZi}Zd=p0!OlEaqZOhUd0a)Yt%j*fa0 zillPXR6p$%j2#~6@DiL18w6Kc1>RPKJqUY^yMmQQqz=XCA}|;N6Iwvaq>v+J@X!3U z0(s2+>acS6@F3u8&qoyD1w!?q1524d-Rq^+OixHS5)j2WcS%lu!+gehOtysaC8{F_ z?I5;6V3x;aq79XGisJwTRwkR2YIghnEHeeR3)HW$m7sm#xjcKEoH7`6x!?bnCW)~j zkmQvdh&Y}AO3p4_2jObPsWm*QPFc0@^$|tPHsp}(DbiCOF$n2kUXR;aiifT{2ZVm@eLau?yuX3 zO$*p_;Q1LB_f}4IM0mGt-;U7gYoH$;^?Y*30|z}Lr+x(sZ>S+?cVPO{_xCy*59GAWcnP%f@EAjvWFi8Y23gEc=%uGm2s)vDxL1p7b`V9|SbOc8OHTjFg zuZDpAnZjU3DMaSrdQs7~{=vyBz$8?;D6N{kwBwj0Vyg`JC41nGocHp!yIAiS3FM3Lhtt9J;zp(JW}v2seP{ zL!PFKvurnT3=Wscn*X}D_YPnLLL`o9n|0!E-6E)n0f9->ZSd=;{-IdJ<{{P?`uNN% zad;up45V@n@P%ktP#lnDitOKLiY*TE;S9^~m>by!rqc;G97 z0jY1-^v8f+h6vqLZUc~q5?#Y|%G$O9#AOM&? z4s)B9cs;^g4kT|Wc@{5u=pv?5^0XBseRQTUWEL-cdHkaj1>Q8zw#M-?{9SjF==}Cg z4%iF|9U${=sWmrtV+?EYt7Y2{tA*?p$Abam&@|TE>IC~bfI==Oqak8vZT%8%ThQf# zK_*jZ`^rcKzDBM?Tb4#+{|=I|P*J&wiVIFkWVU^C^YhfO`X0AMb|Q3zGd3bn@mU+L ze96Ouhtb4;zUOWE;;|id3X9+paA=hEm%F%*PYC2-P;ue}&*pUX7l94X1g24mVPVv9 zCLI#7!fkqtBd4iyv4S+;FRYWmNxl#e4I(Ah{9z9kW_T@ocG8-t3_xd0f8K6KdAoTA zK0e}fEWXYwk8q>(f&t5I-3q*7M5D@rvPS5hv2mM{xd|={1)8Jc^_bdMC#S7WZ5M!A zJ%-54_)xTqYZu)TOH!UNm@Y_8iVNSjXQ$78*b*aCRX zUM!X$e&YxQtYl%4P*zR^iNXLDr=y`wJc%fsHL}W%q8Ih4lh*A+sL=qZgj}D56 zWMza@GD-_s*)x?$h$17qG(<+jsI;Ef&G|jgKfmWVM}5Da&wE_=bzS#$J3zRzeEi;< z@d_ju6pX%MR2OLg%vvy3nF{wX?_YQ~PF|syVL|QeRcbywY$u2civT74sCd7Lkipz~ z1;EdG!&aN!JwL~Hw3$kEO?7Qn`H7fWnt?V=x@!baH0rg($$Qw*<}MGk997K~^k0QK zE56L~H?@e4n>4Yjx<|L*@81G^Ek{ob$k4K!aLGnX*Y1?%#d*(;*~X}FuRK^5vE=&) zt0QAW_Czn*f=*~#xNLb+nI|ziFWuH1r2)m_9r^NQEG^4;`63%ww)Wk$$tKdQsj#6>j`~Y=dXb;lu0=T_5k@u_Hl*&`YkPO?O5|EFu6c7UdBf)BTgO|P z)UnNYbl7p#;ES&Mx^)8A-5+k6IJ}{&#XpXUdh6A^YmIiBUx_@Ow&BOWHZJ;4qiv4c zo!mL{#qunX*Be+>T3dS zbUOCy-`}>1=aw)$xqnwfp)NpAc>XX>M3yh>LN!Yzs8drrHx58MhF{AaSvgOC@ywaS zInY)6_J-Q*Qo~OUOHSx~>7P6^m3T~Jw0uwt=W_5kJuL6JpN2A84Bvw}{+vPkS>4VH z7d`}_9-EPX@Vzk!rU}s1SV|peOW6I%_9|=XF%dsI*VX-AT6;N#xe^*SXpsFodW_N9 zk~#`4moK-EI^v|*U!hRJ)b`k$xrr^an=Pq(CF*qA@{sMCAx)g?)>SuHJnqTthi(~B zTO)=nrpOZ246?y*QEJ;z<(S51&)VJlo|~#?BSs5 z5#RY<`BpzaR-F$&nZlcfvTTZJ&TBj_<+Yt&s5Io>zlGVFA_JUgib~Nc0h56R@6dEt z!fxN@ZThSum}d*O^TgF>PoZj@$P@9PrE!=i4 zv%eG=dRuXQqE`04Yl911^)uG}*4FO7_^mo+SoqW=q+V9we@H*7h6ExpIOXDz~qWP@CXf95H z9XXAIX=6a}N(NW+sFgVNxOp7_}CrgGMu&~l)F`6Ey`XK8DN9*t~MaN?w9 z70s;zv?$c`Kkakkv|*hNT2rVRoEFwiOKaoKdGmf)j!$XSeTKs4in>c$7YCXbPo9{& zX4PPot-drkOA6J@*|U%A>pbqRAx@k)G@D?(8EbfZx+lX>v1ajSm%@i(wjJz%G>A2q6rJ~OvqPngyRRI*6c1zgIJ#GO{P z{TtR7DQ^1c!GiF1t=3O%ZKwG@Ne#?` zMJbxNf|HInexEPcixN8Z{3Z~#7+q?~F_2K=4ntC*^ zs&mrXuE5$}{lo0pP1+gm;jGw_T{R}D122i>r1H;*s z5oauD`_xjCPzy`T9Fzt*AE?Eb5e3Gwy9F{(5Y~vH2GJZvg2AXZofj-P7#px;9hQ}) z<;An4koj=ot`8;~b%R9@PEEVacWEV6Y9>{k%%Ldd$sv4n-m@e;8?B}u7+e;`*+r+% z?J34iq(3uJfsqX6sP67x+hpBXokCZySMv@Ls(ps`+VTF_^Qd{pO`0842(pMiXfgii z*^su8cXmcY)^9W(2+8B;D?w`bn0H~>#4gW~+7JiuQ%Yn`A8XcJ-+|Na8bm}!Hf+>L zysz>tq5k;)Msx0>SO2xzm>~Az9I^g1>eqY=4=rF^7KPr=5<-imYDjDw*uW{Ypi@ydP63MrR-E5<=H0p>7MXC}O)m?XX>N8Cjh&6bjcU6l@i-1PY7CgYrN_nvir~|0Cpx=&Tj1bw3lT-WQ zE5j!xD;4E?X;K3vOAsY=m}ckB>-bKnCosfB=j;F`ICJDd9dXL*2JZFl)v!M0tTB8! zd3uyAoplE_E^7$yoXgH;+nNba>EI5r$Z2UtA0T%pCv>At+}%Rz!@)ye-?O`X@1CX+ zv}WICaooz5!{Wsea>=EkMyzAs(b~6KJZkYQwAbeKTMG+E&2{Uu>dhd?s>z+N+a|0o z+sPV%w#HG@8Xggty_aJ7N&K4CXg`~97wzli*!=?dnrQHBdO2}U`>_PO1gD~*Hp2!c zwmV*F{#?CXi?!Zkh{4mNMBRY+Im&hRp0V1IRNN#%?cy74Ti3v6*L-v=!H$sFsJD(t zA~$8`NL5u8Y{TylFTNZco78Xi$SfA2+S_;Tzc<}XP2F~A$cjg1Ry%fl`t(%!S@m0$ z^j3XStgDF=vouqK);s5mE*gB3Au@c zUE>Q}Sy|MP2mqR_eEF2Z+>mZLsY}E9kJUq`6PPJo{YKt34o%Lpb?mr%{j=6q?qD}k z^53l`DAtOS!w?TJy3#F!%c(J%l$D3rv`=WWr?2PK({@S8r%#I?brLE zs5+>HCnA^AgPZQPrajvMw>Rl%8@cE|yfXH0Z)n1ez4v(|6`V7mH=P!3EoR!Q?=T(E zBTyr|M?ha|?OVw&Luhb0;bO?M$Nv4Nt@p(X8P)>qOaZb>3Me;X1ntyLHUmaXsDpi! zW9Jra!f{uy7*D?#GC?Hv zL(4H+5S6M*c}1NEf`0I0>imuwNLByI{^Gj}r$k|>ymv1LTQzzKuyf7cs#m<$t} z%Fa0bt&DpEc65Z!XP?S1FK@j~9NsMIh|L8BtsBSpy3Dzwa`oUQ3yUj*4K)Jpl}0sz z_ejQHf*_D_)qPH=k!KP|W6bpHm#&nX=JYt$ImXtK>rrgN-Nqdxd~r7-kVMf_(jI=g zrk%RF{?eRLcpqrLR}0*2!BYi1>%)dl7uC)oGCX_)*Bcigs`ah}aRNgo>SM5Fg$K{C z+_7`#2VCCLAZA$ejsqb#xWkYayFW!1_K6&Jb-!=Hhbj#zPhN9yzc@dqIa_Y@^yx$G z%GN-YiG07`J}q>W9F)eZR5KmSe`(g1mmM8e>s2)^HrOlotVedY!-EcoZmeFjP3_Sp z|MS7~6`!sDG<-qtXJu~deCt6kNBmfG&qW`P^>D6Y*b5ho28L@xb*kU|zW(|rck0=T z7n2hc6=0{(7pjwAyub{r&${Ns_Gd~4r2A2H${9b5XebfI7t|3M8!P_`h4D_ zvk#mM(sNE$LP{1s;@Pw02M@NxT5+cTewMq_-@kyKj|jG{UxHQGijnCrdY^y$q;}zT zZ{y&XZCvh-Z4$3FXwW{Y9qTCb7EPKqrFO2It@g2~SHV$9=4M;l#HcoJ?(gM3eJ>O% zcfjhFCcQ{M zrA%A+ou^L)7R>ov_sQ$m2UwKuK+&EF16lJFFc2pPb3gAc!rq%TaA6nC0h2yu^{jq< zDxs>=>$|o2KTBhcp4TnU?Nk(esQTV|jdf)Pj&^ESnpk!)2%8l#DyhQ$;*=XpBW=oZ z!P4Tp#}WxinI7i7t9INaQIKiiEqUqAGb*4^#q#)NTi%bbB1uAn1#9TFAo};^lKsoB z8ch__JiH#GYDx}_QM(8_6n^r<#`5Qt_w!4=tuEDm&h3$VamMR!Vd?q*Hi^&Ye8$P& zg0K5BH!MEuXnys(WwWblamr#_WdK1EfbGO0EzMucbvwW8@Fbg{Yc=X3^WhdCLsZ9Bmg|rqmTRek@EPNu7{Jtzmy? zy50B3_=!d%18H{eZUF}PK!pYP?1&D@qBJDkoSi%)wD$5MF^EA2x4M^mwBg)4)L_Z%D&k=)i)iFoqSjIt9h z{WHf1FNh(wKNB~%k8{$A7bOATg6G%oA`b1Nk&z=QozV{9B-i{3mH+4SXK44xDJyhy z*R)hop)ad?w!^vKO9KPHeKhQ_Ve#YV&$SH=D-Oig&hbpRe2|93r^hFW9oh86h!H|_ zi3^3sts3{&Cs&NFFMZHJlMi?koLo1KT}32=wzjsYiWJ;isbLP=wk>PPhffRM(I-MH zZkJ5{!JL7>qMO;xd4qpI|Dg;=)Hxq9XRKRPSo6a72_IfRdl+8r@+RqMZFSem%7Q1h zngi7QUk}%vRQT#TMH5tOtNqfIQ$dptb@yGk9E-7INA(W&Uf%jQR^npn0%({NHVxs2 zodC(S>1`*R#%0w3hl+D%b_Od?-->#R@0c0UK7^$=2T801rOSi6#%}`tWwy%x?blu`_YvNUm-Sa z6d92g)}_x5NqZoj9r$e&%fF`&OYry#yU@zuONeo8?V@K73kTmO?!9)wfOtgpOSyYr zw(%Ns=X=(wHM3o!4`NxQo5X4H;@GE4pI)sFfzs@UfAqwKHK;Wr$@6qajUBuF%I>>S zEK^CXOqsD}jhREun8~RxkB;vo4X(Tuj$Y;vbjg|_ zBMmfQrFn4Omk-04{ns#@yknI1%NH+p?b=oROfS3M2xZ^eZg+gumT+>&b|W*ky+zXb zA=+;MMz{v41!DH*I`r{2zP5I4d&}|Zfme3cTd_93Wc#+cogNkSNN-*_M5pH8Y2`(I zvuZCU#NJrj<;;g+JG&b6u9>mE#NNKWk=K;7nv=#6oGrdFOd$E6f~~Tu>sJ)D&f?nk z?t3%yXzC0p+NsmLOMa8fbp|>wUFDqjw4C`EDDl4N{0}nVGhF-l)q!v{IqGbOzKXBC zcT{vVT$X3o&Fgope=Gv$VDh&F*^iv<#FGVMoH$PbC@@C9pm<+m^~aC*#~Hi~w}I)h0uQX5Q!B}Hqx33#M7UmLV;PGJ=e3D^)? zNOZ;&?8faSMi)Zi0RLjcCPV~EAa(M=b1EAa>wtVNmZwa20d7RQg7q_zmHomaNv*?(8L=SE|{XKmf~jpQ#W_O zgh^IAOo5{J7hIJR0k}+is&TruM%C$IcV8Yfv;*mM1TG3$v+ zvzfDIjd*))d9tr_$dC88>RSf)eh3<9Gx*K}}@QGXg6Hb3o*63F8Fxv^OJ48tA7R`;{Yo_ zzNDvA`no`$vvm~HWm?}XYDr+q*oWs(8H<>`dk^ykpxqBla4C7&-t@E zK=6v<5kCTy>rE>A@o6Hfd+wvdis(8tKs(dk9~3k>bnj8YS?c}G`+ho9oF#cdBs)PO zDF5%1$m9(ko^*-K(|x4do-Awvo=)$LrtsR{!CUGL~d&eOI z)9-5pT5_2SVM+O&gVPxWCWL-|Dg)blFHr3RJ1#y|nBTJWGSIdk3zD|^L2G{ZM|W_B zVu4To;Q5kO4t%3aez>izHtDC2Xz#!Yq6fq6gE=!v%=r?hCzC?7?ze2+I_QLRIX2|E zPvVCjin2-D_C&+(hSIB;6D`?5X^mSSyB$0nynwa18Q}asqURV`g@t%4Gc)*ygZ=Yy z%D#H0qes(4I1Wl;?%Qhv=ztTKKcqzlTxMW-yE>2X8!;l`Wt`{vV;9C2GSvjyDHo_N zrDFpS_Y0VAL$Uw}jhzspkKNU2=FD9uDi02H*FM#&*IAp1x9kqKzM5nB$ZYzdm0DxY z&3ikT#y;EQUa;D96N!l@Hz&sgevNG$K|=@&Dp;Hlma4uE2-&x(18+ikV?gf7I205D zEHwkEOE<%a-PgT~i(`u(Wu@0tZxZQ0-Cc|aVm;?bex zb9;TMmvj@b=A{{qQ(W@A#%i~0)uVx-CJeKf;}$Fs3p-t|@8N^#-q}il)X%dt&nRSA zmmaKr-?7@cm$TNkY_-236f_nW)9sDb>WwIWReh|%h?`59uRiYJp!Ep}36_C9^!6+% zzp^6xcCyj)|IY=m2sk$NMC%@J-(2Yua2D42eva+Ez7~mIGh;@X9BtNN$&!CZO`O;- zYi|Pl3awRskW(c?ay+TIkSJ=`!M`VDKn0 zGv5Bi#smQiklTb_KxP#Qt!*>6Q?X+Zu0Eul7s!Fi}Azf+!&T7 z>Xs{~9;D`eDp3iig;+;skH*^0QI~OP;WRo!doAj&^j_w>OSvh{htPYdFtT9A-&2hf9jQ$XLmSiK?<<1R!!0*x!1edcB z^OYP!9ZbqgJ{T&=kg6 zS&8D$GG)4*=8PQ&)EWgNwGkI0N)gZCKQUHPWoS5s3>7~qb+I+wMy zSznwtX=z54g!4!q2nU2Y(8l!7SBtmqBfKae^$9KPfjz^HpT6Y~>p1kyo3~e)3X<^k zZR)zzV4Yrp`o_I>%o<){;x$gIPQtC)vnvZ`4fs&oDJATyhN;cg*5M20T6e8XoAQq{ zhLX)ww=ZnH{pS$p)F%DfIQMb3(l-h;aIH};E?X@*3&ob@m2N*Tl=p$YK zB(;mtjyWb0I~!xm@xG4ipTP%dWuACr)pB-;V@^t{-yXpmm_^+&iE!K;ZuW*6T>+lW zn*RvvuiJ|=8n7~0vFCk#csYg6YE|0&q;U|tZ=TEj&EP`FqXn-oS(#i>HZgCI*@Wet zgWoz286bJpkZtZ|qv%J$P)dzV%+%hZ%L``hF&Qy&=uj{BWgivYMjp*+-?_<@6Okjg zobDZ>+saCPpo^n_$K+rCj9*Fv{jp~KzZ1O2})|t&r>+V9XfjMZQ`17SJN?lQOq+G;|ra(qy zZg%!?%m|-QoQb*s%m=dC%oS%8mpwpwrUk9xd!?s@%c8cXF*Q!^NGj-*eRNqa);;LC@`v>o@CT)NQl&wQK+0z4v~4qUnaTFwL0; zXGYouAG&r;v*1IH{n2KfE-oI=`p+sn`78O5O-vp`80DZWY+VH7#=HYT-A&c|&!ou| zN43!;uCvjSv4T^~GI|fR;rE;MJM9Vzx4PW0SIwS72YSH2**Tx_Wv}fOq zL!~40&ob?R$)97B-B_1UOEiQwDgGUt(`I;=JgcsJq2A_^Znw>5x~+Dsdo|o5WK;K! zgQ|*B6zkRkL**dTu@GJx`p1p8^Jg|R=H3ZYeAe7_iS2qZ_afH}zap&TI}1ORj9)Qu)9>Gd>`s_1 zG%)z>v5F!^x8=s6GH?D_z(j2NVnf7tOt2>2;Ej@tOIO(^Ju-@N7uJ(F%urQQ+^?lDINBzC@owvmqY37*^+6~llqWC z8*Q`IB%FZWZN#9eHd2Dv9J)yM9Q`pk2AHa>F?48$jV2>FQX~(B&qhPX&jKzjvLU-I zTAPRctqF!3sw)m=A3h5&GZOk!+Ne8RA?2t$350Jt_qZcfKkBv^Iv@jf`{YGJ2T{X1 z1nIUkY0&V!Rwk>1?v|tcaq`+}V*+)5KG^nnPj9`$FJHfYLc;XV%*Ai7nR3=)b!Xt4 z%7I;pCFLF}0ej5Xlo!Wj_I>tbq+0h=C&s+Xztr|{w)M74jm)ieJ8jg7-gD6VdZ&WE zPcu`lnJD$_F)8WzGS~g~Qy(9-Z1r>BtgA0P_j7Qw_zEX~Bl%joUJzMuptD)~TkGk} zcCslpZpxh|eH3ghM91;mAJV$U&E;{Taz0<2XUHqCVhN z?Z;?NSN=nYXHTZUGH-E0?TqjVs^9!~$Jg|mKOO11AoP~!dSkso!%|_#L$AafwuzY# zV@r-m>v@$(na`H@C)|!0A+fn$`**H%E&*CONF?%z%hj^OC6ICD?SzcT;qHW#GaxaM z3qi|Qt&cCUEYj5{9Z^axxboGs#t>lx;)32eGHO9AA`>6!(f(Y#z&n__gxIau#K6CP z`#{TNKFw2SqbGkmwSW7(Wtny8(1Bkcy=bXd+yJY%DFim+BExsm0tYm{ts1=U+C{^D{Q@{#)KED)c6p@LOzBZ_Ke)prQv%~>ZlmQdFVE~yrlIJ6HVzugDmsDzI;^Ed(y}weZ6>u zEN`Ei;i*x@*}Say_WS@+U2#}E4X@<85XTHdCKJ*myPX3n{m{m^KB9X_>A${ieQ~!# zoJMg*CU{t6aZY)IgEE+NNGe&PuPwo2 z1Dq1Lp>Kn)rV*cq zHfV+c>zLTgp7kfhT|ki%jK2IMaiuaUtPEo&=z zD)N-xc@~IIysL%-xYc2A6PPxDUq7eYTU0K-CC798sWm-rZ|knw2rT`qWelkhyGR)a zE6N4*^;IjN2o6fe)O2so9Rkwcvjw|xzC#>l=jH|!+05KOlWO|GLBjyJZ< z%X&RI{d$nJ&(BO$@VvFvPWo4|@4cj19JW4?(;E?$Ep4vxcL_@VDnDRSj`mMLqR6A? z1|IV`O_z^+Fz0KKblWYX@g#O+NFufaTA$m&@Wd{coF-{|W&0A0Ui@x#(PKg};HwXp zqzsR?5zCKcY0wQ&{^fkS<&W|rHJ$0#`gigwR>aOs_uKW`X-;~6=@sCwAuK^njW~r@ z(A?51;_t6_PGh7Xeuuc$RKe0m0N@P1vhebS3!|cKcJJLA)vDLlF5X;cw0qD}U`U8N zB94Iq9Emj5OYZ=+BbP&&*(G^yWRc1cZGam;Pr{JqYd0eq(uuUZDnfs^cO1H(qSUR* z7{qgkM?}}HZolYlcF*DYIF3gHX_$`M9IlXye?3v+0k#*81MVhg#a&n<$)w#DL)%!%jl;CdG_Z zw&B2m*Qn4k2aC|)^*j|t2`8`L$?Nd)w|in8t|nph-+{M=8mJ$C%l=|5#}H85mlEwe zRaU@H=!tarph5IHe;<7!zk5>Bo3&1d<1?Vx|(eLKSR6X1yW^$7&FC&NXHoq>?cpsZH47q zf;qqt!VLQR`%nI7{5l>DV5Pk*b>@GqkB|RVql4Bt3xPp85W!2H&f9{Y<24qK32o}n zYZmErhs75I3D#|@oB!5b9NSn3V6Fj&h%!bvX;PTRL~=)%Eh#QO3~iKrsqtcj6A1-1GHMT>Ax{A+;=(TvjvalT zoCJIETgJojqGFi=Qs@Jp1kANS7uA$-v+Rym^)Jz2D4P9 zE1lIy9&tOM+3{y7zt|z5xsANoKEw~uMx5{KEj^!ZFVAVA!-R7x*=G$I9v6r4_bhU} z&L~A$Z`_~4xFLA;`tK%_wZ;t^Np-2-A_)PIJJip&*vTZuR%SyAT?*L%7JoCb-@fbD zuLmDHyF)4}3)4CKU$T?AQWC*@AAEhbM; z3W?^F>w|@M7%4#-$Qo(Lr+Q}`ZF5hu5O8)&RFc`oB_SHsKW$=crSC${+p6-Va^8ZK zDRr@=^<&-}u*WVU_Xu?d=!}F!0o~W7QA~U9h#mL6Ljgv;MU$ z68nX%nFPNy7MB=M(N$BPJPF_nw1+c+?;bGQyHz74R0kJ;l6+b$(=ajlqsr}2ro>7O zAm+?*;-5@)Bovz-Obdz%!b5D7t=*^%T~W>)uCF*Xs;vL2mTVCVOUo|W+W#=lTW8Ck zPq`Wzzzo4}z#_wF4h7!n12%}3Z$cfOC1_YOs%dX3pf=!fFq+VW6QAfjY28ydyyecX zreVkB?^FO#T<>nTooVoUzy&~jUSNrVDAW>J?vZT zB_#l1wq!mE1*GY_?Q%C7|8Qn-;zcd3JOXWw7PysG0QjtBXBun7UaX`+ojo8a{M2AY zao=~YTCj5IjKNdm)mKWNGD zX8#(8X1@AL3qGlIP?(SQN52Ew+-Aja8aQFMW0Z{JxLvhk+5{DJr6`d`g>z{?d(1-< z<~Bpr_*15g7!@uFpWMrq8jZ@=zK&mWUG*<28%oV8m{gjNWjKWQi7}KR= z-l7CwARc4M|If~rH`MGs3k>{Zy>@Z_t+#Iv!PSPsbA;Ub`&m43-n_?OA)avc>;7=SI9u3QpPJhhJyO! zSnZ=wL;mBu1w|p3qbc#=D`hGlb%1=7QeMJL7nj>qT_0bbKdPFigq591K_B2i08M{U zL-K)(WW)e^=DxXd_Rcrw5PjEUlTNh881eMJaww%88)XoZupnE^!VurtU%y&$HeWn_ zv2rS{t^2SY!Oh@+-3OU>U$1_{?8OI3bLD<_1+Zd;WcF1%9l!2yah7hHu}#1wib2-u|`FRIDV~9Bg>v+M@9AiQ|LNBk3xDfid-Xq%hXSE6su0bY=}p4NHW+?JHFiL z>e!r^c2|0>fGv+Iw)!{ZHm}xt=r*V09UE_+pmUF8;)^pUCF#A;=0L%6!WqJzb z$dv!ed7mwHzL%Er>{{Av@#18zPaGICw>Oz(7-4xp)@B5ONzK#@tz%wPy5+6~9`hKYRR``PrKR{gRzesRaU(k)G?weG(83 zI^|E8zVtrC`3YZsE@ZOs=yyg1+`$zF!$)>(<(be}+qYRarAhy|d zjG=R_?0y(QwS!@n(bo!NH|@)%BK|hV|2lbD)boh8bGbjL767esD|agzbH%&Xm|L(L z7D&3R!>4qt%n|Jn4xuHwIX1B9&Wo*?=tA?26AO$+<{6E;G`fv(2hbzwdjNs3RnjF+ z_nduo1tqzZtmxQ<(GQW7q^pn%%Ohtsb0cstIe`_#k5;$thHf3p$mQhHC`{)M$DsTX zLoi{y&yfk;F{J&2Kt>2^6OX5ZvI`1E6K_7zTTP~+NSXns0Zq!iGVJd)`DD^yVV8&F z7f+>%gS@)^@S)Y`UvEM9L@}$L$gU>)={+zF%=PlBdpDRA@yc*_O92nC&{~%e1?n-b z{xOle3CI~hH>Ny0Tz(Pp%rgL}NWaKQEj+<-*kMiPh)lsTjNJ33?pHpK@Fg+w6 zgq&D#mlu-^7ysK(?ohd>SsX&XK%KzAV^)d)>M=&pUm7)HGUMqbm0nExL^#LP*YUrR zCr(s8%?$1}?MO|$^e}Q%Q(cnsnJ*Fz&ne6B?8-n~!1s8@t~_5=V8Tp7hvl2J0ZZjHP)CLsdwR?NlbG;iL0g09Ip^;6AP z67ec{#@c&BisEiTHT3>`SFI&K0i(obDgn#Bqt><)v;3msQO4aI8tcGK40vuju9Zqy&S7At>T{s<7&WYxk>&CjpNJUKN0 zO0@-@jKsRUb1Qzw6bZ}=1-xcl2hy-gJd6Y zmcm97*|x0iu{JER|4}Q=jmw9xO5Re(^Ksq+m^81P?{Th{csQTVV4jkS0`$S!!b%yf zG@O)>iT+^r@zu2{1wB2VXLbtv+tjQinR$rS2^WthYcKf>F(#fm#5=@!2V@6 zuDx&{C99wtNe*0p{`~pL-xA2^BXG|Py3nWW^;truFw06)VA}!n_SU=a1+84d$uNs% zHDFw*dalm9cwr$`RHRAGCBn!$TiF;GG-rnIr-b;_K1A}EtQk7q@lo2thiksM*9>Xv zU@a^$C20=*%5X;GFns3_A3>ik<%Zc66S}Kz?=Yk_z5mSlib@&YcT&8UC#5Lq1;z6l zb}{)5IARLRcpICHF>Acv?U|0n)w}R4tbZNVPYUf^^N!l6w7=g`fv5|1Sf&*w}DLh?hdPG}0>8Fb;bXseL{dGZU4<5Xq57Fv(T6bJ^{&55KbY%U?9@?BY8xqs7M{zWofaoMUR#Vv`2eZ_EdKHfB2CiwkoD zp06}be}aXT#ntGUl2weYP;lILyn+G_SS_q&3o>rlnZbwDRlatSJNs z@(KbJ9ET%OEqHq^P=@lsA?cVm6j20+=153M)scCm{pCkbOWLu0yJV7sXo)~*{$E@R znlX@OCf5*HtM#W}G*g9fKa1a!a&`Oq^@`%q!Lc|ZT|c>T-_1keGj^|g`9@3(us79E>oy{8;+&i5wWnkHlGq*e3c#fz%B zGuVPIF#FvD?;q_bgHHt-0$hH4&h$q|c|y}Io)|Gh^Fo3ZYJz%^Dap+t1&U(aI&m_h z$IzwIoJ_kwpS{Crka&YT_3noY%KJMcr5_qn_8?(tPA!8DC0!@vpjCtLk|?dJrr=y> zc(?~)!)ExE?Bl_d@V2E*ZTYrrZlD+3sG_Lx`XQ5u?C%Vcb&ZI1!riCESw_={ufe-{ zHZ~B-63h(^nody{)mh8-KO#(~;6fVN-%4&g@`_2gkSr1d2;|2I3XWh!h2c&g0&a5LO03oS~{0xY;l}`Gi1_2XD|*-KRCcXRcU38g8Qm)< zemIS?$WH`L%VYtrWgVNobZ`!oL`t*mZAS3IqB|>Ha7q>(2}p(e(4hBWL0TOD6- ze{9k(?tYU`ucCONRMe#m9MReI2ShBV_UVE!E3(J(4kWwCj5R^RkRdQU=VfLhO}|)- zc=bmwbZ_MhpN&OoGj@~-i@LW;qC-;Z{uy2+8+)1_nT63!+^-y7d!4ov94C;A3opi|Hj7zadD>PN-njdWwG~<{?{0L^fQ&>C;p?p@NLTJ# zpv^e@#(ptUj1c3O^ZE0(*w|RrsCArC`*4J@6U2RBquiA8ahPlY5M6YeoDf+j++dWP zyPe|JL0sZ~hJ&*5)9`&XMZfRsc0%v7`ODEEF3wAU;^IJI3ie)LB80Xy*E>2orXSpO z90)|UvYE{Z7d+H(so!qFQ@rL382gaN*OfUvrC6XJ$m_@B(R>s*-(LI-iBmfTCc=d?c`2Y|mx;^o<7GFAQ5i!93YQp(E{*AK4t^gPS^ zhzAlQ@)Rev4d+yN00|*VZOfzkP|WP~Vg@ipN5@=4co1>?2pvbgt@L;%cS~1YT|ek) z^S-wymim-Y9Y(d00la&I{zyb6g>;p}ef9>JKlxY3f%=uy{F zJ=YQ;B62?;E~w?vHd0f_6uX`=KUE~a9o{))@bcqhZy!;jAQ{bR&b1JzI?@P*(-%3; zor)hb5I8qV>yI=VrygiYe&;c>@lLHebxNm>+8h?vi6L$8DI#6D=IA-zrjM3Wco7*I z?{rfXu)v0|-ksaNy@{a8MAoFM#VHfXPT;Pbi71KqY&qh>sD!oE1R(nMJgXTR!w$?) zaa4e7{p%A*J|Sg3jugb!Nx8q!B7*qYR1f1(n`j5oz_$cAjT8H=lv^g`=KyZr&sm}Glw z{8RVq=^-Ov5=GSNuj|FMpb$Bs$jFbc`8}WAYKpz{QKt9LiW(jklye@RBe>#_n|gEr1NR*fVaa|BMzbKbc6XQ43Yd5g-Yoxya0t2_kzjC?a6 z)gm6!COgfX7JJD^~IoU(8e*O9ipZ5IW>mAeB zkQ}?cOQw^5|3so1dEAOK;Uy&3*5$?sX`F#cat0L9S~Z2Q zObByhDn^_uh$pJKO469c{qa|tDp|%-rziL* z5{uRf@yUy0h|L_uaist<{&q2a&nenz5w8fUD_}t{)4!qUIe&fGmzNW}7WDBrMD>LQ zQ_L=81=}@>$lu?8u7t6m^z3$OFaX5E&EmT3QdRG9Ogg@NIT)ql(UjQ!9%Ya?82T() z&6o*?CdNZhRO`xx3!z;pfeX|19OLeG3-e0idkNtLoy{P`${mijmvEp^GzC)3GX1k> zx2N;}ElBGi1r4?kfw(@Z8Qg3HJm%9r5#ai!9&x?gA${FHj1|K8yKVyo5~rAk?3v$D z<<*kvNqCV6A|Y%$>}i^)7TZa`VIHp+@d>P1z@H5?9{PT)gbEHxsAH!!q6^q6-%-he(thQ#E#`;@`u zyu@<=L!|fo^(zX!<@=;XoI-duAIl3$(p3L-#fgs@(TzbHa1YzK-R^GvsV=QS(yPoh zW2)VxlwnicSJohq~2u(d7dW+7YZ>Q=nQNRl#S}zT46LbTY zIkh^=D?dkCwY)pVL90gXBD7XIn=6n9xU+Ov^Ng}GS+!$pHWyQ z=lHNPT|#7gI+s;P8JW^b!LaEw#4ZERyDGJyUU4TVMQ4;T_7m!d^*sx;MbK*WjVl26 zUh7JW0ep!2psxmrEH!bb`2BaN4lfv9Es|@QJ6R?vQKz~ z1w|{DH~{5t(*%hzLuXSWLM&|#+{V#u^d`{Hpm5bw>(8%MTf{ge%FfAW-^9bCO5TJ( zbCS~%TlR}o8;uQv9VJr+3~%~%mVOf9)B7xXGYG$FRQHWP0I|Ed;sA+6g*%?_V*o5%{Byo?fhzVd@WXM;ltFU?iw)gOE!Dx9orQ@}*2g;B?*y zG7R{l6=SPQps*FDK^6s}lT-X`%wc(4*pd|Lo2VOL6yv23hH!xcy`efX_2tW#m`%Hm ziTiL1*8pw}Y_)%(Q5SF0TQzlck5AcI-yi-ARR;F_JbqIZg9GQe7GW=d2ouI#2>!vV zKTkB1Y&CGMJmCWeQiX-YszfirAKD7!D25F145XCKme%51##$;DAsl3eiX6y}Hlk8t z@`3=spL=mt#VNR>W3{z|mm>(!cIRZ!(FN5W4zguq(t6`7WOgjwk0Nrv~MH%Xj!$FEe&;|H37SGSGwSexKg#Zef zcx}qKE8Q*s!B#9G8tl`1ckk}zuHo_Xk|_kS!&cv3{t_R*w6aN^&jI4QdGiVH$%M9WK2k6CVA)&qX!t@1rL`?@ zeQmUQM>k*P#eSN>f3nEH+rn5da~+h3UTM_g1%d#x6Why2hWSC$*?6>kWs$UEntdsDb9;uPP6S13YhPJYOT^d#|`tpZih``6<_A_094O zQ_BnfLW#AH5kG&uzsHI~jg7Z`<(*)oI7~;TC@$iMCtX(J*4YjCoyYqY`^3$*9)iye zGT@$H;ii729fGINLB686mXzhYZ zVujI$N{r(oJnm7<+*9unTM9pi8L|AOF=q&>*TJB%-|Kf=9(g|?GiNZQ zCX}Qvu6X1)sr%%Cwrh8yqrc;$%$|h@lmQMhlh_O13z}+P)zWsrHH`Mb7CS@VgNL&5 z1h7#Xa{KGt9b=F9CD8^H#$6_I@`w>_!h~IKMm;rQf!FcNG}1Sf33)PGEBmq41LAN{ zb&E?&y9^#||F*Jgk)dbS_ZjkFU%zhW@$^LFnGhce;y8lxo`_rl-UP|X2rvaSpzP?= zCBDizRvE!6MnwM~>SoMbU`h~I86qE`?JLZMNvO+LJHzX}XY3;*^bA(?C`oa5M zP62uG$g?MOGQx$gL*L^AAA8)ZcB3@l$`#DdVpHt)iHpk1bCDi3j0_BM5Axa45YI-vnZO(@F8^RJ`RQI7UodsO@nVMxtOyF_ z+-!>F*vG;=PQ#za29=)NAAe64Y4-+m8`j!cKZxkqwr$1j4sQ(x^jnQdn^8ft(4>S9 zw9ou{1^EFh6u3Tr)Q89C{*`+f(gu)YKEiPX4g`n@`ZEXGG_lSjC|}0n0d}#lQa63d z8Dpmg54Pj;&PD7y8d_iuB&QPGMDfX2W@RfuREkxGEjyCBNp_s0Vy!CRSGK^ZCtRF+I8@R;yk-Ze%nJn1GOKFvTqk?@K1GqY}8h)@2$p}BD)RXy+t0tpN% zT7;NfVkU(9#Lb;C0`h@c&(7aMHD@_PPmiFEV<^~j;6M+h$6a`Cz{z=U*5)tQ%Obc= zQXE&B)|W*;aNy({Z(e-A7BMW18wF*j?jr~h1s+lgCAqkwfC(;h=D6qHY06UjZELrj z#j(?6tcM1c!$ZuS|4UgXPV~nM)^YXfN&h zlbSlFdWvJ`VPNs~Bd>6BU(<@?Wy+*sq!qLroo>68BW~{rDo%QH_EF{>evTI`A0(vJ z(8&}G_!Z}6bC{W3tKEb*Tfv)2U4w6LEHA`l_+i3&4e8 zE!Ogjmd&K%F|=!{f@>ddEyP99m9GdS4s7qURUq^lRhLmUB6%$9p}z zbt`iCty;%|pZk$? zH2%klKfKL6I*Mr9oi%4p*{ii?DC;OH`8Uem870-Cgeqkrpp&4qo(-s3* zsGWR&CKi|9zLsLrFXX@hS_z{D77~+6`f)cjse4MaFc?q0yot7aac&QAFs94xaHy3<(K**I8h@4E59ROJ_F>R<3d4%PC~2|6!hLvN#sG zrH;TooBqBA6^2UV#C%)`L<@Gpq$yCZHWGB>r z&1B`plfchcSA10$BdU|z8EQ@iw>*8<*si{*jLAXZ4R1|86+3m?g+TilTm6Nv`imu( zLxzdO2UUM-!PH4;5b{g^WEkN^JRZ) zfCJB^JC4#jy(dn#Ee(46U*#Iw;JxYFR%>8wB_Dt!PZz?gwYj#ldyhQwTKaP-pgv&Zd9*4H%%+tCv-yk~)72#nZ{;FcPfY z5U)dL9i4_A{de{-HjX#%=E}HmCsBkcllYt#@3$o){NQVJg#$|WiqpKJU|DpMAI-$i8^~&j3tG~)CB&KLZyO&UX&vF3E=;GL zi`fi>jPCo>__*xMCBW zEVvY_ugW7lKke}FEhI+w)Gaqxk>PXcdypjT1i*g_y;=ZR1 zf!lBTe*dE}C%TJ}frlCDA8rTt5M1N~_ABs7s!dHTt=*`->=!<9U_?V4ar1R)#s+mr zpn(G7Gk$3XuSuiFy3wB}4jD8^Y-vb=B@Z$s4j8|59zV-$Ms01XH_WhnA*urxhr;=f zj##x5cMP10yc2$w@S)tWio162wqBr}fVEi$QcG3Kp{&nR-rs8YlWS@zQJPAR35q~$w*hX6Z5WT@M90~OfNx`+VEr(?JqB2$`9 zY*t6o|pUl{y2|w&V5VY@8|P=U*mPXUa#wO;YA1V{5U?)VrptO-g($ZswVYr z+h^{P$x$CO5CqK1JMyrJk3NjefQ8MP6yI;VCpIC$0ZJZYk0x;N}k zzgsIme>DbUE8I29EA?qw^FN5EasE4ymg?=5>V~f;8tk$O(LM~^p_SF1u!Cl-D}-Mz zvHC~HSa3TruhlS_TVV)sv*YTbqyhHZ`koiRyl*4=VH+6&&?YmQj`6|c{(-fDA&r{B}h9W_|xqk{WP zP@zoCV3Pnqvcu)o0pydGYHHe(9tJv-4Vu7#?sX);{nZ;xU?LCBS|HwL{PXYM59OT; zWt2fzpx#+KK)1M{`$3=HW^JLoW>^}4j8W8IS~@WKuM}}V*3G6+4Em+V)(7;u^!t4` z)(XI^yycbns{cM=FNNZRb@UH+HSqDamT2W&j?T#Rua|6HVBpOKQ_<{UET8BoHfhZ5 zLLOBZL+bdQxU_!Az#-2PmL`ElI!p;r4Gr+LXrQ9;AT5>+Tm<$XZqsUyG4;cP|L66R z2?w1j=*wCn1*Le?%`7|408bJvM8JzXgCMbC-4QY3nMLB*+cJ!gTFzkwRJkn53^8hI zP!$Z1y2i7-m3C8S;<{{WxbOAefvY)!PRZI+N^4#qOGfZe;WzP zIU%107qfAZ>(+hkx8VJzB$uNL*}lkQ7j8JEx$^vmMq`esk`-nBh{@8Ct_k_O&nZ2i zO#kliB(7r@E(aggy2k@wD@Kpnefy5PuAy>h4Cp9}n2bJb@Kj?UrGS>t*0eAaj|T$x z!LbP-NF2buB%S{~o*TLYaTeqYvX!+i#p|Tap<uL~A?%2(?DOt1lty_sJvO4fD zA;)YQwUvTw%P8u){{_?-^}i%IbzH0--)Jm5Dj+Y5SCm`gb$;0!JA@tT^WPDS{^vK) z+1PjsLqD)a*?I-Ti=Cm7>POeK)X)fkz}9c^8(n7QFyhckFyK8@`v#pp*t&m7>k69X zO4Qu;rA3*-RHi$~1-eD@ev{Y%3#}0tqCV1EiGKY3KEK1yvV`u{t02GHy=S+-vz&Eo z?DTn4*QWmG>rwF3yX(y0({vSL1%NjslnzZsD8Dxkdi8n|$|Me&UFU#@5Lggvw;_?s zqQ``U1f^@Xar!&T-F})b1HVK4B6S#O+e*kybZ%Hrjd9oflV(=Qfb+Z8&++5O(_8VS z`bnk2V2JA!V&ZPe$-fmnDC4-Hj)bc&EkAR(7cr_5BOREo;8ymd{6~zGV~>R|BH(5m z9PhXH5;J!1+9k3vur6ezfa3ct{Zs}MT_t-%9tPZD=*S+i&O8f1a2~BSFfm80=vtct zG`PIn3%$yA-n?L+Nx75`bGb&749#hrHh z_;Ra-LeEU|yL~?T_wBs&-ZSgYOZA?34qoD-{?IX6zsSGw%dRhvja^)tS+Du$7q>Tu zk6onTGhu>?hM|AhW`{~g0|T^bsO4AB{?+|+(P6Pzy~%yO^4 zC%$C#EQ?ubh@1GMd+*h@8F1I{&80k7mW79M1_j}f_@QY5rYj3`Ay~8TAV8fBh{*T) zt&s>-H;y0MJ7D=XW+MwRtOiX8q|L;+f6Vc5A$+9K@a#CB8^XeJhx>g&8dc$bUYT`6 zi}99}ogv^_1UO-`!wQ=+f4*q5#9ah`8p&>>6wBT*q=3Z39CiM}*YE$Guhi(PZd`KM z@WlAOPFtE|)|vt_v^EBWwd+x6|tip|J^aIzW2Khf37Is&ImTq0>BH_eY*`<3CbY)*?V%Yfi|h{r9RtT1i?z=WpFbKjsdY|{=h`_GPeN)&&Y z9QNe>lK#L*XBm5mbRA&rErE!E{~msVHETop<}{1#as6`X)RMSRn?zg?-zJ1Z`Hn{f z;emj-Ju*_0-{KhAJFV`+)v0{q|2jksin4vRwY)g6q^mqup|Ejh5WRLpGC1dPQuo(m z1`W}BiG7lg;5Wa$zqlB+`t)h0qq`a%a&gu+3mN4<;AO)nZ5}*5zQ8TeM;*WubaTt_KzWQ%1OX6gh?kkk)>umX-yW z+VDDK8M}c1BBeB`XJ&P@=(Eah0TbDik{=({jJG2D^0eS->~>zDg< z9BruayLhdT1&#V;Ec+C7sTMb)`AiSDK)xXI%IYtn#1uIULDiz%h`FkyRGDi5^$TJL z@GDO5@HHxPlsay}pI(ijKgx%C@)odJn979$XH!I8MV;=*c$0%3)P7&NP;g;cj%q*X z-q8b{W1|1LG9)oRe(Y_n#GP(~`z5ws8kO{7{8~GE@5!y-?W(@4z#7Gm*P3jbgs)^b z*Kum9{`N;>eG8Ju{P=d2@@id!CLSMjF90)DURE1V{T&O4G=kYRAsbU+;~{2Q<|9Tn z-5S@tHC$ApHIfV_KQGw@!Egb#R}1d5dDYJYPKJi#lXV%r{$;O_xK8^0S+@GmCkyOR z8tXb{en418OBc7+x^akKSOw*m(0GW{*LdXA*lJv&ScuU!&$3;yItO;lCVwk8YbMMq zsbwheWxybWdYYNI{Li9l|Hx8{*Be_bF8#f9L@Sl|Mj9g?%{tN~-1I=}&s)YCmOiT2 z*(}8N<&m+se{9w)SXHV*(!6Tq*zwq-W5cLPaVnjAX7}0}QZY0&*K;wOjjy3G0hJJ% z1(O8b&NkO~v}>NT3hN8(W~KqP09%SI80IUkd6!ub`R=w1nTQ0kYlwg@E7FCHC_@rl zZy764VQqy++Wxdv(8rK~L94z6Avie2;~{!!(Jt^wZhN~{(>S*2Ggk|^?ln-QNbX=j zAq^c&?E=T;-lN+KSrWPsoz0K(r`LX=vpC^4a!;&oyyDWO{bN(U?!tSgB>>Z51}ZjC zv31(=fReJQcZP^GK{@4uqjvQW1~`rF@ZmR%{ienOUx+}D+brx~I({DFLC8|VdHE~5 zbkn|iV_!zIr8kHi<4;bh*P;KCm(XQ|rVj*-f-kJ}{B!o*@8P1TwQoOyVrv(xp4qk~ z&$nRyvhS45m9MV`pPpS18kpkY{JC=GGQ}35p}U%_Sg-rC+_v5dg+u;5?d&EtTxD&& zaLFe(!l6}N;l%=Q9J}$S4I~NU%2ske+P!(hkT{`q(|`*fNPND*0`g&54CnQ|J~_?hH1UC5#d+W4t(3%Zp5$RMo-h`k0iMQ^p1aD6Q6W=@1%1@ zn?DYXb9s1hXd{^cz*khV_C3kl5Pukm88~~j|7TS=kU}n~=-hu6pXE;Tq_IukOY4I3 zjTPg~T`Z4);SpYcc;l(h|8DN1 zsiIWV{Um+^6m`iX)Fd``Be6326apF$7aYgxTN<{W!Ek?G2kD z+h-G*BR}^4ATeb`wtD#|PUN@Gh5M8Ng}M#g+pT|x52QLHgBGO?bWq!Wrh3bZrhdq z5r1$h+t7f`#)+Is>3qG;sztRuC-BD3Ok|*Ik znAMAaAIYh38`)~E;(NbtzV^fHuj;DZm;lqHGg_8zt@mdD8;>b%mh##7JQ zRe|tb<+S`Y7W(}yBxi*qy?Zqy7|i|ZQ~Yxnrfd93d^l^@0MF(vuW!F$(PP_fJG&~| zMQu-KWp>=UA+CRiKRA>k%!0t6^{;w!TZgkZEBQsYcK0+8D&>@*w5Mkmb&>HdA}#8M zY4dx0*y3x9^#)PpE*v|75h>z#9%!uH)I`}m#iJPk;w6-zE6<)yuzyt_b`~tzq4b9! z24ca~@L~o4mG1!Gz61Ex^-Wz5Um3gUp`oGw{7j>*4VT(B_Vf8>8uhu1F1_T=?$KSN zbZ+-FV`akQNb2f#`GMvki&odwEPulcluGXx-&U`4bmhRKPNvhYHij*qNdZ@YN?x8( zU%M)xk@08dwh;#qXu&Qj(V2|AXJNTZl_5I}8m4cjTJ{~V#Bh6$$t7oPhI$$GIW$J8 zs;5zKlw-8{l8Pxw6Vsg`Ui-$`9Wa1hw*~L z;2b+T>Ros|6I5Qp8&X%*wtiz!%$F9n0F$E}ett=TNRHvC9H1irD0vsDKx5w~xxKB; zJcI7=n+38?Ntw^y*8x$QXb(CoUww>XH7jDS#$s zm{n1*FgDwwM4^&ySH;s6aUN%>?95hfw<+Wr{F^W^$nFz3+rvZF-0=WgfXSL4@!K$_B)D*Eck|2ydg94wkY< zSF4R|Gb_0rxgDyGt!aSj?#O_R8#c(O7cRt}B!?+@Ep@cr|BF+gwr^A;|51l?>-PW4 zBuYf*lq<7M{~*IM`tk9_?K4-cH{J9c<$eG&qdh= zRd#M4u}=Y8i*0U+_m&0rM_!#Y`QjYalsKBFo{MvIFXfgkjJ!YL(eW7y^@4-v7M)s8 z1Jky|(sE^`(u4`S6`f|7Fpd%o^ieYl#j={3(%;5$*Qw#7zqz#BBGDO{yOb| zn^n345nRGq!IYtB$upbx<38p+n$H{{r|P4{*8EWyO#tQCwDYp0L})U=#bOI^VqO>i zu{aCCdYekm4TXaDzF=a@%1Q2DN5h}WdiLyDXOB_}KZ{^9vn_W5IyS!HKWcEjOr?{J z+ScXo%%qA;x?GM;xYGoSf4e8#%xGr*Dytm^o zpPF_Z&s&zRIWhU>xQOIV{S1De@2Y6aN8DN{e( z*|9sPciU!SUjYe(^%wpG!4#tJY0o&^0;c`C z^o&92kb_P;x(siA+AXoRmMrz*gBuOtBvzF5mFuC-zkrmG;hcpw*Ft>6X|!&iw1n^+l&}{U;$hF(n!6_EH#x%z8o*ItV9j_zHtAg-{1 zR#fP?HD9vpT-;B1Wjl6wptfS^>9-@Nv^`nBZ}T?iswbEv?(E8F6V}wi4~rQ4;f+x;9si#T0I{#~&0QHlgk=~k?Jp|~{3FLM zVxXcAgpQJ{16A9=(LT38KV$#&sinvPMjFZI%3V4DlNF7wV}g+15IZq*2s3F1KtG{K zA=&eOM_gT7WKz4!rdV3tvuDp+L?v`CXEa?x8s_N;)Ms@$j?;hW&KAMiJ zz+3W>w>xTE|JXUTpn#0Q%--tUsBPcg?<&vBD4N;63Y}E}%eQEJz|P7t!gQ|$S9;@J7YKx!u`DNX6n+5W0q^zy4k2M42Dt9`TkJd0nsZ%wXt z>9Hq0I-B3EQOlekI^pYp*yleN+be!noNKpoyNUnk!OdPSczL7gw&i*Ub_FXVVd{O> z7wJc9rc?B1^2%TgOvNVXy{PUgPqqAf*vTi@dhFoWQ8O8yDs+t05Gef&t9a z{Pz7U50e_>(aTq_gxge@KB8xeWy4$Dp2&V@VnKFfsH)ENunO=P=aW;0tZ#OeA&cx) z;r2^XVGOT$VaYYi?_J+h%3&B|0OCfpcfl}ZHRWZCRd>JPw-SJjEiZX@Yw^LGmgwT# zKkUq)rn_2%E8SUJJ@|c$@q%{g7gUclGD&@Yc=7;ug>Sm)3FnNQ_La2${y4kCtij$t zD||EWg#W2a$om>Uv9GPtYP-dCCPqJ3)?exJc|z2iHQg1?hwEOP7WH=iy?|Y=4)(8( z)k_LKmRn}Z5N3gINOv*$LCA!}?AVfL*J*=gFamP|bKT7lpA~L(4?oJXl@>DBKY|z& zP7K$P%FRz^FA9D+)j7+5ot)ZtzyLOR+aSS-)Dg|@^*_1Gyb90E+f(}!#BZ9X=SE^M z9Hf#+_s|R@?(g2x+k-IU;7EAhJmV5p!3_D`ZLB$`i;xO1XAuEYhG99=U827}{k5Xl z3bxD79H*E#hp_vrW!VUeo>#mzzkBERxMTb+AKn_PTLm?B$ijc3Y0~C7ZYP%IG8I;Q z$+i0Z$L0LR)L9Q-g>gAR7)yxzlO|Z!IsgQ(49^}hzR4fW1KPKypXn6(Tzx<4l8%LA zTAkfrnR4xXM~4%WRqcNL>JeAiyVUdPvC3nvwUIw`)*0MNe0wZ>PiTo*g@X2O)#V2J zwyiP$vdd==C-xxklrqSHrzm9*nJ4yHKe7BZso`YLukkd==L&>oqz(MX-DfrYugR-M zxB;mCE?Qi2^UH~!hxJz|ka-`~6=FcifbYL~%38$+lO3L>90RehBwc7fv!9ngYF;V3 zbxlbrzyGc~kM^x|b*S1C`Ze#?AndgHs^t?U-Z+%LduKQuliW7Z0qxFx+coTks)vVT z`Ao;A`71)thqX{ls&{{M?dmU`?x$31l-8=vZp;2XC`t#rFHQmmdm@E%*oVWPg&AJm7web^dd=5D6 z=;qk*mq)9t%-DI&?jP@Q3euFqUNFSex*8^ZCjYz{Z9AagF^AAS%HWOPn=N*ZN4uXY z>o@rsahx@#O?@0^-v}%(NZAN}Ic3hAO#ttN8lma`9$S&Nbl93bc^x&W*?qlMdMw+0 z>3en6%OUZPZUv|o^ip2FaN$eNq)mt8qxOX!)`I%kiS|+qgo{DS_!R31L2dIc;dap} zajXNydCz&B7}nVxx4BX};>of(ZTkk7jDHv%_PqDS+Z)4uvhT0fHR=7M#jWn|4g9|u zeos0Yah*5!$2P@e$MgP|)Xxt0j{Kt(dj7tNc6!1pn*j&Mjy^WApL~0Tl|DY~go>X1 zVFRqpPX07c3sFupTMK3pFNBHTX6N9FRLywiFktF?B9~ak@dC(~0^mRpuDqV~Ww-V} zkxq9aO^O^Qea)RP=Vv7#^-Emu?_YNK zQA~!J=x8hLSnCk}?-3_)6@ufjFR3Rq8_1i<2$qOQj+8mgR=NND^lan4efutbx9myI zbvkCNrnun6H2a1N&KumClrwt4(Wmu46sA45n_THN=$6CQxU@}!$9G=3D#d$npliq6 zW__F7`IXtENm;S(LOQQ6&Dv#KB&aqtTpIsPDp_G-zI?gk;Ijqzg^JH1fkwn#sde}- z<}wv_T=kIo=`N7cl!e7saN=hzoXLc^5mpVu&doc-;2zTmm8~{m+KZ>ho|)eX0Z;Ub z!y~im)pTuJ{!G0PZmaATqBTUfw)vs&4`wf2=Nq5-eM^E~k zQMtn@*^TNMm>tWe>Fk?`jEuaz)VT;c4|C>wdA+mh!Rg&ws;{p*qhi?&h1a(GWA7*# zhI$umYGJfIG_wDmhG#>br@m0B(5X3hWckY5<+JbVue|%UkFH~_hvK*$&;PyHNj3bk znfviEE(a&3H{GLP+xSZ@mIZrZ;uFv$e?tO&{r>&2@|chB-`fkc#RuYFMK9Td8b|Q} z<$vFP{rh+=yT!|v#j|t$cU=UV zi0U#xMlbYAFFYTG`7KDS6zdJg$1kYF%(4OyX^DHnq{dSRnjhDDj$J-8Ng7b{?oMm z;W)+X@y{oBTfh85dA_ypitJB`Gym+b`;&H~?(rFye8=>eXL1KCn5MXz{2JRT%Rg>j zv!CC;*S!0D3Xa*T6Lm}(amPiNUfw9<$DhI$0FW=2KlSj?D=sS-YKjO&^qv?guH@6?b=Zrfi&D02-q}Nt z8(+SNRkobd+afk7Io$fdMf+jl`#M~#sn+dmrByH0Z6q}_^d1YrHvP2}7BBv~#Lth7 z6|kDEAvDC*My#ctdRKUH00hm0)VF_O3Eg4^Sr7rO5{{U|w?U_<7f~IFRbtbaRs$2) z4&Cr_%(-4$M~paYGAu&xoqv+O-VI~pU47@dof~DW8&GvxTr?7k8+KkA3`00&Nzqtz z&ukZbi(gzwrst00D;vL7eVJZeqLZqslM=R&p$;^(%K&NP@SzOPG>CaPOLm_zpUXEX zo(lXhVprdpDm#1h$a6~Q*KL@`^H#%OE{+Q=8Zn}3(bHbS8X2ON6%>1BL;2IxZbKjM zXni93e9x}RX;iIn4~0ohN$yaQTYx{m0Cyn2*HOGG>IB?;uG>EJyn&cY1M@FLu4okK zI0XiiXjJ+7?dEQd#n~my^G;{?UzK@dj*c_A<`F($;otl@wWm`{*Q9t{?DaKKHERXZ zjR01**?X^0aJO}@S>$r zrD;lG3}O`AB9`y}Uz+i(uKL0(_v(S4k0F%b%8mar{==iwjeZ+ySUZ@tZP1~jr;~G> zRoVOOb#V@Q^)$1BfYwkoirhc0_rNd+4g)87+N=r7OJ4~$ZP&n+-W%?z?=QU{*1yA` z>WcaT=A-(;Dsap?odDD0y6vYx$APT53re!-np$6@tv@x`FvNm^>y(Ouleg+R#&p~I zQAe+WiC^E=?uLz*mwiJ1+=y4bW~KT6?%K*P-2TF|(^DTkPfyOHDAWSky?t+gM1h-Sp0s{0 zaVkXq>xe37*gh9U&Yl&h+Au2MFvHpocwmZ2-P!Z!zkM+x;HcUL*q`KYZ z0vj>7et4-9Yoji8=oczkJ7wY2u7(=EPymgG?K?VcZ6|-7uB|p7Pb@2bdm}4)O+5S) z_4eAyyI_chLa`76?&X_P`2JU@*K*IeGppwX1-K=~6VQ2n;1_ zfM6+ZOF&Hk!h%IoMY~l$acsBC>Q-8l)tkS>xt0TZbKd|9Y+J<4lse5;IZh3h1ubPXbLT7n{Krx zOZs<^!6)T7jPlJQ2J!p7L|6&9NrTp6RlFMarRJ@Dw2}Wk_%kKW+rN1fBewL_(BN7Xcpm^WC_gA zwMqWB_&FqBR8&rkiCGbMnHRnroSBwo{GS8xw=fS$(KMdIw7w)^Lgb}6y_Ay(tJ`oX4}LRuGne7@&(kJ4$W1NOiv_A`c(}$z}qg zUn#VQrdp={2y#U$e>--C(5uEVQj_KL;6tJ&gjw92WW_HeJCEAgU6ol4KPPU64A#Z) zn4PFWsNkskBR@5N05<*g#S!PZg~@@nAm8b8$6JmzTsS77EWxQ_K*b2l8|Btb_9|U! z^gM4)cD=PLzq&9!;HTdhPqZc9#Xd_{Ef_a;x~BI5EVSBNI!g-EQr z@$_kwc+tZolg$nU*~od>;zS|VE|{ALtBXB>cM0>~CfzUENSjeuSSW)qU~DIL0?X6` zav%@zRMX2EI(y8jo-W%zaHV)9a9PE^&Dia95XOWd*gi0R#-H6c*?M^az>U~iiNud` zR#pVjcQVCgJJFs`o`>-G2mWk?)nCU!tHpk%>g!Flos-kowXJY5oE87jsfA$!!;+L{ zgZnPjc!GP{&^VoeM#`Rz8XbFlbbWNBCJ^b)&dS>Ubqktd$7)U^eErrfyLF-W(9jx= zIkMyOXLrb;Z3Zke#;ya>U?3HbIi@zUlZN{slPPoGeP6d2hJj_uPDoypprt}wXGDY_ z=IYRS?9|yr&Ha$PAv#2|omfV|@_ma$c}I&4usYjOQh7rxca;>zAPvN!5yE9XW)?Uf zQPX?LhHC!M7X2zZptrSo-DUSn?Br?pRNR9m<1OHaGaQRwJDyHD^)(}nH(9&=h~ov7c} zOl@?;rpnUnhi5!Sa;tl^^3uLKAs<6rs`_LYg<}7sY}-)R7bc%YRgD-NP*#7#@_`8* zGtJ=y?uT=@C%Aduj}3`wThTw5_prbj^`8=d zRCnIA#qjj~fgBk2&qOv!Htsjn;EqfU85GcGh?yJ}C#qZpw7%2$MQFHmZjZd)4VJUb z9mHlhfjo${Ye{?EkBzMyDVed)WG*$h@@%}+-t1P}xUmt{N*u<0Wl>ixeSXk@&KDgx zQ`SR8C%{t>cL-OEHt8A@TS8PuuS0M4bhYLm#GzUeS7Nf%DWvnX8^ie|C5=bP>OP2j z)V0C}QIN`+5O=F|QK`zaBzFUI{e}uYsclJ1>cPPpjW#JZSgSbL(md_^#6|-ZJ1Q>s zZ)cP?C1TT*xX?d#cC(iBHy$~1QTK^jk6q&4{d?LF%;GS{72e6Mt$a_OPAtalAk-fY zn+@QhbV5ADL-}*WnU|TVaOSAW_91*`g^(d}hr@*$!5QiiS!i8Y*ywZ7hdFbk*q5AS z5mpi>Wr*Ly{n7{~Gq9DzySa+O&FMda;m)$RqsY3vxg&s?b5h%aFJWplRAgoFfZcH| zyYozkcd!AL?_R#QxHv07)DW==qhd*S|7nIA>bfs79(S^{&>an({HeY}4IDc%lDS1q z!`T7~g{Qd9>R@Je_9mErH6P~mY6~UF9>%7yC$TgGo`(o|jhKq1kYn>Cx2?F)^Y=-d z;9X&mdbGtk1su@>;gayDJRz?8Vp}Rnj>+-GpCz3Of~jUmab&~cYl=X^uQ08n#jFPe z@*024_US&64{mI?8`1NR*W}cwfhr9e^ln=@EA`3R+MQiWJ@19zIOc!4euKHJ#rvMW zzy24aj2ns%7n~hts1a~9|9Sn626`d8JhqPAxVPV;u1*R!tO94fDhXhl0vGV!j!j!s zb$pz=MhIs~vIYo*gV&1wt13&BSx}L!eq%(ix}I!yCX5VC$Y)&A0VdmZzKOVONkv6< zt02htfx-1yzl2y_RtmvH^_v7V*wCLfM)#ACUQ8F*eNp`tzrPs?GkA6NE|5;R&%(LI z076z5{??lhBfu_1XBeX9I9y{Xz!WKs*3477HJ1+~^CLMPG{6tz95gAAuZSxO(*jf0 z0JYd5;lCp#vR~TwF*JTsAwa=?zd-P)HPv4lA@>Us;0yp z2op_~%)ppwiEZS-|7h3KUPHV7vE@lsRg1z#JzKUs-re}*nbTLZpSJDRuHDCvpNw|( zG-&m;L>rQg1Xa-b5Q_zq%SQDGxaq_>c7)g3=p240wNxkS0 zHnK_i!q>$vC*CtUymrV zYJk&X&i(fDr%u&w5Opu7qk;FAp^ZZ#V93b|Y}vU(F|F4*G+@mR-ia){p+GZoJN<5r z>xle0HR=D34Olq|=wB8=31N&*FqWe$dP|wWfB4XCXkLJ+dZk^BHFZliDY)=_yZpmg z&ceHaYcm$6;)M#`c8C9JF@O6 z0k}Oh@q}zLc9W*bJ*+!B|DzL#e&Pd+A4Kx^sSC1MUz5o~l=yO(=3V zXzW+D3Q9XR`fzg?$DxlZ91Xjr1>DC4lqXZn$!oB0cZ6?TD7Bv0hFz-pF+m)uV6Kbr z&wuYLRd61PD+XI?>t759g_D({oJA`d_I`f9dnw4+o$;-CLOj3hao4Y$hBwfVp)=bK z)>U4Uz=2=L56iHz=3Jr9;_>X|LC{Kyv4Pw?8*^RSAM?o)e^R!Aqf0%ya8gduRkWFK ztktx%8fQX=!4^v!(ENM15lR2g1t83f@ff3N)%c(*R~E@dZ10Jo6=|zr^wIUkbBz9a zEEZ%E^piQ%BCBKh8HN&VyEgvHL63xMbmUL|!a7qe&95kvl$elkvE7+6ftn;yfWNw{EM-z>xI} z09Nc=A&YQC-oxvO8JiI2zx57f%n;@l;wAzE!xMDUWt-Le>FRWaQ--dLCx>}uz=^@6)FPD2O zvI~-0j`E(fqz4KYcH=U|4Ci<*vV{N;$50U!Bb4H}jynW#?jlI|{vhpH}Y8OSD z@V^_H9}A^}Mr(%Bzs6Vtiy z4n}jr$=o+!BbK_{ndy*X2m1UN@i(?zKg7CI>1dcYdLI25N8VtNAjS*)0D{6DM0+PZ z{P83b2gK9cnrK_MX4T48@&TtVVv4W7aMK{wczr9MNi)oh$82_vuF(@?>6&n?6I?P z{8BX~*Q2qZ@X`-~pBdLpzIgFs0i#qFJ5`zf+)A|=4DzM%0;&Ly`2;T*`H^EVF9-z6+2MnK7VO7qHn z7aBzfiv?U{F$!r3+_dC@tnZO7l^ckU&miv|Job=d&}hU&WOq(cEHRk zCV{@N3Zf&$*HA^t)En4b*3!ZvZEiu^+OBkfv?O9aM$sz(83|I%?a|p~l40V`pHsEc z3gK1fa($@>){iF3z9v)uXnGP$E zJn)<~U*QK3>)#Y~l3wEoo4ez@kO>k)#px-~&+@K@j?@=8&M!-succ zvkF~!1+AnE^v1IvWy#||nhV}1;kRYPJ2pM$cWLq4 z=5#_+$aWe%CXQ2az90>NC}U60dnR@|I4hfImZ02xZi+4KBA^R=^$jntXf*qL!bbA^ zax9!or$8gk0xhM#2ni3@t+#SBjkA!~mvk=%G0mbn;(NO!q=nO?gO9d|K*Xq$fsRi{NH1l0=?`xR;i>VJJZ5fglPqvhyeWg zBe?qJEW>4{5vzJF<2?^Y7?NBsS*wLhsq!ZI%X@Iwe$%CY|1G;L#cIF;?dy%ayt$rSLt<^xMi+`*Z~SL$OFeS$o~p}dB_$w;kHA_ z42gB^+7kj)FOmxxI+MAJuGRw8jva;k3b@^8Q_Vg`YxG3SLniTiFOn;w=?@=pC4hh# z;zvPUx6A6e5l$~v`*fbMh?gmwd6GMLs$X!S7TAR{QC$3J2;&K|GG0Lpqa3bk6`R8U zmn}K>drnzsvMDIl(lBR7kJMN1}KD6cbBeU_4ndNwb8nyPKk3fNGiS%@Avl6DQJi ziy;&OljfBz;lvfQ4-a{V#@;GJzV8eQHe&EL4yIJ?%A0oRPMmR^dzQnzefNRx7hBAa z3nkts=3vNJ{W0`>LGP`n1#!oDs0DTG`obH8j(4Q|gs)$} zhJ6jKDnadiwAgRRdTFS2u3q0eg9mvv9hm!IwWp|0-r_WJFWhFG`J zQCSz`zp3-i_+9Iw`eg?<8F}p=yG16ZKeLBz(-?g_P-C~xflk?LyR{l}Ewe$7F-=F# zi2nNLkjY!k*?Nk}lYc(jI$_zp;)#z}?%%vP>GAi;V*Df><@Et$f&XBnDU&BHEyX$( z6v3LVpYZKLwM12L>c}MH+h58vYD@AtzT%V4>cXhEK6^gR1N~;QFcrHlZY^R1oqzt> zN9`sHzQEvFkg&hKzg4(8uBhVfOWSX^`L#R#!%*EP#r|9bOjkvYwz-UpJ#r^HV*Y89jLi&`LvmU08~+2r(|xyLX?wYc}yqYjR} zy)HXYkChOdwFhAQ1=v zw=Z?a?}wQ4W~C1nWEXsYx%6CCaUxN(pl z?%m^$-g`x(|KDn#d#sptD*0j3t=nc=QMQ*|r8B5qd?A%jU)@)eGVcvMAlZ`A%j?ky zE48>+)uS-D*Ws~JE{ZN5-$r#?wH)lH*++!0T1PlKl0NvIg7eD0x}7~H*%R|l{aW|O z;>3W5E87(q^a5GHy$0?kmZtqi?Zk+uws>VZ;jl>Txk8wrV z(D1rRFdOO8h)=?N;G<4nTkh~fImAA0)u$R9i4;zJyiJFvGQ($$w5ST8q3&%?L`F|< z5U6@>Gqywwb*1ksFG)#0qFIdX`!Mg|WHttxlI+=!upLtf9@}!82^02E$BU{O(fEBq zQ#7=Poi!W>ZiZ9pMrw`QnnTnG8WVYmnjo3v=KLbN3lTW6n z;hcPHM{U+}WPiO6K!80N;M_DWKK|I6-(?0%X|eNxQ$~BSVuZDq`GixKL6BsN&d4&W zHIt%dGG;k(F@w-s*T3GqXY6DwsmXw)HbX= zwEk3F>|F8#J>r;VM<~X)<}sX;uHWf0^M3#S{T^MWedm~~`-^TFII0zY{+#sr$*Dt1 z&UfN)1RZK+l?NqyOuM#ikFn<`AL6nT9(rSIC+lAAYgIz2x2ufq$vVwXKbsQvRuVJ0qr!lR;&ok>e`UHgX1&Nl1CR;`MroOHnw-StY&i`+OFFK@K^=SVM%9yvtAUZw|evm zuSu2an+%;}8=L4bT@_LR&kPV#Q{7e>o8Z9!rLxC*1uX~FyN#oxRG#gbEWydcLOagm zS6iG&Lkv2&xrrnXV#^|0DVOh0sbkn=*~4X66bC6q&pTz^9>MO_RnVYve&22$0FpPT^Dcn2I;s zUI-iExb}8-O<5SJRG>M2v;F49@M$sao5Gv`9HyV?u#hn%N$I|9Y#PFAGBR<$6#n=3 zu4&V*UC)It1(P1);SsOoqJp-3C%I7EfvI=Ivky)*sF_BiMvX?M-K7JyA7>S$KAp({ zuI#&DE2rnRDjs%FubQVOp7HWlVVzBEZtEP237R~6#)?ewYQe#i%p}fF0@7k+n2AN{SaSQU{9~;MgWY?d_(g>XDV8Q}zXL+X z?)E`3PsCt>12l~jDvhFg&<>r6B_|*pcoVhQqR@3}V?4XDxVNOpFu>f0e6pcMr+bvh zzgTw;5GZ^paRkAPS9wHZ=eeVf;MzS}d;n*U>)spFN(Psj?=4Yq^K;u6Dh}PyG-SP# z#sF^Jbjlr2%DuQR*#7;pYp}G~&Lo+aw$t=#H&_+7iTvsKm0QezK84~1qk{+R7b6%8 zR})bUK96=;>Q)-F<@++ngb1x(=SFcd1jT~N(oSOZ&xak4$jHsjTg7nGVG^utv?FNf z^o(y}lr^jtvki4RDU60(vQ+DifVMNd7RL`V!e13F#Y-8(1BxoyoFvE$>MC1KmClV@ zw}J%{DKB@A+4zVr$$0B1UH8FHLur>OtEORKD=k!WOXs<>y}V=t$LZ6jXT306bOXcFI@Qd4kw5=s{h(H9w9iltq`Gm_rZZs6a5^UZ_uB>SgTgxQl5cu|c45?i z1_bgrDaIF<<>-mI2mCyHvbJf@o>#CT!F7EnJVj2{mwR`43`ix3IIcSG?kM^ttY&4s z+P7c;6(LzgdPKg9UZXvR&ir>EzM&*MeQoU_e}Daj-2D7&|1q5);PqG=5hQBkyVOMQ*(AAu`01>)fri?N05tQag2 zxtw<)J|VJUe#Q*j&M}$zzB6aI#UGTy2b(UA8jO{70-<$t^?=*U{6fZ_Jl>EKMEEw? z=?Y@Uh0%MTdhO~}4SR$F6r!GC3b!r(MnnkB5#M!~8p zdErAjzSPESoymBTu^)5pWadu;2M%O3?`QDrPImSy6scP6+T~KX_d8-8{O0xRq4fRr z&SI}5VN$#Ra6widWoI&@=;5zPLmq^romj!{&uiocj@K79=Q(tad4Q<>0c2j;B!EjN z)sF`axwzH;p(!%OyLnmHzSr{QKV2^s zVd*U_2>>yY^Ej)nw*u#y%~y^zE53GXd(Gyxif>$$n3o9jAv#cfY^ zn8}(T!ik=$1Y;;dvI~yxJ3xCA?c+b9Jm;3fDibaO9Zu!T%k^<{+0H5kl)T!K5wWaS zY-v?p7uK?ScOho;tfAvUUZu6K-PfuZl=3fE3hXy9=YwrfObj2HN?i$4r}TFwv5hrX z9wH?7U)q(7B7;*!wQHN8(Qs+n0(m%w~qxafxt+SS&Rz=6vk-Ms%1<#(^+Q5yLbqi&mxAP|HCJO zPuHsI3@O)k)vEq5V8hlAZ=8t{GM$@jIR1;t{>}3nEk-zOChn!c_QLr^SJhc()aiD; zW@Sn=Mg%K{kA{XpuxDl9D@P9L0m5=|ej!ua&jR9LFmvH&p^x-wuljEP$j4@>Oy>3L z@q^P^f^U)exNg^|SDLrit`!-U&}I2hU9gP)n-9_+vj_rK8kU&OS|?BeJM1SQTO%)608?cyh|b$nC-O6CiVN&5fV~i>&Z|NF*Rf0>WV8ZNJZ@_ z&3O`6!C@Faol(-aRz~|K&R>M)h75YI%_*Wy+{&?}^Js8pE3!>*M%}oAZ?-zPqrOd0 z20808*B*Y|B5W}J2JOQAq({X2GGES8jk94OE)R(8AzR1@kF&=%5tRdsT86Ap|E|Ci zSoy0mE`Z-}iaumodJ&PP$AhGI2SJ9T0Y+uXiET^Ru zF<3Hspd&6h@7}+E_32Z0!gd|@m&qJ<8N6Yiq${_1g%9U<2;C2f&R#1fPf|auLwNv* z;<+UEaT(w@)tkaAjZ!X@GYb^VcQeCte^9H`r{r1~8Z_0ig zK9V<~T0!vm;hp>fxX}Ik*VSnl`W*kogAjrWz56ULuUx=x3Zh{w0b{9M=Ya#;WAZI9 z1!Fw-HtYRhF0hu(0iqJ%#|*fr6#w*#W~Qb$)0!e&8qY#UkeOH>pSM%vFB>jZS32O& zE;D^RveknOd$F+V&R;Brc{^{sdNbF+Y-*O$U8MyOrA2qa{FRb>9WXy9`wnb6Zq1I} zyNwPiJ1bpW@ns!P=qrOMep@i^MX(Ejr%DYoiGcTMlF0=P`D`q%r7>zI`&`fkO#WkG z$%+iBQ(I~`X3B4Yp7p77=wB*j=t56IrFRci>XQ`D?QeeO&FL-qnT8jI2YuJ^m0~_9 z15Qf2OEXp5x4%OendzILnl3)G3@XQ+oU(RAjy`dE!F>W5MdnFF3ob4$oU%gy{rfxe z!e&4c5PL#uQ|R3GoJ|?mN`@ep$5P-h15)kKp*!C+pmz25Z{MnzK_Z5JczRkLH@x4b zJZ4B#bh1^HR7qJ@pX*bfwQD_Cw#YQ2E_?AHzu)1g0Jag*Wc8|oNH=!weAF$iQ{RPy zzxqICHegpLzC!{W@b0y6uil|=14tCZQFR#tp zw{H*7mM^)0L?sK|h`P!{gVIn4+Y>p*vmFR>BZB)4)LBsS*7H$828s8@pFZ8?OxoJo z?syPjsLOF+V>oUlql9UIy@3L6Zo?ZhklB3SL1$(?RP^)nV;9hDh!9AF>ebFYSbLsV zip9n-mP)*1yrMv_@B8V~nn~)5s4SxV@Cv??ndy)84&$C(2s#vy)jM3ePzx!9iW03; zD5Xal3J-A~p&O2{dun|!#H?5^)?d~$2s;Ddig;mfXjQV77MrXmT_^VZvfF^W#+|%| z%_=Jw$#}yly_g3a;IPB(8nv$Fs}543$Vh%dT63mZ%q@Fg^FH%2SaT09joIbr)6+{= z+;Oj2y4z&>^w(=H)||L_Klywrc)*5rQVeL6K)^ZzA1$q zMr0HlZHSFDgF7<}zT9IVp=<_(APNR*cfF?Cl_6)&^r4@f#u7NLMFHM;#dUGQco1;}+~Q7Ze-5b@y^}ag1GkwLD%fo&0$?jolcZlvYHn(mOJfJx*Ci^VN|qbO|~CRX}^wP zosu6b7Ga>p)Dh`zy*0}^v}n>Kx4PsSH+BWg1`n&T+SY|2?&cnMv}iqtO`8^@q$B6l zlQM>in3UO^bGrO0q&}!Or(nmS3{kA^my`!(8Z=W#Se7hW5Xu*um1sMjc}+mgo-Q+N zkQIa%SGxU7om-&p&!mM_DmFCJJQ|ozTWGLnuO5^^@KW}d9*x`#Q`&!!=d^9xw|jCn z`<*-Xb}Y@GZ(FLqx*BvLwplEBIv(I)7hT zTje?ff;;@DX(rTEfjGn|i+YhkenYBz`e&Jb6ez=OO? zZtC6gP))?<%}qe^?xy+8hBHOMN?#|M#k-#-om+D2>*9p`IXuw~WL!o%J1r&$`O+a? zrJfVVG_=Pz9zJxo5`5tK9z-t?3uSn5=W)xXaQKeG4xs~yBk}-m0F2!t#mUD7d?r#% zpEhkB4$AR{*sCWi8C-2AoI(0b$ai`-K7oJx3LBh*=es&SflDnc>d4Scv zfA$0!n*Gn^FQfK%HSJTNfJO$iBJAx{od_6S}?OCNsttr+RiX~a=;SN;B-y{G%t zBIJ&5Z~ye6@~1G6*)Qv0ZBKfyc6}MEcR`6S*Ot0n5HjwgeP=5TXE!%Dsi@w)3xa6M z9Bd+6<7t|_**nc!vT<{33wFvVs11@YPQl63r^71(ss@2-{*MkrW&hqPqlyz@!|5aA z-v7F|LBobIE=MaT-f5y;dwy3r+yKZ92l)OuAEZV12F+Z>BIyVWUvjXBoZU<_W|vdf z2pC(e1Zg@Lm=$i2>~Wk3*Oj6vG%9Kn7-4+Gb4*MT;@GZ;8PkrWB5eT%QodEi8w1ao zPyj+Rg1~fl;6L{{sv@xl^va^@E}WsQ@`QOc>s?-*FTZpM9ngR1;(mtQr#CclR@_v{ z_i%=e{`t21){*b>I?()5R_cHkkf3#5Bew_1X zOl|lXR6T9@ngA5+0JuyDsCu7pLJr2&t4cinkq+sMLIMozFJLkw-j_hq6OT;q55CCe z_j&t!D9MXu6vfga!=fUq3qzS?&VWZFxX_!as$8kG*Kgm-lJO_0UXjLLI@Wg83XdNi zBv?sqCW?n4tvh5W&VcmUpaL&He%!(b<84zw-&|qg{Ax_1TLd@i6k9Iwjoe+H68$#5 zF|*d|6Y6T&@{EaAEaiH#??Kh_UBqllOUpZZEVk3UyMUo957TJeuSvJdl(|F6u(*J#zhJWY`xn=pe z8mcXp@iO!ANhr#I&q|EZe1t|Ks(?$PKvvAA8N*>yfcwzmuE- z4X**kVk3y-+%^yq1y$f)el!@f1;8VEO=ZTlsA$lF#BhDEi*QCerR=29gjSU3xe>70 z)h#T4PH)*z*r(Wq0vcPi#GM8d=u>E@d@C+DLL+pmosqS6TO>X3l4j7Q2qpAy2l^al z!XYLFzI8P}*$X#GQO#-=HI)=YAXxQ~^eQu-60HcwN)mZ2h_|1lV-JQsR*-F_um>A-=FxZW-d!F#<&spvv@JK-tFS78?ts}B z$j}0jX26tM(gwWE`q8+Nv_A7Wc+E9c4 zCvv~LejGk%oOkPxUEG&8lF7^6Yl_;|4| zi@7W;9UkXWdi#Mf9n1ew^&U_?|L^~QGArJZ$WAE=Wo1i5sE}+$wrr`4NW)AaQ7DwM zM=~RlqLf)e8L6Z)id2%6`rWR4zW;On-*Z0ad+?^$>-l_)>$sxhkb>YH2S8fCZ z2S4MycO23g=$}PGOs!~2=_yZun3rr&#C~v_r)Vu}v8js;l+9N(k zJ}YH^p4HZ-^=Y-$_z6Q9!KlEoB9v-5X~_AukiL`!ZP8JCc2eV6C!a7Cm%xu)dfEfzeim&-Ng+J|{bBdnF+%K# z5t?#v3qy7#FarN#7**h|HhsNGgp)CX(oic|MfU;?gmh*l|4G)G288nP;0lV6>iHUu z%q9ZQcFI@_^XmT1F?iN7O&qjfUypERHrxBYO$=?2Yh__mQ=5C1tvNIpVpKowk@l&UHkU4zua?U>(Eq1e7R@G^gnS$(+;0S8pY6@VA5+qItYC(Ti#sxU2Nr&A%I`IkQ=56R-o6cJuY!waaxydRbvq zGm62joE%RAI*zpNb_gDx_6<=0(7GmFfRdTqH>dH&QzKqk@K8-gi}h)#qUC)Ajwt8r zS1QZ%ryRU}d_Hm}Q7LZ&9FfC_J;6|!K9O?wiH>bipkbbnCe$R%98?2YPJV84G$0@W z=}Ad=qng117TZYk>DK(NuC8t|SkJT$5#iyxC!UQ5>_ZXw_;8nz&GP8b#nm9GCB_>g z|LxWOOfp1wG@Y1s5D~v&Tbr5%Axh`M5G^Sctb+Ua-zo?b?B!`8Bl+5#iN_r{GKOKnMyQROo68Yvrz-3{)@?1>V zfs#9j=2%M{ZsLlmrQOO!rU z8A?#%B8_~^vivvskKaBI^TJvDxhFFiIjN;mGHQPOeU#Gml`yv-3;!{EGLMdPL;{AF(~PS3MhiVuGF+DLD#* z*0>SHRC=;e1kK@(Pg@;mGhM#T2lU2AU>ZL4czF0z@HwOVU3nPfP4$q%hl+;rjaL7* zs!VwUXSt6e9g24q24$=gc{_sN!tly$tG9V5qyrZ}Zsj4F$qIGQ?${G$0`C{uT>& zCRb2iMU#RR5R#j_05EclxjEyHk8*71|6a#!MN`QiZ>EoerLW-2W-ira;H|!GWOpum z5toDJ=HC3971Qmi&s@^65O?LrAe^Ek6!@lVS7i@s%Dm|UO||C;B*NyD?$8@quej1q zqB9KI^eR1l^x4T1BbU!$Z!t+T0U&i!EZQ-a)-Ws+J&lZ>6i;fir@xkRDl~;k5$DcJ!8Pc-4 zU1CM~^&8+GXE+#wOK(E^phrd?pQowm^z*ZIMn;BF<^JH{@nF&lRCiq{fT>WPCT!zn z@jm+2cmMS<#Ke94(xvA*zu+B>Bdpz|%g~|0@-o4YfRnpv)V&hGAS5v0ah@w#8Tzj| zTYN8H?!~7#ErW)7%(g4-lyJMLJJEzgxlrX&owyY_OTs&tIjNQb7Tpl8BcV17#f+IH zy(JQJ$p0Vq{eRuOIRG42?4lHTRE>OF5>bF|A?Aa2Zo-r7X1LvqRWM)fMj zDd^TONAE@=7De-KipB*+SU%_&rsu6EPUrx)kC(;Ng+-HC^tGTs@F9pPg7+kZ0I4yL zlnJ-biucdTURa}^-?*&8+N18OYuBc89L$dTz4!h5XueA`9ZE_IA4)$z*{oI1}Z0I*SPBQg)MVv zxw$f13uu8eyfpk(FEqu#yCpw9jDXvY=g&n+Bavd}Q$3vbNWQB|a|P;g%kW}_R%qzfR8+PimeEMg?1yQjm$ z$5Gq#>eJ^qkBAA+OO)5}PmvWQF}VlUSYKH4CAB_fEG2LPiip~7XTA8<6A>R!8a}6h zR~p$!%keCd3-E{ji5K2M-pN-51uUf!OkbUzeD`;u;IL#Q!W>9Vo2Vut?eYh94v#IK_SH5g=2);VG@O4RMT!ACL7&Zg3-_C zU-i9Dp|u)0o$zpwm68#GfLU-bQ2wX!E?jhy8##Bb4Ea1WGpaZLe2^wYz>;wqX1Bdk zpU(ve68}AYam%I29cWDT9^kdWrazCq-}TRz)2oI&-;3$deGyBJyPBRJjHgRt_jeJ@ zlwQyK0j3Irc;I53zz4)aF$OG++V}$sOS0b&vO(yUoYdcNkxK>>ip60sCWidoZ`^r? zSJhN3cu!aVS|xLRDC+TOM&^H_Bf#3bof~tRw=>WIht)1#I#VZjHp{A)0S|ckXwT)r ziS$w^L3|4T$k=DJz&Ok#J`Ih;816$aS4WDbn(nbRHePj^BaO3m510)XTWm;ywG83~ z@W)2*FAbq3rn4_?485bN@&NtK80z!mZ6)qZql5|+HGR9OgG>(Vvs`9o1Z@`6E$Gq& zItJ4~tKt4D**KzD1;jGDNskn5O#Fn1(LhviyxedOm(O!->Ul=ctac>UB>mBQs< zxlrcvmMF9qXWZ^~h4GYlOG>^(Dj=t|v%wR-@w6vJx3b28!sIQC*T}DVZJQs8AQ= zvQHvnvT;_f`icUq#mw!@>?~u>cK$E*Q)r0vq(sfSqA3SJ~T(wFQqp#B!^`g+`XAYAo?6qX~U~yK{Xpsg16x&9& zcy$8j(7=CbJ)N)>Q{aU%$9ZGXozZX%OHxlL(L)^AZ0ogdB+V8N-Wr&Jisiqmf!%e` zG|-VqE)++l4BlZwvIp8rU?y4&pz=lS*J)*rM6wi!bOmQB;$O?!%I*f&W+J;lfO?Qf zfNof~M#b*HR5B|X)J*Cq^ovp_AdHpPoKJ#sc1qgtmSK&!y#9gXX97oxQ=649;y954 zgBMwxRHHn@;$e);c55v(TOB~Qm_&vlCaApt%`(L-KshD`EQ*6po{h9Du5-8U(KLAm zGkg>70;pA{KT&Z=;0{N@0!2M7??9o^DA>h)kP+qNLdFYv$Pc_)0q7(mm$OwK01XW$ zsNNWOWjM2NE?hH?A4W;r5{&2h;XYj^GCH7ZAl^=k+t(rhZjlIG%f*ZJ$a!H<+K-1_ zLIkPd$?IBbD_JZ0lyG^7_W_bx3ONpAD&A5Q6mo#FY>^O8Q~%}qe_DVRtZcPIDM{7i z8HF=XBxj=CjW(vl5@w(BTV#zh|F$soSEi1noHLeiEY^EUn3;;GpF|K(`BM*r12-IH z>I3X2NDO5F3Y6~aTWh+Z6OAQ!f&1h+)L+-hXF7TYI5OKz^bdd-=FJSN_fMXjV`6}B zl>$ZNJW3-PS#ck&BDVYILGBr3jOe?q(u<>PZgZDXWn2 z78c$hd3x47KUVmB(mN&Yl$+d%SbvywnMubza`M(^gPq$Bsh4O+$T|`#mSNA2sW7F* z;>MTWnIqhC*vZK(DbU9G;_$7NOugqkboo`-n&LG8NJe)QB2o&?0oG4?<40|x|YK)N6N?`c^^1y`%V0ZQ>msGd{%9gaD*oHJ8TU72RE;*rz0jHu?@Og(*kEMqz}H2W2AV<`cdB6(tSCgE@m{!~6* zy@w#6q`zTaP#RIyDX{&l=w9@<5?%ut)@q<*ZnxQIOdDCh0v54> zID~^g#uTO_sI6JIww-!?YG=vd1&Bkf(HIHQg0txpMOPxZ3bi~-&UJiyxe|UOEZz-4heSNqzfsvB1w#wg&0$E3;L#X{?EzWL zyw*VUjJA1Kh~<${>O53Tpp8-RNF~#{bznhxb%^&GY9tZw!0PA!!mKF~?jYAxgl_hU z!>vWD4tq)kj8R*c$PTNaTP>}dRli$xm!Nfn+5@24HB`V*u(#&z+I8oo_ukD(4}K+c zE|6^QsaLWcwU8w@4NdCP7o`BKiZCf7IU@9#Cj>zamd9_Lh#Ip`Bg?JQ0Ig*h)iK?Q zD18)Q{FPlzYPy~7|NaK$;$?JZlmRI(UL2LabY={IKIh3k+p6MPEi=w}+`_2yZn0EVCf5@?{%*w*= z`S2*uGY_T%SrZgZ9j zC;||{FW}U+IWEu5zf<#16ew;Mt^r5%Uq~NhRx*dC`)L*mkfi}6i{q}x8oIWe9$b9w zAPx@p6j{7GZ&ECJ6iglGAK^iolJjW!Q_bfM!7r1~rDv{MytvViNA|z|XJI^OT#p!* zl7P#vD+X>ij;ORdRN1jcJuDLAZvO;?SCFey^TEk@Vtc*l;vr!EeYzUyKPXFQT){ijT@D61 zAj6qAC*W}Sb?6;E+nq?==V$>G05Yj7Mdgh@3m`3GSY-J+LVOgCfuD%1D?TxCKUF22 z8Y-8z{0B1jX)7xjhXr3E?1vxV^_Qo%sjdGb#WH`+10s2;potmsn7G(CjO> zkNz)eApd5#UWbN76TvGvN$LB|hwiczMwF5zA#}tJoyNt=+ivODtyC|10@GPycP^}qkz7FOta zAYm)PKajtE`LC>}S*spD{bg5P$?`x-S5;8(JL1K@bmOAC-wF!8S4M0~e(F{|ZG6<5 z{*(`>gvuU$-0-VV2_DmW3_Q2s!e{WPCJ#vH+J+-Qm4kltq zXOlf!a-=pT{#$y=oc#RGn(-A$@8ADD>U#HXI5G-Jxur4Rb>P6YO)b+)Z*T56dP~xN zu7c;7PDC)0yJ1h$urqDJUO9Kk+lq-Fw^!zu;d&Cr2|PR^Ba4<>b~`5szZUh}r%#Do ztypIxt`eV01~*o(6BtHd0#V7aB>&?T%eyhtK7a466=rdQ2+VOL5_KpQIls{lv&!jD zeNoMl1kHQ-5BhEbEuaWi=Br(=DoLMO>Ap0sM_1OOLv@vlRJ@cEClz{ZL*f3EHla2M ztFd3H;^u3V2@IDELrYFsV7{n*OS94~<8AIG%w~1!M^dwZF+|PTiUhP9^tmjvtuA-$p z%G6X5-6IY4C5RAy{A(yy)ahz_Vu`Ne6VgcO|GTY7uRcXlQxTK96OWz_eA?4`3ju}b zSQ^s7Bi^)}IrB(8ovrJ)*@WlAy0g($32K3=`AnccWhq;1XzJgYi@~{?0f^tbe;<7y zyu3TzID$w){7?ac{J@paSW}1bmlF9LXj}LV8KTB$0R#S(fW2Gw@@TQed;s_*OCL(K zH^dNCmN~5*^54CC8CS|QL4{q4( z*&*0aKn3DN6DYQ_Uxk77bVuq#Qa^?{lweIr6^XQ_RblZyp+u*J3cu#+Q`7w(RiqD> za?g4SgVa2eR`H1z(FsT;PgNqXW1czvDtjjN(%ccCb>bG_E|-`(sA@^{1LGZw0*4`2 zo?~NlMky7!BjRB}+K($BF!oI)w;si*-}9yuDa2YGksJ2q%MKSo}eEZ7D9q zpNX1qf$NQ={;V9hLg}(0kh*$GUlo^X>q`X zb5SBwUCYckPUDlHGqM8+v8khM?SW`qupBaRrNxEdY1*>ocHD(9u;NyvK?EqHMfZyx z=_zD1VLMmriQc*XV`xW&nwvqKN}&zruUsi$xy3KdDI-S6K~ho@#;c3nva!)@-Zm|6 zDY-&Dsa8n(1)Hp$426Y{?r|a#HKdK(0d?!?$6I>&_;f;|k8Z~SU7xi62n7FS{PJ03 z0vl!S1A-kjr=uX_XQL^=^JmTq1Pl$MMwQ_UB3fo0j@9xY{Fh563jRbQR{2VR z!_8Z?s6XWMAC4JIPCePn5-mv$oLV|EdMYauM$d^a0kYc3lQVK;+-e7hIk-?|DlodK zY-|`jw4Xs7ofzl=67z?ie(?oju`?x`FrOF&#cGB4l;jVtHVQ*a;6cZOI})Ox!qqjx z;veFyV`~av29t!PS`2l3M!FzP1;0CL@D(h!0MI_*T{zXM2T#3PW%Z&D!v zdk*6}EZp3;*oO7t2mUC25B+<1Zo;w&T{z0c#JteivZpkB0H!q275vj#f?%%xUJ6x6 zJ4e+|x1P=ECDCv!xV+5s8LwZr;=}^D9|?z{H#?@PlMmC|>vi{T2X=@21~_uVf+&m% z%#8y=Yjbq`4%Q&g5r=w9@B!%#U|(pQiCY|$T)q}nIsfzmdwVB%PI*As=i@lq7#^J$ zej_uCI29nsX`RQaZyN>IMk)wjx=Za%@?qvop1eOtr`WvEz6#0|Us~nY&nTXt+8e)r z-=GhYuK^JGKSUem$?m!jw&W&5@DcvLCv`iD!IV1CAbjDzaq4 z?jV9VHg+(ykjp}z3a^R$;k@AEpec}Z4R{%nb3}2>U&blO*;<{t?Fy>_Md)ct3Gs=^ zG608>Z%MHTU-3!4;XeRAX}~P&YzegEh}W&XX4)Y(n&W_I|eQz5O; z|3}&0@uxc(8t$RcK*WeXy%X*RZiWo)!N8?}wmh|im77=|I7lJI^=I8O!ty|51g_y- z6aWZIYkGTk+q7X;`<>=L~-veYNoUV*`>9#*|r=m43MjFD- z!{wCyAgX^PaCw}x^H|xk$JLvjoESJ*D`UUdaEdaG1xrS|Md^DZiMX~l@FZ|Ad;R>1 zlggRrl-qI=NF*p2QpCNs7v8;n%hZM4yLXTK&i^+E-(`l4NEwRfA9Hr7@;7?8s86w+ zJNC+tKz$AjnEDjp(#)SyP2pB7MMCjb-w=WQ7) zr%fYv!W^+3P*9tct%2EyO0cmrdqEWC*8|0^DjU(F;PL8`AQjyW|uE@)1!0b>G2N7pelw7=gX6oIhbnDZ(hlhsbw`5S_OYiWkg8%e?n3mRZeb3B_tXUPMZ`TD_ zl)WzP8sH80oITv~n#D=Jc<%U12jQ%dPAGs-MQM6e-RNR!hxU>mG2O-{|9b(ow74LF ziBnU1muIOSbg!}fZdG&F>FXBNp{TcpX0x_L|IXqP%Oq=P9MvYplG(~$y74~i2P6)k zKK+pT2T*$Jn{eK~$lrzOzy~A(q_C|n(_h~?$FU`&UjQw}@SG$!16&D?HuTj}C)@W^ zx=va7JovzY?c-062%q|?6vs`q1U&3m(H z#@WgFZWCnUVN{gM_Y1#&%$Zqze+4JO$j}LM7A%kn(bRl`XhL9Sq#L_Nb5x2(31EwD zfPVHlj8tIRU`7&7U;m(ncpSeyC#rRA50;RTH3FbSOIgD7 zk=BA2?W?K0TO$+q)V!f5N0VoPbbJiu8Cdg2;)|Z#D376(VR_o4eqt}F7aYNd_M6CP z34-O0nMVku^!xIWI7%{Fx)y&;H>3f9{gF6U0#0gEy{Yo${H4Kxlp7YY-Q4S7>4Ubb z7;D^h_~(JGUu)a>a5l?G)%O2o>MKOX_7u8hzLX{Y=aU58jgGllVeH<{> zBA6*omtGrxEf>0m1Rpe~`}WpT;9AilD{wO~pjWzd%5cYh_!zXfT0dU`-iSvIVo=Om z>oQ$V$EIB;p@3QyiH(7E>ohdKR0Sr*k#mWCWlj*gaXQncDT-YSW3se04KD;($XUeO zNn-g3Y(uhA4r)gq-9_Kcy=@ zG<4G-@Pqn8uR22yqVfSmKaeyQz1djE?chVl5zTTUN-syZJWxrcV4m&ST#zMTlz}2U zmkNp{IFtd*LgNA{>gpJm{D|C|syj=*`E5i#z}c;hH(UeaY^r;Rk!C2_z@dZQ|1HByQ&X-1o)!Um# z615^ASZ~TXbv|dRnxWavrV%0BF9Lhhx)ORkQq%xydZ7u) z>FZB+nAO<@P%-phfwX60)E;Q=(~(vT4RbtaniR~GO&?|F-N zD@ZF6`AOr`w9ld26Wc50l?)sNIUt)yg}5odm0xv_H9GkHvwZgxuxw;&i52RfS{TIK z%O-pglpj*l&FoT1kU|zXyPzNEv=eB&goIB^6g7v-&{Zo}J`>$Kc{6%?jgF4*(q(0{ zMQjvA_#&O4nVd)C$W7%08lwN;WFrlJsGmDcm5d;gyaB`nwRLb^3VTWR<_q;gAssqS;-tX{Xyp6*flVJbTF0SRXs9SrR#k5h-dYf)n%+p8eO z4Y|fvz0GLg0huL@_!#sZH*Gz#P_(b_W<#YaSJxBzNE`w@sNUALy(Zz}lEa#_h6IyE zsD~t-YVbGrZbkp61t5%SK!Qico@w&UE^xsLQ7C_hA6vV6O^G?vM6vdm5={(RX!;(! zetk1(g*_En=I(~>&1AS3odh;q^W>}ZHiR3o{sn5LucfpQ#FT1L>9ki@<@?UEQ-_r1 zWJ?Ff$&4*j6er#uT;QohY5;_Z6Pt}^xbGe0Dox~Czhy^zNSzwU8^-(FduVj4^C)4s zfx}=v#YD&6DP=1>ZrII??xq<>`M(ok7Vs1=S1KD0EcrI;cT*jU=mGMYB1ld|tbN{v zzaAtc_#HiJ@G&kVj>C`_e1!~LK|}Y>dxV1=%;;RGTPBm_aem)#Zs2@sh35G8>GZm!1g1#Ry5N+z%4DHNKey)qi=f&qhxe0nF+`EqBtElyCLkT-VIM z7)EL+Wwy-iK%^2eefdUO1UeV-1A4mtC!r%q@4;v+TIqUFSmM^>$?U*DC3h%y`U#&6 z&Bzea(-N@e`RNII5*`DV@;_6Am#IX~!6Hdf#>*9n{ouj3=Ux3Pq?uw>eM;yTWE073 zIx0xry=r0b9LQh1`G1$WEJ34j%m>V|`&cwZ&Z~q`+-E9I5t;m-^Gk~gyu9!mvgygIY*f#dT z7Kv%QET2id!Vvt1<9VYC6^ z|*isJy1QEvg|Au-4(aL8>Gg#jU zMd{grBi|Rx0G0#-ABgX2s;wyn0oC0;#GVv`j8fTjIqNhL1xu9N&AGI@fKIT#c;)_h zs;F(|Q9ue>223Hqiqa{Mlc!H>+zvIC<4r;G*AeB1QfbirVmoeBybpQ4WHWJriqe4h zh^WZoNlXh}+E_6(*Z=jWYBG8bskv#vW_MpT)toa4g0L@u!~ZfQt#dc{ ztFHq$6_If5TKeTXGp2FTewkY7 zO+3#?8=IgfabcrH4MR~XK4}OLZE6KJrc!53ABQDM>8rM@dFm-i0l=#!i_4zZ%!R9D z)<~(#WC+=rNKH*XN}7APPjH@nB=wrHS~~sU+mbQmS=u`F8(e zG6`q%&`Ahj$x+Sdc~uW42vMs_!-9wY*o!Y@VJTw)4Fmf3jXubqa!(eLn~co?@xgDB zZz{xwHS^Q`v>&L^98lCq%{aIyx-@B-YP$iVM_^W%8prmdO>Tf~5|j%Bz5nP@fs3Tm z;_W8)U2>E)%5(n669~gH;1ZLc_5NPXJF}_}ku~dP2SuymtnAK+?8IURNWl_F}^3Y3V1KQ-fM6ubyu6#yia z{?GYEa)C+^6q;ZVVcsBwN%0fKlDW;IQL`n+{DNgt$Zc6N=;$6wLBR5R{8gB#5w)Y( zB)>w&R?|l(2h8WiG*VId$o;*|t#Rq2%7wZ19sa;cZ^Nbg1FKJObc!tvXHsPyn8Ps? zG0f(k*-q)prT)YVD)sc4so_BanGJ*97tBO93Z(`oX2H`Z(Et?ivkWUNjn>_Z%nhbH z_T4*?nImDA@FMEW`2Y+qZW9<`*z;nBaWqS`6JV$a`dBA&8Pmr7ebNrK1We03)+(B?vda;g=!ni5ypPb1h@QP@ zQ$^s8%odFan1u_a)D{9=MnrI2;Ga@^W;tg3_){=FYFj((Kj2^;CeC&Ck;H@s4ufO% zKmaCHAeDR|qo8@cDd%lJ3MOAGxm_MSyNG5KoTCp1=7;y|vJx;QaYxh{a=C3KXFnYF z_?5#O)hyfAf%XldjC@u0Jv0tWc0-}DoN~-_-$~WNiy-Yu(}Nn-WR1HF{wgjW4oC3Y zoy*-_RnJCf_JyORFh@=_9H|Kk3jeJoYM7=)4A}AKA=xix47H``0N#j$-*oD60zf6} zv-6;sn(B?j9GbKJTN!U~Ub*^|6@`fccz*1|hhO|6zJcg+x;YP;6`WOhV#X_ix+-2f zk?+tTn{L_d;=NPJ#`2Y9HC{Cxyxk@5Z|{=9s+;l%6eswWHlrWtgvc%d(}pAm zch+9TW1F}YCUoROnRO!qF2DlGZ$ts$$395uM#m`LA^6wY$n7;>Puyre6`68Qc6Kc9 zvh?-HWxRyZOcg}uLX*FPOa)4eGq1j87`}HQQD9Tvw+#WAjaJ<$YYD_D9(x!L1vH$S zIaY|@^Ho0>J1NonZqjQzUUufD@>yWKdn3$ljymr0HNCV`(dMKPA%Q_b5=|omuF(f! zdr}q`D0c39E)Ae32t0u@P}@$u?BAW0vnbZ14nF^*^!C|!>Vz~i84S3%1Br061V)* zg|zD#ON`n`yF;puoG`#J{)$6J)Af}cc~v>LyowtF*l;W;Gq?OgV51(7apAdYPHfxFu}>3*oQn z>4o;izkV^USwDH>c9SRKW^swsn|5xpk_Uf&fMS7Ge*pciW!~Q*|GuMeg)^|gA~--h z0qLmYe;VSpF5skb7lRLB@xA-?TLdvHMgR(=7D;A!O4RG)M1s}Z#?vLsU;Dm8HOFQf zfnUv;KL)KpvUPH;>6 zQ}QQU`)+AdRmyi+OHyS3o3tX+m8gKdbAo?V{+YYT)E)3u?>W|#Cv5A-z_SBMrU%f| zTTNhExMIDzO4^X}ys5qIRIwej$lc-Di~CE7$}Q zd()+~v#_YS72bJU#Q|dZWsRW?diIGXPg*aydhW-4eanv-9OG(AG zUlxxYi_|CAyQ42Qbzvcj+l2J-oh)fa<9WCqjyzfPBp0gJ}*e1#uTe&Y5|+l)PfE1?6M{!@C!qe%r&15- zSu&i0Qbu?U95~}%QC7c@+P6Ym_&xv(unqUFN zhJ;13S0o^axoKlo-u%mu0>qGi{800J!iL3A9-1-QfbFg842$|?#ISlx0IX;iG9VcNZGDEH(Hz8 zhYOn7jlj0SkLdbjz8mZ;ClhX2Kkb>`4*K_W6+^AqZK#Dh(NS{*$TP}akrg8Y2Px#G zZ{{jcdt`YfGV&D610zbUhk88~J3DZ}5HNBLh%75OR14hXrKaxZCz_7M6#X&goj zL1WNdqGqv?Qhe>-yZ6|-)*RwtFTS8bh(01sOQ^dD-w~0u9^lXsLIMCSY>ecuQ0&O7 zV@*p!`J5ZSpZDjo6Y)J@;5LK~O1UlInC8%d@!z26oXPH#cL888S-r?b?!2v$rntIL z-g=59=))<_H|N6F)~jxtDQS7P44?X##VS9UYJ34t1|!F$%@6HETuaPXobemi+PL*6#Ngu%(7#|Frr3#1I$Wijh$R3V6JL20;IQdc((M# zlK6&1AJ9q+DLkf}>!L#?C{8?2wDA(p40oz?NO3ZH9^o~JIKeq`aF|_aBVRACy|z|6 zH5T!>IL)6gHrn7+d4E73>zG~pbEJe2 z^xD0q+?{ij=zK3l4NNyO0Rx2uD35D{!b0Vw6_`slH0YaKLR z2c=q~8xUp@6lwAGw`x%#^gA4`LeT+;x!aO=Kn)|7O1_JHr8#J4MgpEOx8@iJysjB9Jm z*1(>5!(`L{j^GJN{1Ww)Df1Y(T-Pte?{SLG5zy!o;~UUaYKN-#q5~TnJS32JsAg<< zz2oFF6Y8*=C|^6x^_rH({U@=%uv8W=x4LuVhoO6M^pGVmxZG9fsVo%wmEk@q!3u+RVwVQK_ZPIso3B7;e*d;7xf)!4I&S@zI@r zWDf_t*7F2-@quomXqDCv?GE>_8P}7}p?S2sJA@4bTIM%(J%{X|q8ej)Gie&4Bq`3Y zpY&Xn89nb#4@xV#a`Sc0f>tFTN4WX0yVtRa+> zM6HiSMair~;HsaK7c9t~bi2#!n+~2}KWH+aak2>eYH_kE(BPR!s0k2H!h(pBZ(MN1 z&OLh$tSk_Z=gp1f7mpZeXHf^HFfU#%9A3cXj2+YyL?G|x-fTCa_RB4!ffU}TxAozs z8!V1LmqkcP@&Gtavm$8fc^Pvp;p0F@EZ^ar)wc_!0Y}|xr33|OSdDarn$NP+IO1m+KuH<&Ksu8%3t4%dn4+7GnoKl z)Mhu0S_eT+NuLJf5;g&umI5`IFn}`Y384VFM3*`GV=WG$W^CT7Rg!4nNX>98+Hr@_ zN19t0L)o;{?*%JH!QCmq%&L!#+tBp=9DRW{r%(FteO=mW(Y5di$#xpHbetw z6a)=MKE)|$i{xaG`9y4h%dhh}-T7mX%e{5=Wc@AVOYdQSNLDbKV=d1{)|!7GWNh`Qh6=WgP>XvH1(c`3fanF?S4 z?b5ZM*-e~g5FCy_Y8iTOd`v%{kJfz7CYo1jexYejf+$O+G1=)f43?I63?#Sz(oYc_ z$bGx?`k>^9Wkzj4b5KA;DWWz z)_OFxqfw#N@phi?HU9KRt}F2HN5h+Q9r;Pn5E_r!vfNw0*rqpE?1D3AX7uslhdc$x zkc3;EYifpwA3#x!(~CB6VzX&)Vs5uvxM2SL^_QjL0Ke<@w=BJj-AklHWEnekc`NrRINIK9i$C@EPkadP zy`HniDRFI^Hf@HMZ6wl`5Vh39yAWC#ynk*MT0QpOmN$tEDw(8nqnJb#A`~WZlnk7c z(+t?TX~ZeA;6Rn@px*=eM@DjEtM<=*qmC2E?gs)a?xL|{Mb!a(48z9#-jPt5UAr2m zGF~EOsnLnkr}ATVOg*n_VBCIwr%nV@IfnG_)i*)!Qiiw@b^NXP}z5C3v}=JN|s zV4MZ$3fr`8dw6u_M;Czqo+L{8TjX^L&ql<;mXru*hS$+uV1-IDjw?f$DBg6@?11c3 zHOl-$@J{XCz12_~f>3>TIsSaH)GR_D;JwF^9HE!vsrNJgyF2*%?cYB_Q*V#38_wSpfiH8map$Ge! zJ7o9Ro_%|5s&bj%{K+oOv1=F4o$HCBBPFHhI>nKqHB`L^>F?;f8=WiPII5<9pFWQ_ zEnm9y&&w6f5^U*p$nfhe{UH>g+0u8?T?4m=RCpexuxeDSCdDOpjTq$#j{P?#PTd~?< zb?7c16%n!GE(4-Pnxq*uAcXO*E7!faN|`*) z*>()nqBj9~#w;={e5xp`Bzhw0+sXYc%SM`r+8_&p>TS8ig2edru_dTvO&1${L;Uk)< zUG&ScANAg*K1MOPUr?tON$2+u8}uXFsZL`x$4FyaPwl?;3HlywQat+Q#VI>~8FcN9 zmFA$Z33@#Z7J6Ua*$Zf5`hhF)@vrMyI$2&YxN!N>rPzO&f3T1NmA|umm@vVAR8n_B zJe_0^ZLFx0ur5zO5&4bXBn_?O-dd50KO46>QPC_)!C*7`Pgl2WvwJ)Ep~NFozTJOX zkiIi|^%k!)dRH75ZRt0)>+1eJk8Pf6xng$%jo3O44ca-iw2!qpUHAH}IsGP|-*l;J zbY}-uuLkoxH@Us5O`#f@iV_zS_LRA-$ z^?2+a>p`80f9@=6HRu1d06$&7)>~nUWOE{6EOgeSnsp$QvkPq?vNJM*TS=#OezHz4 zeZQan&b=q~DRtGXt5VlL&Eis>qr3kL-`#SBi^25bIlUT=b@HBj^+;mmz?%M#^@GRN z@i~&-)?<6l;MosVR&8{A*2DAVk$J{_KGmE(_Bhqg&wu!+4Gk8UE}z$Z!I0k_%=Rg6 zscfKEZ9PnT@&0Rz_ZjU?tlGG}^Y-b}XUti+V%yQ{?SJ+A1?l$5P=eB4zaCPOaew8D zusNtTr6Wf2C(!`?u(Dy@hoYJ0tL$WNKjZ+g|I>@$L@dN-{YD z65&P9YZy9o(pC%@wI$SaxXXwU=av|`hP|5D`iz2q!!b}T<4kVKQ@snKmK_q)g4ns zP)G=^h{yShc2&mna^`#a`gAeq)nvC#DIie9)}UiOCmgd(Uw!;%lh6P8Khr<3!clqg z;UfucjJ5~On`Lx&t*Xfd2iME1UWASq*Fm#Y=Q=wFT0Wbc}3L&R#SsxX)(j3>{#+I;WmZ_|!$Cr0ud!0rHhzk-2p}DR{!8e)ebohG@?^IO)jjI)(`k z9CK2%+=e=CP3YR9frV}hQo@#!pF+n=xy|TVgDz8hk#k&@l;TYzq_(Ml(Cy-NJAHgR z=^K9W`<~h0Km)y50iS0&pXqk)!-`{#=Y(7x@VDslgUQ~QdIs6+mn+tfZt1nr+1$sx zlR@X*X9ipA9rtfFv*$34<0Cc|&CWWl&~0$H!oqJ3UB8WNomkW4u**#mOZ0N_26LQn zd+iFs6=k-hUF9$IboHpVs0DJfYjLa}ci*I?NqR1R$sdg410ZVf6+U;+)3H#noXGJej9@>owNt2 zwMcl`e(;1Z&g*a9p7c1e%MHDQ2TH%-2KF*mdV2();!3|U1OJD)o{q6&pCz;ZN0YV4t}UJ zx{Fh{uPwLNvUjU4RW44mUG*?o9Oqv5az1>BV=pbHDnv7OpmNR zarfrILt_w@*_?UR`1zN`4F-I9Zm_?^vh}eQ&oj2;!rk6Er%^+vr0k{G zC|JmJU`EVED zJPriVm@#@`vDL2;iIygQf8TDf>mh%pL4a;g4nJj_t#7t2rcYg)-0qVmcl}=X?%B!9 zO;&GqN%Za0{LHL7gT`fTItXC#Dbvn8IvSJ!0b&9Lcq3KSE>$58eBUv9tQrhkaxOl* zN!^!rOPzN+t3H@Axk0AF^#JuZVXbDLH7N|6TtbT1_=uTDdMBqG4(nJG{d# ztA=B{k9An>6nFn=U6w7l33U7DorReaNG$m!V!O5R`9aH0+o1C?uD1914|UC4^fNM* zVA`~F?dQf8{_W7li_Xf#t>);V-|I8`87&C188>Wu|mV} z(&rqrmOZ8q+rKU|bm94_u)ia=pB}Pf5)~+A$A^m>ThL2sJH+%lsXi(z|O(wjY zaO~HVE$-t(XSKfe-R7D__|`5zeR^dFUah(@%(tPbR>9Ji{qL_VNzQX0aPwowbMI7# z9=g8zTIj@W7LzYd4(y+~UcMUHNIrj;e}`7cIl3-WOGl?(j~)@}SDL`x$1w*-a+n5o zmEL9z;-HRLDj6AIbi?OX(+F*Rb_Q9S63U){UsvBNgwtYSwf7iq_}Tio#4i!Z;Y zp8i-_xIC?@qRWg^c1p1m(?t&+HbYk~FiV;+I!v3io?BP)S$ZX%o4%b$;4d`P! z$#zTHY?W7@Hpk9({CH*Cdc%{R4F;;zwDEP*9H6@MliEP1>c$$+W(~VDZnvTBj?00) zf~O2QJ1e{&4#~K8?{2@oeyIPF)UABEEi8qY$nV+WQcF+QpoAT$TKkp^Ie2v`YN^Mi z+oY{F4E55WR~1Rpn=5ncYJK=%)VF8$w4-|tEbUv;CFb*$1{><$dA0PSV(Gkjzt?6C zNxL|6XI7Vx6mT)D8D(Afa~~Pbj(I}+Ghkg_yYvwe$Dd7|KRK<@nofOh{dcS3i?sB5 z0RfFqHr390qd3l~<*;qalVe*X4GnpEviZSAe+R3Vb?!T8_kUg1by+$k+^la=Z5S*b z!WLiB90^<@ff_DA#&AaDrL7XzE#JWLqLOs}@7KUfd{YDLYdgSD?@pFR}k9ZuK->YzN zjIYVGnw@`Q+F06Jx_}5tzfI)v< zSy=e{OP4=Ci>z3`7V$$RErtrxsqkkn!{EDKqr&4ZUDP^f*mb=IF`Q?focP=_Y)m(^ z?~VP*@UDB&^lRMY|Gp1Rird!k`H5wEMKd~Ny^WFR8dO&rJ5|*g)+FhcuSpsR)%buc zfG*@xeAUp}o4gR)`=`ti8Hg4D10UX;P~3`+l;u7Bj-&1?e~bSV#$E6IVX}F2OcTWo z%dWe&Z~OT7>W_=F3m$t}ANS5{=!|jW6U6(1WKZMx(TVeO=WV?e(B}Pw_R%U`2kYP3 zrkAEZpkmy z-Y%QJ4zGjtLNh$a*%-Ljxk0D|LOo#WQ%E5}kQfr*KE%CvAmm2&j{PQ0v$}P7w&m*{ z<@d_%ofQjn2dG%*w5mJKLrEIclIMwO)b8WoaANn2u^Myz*0jQ@cK6j$Jw|{+Xc=Ue1xEa@j2C40|yV4+X@paepFD}n} z(|c<8%oz(J4QT6;Y@~ds8Ps$@-@Ox(C%jhqI;QWF`d=RxZ``GAuYV?8*OM-H zcS|f?<4}M5=0Dfkm}*rTi@JxU&{NJp?`;~JRSbsuglN;hLfV(SG90RX9lH*zlszrm!=fRBdKCg@d?95WroKbumdacW+{xcsyi zSFOKw?$w2~r6+@%MGdO+?mC=8<`ZDS*^m2)oWI7dy8Em<{x64~U0nS<)_K_XOVfKg z545y?*I`z(jD7=y6MXek>b{)za&h|NHyc;?2wC%Y7f@-7*N2MH-vX>j4t8i^4lnjZ zw_8$M<*?9WL`Nl);D9zPI{tCobm~Q~PCfT3fBDk=<(CtlqlcX8tGa!9a${xfe*GdR zBtL~?>$U##oUrKQDR!y%jE-Gv*IX;UTkm?eJpP;ux-fS}ce|KFA7-oe*jt=K{enxg z<)hr?;+KxubeNjpQLmO#=Fb%Ct;7t01J>G`MyggJpqM#m-JYT;UG36$f9>$=``9`$ z*>Q(*Cl1?~o#t-0@ps&s<_qf_KZ3F;+qWiqe3K^)zUD_CY?iI)T<_PN*kyLkl~+7l z4>O1zVt=;2Yuz)!m$J@cAD|mWgxnMm3WH9hdcL$;ktE8aR!=c`A0#kW9-ayFSGYm@&UYM*fng zlhP_CgdcHV_4MO)Tc?<*1t!Vv2d}p8=b@BcXP3f0<$(pC{%W~r&Mz5aa;S%+Ye;VA z6vI!4-YOV7J*#7v(@x(!@oL9Ib@mV2IP&qf-zxPQD|hVNxpUUEQgq5KAGPlyj!hJ~ zXJ*gtoYXYAbGed6SR+AX5uUr2JnVuja?&lMfH?q5YaSoyHTHO@YqQtE$)2-o%8Eif zU&OjpuUmI^R82)NKA>l1Ki5(Qmu~WSRpYY4=*RqLVH=;NM<@MSm^X8o=AY#9qscSY z*4gNI@o`Ofse9%Py@`sYlh4)I7scEU8uYH=9`_OV>^5tADX&a0b~x72d!z5IhB@&z zw$qlCt$AhqC#}pW?c5rzw`o&a+!(NW!U~_XJ1NRuDhto-%`G3TeKIY5^Y#|slh@=a zZ~fu$Y*Vn)fOWxUd0^Zw1uZ#blV9GwvHs1x|8}$=b=9hF+3v!4%T>D7H!SuwzUF4> zxNc<4`m2iNb<<8(R9~s?+xA%L;K&=QJtu~Et#SEiAMsda{U+<@A$r4CMHg+a-hbwO zta+?q-QshNXDS}?JaB94PSY<{?|%m87CDV=?Be=j!`6$>vMfvAT4ZMp3O!M7y6=5r zpVK8<{rl+0_w!vCf3VZ^>&pX%Cr&=7KdG~&%{^p_TYY`4A}{VXbW~lUq@le-1exMa zVmpx*Jyvnv2@dPnqn{W5$=kaor-$O{vWJEG$8J^+bzU1(el4ZTRjcQH9nI?al$CqG zcoDtoc(qICrcKV2l}DNsG|&4P)1}k9{USz;d%=2hJ`z)#JJol64UO%^MEB{jK(6*W?`c^ExVJN;eHIDXu?LQ(^8L zerDC6Q&tUTf38{d$tbVN*73CBug_L{(>t81n0(egXhE+=dZ%jqzv|!muJvJ<@n3VJ zPFsq-Qxd;e{dG&Lmpb_DnhjPZKRfLIdE`)$lk1D1<99Flk9CSudXe~HR_4^vjyc6o z9=6k3=&pSE)aC9oV&adPA*(ndH_+F7z-JOLyBBOxF~7GhDICoL`bi=Gt}TAP8MFD6 z&GWVr1*3m{V&0*O@B_ze=RRtDCcLIYb>`Kz{?%oUpYzl8>$&dBxBJ^y<8Ns*XQ%by zqtZLRvT+Xy%q(B9pwg~?ON08iLYK|CR_9hhQQV7|F5wG3LSizGT%6(9`%~|VEm1{j zn|92*loMj${XqLp!0)WC4{sT@SgCMpWzr%Kjax0#`-j(WbG%@D;pUa*{ZrPwSbuQy zpZMObU;F2mjr)^sbIkT|>+IV`Cd+oLEZnX?q0`f%mFD(G-hMTCwV}?C;ptP}P6^p% zWntg?i}Ay(PlsBS{62O4qk+beK}!q=EI76&_r%9Q+t(vXt}IylEn{KIj2AvFx{bno z!Jxn=*T&r?&l83zb&K^FHN%lBPfhMxfeTm2QA9%!$7Y-7Ef63z%HO};RpV9pGb7p# zUv)A{b$OW8hU3rs_C6V~w)o(b&)Z|Z4^RIyL_sh7?p@ctp7K;Kehsm$@pV zQ9suRl;y`7-wF=?Toh3=?b!zv z>H*KM(bZ*tTf16qxDm0dx?;s{=fP#k3;UGB?XDi!<+_qdy$esD_W8E`enLY1Nh*`x zpP2hV?|8uPS$5M-T{!eAx$*m($|=_m#jNc6EiUV}$+Q%c(La8b-FdMm#4V)k?(7~@ ziuT2+YG$t4XaDn1bV0mPe6c}w4~-*7HVoNqu6CyA3{J5A8TL9;Z&{hc@*dDXe1Upk z!_PC$X36s+d_NU@?8XbbcMP7r5#gpIcdB7vGS{x|fGP$QG1>av5n-q*SO)t2bZ2Rb3#NCMK9T)3-Qo5BHej>_#Z1Tq5 zS2h{{OxCPxqM-CAvGbgywngT?>6N#4A9?Zmr^gS4*ZEfy z9zT9O$L#I9cXb(Rdr1GV>z|)XNX+hPzP{x~;|E!3|F5w(f#$kxzemwTDv}UNGLIRO zF(ee3LMmj+5GgVg5)vXwA~J+RNTeiVQHsnJk}?mOG7~ABeb@6l|KB=it+UR%*0WYm z?^C|t&*#3cVPE^&`?Jw)XHM1cJiUY+frs&F*K*=it-Xc^hWx=tv#JGVx-KP;o?Z0X z@gROEXuD75$}ftg8Ai94yxFt!Sx#*0J++cj7}Z+ozf<4v;JT>s_Zd`t+}P(GkkdLk z8q!ugJh8+^Q6WJe9=$jiKI%NUpp~% z%Eso7{k~olh6f-r0nz=0c4wRWAkkA`3duvmtYUp`*ER+Q6_l<-Ttn+~G$k@RQLvJA zh+oen91Nwtx<<~?ket>*hD}42I|>f8u9}%+?b$Ln->Eh`e}!lENw(+V6PC)pZu{hq zE>0_atgnC0y=tQR{md z8uS|^<|Y{V_54Ck@mWph(_D{Of0Fj$)AxsXmZcvKMZZ~;(+Jxxo~VB z1A>K%Lf!{36a6tbN2~nc_*1K*bHCGgOoL4;zkZFweivjX1nf8KreiJ41?*e)f|H2( zvm9c6%Yt`~Mpac7j9AsF72bp zc$6V#a=LP8x4NXu+xJ@WClxq3DX|59HKqhY0kECDv959Lr|l=HBhH3+;71WjEs({d zlS>bPs10fS^AB@2FTN_iFslCR(NX&x{qwq;?s#xfa-6n$vm0cxcnO}`m{$Tv9;_CM z$$ndwr}VH-`^X)GVP4L|5vRBU7AP!}9diPB^=dPj&K%3GpM_($fIzIvMtsTLot>RU z7hH1Ne_(=!h;U2E*9Qd+{fYQspo@A4eVF@jZ}wdWf(jyHQ-`*K5Ef`FAtZ6`oJW>n zdsGF_2PwIEF>%YhOo2$dhE$8!{%y~;WBv*RH`iy(EAHs&pz57l4AYK^F?Xa|LnVF( ztA#xOQj{KZjryGdtZ7(QHgC%u^{|Vsi&HbnYIz3dLo5{vR!OsgR0+C?KNxp8?c`&< ze+MQZ3De*cVNXNs;Dw?F8UTj)5}13+96XnF@7{LTm5G(y-3CEkf?E$hn{*nSO|uPi z7ihm2LTMa#H@E7I|4KuWF0`(mU5${|qzs*vY6LQy5kM=^#R4#5Si3#xLw8VT8GrD0ioa>)>=)ra-4A&)~p^ z0UF&ax?9&9Q?;<}wZ?Ib53U-qj#1$DzFOf@oi~8hHX5J9U=eff{rC0W zPc^Oum5Ncscu(uOGR;l*=_?8F4Ag|4o}Nh6e}StvmN=rzhl+}edtqt`!5^@LWxY(k zK~fW|1hJB*)+Qc%!a$+bPE9VUrc4rpGOItH8WF5MxRViAw-QD5yYo{O% zYzt%<%5v3%B!P;L601WMV?L(r;2l}Lzs0G~j6=~_FVcf2ZmCN4unW#@2M?D9j7Mw0 znjnszq)KUp8$1zy#a^zYO(w$u7i_OX#T1K2!CMbUPc}8Bd*3&jp}D2I$tEYA@&(0~ zqk=hS^4|}sTNQFQTAZioyo3j( z`xOR#=79H78W-=5-#8G z5{V~*Ok}^(D|8)Pbj;*>MU7s+ve>P25K=euf3}KllK#mdB7NuFdj+N zldZ*0&HI?$9AsA%Z7=?fx3TIiB%{H4m%tDlYd1*)qoTg=FRWX=no$3UBF^VE>+-}z zZ=N$2S@qo5n_^o;bY2I=XNs;zh6-!q_Bu6ON{w3jA1*-Dct@DTFCR9D3Ifo?9WTd{ z6&Cq$?}&~kuF%bp zjgW!CnxC4VAH+)}D;-d9v+Il4ZLUakfw4fVjM1}8-COZ(5V4H=S}%<`DE2wNPqbGb zo0@7i5FEaBTjf%6q2Y?~JUcbz3*?Q%_q%!Yo}4>r3$`7)TYLN`~qx_mehjMOhu=%i0;X|0Dbfm|1g&(5DKwwJ-s>K5k|6QA6{L)^!N0dfcS_N8sf}LoY=8+SXVbaesTl>y#c!TRrn`nw5qlQ z+iFn|e^puvoS5!n}l0jQ@)XnPZJ?j)G(-PLl_VPMYe0gZ-qL;NM(0`=m zk@)*oW+=E|34{C4M5f+-6-GhBVBD>=SJBvz2Z&1eUkKOj#U7#WUq&e0@Cr6cQM<37 z>~ieBXu5je{VSh_Y;AV0E#x}Ypf;Yqzkq4}YixN} zsN~3lLp0Z42w#UdBPhyZgld7IRA z>;O8W)-nnTnr^we?u+1^1*)(X2Ue;(x6b!Z2(6ddz;4Jdlc9f}Pj;L9;lmD}Gb_F< z&JpoaBO{4m**s8Z$R+{tLn~;-ibxT5hDOMQJ-)-gM2bmC z729OTp)rBY<5P&$>iJFxoIpC``KOaGrZp~%VlyjV!K@n}XRtA_aIT*9GH#%PF8 z=hW2T4p6@&KVNk5`wqGXJ0^E6zcbhH+P`%@^O})24tsJ~`8O1fa0JfrU6-5|i1tuA zU!Jtc{d?J*?p@M)L*GZ;J5m)EgsP3=1gj@*a~!ifwqBmOGVMMsNb@*@)L5RF#>8cC z(91Ihs9=E+UDb*tv~U(Go-xTc$~+h67N1RNL*^bdV>IDf%8*)eea9wGBY++%P% zt^j5+)pMaab7r(zs;=uh#7sakL}EAqaS?Ppy*(Az-u_{&xO$b&gZ=ly6V>f{cLnui zGn1(^?lqrI*rTVMlpufqcQJ*Y-pb1`_Ejl!>pe1j-%>oae-*w)?FN<2V+OV7RC7rV zdo<$8&b_HWD#PfUtbB#4u>T^bZ^R#&qpoj0e%#E=9MjBgqS$>&iMWYju!YUaRtp=D zVz%SSoQM-<1OLB~|Fc~Tw`s6K#-WH^L>2?2>LX=7mx<6hpWUxn=s-a>3`P#U0|SAP zkp_uhoG>$k^p<@`u@nV_61q{45(Y9Bfb+%{-M@F0RMqxdN3CBoF1U4|*+KL4o|PdG zA(+SJX_rUZJZI)w*RN?kDkz%SQu^Vv*b#pIAN>KBs&5+#9lbt2P(wVw@y2&EZ{o5A z#|kw+NRz+1<_6!LcsX(U&#>!lbMBR~i`sSC zdQ5u$TTZV(ttX$s$x(c5-^w4kvAkLShPcoQ1GnsE`=l=Kw^ypSY>Cp1qNj~p$J*T3 z;;ZyobBiOpV{W6Ju40L|%z*2g&gvc4BsRg{j~?p~SetsEq*Ug{+-MCM5jcad#Y4Or zmHzQGx)#f~!J8S{9r2{LV8*@B=)25C;?>1x`S~{3BPN3(bg{WudcfMj-N9Zl=(ytu zVwrs(+#z%m6Ha4+sF&lEBXsvL_aml}I7Ix(05Nqm#D9s02S{@lK z(>r;1=H|z;vjq%ucQ&Y9?>_RJU*!7M6tu^X=s0ywRPtg-K@uOzG@aJ3<9M?lcm~5c zw%`;kdcd3sEj^dw-o*5R#A)H4ZIIpPR?yukph)BXY%wV?*6))}NJuu#@gW`V?0Zbv zX-xUq#uxQZq{>zc&+^4e6;F*_K!z|J?m7^cBrwazzt~l)(P5@P%O^Ed8CXj7e9?Nr zz4tN4h%S5axUlZ0(!`&m=@!#(R;&3_MW*u>w{B7XZ5qmJcJ+JO_w0BJd%KYQY$@13 ze=+`MWq0=4y|C?j)f{u5ScF3ugZR9D`E;CqY_ecu987-;?AbsYTU&5#@%e!# zTQ)g4O_X)9rKdvoWEBzWMS#VXR=B(v1q{TMYj1QRpRUJjKTYH9+sk&pKxB9#fBb9Q z-PDHmWjVe>tIZyIu1IDxg(*&jx1K)q(RYd0)xco-TU*avC8eLgX8LX^DJ9;#*^gbe z{7dPO#L`OOmDp!lkemB{Qn(p!W+x}-6Z8<3Tc+KSA$rdDht786ufPy*#zwh;m5+cf);_m!LgPA zUj<7Wo4|kMbKgru*+s449TxFWZuUWHSK@f=lfjRfhMBil%fw?8zucfUON8~ah{ z%Za@-06h2&9MWp$S&!w3)8>3F_OFbxTCk^OX>0qlKn`wovBg1WzsB34L;w<;`~3M^ zP?6Eg$tv$^<(mj%Uxwu)#9KIdcmlbX)}4ZWFNpH^0yyFnrQW}8g($bYM(v*xG`5+N=kcQ z7aWDHi}wkAVAc9JU zWLK{qNNign3{#jw5MesBEEy+DUKn1dOEnby=SgVf&#AgsjI-~dj z)+j}=QoUe)sYioBG~oRmNMIkrLK{FNvcQQxVlehHOa+PH6+R;5YiYsp{rdxW^Kx^O zac!}rdj|pr8p?0iL8+J>g|Ou5&|$)!Y%hGO)gG&j_fOCQ?8xnnyL_7)bbmT*0kF$o z*Z{H@(XfJo&`jN{_}JK|V9bNNWe@vA81h7|hk{b+shLBSwk1}t8X z-FrBg2yexsl8C@n!KWkM3MiQHMJ{2R6@GI~So67h#}iGNS*gc<(1>`-+pMwQ zonf>$f_xPv0?9#H5#JY_cn(Mz*Gfts#BNgNibp&a8&7fv((j7QcWfhym{_yrdQCrJ z10E*o4eY}?a zstg+5M92m=0Ba)7AZ}%D0Jx`tttX(XHs~&Z-f1eU23?--xQL$e3e}qREPH62;46WG ztPXT>yh>6;kSp#Pe(l;i;#Gqp4cwLQvC-Ed`*t37eE|G2jy>5y;Lw_#coV6>svx&9 z1uEEKF?5b^-xBR1sM%QLN|22&bcCq;8M+}maQ0gesJwVIkrZ}U5mQgu*&PJSF6>;Fp^FE)P;B%QbsO;3(tKBzO-nr@ zK#Bok80J+5Wym%_Z$!5a7YR$+e;U&y-~y2L0Fv>SJ~`x*)(S*v7B_zp%TkoF#1$U9 zp0eu$-&hQcgDwX)8S8Vch~-no(3QYZ4Aivo7RXY`YH=74qvhf8bQG?LQ?2k4B!616PJ2k-O6y8PQ}ZaS)DjpV$ZLafp@Cs!F_P89Hw_{3r|)Qi9J&wSa64OYK#GOk{|I{d9*H{OE{ zATDGmUGh9xZozgO8>T>tG{xS^AH`}NKKK;KFKmzNuw+M!=And7jB|I|abisj7%kbx z+`^}`r{;2$INl(gey+uS3}-5^j!1T@A(POH4qv^j3lXFbA0k2i;!TvAaAp;~v!9xC z!)XK~plF9~J5TAr5SaHLNwqn702^%)_#)U`*+_JFNZE+C@WurWOaQ1YKllcK%7$$e ziAFcwaA>!W(d0r?4SaO&t>mvw#VRey;}Bq_3Gxs^1i`Lr)K`%XdaRoeRXKuDqdjHl z7mkAFIW|U#jwbfG)k~5fC>tPl7xUCy&>;e4 z7}%DFZ18GU5!*G~kMC&Zqb-7cW&U}D z!65}_M-~$wGcz-WA>lm`T!v-jdr%o~_?Z74#XV|0^)Q@W_6<}r>VZ_wy!DF5_Ok85 zWDgMID55=x9cmJkrV{@wE#W$Cz)G-Kjnq}O%3q6%y+3}`0pc>UwVliA_TD%Rv@RY3p*;$NpB1chEnY;kEEhrDcgQ3#2I=+S_(Nh*y@ zysxPpMC%1oAAo!Btv*1c^@y7zG*}!yxNsoTi|lt3byQnhUdrke zR6cmT4#;<9H(YxDo}gCjDdX32udiXg2WXBIUghch-=wqFXJH4 z_iAxO=xTI4oXwyogGlg%yZZ!g@?o;fi`9pL%)-Oudzz3P58&0P!78Ci1FErk;ukp3 z70f_{gLuW0V1-O+^TM1n;7~d$6$W3 ziGI@m-WqTw2-Ov>p(Rd|r$g63gD!Q12<-mIP7(pTvCl|Sv!_*^8*CQxY6e0O-VGUK1Ijl$lpPE=6iy?b6O&;mN?eUbxQ)flLhy-Ptw5`NSL_PDJzj0h z;Lnp}j{XAHO1+pjTHynb6?a?}R}9=!x}0Dw1*^V#gz0J|CCn@!-w^>-4~;bZ$cy~P zb9WMm2B8iQOb4^$hTz7N%mxrU&!4Ee*78A zQ@eKU;@3~5MouH@%8>C->6$>G$^|rno~IDnSV&ugk-CbcFVPnWpzrl%k#bySc zjXvf?>EJ)HI3(yBtL16O=b)~o1D8YW3Gm;kQP{Q)R5g`;7AT8i10f3NC$T2NstJSx z_n_WaFPngz6fVPgFj&-Yu)M~OCr111c-(3!9}f-SiSI!?4#K)2Y~&*>Sc;G;BrsmY zZ`w|-HpSLHtmcR$5RC^DV)=TA4S^iY*~r^X=4dXdWI~MFO~m+6aS&B(L}X8^uCtDg zcVWXyx-Jl(hzuR3sCcJbTZwW!>cegF7lettHY5zFtlnZkeiO>ufUu-I+Hei~fW}BL zDTES5K;pTCtpx}-M5mu36r){sW(itaMC!i?)R|C3+B#{kuh{`#z5ykLNnFAs z�QF7N_nV86ULYY7n{(i)GurfI^lUIMiw77XeUkx5~QfMW}hc?D!9-5m3EF>#V*n1>P_uPFd!3y>N6esc^yMe_V9*D-E86ktf4gW;Kuf`}rpM(OZW%ug_ zYaobZcMi2J5q6`YmJpAJqc=J*OHjN?Aj82X!!wBMp=PO;6ep6Bv9M4hy6)&3wQw!* zhjCt#GS_oXe_sj2(d_B#BMPcKJJq?NNP=>QXbS_{-eCL_kC}`d5SWRx8^aH+S+oYt z+dd%G_!kuwtyUv9`y3<6OJs~#X{Q~iY4E&YdVEVkD$)bj5>)F!;y_VRaTOp`6Z$5N zO4PUwI8Xm=%HR`dDBmc@wIhz`c)qEKi)(k6AtnQ}a>433;aAXk>uVRlx|Xkl>V-{SNi#(O7i6+R$jl@6`h ztimFKtyGjU&gakfkaEheUmwna#IuuBxlo^AEBymv3r67L1XRa0<>`*KhORg{YZ%*p zK*LQFOd5*PZZcjnf0ckpH3L)+u<<*bLb%v^Cx-0D4@QKsfp7y^ekF}LluO_iC*0Q+ zK(9__@|bTXpeoa>H4#Nv-S@~STN91ziBgZJuWRX0L862qv<3rP6J|Mt36hzh2L2J} z>A#=<9r2rhAW+~8!g)n^%|@IBKr^^)$8ZEoCwN9gI|^82GHwFV6afd$%F>eajw*Eo z53yf|NTTG@L_kmw7PGmlYxYAClE`#*{o0Xl9ERl|&;)zX@2y=~f$;J{sQcKX{K1F0 zef74+UndQmVPXveMG2mrBpgmlU}TFWMe=hIpvf6Tj{ghFhfCL1BPZR(ok8D$ews-B zqNomtkB>KG;8cCM9_#@$^O=XY+C*am1rP)wYB)PP-zvAu!*((H6J?;GoxffYIbcYA z#39&19=Qbu=4hXOp(&hce)jN$J%TJ&F`_Xm(xehav{d=ifnSrw10sinX(EC5VQ3?x z>wfY^NC=4}L@f~ga8s$LE4~X^nbJ(t7Yw1_)pUFE!-o%AD<(v^Ur{j;1F1vZc_&aR zEKAWSVa5VAV{!^13V;r#+tS284Z6uh_7}mBoHH``P6@w4 z|7HQ2Mr(oBLG&k3Jte}gUk~*I?vIs&Lk()ajb_U@nZuALB)gwT5Dd<%44`JSO(KAt zS~Mg?5ryVDX&)kd@Ikx*7ZCdvbc6&{05n6UfJE8_?@Mfv1u(QEvI7BNi84D}FHkY2 zdrb9`>xr?Th8hH`hz4CBpurQLmQI!?q4a$UQV}3B7<5Q`{37)S0CbcA>PBnCbFrd_ z!F40(TVGyXmd6c`N(OH^fPMo0WQmYHbcW;(+7xwkGQ>0-(X$+Sk!S%yv^di&j_sB- z#5j~N*YJdiBs!XNbaexzINC)egMNu-GMO5F8GFftjg^+j=t>E-sjUtJdDV~R_}+fa-hI;P91 zdUGpyHK1|hG2XH0-${6J_$4`5o4M3kO8>x;AP~o#7_Dg~g(*Gp+!w9`88~oV9^l|G zFcK)|%_N3s|3q{Wpud2ZfV{K~Eeb{?w-8Nj%dbB}5E6q4lbQ zd&O^7Duc$N8{LEci2!=p>%dtcyJCSV8s=Vym0kQGj)GSfUQ%-D;f+<{H*XRpz~^Vb zMI#R%D&IJSrU5$rbiis5;%=6dl>ANF0q?zmMMJ^!4pERq3xgxOX2`bJ{lbNZ05gHL zLCVkKxwtbtX-QEGxf@b;qZ1?gy`XDk!2B1XuNO)E@8|Nz`X%C{CF|%Z>W>KCf@*dG_0^44(#a^UKOxFs5C0vaF^O*f&fAGbsBHm> z$b~mEfDjU`z|#Ybt{V=M75xMds-=bR?X3z z0te-q=GfB)pp1M>S%({)isMA|6nWjYs91;MyMl%M9!DhaEF0qY%&gWJ2KSaw&-8}; zIQA{Ox(6E3G!khrD1Zpi_?Wp3Hf%@XRt1SPBCh-JMi&N?aLU>bn2F98HdlyCJJ4VQ zx8OG|6odcKeax>fO(tH-Q0Gf|QVIj;TDzOzqC__mdAf(xCDML4hCRc>G|DL0kQEph zR~cYQAUgn2%kteJDkUyE9R68(UWLUbqED+cWznUOW)g zv-fdBNV3BBAVQ6NdtX_Q;Sg8+Jl~jc7=r zk|y}`tEHbNfW&#kVxfc%D473D^0lqlp9B5gq+1~eleLRDes14$zf zPC;;r+`UzedTKw!l7=9=MoMy&4;+S{aSJGkF$%N_`7A|ROY4ztdDiXVB60so7LFSrQNWa7R_8Tr8G#B0>h{yuGMICH~ z)J`rO()S1A$+~?z0lX0Mdhp+-{PDA8Y{iH?Mamklo5NJo|OGsDk&XTKP$|^4L$M@(6ERBEZ`8!56>o` z`Hdz4lO_mRu$h>cWWJx^gxVVp2XR~>YI~?QRFNKd#7J`xaA-FwPcm-9@;GrBK4 zFB50xCcwu;JfxStdZiAxfeZ)z)OFjclAB4g*20J~dfngq5RaJFO!lnjnAR_zEE!O6# z#5GF3-}&n|Zg68U9j^qojx|7l0fi@nE`zDoCjhoM&}a7?z6G&NqN#~9{{?d(NZ+M6 z_q-vSgSewj=r1vYPPL+r*!b+yy-8^(xPy19i;D8@9((;D-xb_bfJrMHFP+f1!^q-5 zrlAW+2F9KcH?;|-E25Kg`~+^a<= zo90g)5#!{itwq8E(Evf!Qc*SmL0mMcICh&rM+bEPWGq^uKJDk`9rLT z+%PZu&E$}u?J9EMgjc*AOIgpcY?eGDA{FzTLml_QVtU{FA((53=*APBrD#X zJ1Ni%643+PLEI$Y91i*20SgyV)}bMSmwP38Xvsgn@n!q)a!{3{bS@{^4m~Xe;B58n zCD@e}gWV<>t>Ln7SmgYUK{=qD&7GZ{sDchKOl}}g5L=U$_*e1};FWliVKTvMDIyw8 zD!o%AP9XxbEOfHKnBhZQ0VqI-y+40GMD0#yF0gt3J`&;r1`FXrBBc+a(15iL4!{*` zGCK%)qDC{YlVDP@Hq~*Y@jVOAJFK~gL5Oz4Qs?OTPA5T3eW5T~+1%XBA8vzok~C9f zSb=R11|H2)N%d{^G;{zaFfk?%3&q~R*K>;}#t^HBj5De=!g@diIza5C{)M^sW8j8h zxsUCBz>!zpo)B%cv&#aUhM{}XB0Sn*m=FSvb5r5@ zQ%Dc|0MHGoId5>&Kx(q<*N?nLBBhO2raM1@`R8>~?t|_SdSY>OmlE}VRsFyOdmi_-M%35q6Xpt z;fNeLL_G}D?#i9N1+E)!59xFNgHhhUUgQA2h@za(1z^R%b&}(FDjyBhZi%xYfR}>t$xNLIfHhRTzeBC@Ij7C(p9~t%_95QjM|**BsDcKjd8l0t3m;Ern*Za0vFh zz|A5ZJAw{o^f6gKccZ#RGIR!~0!20rEiIRTKnQ$K9`g2+FxJv?6!YpLews2M>u6jF z_Kxd)xKnQ*nDD}4i zYezRrsFw^0rs|5Kn^mYS#SmH`9Yw%bVsuJ@99)M7Ou!f-c!Nj6BPh5YWKiP!ot|zt zl7iGP>m}j@vu%8jgXq!7NSV;#@qO7L`iQKQcdkRu?3K3$W)!DCBvT@|!DpCU}aQ$ZD1OEa|syirJR7 zqFK%i3?a_%(xppmsqH9io4|a>p_hTDKwaJxF@YdlF);a2VQ8WUBjk+ozgUin^$mas zpv|P9y8u~VT^)MQiE($)Psie{-q+8BbicQQz8PXJ+%p06d*1xE8#{Wqf%;C}coDJ~ zY#=ES_A9AW0Nm+6KYfJkcHvzRy;QW}5;PI?ZEzK^A_fuoLYS$B>Sso+_I^!;TSENj zH%9xd1B8o+N^*ssp5D#zrKo$*nLS3#AyLW`(;-|2`~}DV717j|GN8JF@uJ4cGxtN8Dw_yFoFfjp7k*t8Z#+g^oRWC%Rnr^7p3jM-B5b%Zax?Y>8kx#Q8Fk ziX}B;GmZZtdf_vc7w@S3uo20)c&znVNsXCX#6ilr2h6KCZ$7@6jcR<203|-{ZM{!4 zoSYU6RyEeP;~W|dyo_oN42n+%Wg9u|Y#ZtlT-669yUZg#|FuR#0sy**Bj<-+7WD-& zGA75FOp9m7JAU7$?j{2{Sey)wWIa(Fy&N4CVlJGvkm9SHe1DsqbgY71*LVDq z4KI}os)~w6R;yyQ4WM~DMiR5q@M})NS ztA9Pb4~L4izP>9=Q|}%QD_;m#n5L#68IgP(JSO;~+(p=LyBm5L>=FXNBAX)q%?)Vv zD=aKc_%=#+d6S-5KveWQ%HZJS5i?(8xy6Y~CM$|50yV0tsUqMC+5x=+-;JdfRa5S_F@k3R8sv&P zsHXN2e#J~}(oa)U1JD2y12S-+-(tR<@%V8+wp*kh^@nMHk%ybqmfa%yhm#&W&V2Sv zBJt9=EcAuMW3`%FS|}~@jXlA8`}Sjx@uq9U0h_G8n0fRBx_(-{Pf>9PT4rn{9|toA zC%NU?{hs4bjz9HpHMPhR*s=9XGV2 z(RN5;2kJ7#R&sp4KE#j1%|8gApuDlRwl;NsT})3qOFgY^Yy6KuOr>UxH|EFdd3ezUr zhg-L9O?~)~_2>)=vFX?AIGi!J6K0}C-Uj~=L0$&~StJHXC3cUDc)d3F1>J|BrLYB) zer1TF?jG9STyQ?g_#VfeTG_(FB2RAz8{4IiK6>=_0vH?Uj`|=JWlt~NiHVtMke{HK ztPy%L2|ljX-uTQ%k9vy+6S-gpgIYDoogbL7EXWQJ$G(lsxi@~SS3fE)t_GZl<8bak zEN}N+SyEW&zKSqDbl_r$L6S3QI8FV|GNf=qnS-x$J36`&z#b6J-8fOWaTrO3>5(^t zZ-dD$Gf#rFJH`@RQ4k!sgD;?GYU-#pzZWl^>+ZFnAhT&#?%JXj*2MsH=rMeW9{zy5 zhXWoCh2haaquaqfDZHT5D-_k}se*6f@%8Ngic=9RcvjYFO7@OR^aZ)3=K{&-J>?g7agPYm*7L3=>y)9SlH_3z6v z&A=2Dj-42^v%OXj5eWmvwyj5jpFiV-7&fHKhI$nE_^#nLa@{3wq^3p_(_bwFoG{x7 z;_`}V^<}v0zV}1Yhao{;F&!#I$ooVn92%LQIU>9jT(VIY@zCVcVTzAC59Vl}4bKR?@ zX=Xo(GbW6+;!l_@qSU`wH70sT6rni35Jt|o6O;}a0R8$4BR;lz<$S-AYd1+7~utzmWq)v)k5^_b%Y(9 zo!+NaI|%EPZ)5M8HEXg30~%u`2LtF~8c1R@7`}=&6ubHPu?RLA^_vjK(eryxy<;Zm zWz~*LKW-=7Cq|;={s7l>X|stD2f1<_-FK*BMmj%&T(IeMPpSCO-k=$xCg0Sgp{>mW zK{|}s?t*b^i~ld`Ag&LrB%GZmJ;%zF9nWK z*7kFD)@_GJH|6r6@A9?s^71V^V!$24UVx^y_C{8ejszkWLrzXP4AL=FOY!IUUZN(a_c9+qbVHBVmGs?m^qUZKx}Owh$XFZtl0o z!A99VH0sD&2NrK7s;;MPj;OG}kJHf7+5j^-vWSI$0iD;~f460}ed44#;*7#vvw?c; z){K?=4#1Er>sE`zHVyY*S+~Oi!~FCgedqSZqd6ad3AeH% zd?F8!EI2SQa9o%PaV>2AGs_1lmE7mn1la_|L1Dh#72(IK;9F)^78m4Rk3zaA2ZgMQo-{6oca&4xPvg774Q&TN3FS*#*STah*vBZ}|5lCxLqo$oCaoBa=pn(7u)nP4XOFvdJ@+ga>OuTIhV!#LD#bQ8J^XJdcFizOvHN**~R+nv+x6Fkf zEI8YJsFJJ@14x;NWFAWjEi58!R1sL;1Nz7G43052kNFf7{sequ%^E~43peMPfDu_g zH0wy*;PNnF#)W7yHQ%;&7BPc?gF_c6{L^~dyr)>J)i*E*`~HL66wp~IfP`({vgMRv z-6<<8tUfe_M9(}bElogE8W|sd7=sDYz2b@tM4#`I@%r(FW`4r@DYO&CVo$fy8?h;g zC`mm&lc4h`LC4{qj*p9s&Xh`aWQ zjQqoy6ckKz8mjvZkyB7^ek|>D={#+*7swIivcbMCz+*TpXI}IV4zih##IK$0i$SnT zYHD@sE~iYf>O8)$tH6n4)3sHYbfcP9S7&W1U20c8;&T3Z(3Lha8a6g&#S6CbwdHL) zaUv=@rghz^tUIrGx4d$9@0Y*#mMX>XT=~Gp!_wt*#}w^0I;U3aukWXnRZgr{z1gC9 zvq{6gqTIQoqkWQ=x?D--E;1cnVBw$X+W0Y7jE4zlPx*4BYe|O=a&;^K2{iwT_aB(R z>K(KYB4;1BX8vKFtt3WDhyWAEj*gi0mOH{ijuiU{B!sSn&-n)2)w`$}(1Fa*R2lre zIjW-9hUr82c6oJtJ2Hy~ob%5hg%zA1%6}>@7VetzJPHQ7Pkw|a*6N+uHxY-ysM>ZlKu9pqbgN|vQl!@RQOryPG(K< zTkC&Nee$zrX4W4W$(6n5@pEXus@tBWb(`K|M+1gabbbh*#K{tv$fcTJud=eT2pJUh zG2gu{OA9-F8u!gle(PF((Guuf($N4$b7R@^+-^b!F*DnN2CNSuz4xa4czIRTK@R;R z)ClPjpV>adi+Xx`s&ARE^z-qSnV*@F=W7}D_opH#6Hb0yHrqijBuE9$glmC;J>%m! zksdSw&go4W`0^M+_Fmh__9Zg%0CQ*{&Tx<%`v%IjRM|GAqk`S*EVy!CJARjs3^}zi zDIb%J^Po5Z$#AXV?Z+EnHuJaw3QD5O`aFYswO~0sU!XTLFsMK_J@LYE8`vtzg@tzk z>JYXS(6@h@a24G-|+)UX84a$PsK^uV_uPh ztR`1<_Aj)Ih<2uIIqm`1LF$X=IX~L=t+lw@(dkgO;UYzj!*OSs3<@TSwsK|+@T*d} zy)KScP98}q94{WlT+-?10T|KNx{jPAT5gO>vv1zn44TRbi+{ZK7YO^#U&9g(fBC@AFqGod1}o|V6$?bWuF3b z0+6ip&_Ww|FADsgo~C|oT$Gh_Q{L||Pmm2os9&I{=#PRn>XC0T=L=BL zu)5457K`Xzd3qeA1URA96m~BC`P1NGJxP0nX19pQ0BB}*;Nm_u&R>si5-F%cTe{;d z03qErM)QBytX~EYoT9xSHxDXvag9M$N z!nffoOx-6VL)P$a5yj?RR?gay-(d%l#kMI4_LrsBO&53 zi8U88^TQ7xJ;3)jiR>D@U2ls5Wg@`2ljSShpo(;Sy@#hKL{Gw>JbCgIq$Z+Of$>~BekUt*GUPi4V#Ll3fwuxYMejfBJ?Qu9u2TV|BG92;=?z{dygGG|!I< zdw6*z(l@ZJi~)xLYH6F-#&QS?OXuqovobQU;kE(L2WSFx_%>vPGa-Tf#|3%RxcJ8S z7V$?L52&c9Ss}{a|VGY1-&SRt{9ixD=HTfGMa== zQoHxJv#4FB66!*6W`6;9X&hO{oZeK8kQaBfLF5)uWeIvhMtDLZW#9^6Hr&p4RE@CvN|&IHAKk5Uf)3hijP{5&zD-sHNV1=Hm9 zcPi<%y9cMzlXyW8xiKFT6H^7~8Xi)TFYcY=`Gd9zjDbq(P6)D~!zN4gSTj>oQStvi z$rOlIdvnB#N2#fAsfV;HykJ*{;9L3JjyEy{=qDTHOJs1YGs0}~?4FBCZGZZKh{)ad zdW947EH5u?#1mH{Os~_;LE7BBtBFVypn2}*G5nPw1b?=c>;|(Ti?{y)DV^)YR-X@iC3q9l-;ej zCG5?O((UEgN%OgIU25~c&wcoiz5TGjDlap$#E*R2N3uG!5B!?q?0_5ybxVGJKI+vc zCzPEMzGfE|vSe_Gyux=;=XY|9=3rzDC@b>~{o(7AqvC<@ynA_A*N{W*?aJ4&8=ulV zbS~3qJu|lN%Hw-d`jZuaYuUnG6r6;%fstKYpi@?h=E;-yHopP3j9Ks1&BbK-5qC;Z z@Hz@Rq%}EG*F*2Yeb>L3qUv)GjJgOMigV{gfBpJJ3K(BzM#fu7NxG)hxyNO93)6pR3WQ02nIMonZ4}g zEW@fF7Zjg<`*?hIu!^$#LTKO^6YcKOKN82xJO?x;j+vP?ppp0ma8fM3p07^2v+yk9 z2w2R-5(_B{7vBbqYBGu+%WrkycKw@?H%H7g4!h|a8d?WTSFZPJDq|Dsq5viSw2|yQ zyv#Arz+uMm2rDW1EpWhV|9;9a^LezBx~v~SqZ@87n(_2}aduZtW}N6 za4V*#r)9uYK|Ew+WYjx;{E&~2BFZHq#)8g?OE|XP7IQUH5s;B(Q&SU*U)GBk(buo9 zC#5TjCPa{^n3z>_21QbPBb*XkU)oy$&zjdTPwF^V_3qt3K<;OjN(4LbA{anGlN;^> zS<$n*^OEPI7tt}s70jbS`7^urEqgCWilv;C4>G+hg$JDdvGuoG2imk!RCSBF*Z<-n z!KSQ^d)aXKa8zAMb$#2%`}mX0Oz|Ckvj~P|XU4-70JlV)4Ru2>z0E8Oj?1_Bo7Qo;eK84WQGO6g%OPs)_GxoHLK^y3aVjqj4H{xq zj+z|+1QuCRukt^jCplx+pmEdp38I4su=o46iBZ{Zp7H^OSDnV)N#(mNbSqw1bc=RR z^(B@(E%I11$(XyEf7Qs%l9$6Vqf6x+SMvSV6t1~TS$MDi)x)GV+f`<3zfmD z#*u!<{N4n$Mt#gZTfIi1AoA?q&yUSI9RE;k{_n{#S)nH?V=2xy5&{qD56a>}I39On z6qJ6it&VCW>*Zk628-<pa(RkH52hNl2yX^MYf5h-YwZHWEOD`$Rb+DJt;0GCw zq>bGaO3_gt(gVZU+NX9Y*d5MHm^&{gAn$#gxgc1DR9V5#$z~$ohQgnR<=n>W!kLHW z)<{1tQ>Z>G#P*rf?x}BA&|Fa83#ZfX24|~+T)F?}x~+AaXbGhX9q?B;*7x=VonBX2 z@iWU48x%a+FE^;3lAK?6g(HKb<KCU;Tml=xg(`Gj|v5 zNcYLj)9>D!>{{U>@5~syZijs6l5Uz0eg4bxt$t@Jik}vDeqC$lLLq+z)W?$<|MLNu zOx;_KvcLG&=0Ta5wSBJ1=jB>e%CRe(Hm)SD^=_h1Xf)IvNmph&a9PPkx_d`=ebBd( z7tdVtcF0&2NMid5f05n(loYfm!G}UWg5vz(&70#15wCM;#k5bzcgF?|#K<&u|J=FU zR95td3q-oN@}vptrrtXNpsY9l{??m9gqX~93I-LZQW zufEco1qusI{2ATj*{0$a6j=lm?XQCZ-ml4&Z;5ha4mdrVJaIoZd{?9A;yzkFY zim7{tyQ$OOJoRgR{=e3;oZY)c$%UIPb6fF$=gV}Lxlz6CL&bUa)-cw0{E;Ejg_8!Z zCp8nArDMuIT3hvpNl!O7XWp&9z5$)ecY@w-j$o>{mYC{q((UUTt!(u`{vZ;w+#Tg`d25obCrC~hL84#=0j@SLK3wi`0h=?1$poTrn~Q5B)gxOU%G7$`JcanbW(7YP6Sz&NVY_an$_Lyw=R8bG@g;w zr?YdBL7rFMFWgVfNNC;2NZ~HIxMNNwVHkO9%k%!1gj{|CnL@S5L;gXPioPL8&HVB^ zC0N_my$dvP+*?U;?88sN{tFhgb4Mq1PnsSoXq%jxw18FwI8fcg!||9>a!ec)DBR!#6@=oBU?P?nJ0L=iUWX+0N-J5o|^1;*QcIE%gQ z*~t`4QEL3<())E6R#lg)=f!H{R5 zY*UU4r7dCKR5NygqSR}4r-EEuNJIoNgGbddH^uZX_ieWSg(pi%Nr_IEpeK|lB2Xz3 zyaZkDtx{hlb7m%eL*wh&oYW@Tyv4@CZc?#@>_fH%O&vYSCo7nLRJOhNCjB^z?ekXU zlNDWQwvuT~v zY-D#915;eS`WN!-n@Cl8 Date: Fri, 4 Mar 2011 15:23:04 -0400 Subject: [PATCH 0970/2835] releasing version 0.22 --- debian/changelog | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/debian/changelog b/debian/changelog index 79be983679..56f3bbdaae 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,5 @@ -git-annex (0.22) UNRELEASED; urgency=low +git-annex (0.22) unstable; urgency=low - * Fix test suite to reap zombies. - (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang - for his help eliminating the infestation... for now.) - * Make test suite not rely on a working cp -pr. - (The Unix wars are still ON!) * Git annexes can now be attached to bare git repositories. (Both the local and remote host must have this version of git-annex installed for it to work.) @@ -16,6 +11,11 @@ git-annex (0.22) UNRELEASED; urgency=low (Supported on systems where corresponding shaNsum commands are available.) * describe: New subcommand that can set or change the description of a repository. + * Fix test suite to reap zombies. + (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang + for his help eliminating the infestation... for now.) + * Make test suite not rely on a working cp -pr. + (The Unix wars are still ON!) * Look for dir.git directories the same as git does. * Support remote urls specified as relative paths. * Support non-ssh remote paths that contain tilde expansions. @@ -23,7 +23,7 @@ git-annex (0.22) UNRELEASED; urgency=low * Bugfix: When fsck detected and moved away corrupt file content, it did not update the location log. - -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 + -- Joey Hess Fri, 04 Mar 2011 15:10:57 -0400 git-annex (0.21) unstable; urgency=low From b4bc6b6385f503c16f9472a6fbec8a98afb74c83 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 15:23:51 -0400 Subject: [PATCH 0971/2835] add news item for git-annex 0.22 --- doc/news/version_0.17.mdwn | 7 ------- doc/news/version_0.22.mdwn | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 doc/news/version_0.17.mdwn create mode 100644 doc/news/version_0.22.mdwn diff --git a/doc/news/version_0.17.mdwn b/doc/news/version_0.17.mdwn deleted file mode 100644 index ddb59046eb..0000000000 --- a/doc/news/version_0.17.mdwn +++ /dev/null @@ -1,7 +0,0 @@ -git-annex 0.17 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * unannex: Now skips files whose content is not present, rather than - it being an error. - * New migrate subcommand can be used to switch files to using a different - backend, safely and with no duplication of content. - * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.)"""]] \ No newline at end of file diff --git a/doc/news/version_0.22.mdwn b/doc/news/version_0.22.mdwn new file mode 100644 index 0000000000..4848e861e1 --- /dev/null +++ b/doc/news/version_0.22.mdwn @@ -0,0 +1,24 @@ +git-annex 0.22 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Git annexes can now be attached to bare git repositories. + (Both the local and remote host must have this version of git-annex + installed for it to work.) + * Support filenames that start with a dash; when such a file is passed + to a utility it will be escaped to avoid it being interpreted as an + option. (I went a little overboard and got the type checker involved + in this, so such files are rather comprehensively supported now.) + * New backends: SHA512 SHA384 SHA256 SHA224 + (Supported on systems where corresponding shaNsum commands are available.) + * describe: New subcommand that can set or change the description of + a repository. + * Fix test suite to reap zombies. + (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang + for his help eliminating the infestation... for now.) + * Make test suite not rely on a working cp -pr. + (The Unix wars are still ON!) + * Look for dir.git directories the same as git does. + * Support remote urls specified as relative paths. + * Support non-ssh remote paths that contain tilde expansions. + * fsck: Check for and repair location log damage. + * Bugfix: When fsck detected and moved away corrupt file content, it did + not update the location log."""]] \ No newline at end of file From c4dd0e377301b8b2e1f7637bbd68dcad860a0646 Mon Sep 17 00:00:00 2001 From: "http://batchyx.myopenid.com/" Date: Sat, 5 Mar 2011 07:30:19 +0000 Subject: [PATCH 0972/2835] initial bug creation. --- ...t_choke_when_remote_is_an_ssh_url_with_a_port.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn diff --git a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn new file mode 100644 index 0000000000..031956b9a4 --- /dev/null +++ b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn @@ -0,0 +1,11 @@ +when i want to + + git annex get file + +on repo ssh://host-without-port/annex, it works, but if i want to get a file from ssh://host:5122/annex, it tries to run command +ssh ["host:5122", "git-annex-shell 'configlist' '/annex/file'"] and fails. ssh needs the -p option to set the default port. +this is confusing because git can handle this url correctly, and will happily clone/push/pull to/from these url. + +temporary workaround is to use remote.name.annex-ssh-options, but we need to workaround when doing get and remove the workaround when pushing/cloning. + +if i had more time, i would have learned haskell and provided a patch ;) From 5c4f90b2d0188abf2aa40e1e5f6d3ecc41e5aa5e Mon Sep 17 00:00:00 2001 From: "http://batchyx.myopenid.com/" Date: Sat, 5 Mar 2011 07:33:05 +0000 Subject: [PATCH 0973/2835] clarif --- ...annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn index 031956b9a4..0746417fa2 100644 --- a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn +++ b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn @@ -3,9 +3,9 @@ when i want to git annex get file on repo ssh://host-without-port/annex, it works, but if i want to get a file from ssh://host:5122/annex, it tries to run command -ssh ["host:5122", "git-annex-shell 'configlist' '/annex/file'"] and fails. ssh needs the -p option to set the default port. +ssh ["host:5122", "git-annex-shell 'configlist' '/annex/file'"] and fails. ssh needs the -p option to set the default port, it doesn't support host:port notation. this is confusing because git can handle this url correctly, and will happily clone/push/pull to/from these url. -temporary workaround is to use remote.name.annex-ssh-options, but we need to workaround when doing get and remove the workaround when pushing/cloning. +temporary workaround is to use ssh://host/annex as url and define remote.name.annex-ssh-options to "-p 5122", but we need to use this workaround when doing annex get and undo the workaround when pushing/cloning. if i had more time, i would have learned haskell and provided a patch ;) From acde7a1736fdee58be0af0773da6e2d9e0c2d220 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 15:13:16 -0400 Subject: [PATCH 0974/2835] improve GitRepos functions for pulling apart URL to repo --- Command/Map.hs | 4 ++-- GitRepo.hs | 48 +++++++++++++++++++++++++++++++++--------------- Remotes.hs | 2 +- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Command/Map.hs b/Command/Map.hs index 4d0f900038..b3005e482d 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -161,7 +161,7 @@ absRepo reference r {- Checks if two repos are the same. -} same :: Git.Repo -> Git.Repo -> Bool same a b - | both Git.repoIsSsh = matching Git.urlHostFull && matching Git.workTree + | both Git.repoIsSsh = matching Git.urlAuthority && matching Git.workTree | both Git.repoIsUrl && neither Git.repoIsSsh = matching show | neither Git.repoIsSsh = matching Git.workTree | otherwise = False @@ -210,7 +210,7 @@ tryScan r "git config --list" liftIO $ pipedconfig "ssh" $ map Param $ words sshoptions ++ - [Git.urlHostFull r, sshcmd] + [Git.urlAuthority r, sshcmd] -- First, try sshing and running git config manually, -- only fall back to git-annex-shell configlist if that diff --git a/GitRepo.hs b/GitRepo.hs index ef8ad25baa..a62d765961 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -24,7 +24,9 @@ module GitRepo ( relative, urlPath, urlHost, - urlHostFull, + urlPort, + urlHostUser, + urlAuthority, urlScheme, configGet, configMap, @@ -131,7 +133,7 @@ localToUrl reference r where absurl = urlScheme reference ++ "//" ++ - urlHostFull reference ++ + urlAuthority reference ++ workTree r {- User-visible description of a git repo. -} @@ -235,29 +237,45 @@ relative repo@(Repo { location = Dir d }) file = do Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo relative repo _ = assertLocal repo $ error "internal" +{- Path of an URL repo. -} +urlPath :: Repo -> String +urlPath Repo { location = Url u } = uriPath u +urlPath repo = assertUrl repo $ error "internal" + {- Scheme of an URL repo. -} urlScheme :: Repo -> String urlScheme Repo { location = Url u } = uriScheme u urlScheme repo = assertUrl repo $ error "internal" -{- Hostname of an URL repo. (May include a username and/or port too.) -} +{- Hostname of an URL repo. -} urlHost :: Repo -> String -urlHost Repo { location = Url u } = uriRegName a +urlHost = urlAuthPart uriRegName + +{- Port of an URL repo, if it has a nonstandard one. -} +urlPort :: Repo -> Maybe Integer +urlPort r = + case urlAuthPart uriPort r of + ":" -> Nothing + (':':p) -> Just (read p) + _ -> Nothing + +{- Hostname of an URL repo, including any username (ie, "user@host") -} +urlHostUser :: Repo -> String +urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName r + +{- The full authority portion an URL repo. (ie, "user@host:port") -} +urlAuthority :: Repo -> String +urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a where a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) -urlHost repo = assertUrl repo $ error "internal" +urlAuthority repo = assertUrl repo $ error "internal" -{- Full hostname of an URL repo. (May include a username and/or port too.) -} -urlHostFull :: Repo -> String -urlHostFull Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a +{- Applies a function to extract part of the uriAuthority of an URL repo. -} +urlAuthPart :: (URIAuth -> a) -> Repo -> a +urlAuthPart a Repo { location = Url u } = a auth where - a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) -urlHostFull repo = assertUrl repo $ error "internal" - -{- Path of an URL repo. -} -urlPath :: Repo -> String -urlPath Repo { location = Url u } = uriPath u -urlPath repo = assertUrl repo $ error "internal" + auth = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) +urlAuthPart _ repo = assertUrl repo $ error "internal" {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: Repo -> [CommandParam] -> [CommandParam] diff --git a/Remotes.hs b/Remotes.hs index a7d6be67df..aeaa5874f3 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -318,7 +318,7 @@ git_annex_shell r command params | Git.repoIsSsh r = do sshoptions <- repoConfig r "ssh-options" "" return $ Just ("ssh", map Param (words sshoptions) ++ - [Param (Git.urlHostFull r), Param sshcmd]) + [Param (Git.urlAuthority r), Param sshcmd]) | otherwise = return Nothing where dir = Git.workTree r From aad1372880ba32f1161a0d05422008cba38bb412 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 15:31:46 -0400 Subject: [PATCH 0975/2835] move repoConfig out of Remotes --- Annex.hs | 15 ++++++++++++++- Command/Map.hs | 2 +- Remotes.hs | 25 ++++++------------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Annex.hs b/Annex.hs index 62e7e023d5..dd3362b29d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -16,10 +16,12 @@ module Annex ( gitRepo, queue, queueRun, - setConfig + setConfig, + repoConfig ) where import Control.Monad.State +import Data.Maybe import qualified GitRepo as Git import qualified GitQueue @@ -115,3 +117,14 @@ setConfig k value = do -- re-read git config and update the repo's state g' <- liftIO $ Git.configRead g Annex.changeState $ \s -> s { Annex.repo = g' } + +{- Looks up a per-remote config option in git config. + - Failing that, tries looking for a global config option. -} +repoConfig :: Git.Repo -> String -> String -> Annex String +repoConfig r key def = do + g <- Annex.gitRepo + let def' = Git.configGet g global def + return $ Git.configGet g local def' + where + local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key + global = "annex." ++ key diff --git a/Command/Map.hs b/Command/Map.hs index b3005e482d..fbc48392ae 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -204,7 +204,7 @@ tryScan r configlist = Remotes.onRemote r (pipedconfig, Nothing) "configlist" [] manualconfiglist = do - sshoptions <- Remotes.repoConfig r "ssh-options" "" + sshoptions <- Annex.repoConfig r "ssh-options" "" let sshcmd = "cd " ++ shellEscape(Git.workTree r) ++ " && " ++ "git config --list" diff --git a/Remotes.hs b/Remotes.hs index aeaa5874f3..faee8ace5c 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -15,8 +15,7 @@ module Remotes ( byName, copyFromRemote, copyToRemote, - onRemote, - repoConfig + onRemote ) where import Control.Exception.Extensible @@ -26,7 +25,6 @@ import Data.String.Utils import System.Cmd.Utils import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) -import Data.Maybe import Types import qualified GitRepo as Git @@ -182,7 +180,7 @@ reposByCost l = do -} repoCost :: Git.Repo -> Annex Int repoCost r = do - cost <- repoConfig r "cost" "" + cost <- Annex.repoConfig r "cost" "" if not $ null cost then return $ read cost else if Git.repoIsUrl r @@ -194,7 +192,7 @@ repoCost r = do - annex-ignore. -} repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do - ignored <- repoConfig r "ignore" "false" + ignored <- Annex.repoConfig r "ignore" "false" to <- match Annex.toremote from <- match Annex.fromremote if to || from @@ -282,7 +280,7 @@ rsyncParams r sending key file = do ] -- Convert the ssh command into rsync command line. let eparam = rsyncShell (Param shellcmd:shellparams) - o <- repoConfig r "rsync-options" "" + o <- Annex.repoConfig r "rsync-options" "" let base = options ++ map Param (words o) ++ eparam if sending then return $ base ++ [dummy, File file] @@ -316,9 +314,9 @@ git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePat git_annex_shell r command params | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) | Git.repoIsSsh r = do - sshoptions <- repoConfig r "ssh-options" "" + sshoptions <- Annex.repoConfig r "ssh-options" "" return $ Just ("ssh", map Param (words sshoptions) ++ - [Param (Git.urlAuthority r), Param sshcmd]) + [Param (Git.urlHostUser r), Param sshcmd]) | otherwise = return Nothing where dir = Git.workTree r @@ -326,14 +324,3 @@ git_annex_shell r command params shellopts = (Param command):(File dir):params sshcmd = shellcmd ++ " " ++ unwords (map shellEscape $ toCommand shellopts) - -{- Looks up a per-remote config option in git config. - - Failing that, tries looking for a global config option. -} -repoConfig :: Git.Repo -> String -> String -> Annex String -repoConfig r key def = do - g <- Annex.gitRepo - let def' = Git.configGet g global def - return $ Git.configGet g local def' - where - local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key - global = "annex." ++ key From 6c1607ce66fb456880495d9026fa368ad48eda3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 15:47:00 -0400 Subject: [PATCH 0976/2835] Support ssh remotes with a port specified. --- Command/Map.hs | 7 +++-- Remotes.hs | 6 ++--- Ssh.hs | 26 +++++++++++++++++++ debian/changelog | 6 +++++ ...when_remote_is_an_ssh_url_with_a_port.mdwn | 2 ++ 5 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 Ssh.hs diff --git a/Command/Map.hs b/Command/Map.hs index fbc48392ae..6c3e0b3df5 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -22,6 +22,7 @@ import Types import Utility import UUID import Trust +import Ssh import qualified Dot -- a link from the first repository to the second (its remote) @@ -204,13 +205,11 @@ tryScan r configlist = Remotes.onRemote r (pipedconfig, Nothing) "configlist" [] manualconfiglist = do - sshoptions <- Annex.repoConfig r "ssh-options" "" let sshcmd = "cd " ++ shellEscape(Git.workTree r) ++ " && " ++ "git config --list" - liftIO $ pipedconfig "ssh" $ map Param $ - words sshoptions ++ - [Git.urlAuthority r, sshcmd] + sshparams <- sshToRepo r [Param sshcmd] + liftIO $ pipedconfig "ssh" sshparams -- First, try sshing and running git config manually, -- only fall back to git-annex-shell configlist if that diff --git a/Remotes.hs b/Remotes.hs index faee8ace5c..3c9db314c8 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -38,6 +38,7 @@ import qualified Content import Messages import CopyFile import RsyncFile +import Ssh {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -314,9 +315,8 @@ git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePat git_annex_shell r command params | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) | Git.repoIsSsh r = do - sshoptions <- Annex.repoConfig r "ssh-options" "" - return $ Just ("ssh", map Param (words sshoptions) ++ - [Param (Git.urlHostUser r), Param sshcmd]) + sshparams <- sshToRepo r [Param sshcmd] + return $ Just ("ssh", sshparams) | otherwise = return Nothing where dir = Git.workTree r diff --git a/Ssh.hs b/Ssh.hs new file mode 100644 index 0000000000..04cd9bec83 --- /dev/null +++ b/Ssh.hs @@ -0,0 +1,26 @@ +{- git-annex repository access with ssh + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Ssh where + +import qualified Annex +import qualified GitRepo as Git +import Utility +import Types + +{- Generates parameters to ssh to a repository's host and run a command. + - Caller is responsible for doing any neccessary shellEscaping of the + - passed command. -} +sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam] +sshToRepo repo sshcmd = do + s <- Annex.repoConfig repo "ssh-options" "" + let sshoptions = map Param (words s) + let sshport = case Git.urlPort repo of + Nothing -> [] + Just p -> [Param "-p", Param (show p)] + let sshhost = Param $ Git.urlHostUser repo + return $ sshoptions ++ sshport ++ [sshhost] ++ sshcmd diff --git a/debian/changelog b/debian/changelog index 56f3bbdaae..6d77caf4cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.23) UNRELEASED; urgency=low + + * Support ssh remotes with a port specified. + + -- Joey Hess Sat, 05 Mar 2011 15:39:13 -0400 + git-annex (0.22) unstable; urgency=low * Git annexes can now be attached to bare git repositories. diff --git a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn index 0746417fa2..92cc9170f9 100644 --- a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn +++ b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn @@ -9,3 +9,5 @@ this is confusing because git can handle this url correctly, and will happily cl temporary workaround is to use ssh://host/annex as url and define remote.name.annex-ssh-options to "-p 5122", but we need to use this workaround when doing annex get and undo the workaround when pushing/cloning. if i had more time, i would have learned haskell and provided a patch ;) + +> Fixed in git! --[[Joey]] [[done]] From 598027d82489aded053154f7d8f7b5dcf4d79ebb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 15:58:57 -0400 Subject: [PATCH 0977/2835] update --- doc/future_proofing.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn index 4d4939b5a9..d9a36ff733 100644 --- a/doc/future_proofing.mdwn +++ b/doc/future_proofing.mdwn @@ -11,6 +11,9 @@ problem: files from the repository; they are there on disk in the usual way, with just some symlinks pointing at the annexed file contents. Neither git-annex nor git is needed to get at the file contents. + + (Also, git-annex provides an "uninit" command that moves everything out + of the annex, if you should ever want to stop using it.) * What file formats are used? Will they still be readable? To deal with this, it's best to stick to plain text files, and the most common From 41d5c4acf6810966bdef25e2fc7a4cfb78b6f972 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 17:05:19 -0400 Subject: [PATCH 0978/2835] shorten a help string to avoid column getting too wide --- Command.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command.hs b/Command.hs index 09adc09491..eba7f2cef5 100644 --- a/Command.hs +++ b/Command.hs @@ -216,7 +216,7 @@ paramPath = "PATH" paramKey :: String paramKey = "KEY" paramDesc :: String -paramDesc = "DESCRIPTION" +paramDesc = "DESC" paramNumber :: String paramNumber = "NUMBER" paramRemote :: String From 0de3005c648400e67ce4bfe88ac7999e56e3b56e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 17:23:55 -0400 Subject: [PATCH 0979/2835] whereis: New subcommand to show where a file's content has gotten to. --- Command/Whereis.hs | 41 +++++++++++++++++++++++++++++++++++++++++ GitAnnex.hs | 2 ++ debian/changelog | 1 + doc/git-annex.mdwn | 5 +++++ 4 files changed, 49 insertions(+) create mode 100644 Command/Whereis.hs diff --git a/Command/Whereis.hs b/Command/Whereis.hs new file mode 100644 index 0000000000..de51923855 --- /dev/null +++ b/Command/Whereis.hs @@ -0,0 +1,41 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Whereis where + +import Control.Monad.State (liftIO) + +import qualified Annex +import LocationLog +import Command +import Content +import Messages +import UUID +import Types + +command :: [Command] +command = [Command "whereis" (paramOptional $ paramRepeating paramPath) seek + "lists repositories that have file content"] + +seek :: [CommandSeek] +seek = [withFilesInGit start] + +start :: CommandStartString +start file = isAnnexed file $ \(key, _) -> do + showStart "whereis" file + return $ Just $ perform key + +perform :: Key -> CommandPerform +perform key = do + g <- Annex.gitRepo + uuids <- liftIO $ keyLocations g key + pp <- prettyPrintUUIDs uuids + showLongNote $ pp + showProgress + if null $ uuids + then return Nothing + else return $ Just $ return True diff --git a/GitAnnex.hs b/GitAnnex.hs index b26714a598..da91f6e74e 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -34,6 +34,7 @@ import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit import qualified Command.Find +import qualified Command.Whereis import qualified Command.Migrate import qualified Command.Uninit import qualified Command.Trust @@ -66,6 +67,7 @@ cmds = concat , Command.Unused.command , Command.DropUnused.command , Command.Find.command + , Command.Whereis.command , Command.Migrate.command , Command.Map.command ] diff --git a/debian/changelog b/debian/changelog index 6d77caf4cb..e8b094607c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.23) UNRELEASED; urgency=low * Support ssh remotes with a port specified. + * whereis: New subcommand to show where a file's content has gotten to. -- Joey Hess Sat, 05 Mar 2011 15:39:13 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 76f1a577b9..4998a64911 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -154,6 +154,11 @@ Many git-annex commands will stage changes for later `git commit` by you. With no parameters, defaults to finding all files in the current directory and its subdirectories. +* whereis [path ...] + + Displays a list of repositories known to contain the content of the + specified file or files. + * migrate [path ...] Changes the specified annexed files to store their content in the From 80fdfdb72bde67977da950f70a7a058f53322ba1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 17:33:57 -0400 Subject: [PATCH 0980/2835] note current repo when prettifying uuis list --- UUID.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/UUID.hs b/UUID.hs index 67cba30313..239d373f14 100644 --- a/UUID.hs +++ b/UUID.hs @@ -106,13 +106,17 @@ reposWithoutUUID repos uuids = filterM unmatch repos {- Pretty-prints a list of UUIDs -} prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = do + g <- Annex.gitRepo + here <- getUUID g m <- uuidMap - return $ unwords $ map (\u -> "\t" ++ prettify m u ++ "\n") uuids + return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids where - prettify m u = - if not $ null $ findlog m u - then u ++ " -- " ++ findlog m u - else u + prettify m u here = base ++ ishere + where + base = if not $ null $ findlog m u + then u ++ " -- " ++ findlog m u + else u + ishere = if here == u then " <-- here" else "" findlog m u = M.findWithDefault "" u m {- Records a description for a uuid in the uuidLog. -} From ef92bd2b0bc2a85f42594e92d295230421186b72 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 17:41:36 -0400 Subject: [PATCH 0981/2835] add copy count --- Command/Whereis.hs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Command/Whereis.hs b/Command/Whereis.hs index de51923855..5b0bcbbd26 100644 --- a/Command/Whereis.hs +++ b/Command/Whereis.hs @@ -12,7 +12,6 @@ import Control.Monad.State (liftIO) import qualified Annex import LocationLog import Command -import Content import Messages import UUID import Types @@ -33,9 +32,15 @@ perform :: Key -> CommandPerform perform key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key - pp <- prettyPrintUUIDs uuids - showLongNote $ pp - showProgress + let num = length uuids + showNote $ show num ++ " " ++ copiesplural num if null $ uuids then return Nothing - else return $ Just $ return True + else do + pp <- prettyPrintUUIDs uuids + showLongNote $ pp + showProgress + return $ Just $ return True + where + copiesplural 1 = "copy" + copiesplural _ = "copies" From e082bc9aab5dc7424f2a8e79397681af96603555 Mon Sep 17 00:00:00 2001 From: "http://m-f-k.myopenid.com/" Date: Sun, 6 Mar 2011 02:34:51 +0000 Subject: [PATCH 0982/2835] --- doc/forum/rsync_over_ssh__63__.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/rsync_over_ssh__63__.mdwn diff --git a/doc/forum/rsync_over_ssh__63__.mdwn b/doc/forum/rsync_over_ssh__63__.mdwn new file mode 100644 index 0000000000..dab8e0fe8d --- /dev/null +++ b/doc/forum/rsync_over_ssh__63__.mdwn @@ -0,0 +1 @@ +[Walkthrough](http://git-annex.branchable.com/walkthrough/using_ssh_remotes/) says that when using ssh remotes rsync is used for transfering files. Is rsync used via ssh or unsecure? From cf25bb296459549843eaae1759dce5126c07df0c Mon Sep 17 00:00:00 2001 From: "http://m-f-k.myopenid.com/" Date: Sun, 6 Mar 2011 02:36:05 +0000 Subject: [PATCH 0983/2835] + signature --- doc/forum/rsync_over_ssh__63__.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/forum/rsync_over_ssh__63__.mdwn b/doc/forum/rsync_over_ssh__63__.mdwn index dab8e0fe8d..9c0c9add63 100644 --- a/doc/forum/rsync_over_ssh__63__.mdwn +++ b/doc/forum/rsync_over_ssh__63__.mdwn @@ -1 +1,2 @@ [Walkthrough](http://git-annex.branchable.com/walkthrough/using_ssh_remotes/) says that when using ssh remotes rsync is used for transfering files. Is rsync used via ssh or unsecure? +-- Michael K. From 8617847ecaf924ab00f53e3e7d2be6badcdf1067 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 6 Mar 2011 15:59:37 +0000 Subject: [PATCH 0984/2835] Added a comment --- .../comment_1_ee21f32e90303e20339e0a568321bbbe._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment diff --git a/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment b/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment new file mode 100644 index 0000000000..2b9fc9552d --- /dev/null +++ b/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-06T15:59:37Z" + content=""" +Everything is done over ssh unless both repos are on the same system (or unless you NFS mount a repo) +"""]] From 9c98901f1622e9d70b5279d7e886d53d59b6220b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 6 Mar 2011 12:22:56 -0400 Subject: [PATCH 0985/2835] split out summary and inline raw --- doc/index.mdwn | 13 +------------ doc/summary.mdwn | 12 ++++++++++++ 2 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 doc/summary.mdwn diff --git a/doc/index.mdwn b/doc/index.mdwn index 0838505e84..47682349ff 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -1,15 +1,4 @@ -git-annex allows managing files with git, without checking the file -contents into git. While that may seem paradoxical, it is useful when -dealing with files larger than git can currently easily handle, whether due -to limitations in memory, checksumming time, or disk space. - -Even without file content tracking, being able to manage files with git, -move files around and delete files with versioned directory trees, and use -branches and distributed clones, are all very handy reasons to use git. And -annexed files can co-exist in the same git repository with regularly -versioned files, which is convenient for maintaining documents, Makefiles, -etc that are associated with annexed files but that benefit from full -revision control. +[[!inline raw=yes pages="summary"]] To get a feel for it, see the [[walkthrough]]. diff --git a/doc/summary.mdwn b/doc/summary.mdwn new file mode 100644 index 0000000000..458eaab56d --- /dev/null +++ b/doc/summary.mdwn @@ -0,0 +1,12 @@ +git-annex allows managing files with git, without checking the file +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. + +Even without file content tracking, being able to manage files with git, +move files around and delete files with versioned directory trees, and use +branches and distributed clones, are all very handy reasons to use git. And +annexed files can co-exist in the same git repository with regularly +versioned files, which is convenient for maintaining documents, Makefiles, +etc that are associated with annexed files but that benefit from full +revision control. From 232553af01f4194c9e0e2f3f603740aa1e1c16cc Mon Sep 17 00:00:00 2001 From: "http://m-f-k.myopenid.com/" Date: Sun, 6 Mar 2011 16:32:16 +0000 Subject: [PATCH 0986/2835] be more clear about the rsync usage --- doc/walkthrough/using_ssh_remotes.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn index 831746ac0e..6af9e1f477 100644 --- a/doc/walkthrough/using_ssh_remotes.mdwn +++ b/doc/walkthrough/using_ssh_remotes.mdwn @@ -9,7 +9,7 @@ to clone the laptop's annex to it: # cd ~/annex # git annex init "my desktop" -Now you can get files and they will be transferred (using `rsync`): +Now you can get files and they will be transferred (using `rsync` via `ssh`): # git annex get my_cool_big_file get my_cool_big_file (getting UUID for origin...) (copying from origin...) From b945beede9f3e88efb55283da19ec8fb40117ff2 Mon Sep 17 00:00:00 2001 From: "http://m-f-k.myopenid.com/" Date: Sun, 6 Mar 2011 16:33:31 +0000 Subject: [PATCH 0987/2835] Added a comment --- .../comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment diff --git a/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment b/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment new file mode 100644 index 0000000000..49003937b6 --- /dev/null +++ b/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://m-f-k.myopenid.com/" + ip="92.194.43.135" + subject="comment 2" + date="2011-03-06T16:33:19Z" + content=""" +Great! This was the only thing about git-annex which could have kept me from using it. --Michael +"""]] From 2deb35e4b03f21d33a56b437bdcb7715f369f50b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA" Date: Mon, 7 Mar 2011 17:21:54 +0000 Subject: [PATCH 0988/2835] --- .../Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn new file mode 100644 index 0000000000..f458ba72e0 --- /dev/null +++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn @@ -0,0 +1,3 @@ +FAT32 does not support symlinks, so I wonder if there's going to be a problem with that. + +Generally speaking, I am wondering about portability of git annex on windows and on android... From ea4932d2bfe2c99bd120fd810a15e6c3952ceb6b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 7 Mar 2011 19:13:14 +0000 Subject: [PATCH 0989/2835] Added a comment --- .../comment_1_426482e6eb3a27687a48f24f6ef2332f._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment new file mode 100644 index 0000000000..7b9935f7cc --- /dev/null +++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-07T19:13:14Z" + content=""" +See [[fat_support]]. A bare git repo will have to be used to avoid symlink problems, at least for now. The other problem is that git-annex key files have colons in their filenames. +"""]] From cd02f05eb11fd6a8eff345c76deeed6c378cc2af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 7 Mar 2011 15:16:53 -0400 Subject: [PATCH 0990/2835] fix --- .../comment_1_426482e6eb3a27687a48f24f6ef2332f._comment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment index 7b9935f7cc..119c9e535a 100644 --- a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment +++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment @@ -4,5 +4,5 @@ subject="comment 1" date="2011-03-07T19:13:14Z" content=""" -See [[fat_support]]. A bare git repo will have to be used to avoid symlink problems, at least for now. The other problem is that git-annex key files have colons in their filenames. +See [[bugs/fat_support]]. A bare git repo will have to be used to avoid symlink problems, at least for now. The other problem is that git-annex key files have colons in their filenames. """]] From 4cfcdfa9ba1f88e7aa9a6e7878cd86a2705541ab Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 14:20:11 -0400 Subject: [PATCH 0991/2835] mention mercurial bfiles extn --- doc/not.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index fe6e1b37d9..1d38b848c6 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -38,3 +38,9 @@ it does not support keeping different files in different clones of the same repository, which git-annex does, and is an important feature for large-scale archiving. + +* git-annex is not the [Mercurial bfiles extension](http://mercurial.selenic.com/wiki/BfilesExtension). + Although mercurial and git have some of the same problems around large + files, and both try to solve them in similar ways (standin files using + mostly hashes of the real content). + From b85c91743c73597f41cae73b042ffa86be929ed3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 17:48:25 -0400 Subject: [PATCH 0992/2835] update --- doc/bugs/unhappy_without_UTF8_locale.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/bugs/unhappy_without_UTF8_locale.mdwn b/doc/bugs/unhappy_without_UTF8_locale.mdwn index 6f1df4fab7..7b0e5be661 100644 --- a/doc/bugs/unhappy_without_UTF8_locale.mdwn +++ b/doc/bugs/unhappy_without_UTF8_locale.mdwn @@ -4,6 +4,12 @@ Try unsetting LANG and passing git-annex unicode filenames. add add add add git-annex: : commitAndReleaseBuffer: invalid argument (Invalid or incomplete multibyte or wide character) +> Interestingly, I can get the same crash in the de_DE.UTF-8 locale +> with certian input filenames, while in en_US.UTF-8, it's ok. +> The workaround below avoided the problem in de_DE.UTF-8. --[[Joey]] + +## underlying haskell problem and workaround + The same problem can be seen with a simple haskell program: import System.Environment From 26544de9463291b8185fdd1a7c1b33710ef7db3c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 18:05:20 -0400 Subject: [PATCH 0993/2835] put in utf8 forcing workaround Haskell's IO layer crashes on characters > 255 when in a non-unicode (latin1) locale. Until Haskell gets better behavior, put in an admittedly ugly workaround for that: git-annex forces utf8 output mode no matter what locale is selected. So if you use a non-utf8 locale, your filenames with characters > 127 will not be displayed as you'd expect. But at least it won't crash. --- CmdLine.hs | 2 ++ Messages.hs | 9 +++++++++ debian/changelog | 6 ++++++ doc/bugs/unhappy_without_UTF8_locale.mdwn | 2 ++ git-annex.hs | 1 + 5 files changed, 20 insertions(+) diff --git a/CmdLine.hs b/CmdLine.hs index 475ca99e78..1c01aa75f6 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -11,6 +11,7 @@ module CmdLine ( shutdown ) where +import System.IO import System.IO.Error (try) import System.Console.GetOpt import Control.Monad.State (liftIO) @@ -30,6 +31,7 @@ import UUID {- Runs the passed command line. -} dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO () dispatch gitrepo args cmds options header = do + forceUtf8 state <- Annex.new gitrepo allBackends (actions, state') <- Annex.run state $ parseCmd args header cmds options tryRun state' $ [startup, upgrade] ++ actions ++ [shutdown] diff --git a/Messages.hs b/Messages.hs index 90857280a5..83b3ecf239 100644 --- a/Messages.hs +++ b/Messages.hs @@ -64,3 +64,12 @@ indent s = join "\n" $ map (\l -> " " ++ l) $ lines s - non-decoded form. -} filePathToString :: FilePath -> String filePathToString = if SysConfig.unicodefilepath then id else UTF8.decodeString + +{- Workaround to avoid crashes displaying filenames containing + - characters > 255 in non-utf8 locales. Force encodings to utf-8, + - even though this may mean some characters in the encoding + - are mangled. -} +forceUtf8 :: IO () +forceUtf8 = do + hSetEncoding stdout utf8 + hSetEncoding stderr utf8 diff --git a/debian/changelog b/debian/changelog index e8b094607c..a414b3befa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,12 @@ git-annex (0.23) UNRELEASED; urgency=low * Support ssh remotes with a port specified. * whereis: New subcommand to show where a file's content has gotten to. + * Haskell's IO layer crashes on characters > 255 when in a non-unicode + locale. Until Haskell gets better behavior, put in an admittedly + ugly workaround for that: git-annex forces utf8 output mode no matter + what locale is selected. So if you use a non-utf8 locale, your + filenames with characters > 127 will not be displayed as you'd expect. + But at least it won't crash. -- Joey Hess Sat, 05 Mar 2011 15:39:13 -0400 diff --git a/doc/bugs/unhappy_without_UTF8_locale.mdwn b/doc/bugs/unhappy_without_UTF8_locale.mdwn index 7b0e5be661..8d22b9ee44 100644 --- a/doc/bugs/unhappy_without_UTF8_locale.mdwn +++ b/doc/bugs/unhappy_without_UTF8_locale.mdwn @@ -8,6 +8,8 @@ Try unsetting LANG and passing git-annex unicode filenames. > with certian input filenames, while in en_US.UTF-8, it's ok. > The workaround below avoided the problem in de_DE.UTF-8. --[[Joey]] +> Put in the utf-8 forcing workaround for now. [[done]] --[[Joey]] + ## underlying haskell problem and workaround The same problem can be seen with a simple haskell program: diff --git a/git-annex.hs b/git-annex.hs index 878d8bdbbc..9d6012f2c1 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -8,6 +8,7 @@ import System.Environment import GitAnnex + main :: IO () main = do args <- getArgs From 876f0c6fbce125b946aa37eb1c409d6030785d72 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 18:19:52 -0400 Subject: [PATCH 0994/2835] add new todo item --- doc/todo/support-non-utf8-locales.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/todo/support-non-utf8-locales.mdwn diff --git a/doc/todo/support-non-utf8-locales.mdwn b/doc/todo/support-non-utf8-locales.mdwn new file mode 100644 index 0000000000..26f600be4a --- /dev/null +++ b/doc/todo/support-non-utf8-locales.mdwn @@ -0,0 +1,19 @@ +Currenty, git-annex forces output, particularly of filenames, in a utf-8 +locale. + +Note that this does not mean it cannot be used with filenames in other +encodings. It just displays their names always converted to utf-8, which +may not look right when you have a non-utf8 locale. + +This had to be done to work around some bugs with haskell's handling +of filename encodings. In particular, + +* [[bugs/unhappy_without_UTF8_locale]]: haskell crashes when told to output + a string with characters > 255 in a non-utf8 locale. +* [[bugs/problems_with_utf8_names]]: On many OSs, haskell expects + non-decoded raw char8 in FilePaths. In order to display a filename, + though, it needs to first be decoded, and git-annex currently assumes + it was encoded as utf8. + +git-annex's behavior is unlikely to improve much until haskell's +support for utf8 filenames improves. --[[Joey]] From d7b4c8372b3901e09a0268d55b0a567a878166f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 18:23:13 -0400 Subject: [PATCH 0995/2835] update --- doc/todo/support-non-utf8-locales.mdwn | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/todo/support-non-utf8-locales.mdwn b/doc/todo/support-non-utf8-locales.mdwn index 26f600be4a..60f35eec81 100644 --- a/doc/todo/support-non-utf8-locales.mdwn +++ b/doc/todo/support-non-utf8-locales.mdwn @@ -2,8 +2,9 @@ Currenty, git-annex forces output, particularly of filenames, in a utf-8 locale. Note that this does not mean it cannot be used with filenames in other -encodings. It just displays their names always converted to utf-8, which -may not look right when you have a non-utf8 locale. +encodings. git-annex is entirely encoding agnostic when it comes to +manipulating filenames. It just *displays* their names always converted to +utf-8, which may not look right when you have a non-utf8 locale. This had to be done to work around some bugs with haskell's handling of filename encodings. In particular, From 42b7f244060bb9f49f9cbe9e93ee8024a678771d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Mar 2011 01:56:24 -0400 Subject: [PATCH 0996/2835] update --- doc/future_proofing.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn index d9a36ff733..4fc8246b6c 100644 --- a/doc/future_proofing.mdwn +++ b/doc/future_proofing.mdwn @@ -25,3 +25,12 @@ problem: * What is the hardware interface of the drive? Will hardware still exist to talk to it? + +* What if some of the data is damaged? git-annex facilitates storing a + configurable number of [[copies]] of the file contents. The metadata + about your files is stored in git, and so every clone of the repository + means another copy of that is stored. Also, git-annex uses filenames + for the data that encode everything needed to match it back to the + metadata. So if a filesystem is badly corrupted and all your annexed + files end up in `lost+found`, they can easily be lifted back out into + another clone of the repository. From b5134b4716c266147a35353316686cf29658350d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Mar 2011 01:57:30 -0400 Subject: [PATCH 0997/2835] tweak --- doc/use_case/Bob.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 18cdd4d8f4..42d10ea975 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -13,7 +13,7 @@ they're on, and easily make them available. Indeed, every drive knows what is on every other drive. [[more about location tracking|location_tracking]] -Bob thinks long-term, and so he's glad that git-annex uses a simple +Bob thinks long-term, and so he appreciates that git-annex uses a simple repository format. He knows his files will be accessible in the future even if the world has forgotten about git-annex and git. [[more about future-proofing|future_proofing]] From 9229d182d32570f6829ced655aa673ceddfe7693 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Mar 2011 15:59:44 -0400 Subject: [PATCH 0998/2835] update --- doc/future_proofing.mdwn | 3 ++- doc/walkthrough.mdwn | 1 + .../recover_data_from_lost+found.mdwn | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 doc/walkthrough/recover_data_from_lost+found.mdwn diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn index 4fc8246b6c..3937e92656 100644 --- a/doc/future_proofing.mdwn +++ b/doc/future_proofing.mdwn @@ -33,4 +33,5 @@ problem: for the data that encode everything needed to match it back to the metadata. So if a filesystem is badly corrupted and all your annexed files end up in `lost+found`, they can easily be lifted back out into - another clone of the repository. + another clone of the repository. Even if the filenames are lost, + it's possible to [[walkthrough/recover_data_from_lost+found]]. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 1e8ad7e988..3b4f7d56a4 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -23,4 +23,5 @@ A walkthrough of the basic features of git-annex. backups untrusted_repositories what_to_do_when_you_lose_a_repository + recover_data_from_lost+found """]] diff --git a/doc/walkthrough/recover_data_from_lost+found.mdwn b/doc/walkthrough/recover_data_from_lost+found.mdwn new file mode 100644 index 0000000000..6e2c241485 --- /dev/null +++ b/doc/walkthrough/recover_data_from_lost+found.mdwn @@ -0,0 +1,17 @@ +Suppose something goes wrong, and fsck puts all the files in lost+found. +It's actually very easy to recover from this disaster. + +First, check out the git repository again. Then, in the new checkout: + + mkdir recovered-content + sudo mv ../lost+found/* recovered-content + git annex add recovered-content + git rm recovered-content + git commit -m "recovered some content" + git annex fsck + +The way that works is that when git-annex adds the same content that was in +the repository before, all the old links to that content start working +again. This works particularly well if the SHA1 backend is used, but even +with the default backend it will work pretty well, as long as fsck +preserved the modification time of the files. From 4163341c9d09f92f4bd5dfb5d2f34538bee51bdb Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 9 Mar 2011 23:47:49 +0000 Subject: [PATCH 0999/2835] Added a comment: use mini-branches --- ..._0531dcfa833b0321a7009526efe3df33._comment | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment b/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment new file mode 100644 index 0000000000..8fec6bad72 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="use mini-branches" + date="2011-03-09T23:47:48Z" + content=""" +if you go for the two-commits version, small intermediate branches (or git-commit-tree) could be used to create a tree like this: + + + * commit 106eef2 + |\ Merge: 436e46f 9395665 + | | + | | the main commit + | | + | * commit 9395665 + |/ + | intermediate move + | + * commit 436e46f + | + | ... + +while the first commit (436e46f) has a \"`/subdir/foo → ../.git-annex/where_foo_is`\", the intermediate (9395665) has \"`/subdir/deeper/foo → ../.git-annex/where_foo_is`\", and the inal commit (106eef2) has \"`/subdir/deeper/foo → ../../.git-annex/where_foo_is`\". + +`--follow` uses the intermediate commit to find the history, but the intermediate commit would neither show up in `git log --first-parent` nor affect `git diff HEAD^..` & co. (there could still be confusion over `git show`, though). +"""]] From 72d268401604fbac93ca4701ab53d32880483686 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 12 Mar 2011 15:30:17 -0400 Subject: [PATCH 1000/2835] Rethink filename encoding handling for display. Since filename encoding may or may not match locale settings, any attempt to decode filenames will fail for some files. So instead, do all output in binary mode. --- Backend/File.hs | 6 +++--- Backend/SHA.hs | 2 +- Backend/WORM.hs | 2 +- CmdLine.hs | 3 +-- Command/Find.hs | 2 +- Command/PreCommit.hs | 2 +- Command/Unused.hs | 2 +- Content.hs | 2 +- Messages.hs | 30 +++++++++++--------------- debian/changelog | 13 +++++------ doc/bugs/problems_with_utf8_names.mdwn | 3 +++ doc/todo/support-non-utf8-locales.mdwn | 6 ++++++ 12 files changed, 37 insertions(+), 36 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index d5691595a8..d76cd29391 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -193,14 +193,14 @@ checkKeyNumCopies key file numcopies = do missingNote :: String -> Int -> Int -> String -> String missingNote file 0 _ [] = - "** No known copies of " ++ filePathToString file ++ " exist!" + "** No known copies of " ++ file ++ " exist!" missingNote file 0 _ untrusted = - "Only these untrusted locations may have copies of " ++ filePathToString file ++ + "Only these untrusted locations may have copies of " ++ file ++ "\n" ++ untrusted ++ "Back it up to trusted locations with git-annex copy." missingNote file present needed [] = "Only " ++ show present ++ " of " ++ show needed ++ - " trustworthy copies of " ++ filePathToString file ++ " exist." ++ + " trustworthy copies of " ++ file ++ " exist." ++ "\nBack it up with git-annex copy." missingNote file present needed untrusted = missingNote file present needed [] ++ diff --git a/Backend/SHA.hs b/Backend/SHA.hs index c074ab48a2..4eea890ce4 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -83,5 +83,5 @@ checkKeyChecksum size key = do then return True else do dest <- moveBad key - warning $ "Bad file content; moved to " ++ filePathToString dest + warning $ "Bad file content; moved to " ++ dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 8a6412eb11..a0d814aa08 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -70,5 +70,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - warning $ "Bad file size; moved to " ++ filePathToString dest + warning $ "Bad file size; moved to " ++ dest return False diff --git a/CmdLine.hs b/CmdLine.hs index 1c01aa75f6..b8fd6af7ce 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -11,7 +11,6 @@ module CmdLine ( shutdown ) where -import System.IO import System.IO.Error (try) import System.Console.GetOpt import Control.Monad.State (liftIO) @@ -31,7 +30,7 @@ import UUID {- Runs the passed command line. -} dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO () dispatch gitrepo args cmds options header = do - forceUtf8 + setupConsole state <- Annex.new gitrepo allBackends (actions, state') <- Annex.run state $ parseCmd args header cmds options tryRun state' $ [startup, upgrade] ++ actions ++ [shutdown] diff --git a/Command/Find.hs b/Command/Find.hs index 3e9125b9a6..1ca6ff1e7c 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -25,5 +25,5 @@ seek = [withFilesInGit start] start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key - when exists $ liftIO $ putStrLn $ filePathToString file + when exists $ liftIO $ putStrLn file return Nothing diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index d2f6964343..6f9adb79a5 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -34,7 +34,7 @@ perform pair@(file, _) = do ok <- doCommand $ Command.Add.start pair if ok then return $ Just $ cleanup file - else error $ "failed to add " ++ filePathToString file ++ "; canceling commit" + else error $ "failed to add " ++ file ++ "; canceling commit" cleanup :: FilePath -> CommandCleanup cleanup file = do diff --git a/Command/Unused.hs b/Command/Unused.hs index 9f3881d595..a614ce5d94 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -68,7 +68,7 @@ checkUnused = do dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"] table l = [" NUMBER KEY"] ++ map cols l - cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ (filePathToString . show) k + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k pad n s = s ++ replicate (n - length s) ' ' number :: Int -> [a] -> [(Int, a)] diff --git a/Content.hs b/Content.hs index bcd4ac0e13..895a8812c2 100644 --- a/Content.hs +++ b/Content.hs @@ -50,7 +50,7 @@ calcGitLink file key = do cwd <- liftIO $ getCurrentDirectory let absfile = case absNormPath cwd file of Just f -> f - Nothing -> error $ "unable to normalize " ++ filePathToString file + Nothing -> error $ "unable to normalize " ++ file return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ".git" annexLocation key diff --git a/Messages.hs b/Messages.hs index 83b3ecf239..733638ce12 100644 --- a/Messages.hs +++ b/Messages.hs @@ -11,11 +11,9 @@ import Control.Monad.State (liftIO) import System.IO import Control.Monad (unless) import Data.String.Utils -import qualified Codec.Binary.UTF8.String as UTF8 import Types import qualified Annex -import qualified SysConfig verbose :: Annex () -> Annex () verbose a = do @@ -27,7 +25,7 @@ showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")" showStart :: String -> String -> Annex () showStart command file = verbose $ do - liftIO $ putStr $ command ++ " " ++ filePathToString file ++ " " + liftIO $ putStr $ command ++ " " ++ file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () @@ -59,17 +57,15 @@ warning w = do indent :: String -> String indent s = join "\n" $ map (\l -> " " ++ l) $ lines s -{- Prepares a filename for display. This is needed because on many - - platforms (eg, unix), FilePaths are internally stored in - - non-decoded form. -} -filePathToString :: FilePath -> String -filePathToString = if SysConfig.unicodefilepath then id else UTF8.decodeString - -{- Workaround to avoid crashes displaying filenames containing - - characters > 255 in non-utf8 locales. Force encodings to utf-8, - - even though this may mean some characters in the encoding - - are mangled. -} -forceUtf8 :: IO () -forceUtf8 = do - hSetEncoding stdout utf8 - hSetEncoding stderr utf8 +{- By default, haskell honors the user's locale in its output to stdout + - and stderr. While that's great for proper unicode support, for git-annex + - all that's really needed is the ability to display simple messages + - (currently untranslated), and importantly, to display filenames exactly + - as they are written on disk, no matter what their encoding. So, force + - raw mode. + - + - NB: Once git-annex gets localized, this will need a rethink. -} +setupConsole :: IO () +setupConsole = do + hSetBinaryMode stdout True + hSetBinaryMode stderr True diff --git a/debian/changelog b/debian/changelog index a414b3befa..90b4cf6b25 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,15 +1,12 @@ -git-annex (0.23) UNRELEASED; urgency=low +git-annex (0.23) unstable; urgency=low * Support ssh remotes with a port specified. * whereis: New subcommand to show where a file's content has gotten to. - * Haskell's IO layer crashes on characters > 255 when in a non-unicode - locale. Until Haskell gets better behavior, put in an admittedly - ugly workaround for that: git-annex forces utf8 output mode no matter - what locale is selected. So if you use a non-utf8 locale, your - filenames with characters > 127 will not be displayed as you'd expect. - But at least it won't crash. + * Rethink filename encoding handling for display. Since filename encoding + may or may not match locale settings, any attempt to decode filenames + will fail for some files. So instead, do all output in binary mode. - -- Joey Hess Sat, 05 Mar 2011 15:39:13 -0400 + -- Joey Hess Sat, 12 Mar 2011 15:02:49 -0400 git-annex (0.22) unstable; urgency=low diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index efde1c9a3a..d6dc6ca3c3 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -63,6 +63,9 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > One other possible > issue would be that this could cause problems if git-annex were > translated. +> > On second thought, I switched to this. Any decoding of a filename +> > is going to make someone unhappy; the previous approach broke +> > non-utf8 filenames. ---- diff --git a/doc/todo/support-non-utf8-locales.mdwn b/doc/todo/support-non-utf8-locales.mdwn index 60f35eec81..da40118d52 100644 --- a/doc/todo/support-non-utf8-locales.mdwn +++ b/doc/todo/support-non-utf8-locales.mdwn @@ -18,3 +18,9 @@ of filename encodings. In particular, git-annex's behavior is unlikely to improve much until haskell's support for utf8 filenames improves. --[[Joey]] + +> [[done]] -- I just turned off all encoding handling on stdout and stderr, +> which avoids these problems nicely. Git-annex now displays just what it +> input, at least on platforms where haskell does not decode unicode in +> FilePaths. This will later be a problem when it gets localized, but for +> now works great. --[[Joey]] From cf6a13a05710e53a9a1b28b7474a5a31494349a8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 12 Mar 2011 15:41:48 -0400 Subject: [PATCH 1001/2835] add news item for git-annex 0.23 --- doc/news/version_0.18.mdwn | 9 --------- doc/news/version_0.23.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 9 deletions(-) delete mode 100644 doc/news/version_0.18.mdwn create mode 100644 doc/news/version_0.23.mdwn diff --git a/doc/news/version_0.18.mdwn b/doc/news/version_0.18.mdwn deleted file mode 100644 index f2e0edb424..0000000000 --- a/doc/news/version_0.18.mdwn +++ /dev/null @@ -1,9 +0,0 @@ -git-annex 0.18 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Bugfix: `copy --to` and `move --to` forgot to stage location log changes - after transferring the file to the remote repository. - (Did not affect ssh remotes.) - * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ - * migrate: Fix support for --backend option. - * unlock: Fix behavior when file content is not present. - * Test suite improvements. Current top-level test coverage: 80%"""]] \ No newline at end of file diff --git a/doc/news/version_0.23.mdwn b/doc/news/version_0.23.mdwn new file mode 100644 index 0000000000..e045352ad4 --- /dev/null +++ b/doc/news/version_0.23.mdwn @@ -0,0 +1,7 @@ +git-annex 0.23 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Support ssh remotes with a port specified. + * whereis: New subcommand to show where a file's content has gotten to. + * Rethink filename encoding handling for display. Since filename encoding + may or may not match locale settings, any attempt to decode filenames + will fail for some files. So instead, do all output in binary mode."""]] \ No newline at end of file From 175d055d4d632c0fc0d2080c7721bf7b3121ddc1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Mar 2011 14:25:32 -0400 Subject: [PATCH 1002/2835] Add Suggests on graphviz. Closes: #618039 --- Content.hs | 1 - debian/changelog | 6 ++++++ debian/control | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Content.hs b/Content.hs index 895a8812c2..dc675389f1 100644 --- a/Content.hs +++ b/Content.hs @@ -34,7 +34,6 @@ import UUID import qualified GitRepo as Git import qualified Annex import Utility -import Messages {- Checks if a given key is currently present in the gitAnnexLocation. -} inAnnex :: Key -> Annex Bool diff --git a/debian/changelog b/debian/changelog index 90b4cf6b25..6e70ac67e3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.24) UNRELEASED; urgency=low + + * Add Suggests on graphviz. Closes: #618039 + + -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 + git-annex (0.23) unstable; urgency=low * Support ssh remotes with a port specified. diff --git a/debian/control b/debian/control index d0f8805d0f..4b31a295f9 100644 --- a/debian/control +++ b/debian/control @@ -11,6 +11,7 @@ Package: git-annex Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync +Suggests: graphviz Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when From 4ddcf26051f6fed9e846f15bee9e1c02b1b9a9d6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 13 Mar 2011 20:32:12 +0000 Subject: [PATCH 1003/2835] --- doc/bugs/softlink_atime.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/softlink_atime.mdwn diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn new file mode 100644 index 0000000000..ad91cf8a9d --- /dev/null +++ b/doc/bugs/softlink_atime.mdwn @@ -0,0 +1,3 @@ +When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. + +Optionally, editing the meta-data should change the times in all annexes. From 61afcd2d16c27430f1a6c6ae5cd468b16243c966 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Mar 2011 17:42:07 -0400 Subject: [PATCH 1004/2835] response --- doc/bugs/softlink_atime.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index ad91cf8a9d..e60b8ec98b 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -1,3 +1,10 @@ When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. Optionally, editing the meta-data should change the times in all annexes. + +> Thing is, git does not preserve file timestamps much at all. +> It's not uncommon for a `git checkout` to or `git update` to +> mess up timestamps. This is why things like metastore exist (and +> metastore should work ok with git annexed files too). Trying to +> make annexed file symlinks have better timestamp handling than regular +> files in git seems pointless. --[[Joey]] From 7dd8cde63c6e41c78a06f0f03bd555cfa780864f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkbfH595UQMdPtWbUKZtjsr-nR6AR-cxek" Date: Mon, 14 Mar 2011 08:10:08 +0000 Subject: [PATCH 1005/2835] thoughts? --- doc/forum/hashing_objects_directories.mdwn | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/forum/hashing_objects_directories.mdwn diff --git a/doc/forum/hashing_objects_directories.mdwn b/doc/forum/hashing_objects_directories.mdwn new file mode 100644 index 0000000000..4e5f4b1c98 --- /dev/null +++ b/doc/forum/hashing_objects_directories.mdwn @@ -0,0 +1,17 @@ +I'm wondering how easy the addition of hashing to the directories of the objects would be. + +Currently a tree directory structure becomes a flat two level tree under the .git/annex/objects directory ([[internals]]). This, through the 555 mode on the directory prevents the accidental destruction of content, which is _good_. However file and directory numbers soon add up in there and as such any file-systems with sub directory limitations will quickly realize the limit (certainly quicker than maybe expected). + +Suggestion is therefore to change from + + `.git/annex/objects/SHA1:123456789abcdef0123456789abcdef012345678/SHA1:123456789abcdef0123456789abcdef012345678` + +to + + `.git/annex/objects/SHA1:1/2/3456789abcdef0123456789abcdef012345678/SHA1:123456789abcdef0123456789abcdef012345678` + +or anything in between to a paranoid + + `.git/annex/objects/SHA1:123/456/789/abc/def/012/345/678/9ab/cde/f01/234/5678/SHA1:123456789abcdef0123456789abcdef012345678` + +Also the use of a colon specifically breaks FAT32 ([[bugs/fat_support]]), must it be a colon or could an extra directory be used? i.e. `.git/annex/objects/SHA1/*/...` From 1033b4a555df340dcf80ce2e614540263a728fdb Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkbfH595UQMdPtWbUKZtjsr-nR6AR-cxek" Date: Mon, 14 Mar 2011 09:40:08 +0000 Subject: [PATCH 1006/2835] init --- doc/forum/hashing_objects_directories.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/forum/hashing_objects_directories.mdwn b/doc/forum/hashing_objects_directories.mdwn index 4e5f4b1c98..715e972ca4 100644 --- a/doc/forum/hashing_objects_directories.mdwn +++ b/doc/forum/hashing_objects_directories.mdwn @@ -15,3 +15,5 @@ or anything in between to a paranoid `.git/annex/objects/SHA1:123/456/789/abc/def/012/345/678/9ab/cde/f01/234/5678/SHA1:123456789abcdef0123456789abcdef012345678` Also the use of a colon specifically breaks FAT32 ([[bugs/fat_support]]), must it be a colon or could an extra directory be used? i.e. `.git/annex/objects/SHA1/*/...` + +`git annex init` could also create all but the last level directory on initialization. I'm thinking `SHA1/1/1, SHA1/1/2, ..., SHA256/f/f, ..., URL/f/f, ..., WORM/f/f` From 5aac014a677ff636f61f4ccc70cc61bf920a14c6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 13:47:38 +0000 Subject: [PATCH 1007/2835] --- doc/bugs/softlink_atime.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index e60b8ec98b..ab13863f1a 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -1,4 +1,4 @@ -When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. +When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. -- RichiH Optionally, editing the meta-data should change the times in all annexes. @@ -8,3 +8,5 @@ Optionally, editing the meta-data should change the times in all annexes. > metastore should work ok with git annexed files too). Trying to > make annexed file symlinks have better timestamp handling than regular > files in git seems pointless. --[[Joey]] + +> > Improving an area where git is (not yet?) good at still makes sense, imo. Photos and the like need absolute timestamps more than source code which is fine with relative timestamps (local builds & updates). Maintaining global timestamps for source code could even cause a lot of unwanted effects. As it is, this issue is the only, but a major, blocker for me before I can start adapting git-annex. As I have three different use cases for it, this is a shame. Unfortunately, I don't speak any Haskell so scratching my own itch isn't do-able (without major effort and not soon, at least). Is there a realistic chance that you will tackle this nonetheless or is this WONTFIX? -- RichiH From d86357a4166e9fdfd45ea2b47d62e3c09ced13fd Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 13:52:07 +0000 Subject: [PATCH 1008/2835] --- doc/todo/add_a_git_backend.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index 91a5001cc9..6020db86dc 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -4,3 +4,5 @@ repository! This way, you know your annexed content is safe & versioned, but you only have to deal with the pain of git with large files in one place, and can use all of git-annex's features everywhere else. + +> Speaking as a future user, do very, very much want. -- RichiH From 9b353aa60e3828d354868d2024b858b7155622be Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 13:54:07 +0000 Subject: [PATCH 1009/2835] --- doc/todo/auto_remotes.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn index 2ac4790af9..c12878950a 100644 --- a/doc/todo/auto_remotes.mdwn +++ b/doc/todo/auto_remotes.mdwn @@ -19,3 +19,5 @@ otherwise ssh:// ones.) Question: When should git-annex update the remote.log? (If not just on init.) Whenever it reads in a repo's remotes? + +> This sounds useful and the log should be updated every time any remote is being accessed. A counter or timestamp (yes, distributed times may be wrong/different) could be used to auto-prune old entries via a global and per-remote config setting. -- RichiH From d7ea568f50026ea581b0c91ae97495a4e89609b2 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 14:02:47 +0000 Subject: [PATCH 1010/2835] --- doc/todo/hidden_files.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn index 2e5ec4d9e8..14c4f0da5a 100644 --- a/doc/todo/hidden_files.mdwn +++ b/doc/todo/hidden_files.mdwn @@ -18,3 +18,7 @@ TODO: * What will `git annex get` do if it's asked to get a file that has been hidden? + +> Unless I am missing something: Make sure the data is correct (for SHA1 or other tracking) and restore locally. If that's not the case, delete and restore from remote. -- RichiH + +* Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH From 1a2c9b61f748760176bb61193b2aec2babf0e169 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:08:38 -0400 Subject: [PATCH 1011/2835] note --- doc/bugs/fat_support.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn index 2a7187a7f6..8088e965a7 100644 --- a/doc/bugs/fat_support.mdwn +++ b/doc/bugs/fat_support.mdwn @@ -8,4 +8,8 @@ be VFAT formatted: - Use of ":" in filenames of object files, also not supported. Could easily be fixed by reorganizing the object directory. + (If the object directory is reorganized, should consider adding hashing + at the same time.) + [[!tag wishlist]] + From 769d2a780d33fda9dd6e3b4242d31edf5ff2e232 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 14 Mar 2011 16:12:49 +0000 Subject: [PATCH 1012/2835] Added a comment --- ...mment_1_c55c56076be4f54251b0b7f79f28a607._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment diff --git a/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment b/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment new file mode 100644 index 0000000000..3a19310b63 --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-14T16:12:49Z" + content=""" +My experience is that modern filesystems are not going to have many issues with tens to hundreds of thousands of items in the directory. However, if a transition does happen for FAT support I will consider adding hashing. Although getting a good balanced hash in general without, say, checksumming the filename and taking part of the checksum, is difficult. + +I prefer to keep all the metadata in the filename, as this eases recovery if the files end up in lost+found. So while \"SHA/\" is a nice workaround for the FAT colon problem, I'll be doing something else. (What I'm not sure yet.) + +There is no point in creating unused hash directories on initialization. If anything, with a bad filesystem that just guarantees worst performance from the beginning.. +"""]] From 5e9f90240367e9b1006402fe776c24f2488876b2 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 14 Mar 2011 16:12:51 +0000 Subject: [PATCH 1013/2835] Added a comment --- ...mment_2_d095c769febedca2d69fa5d099bcb520._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment diff --git a/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment b/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment new file mode 100644 index 0000000000..41c6ad5d96 --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-03-14T16:12:51Z" + content=""" +My experience is that modern filesystems are not going to have many issues with tens to hundreds of thousands of items in the directory. However, if a transition does happen for FAT support I will consider adding hashing. Although getting a good balanced hash in general without, say, checksumming the filename and taking part of the checksum, is difficult. + +I prefer to keep all the metadata in the filename, as this eases recovery if the files end up in lost+found. So while \"SHA/\" is a nice workaround for the FAT colon problem, I'll be doing something else. (What I'm not sure yet.) + +There is no point in creating unused hash directories on initialization. If anything, with a bad filesystem that just guarantees worst performance from the beginning.. +"""]] From f130c413a9c8d82b7813e001c7e561609cce4caa Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 14 Mar 2011 16:13:30 +0000 Subject: [PATCH 1014/2835] removed --- ...mment_2_d095c769febedca2d69fa5d099bcb520._comment | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment diff --git a/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment b/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment deleted file mode 100644 index 41c6ad5d96..0000000000 --- a/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment +++ /dev/null @@ -1,12 +0,0 @@ -[[!comment format=mdwn - username="http://joey.kitenet.net/" - nickname="joey" - subject="comment 2" - date="2011-03-14T16:12:51Z" - content=""" -My experience is that modern filesystems are not going to have many issues with tens to hundreds of thousands of items in the directory. However, if a transition does happen for FAT support I will consider adding hashing. Although getting a good balanced hash in general without, say, checksumming the filename and taking part of the checksum, is difficult. - -I prefer to keep all the metadata in the filename, as this eases recovery if the files end up in lost+found. So while \"SHA/\" is a nice workaround for the FAT colon problem, I'll be doing something else. (What I'm not sure yet.) - -There is no point in creating unused hash directories on initialization. If anything, with a bad filesystem that just guarantees worst performance from the beginning.. -"""]] From 90eb1185679b70f27a4659155e5ec22944ff2783 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:17:34 -0400 Subject: [PATCH 1015/2835] response --- doc/bugs/softlink_atime.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index ab13863f1a..7ab0881df3 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -10,3 +10,12 @@ Optionally, editing the meta-data should change the times in all annexes. > files in git seems pointless. --[[Joey]] > > Improving an area where git is (not yet?) good at still makes sense, imo. Photos and the like need absolute timestamps more than source code which is fine with relative timestamps (local builds & updates). Maintaining global timestamps for source code could even cause a lot of unwanted effects. As it is, this issue is the only, but a major, blocker for me before I can start adapting git-annex. As I have three different use cases for it, this is a shame. Unfortunately, I don't speak any Haskell so scratching my own itch isn't do-able (without major effort and not soon, at least). Is there a realistic chance that you will tackle this nonetheless or is this WONTFIX? -- RichiH + +>>> Not quite WONTFIX. git-annex should at least, when adding new files, +>>> preserve their timestamp in the symlink it creates. +>>> +>>> Since it doesn't have anything to do with maintaining the symlinks +>>> during an update, or a clone, etc, maintaining the permissions of them +>>> is also out of scope, and it's best to just use metastore if you need +>>> it. Otherwise, git-annex would have to reimplement metastore, and is +>>> unlikely to do it better. From 5ea4f0c0f36fc70c905b452ecd1aa35201d8554c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:18:23 -0400 Subject: [PATCH 1016/2835] response --- doc/todo/add_a_git_backend.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index 6020db86dc..25e780269f 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -6,3 +6,7 @@ have to deal with the pain of git with large files in one place, and can use all of git-annex's features everywhere else. > Speaking as a future user, do very, very much want. -- RichiH + +>> Might also be interesting to use `bup` in the git backend, to work +>> around git's big file issues there. So git-annex would pull data out +>> of the git backend using bup. --[[Joey]] From 16845a1f663a38b7d82750db48424011c9ca5a5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:21:58 -0400 Subject: [PATCH 1017/2835] response --- doc/todo/hidden_files.mdwn | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn index 14c4f0da5a..97eb71e7be 100644 --- a/doc/todo/hidden_files.mdwn +++ b/doc/todo/hidden_files.mdwn @@ -21,4 +21,8 @@ TODO: > Unless I am missing something: Make sure the data is correct (for SHA1 or other tracking) and restore locally. If that's not the case, delete and restore from remote. -- RichiH -* Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH +---- + +Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH + +> Nah, `git annex unused/dropunused` already exist. --[[Joey]] From a021ae92913c0eaca1509fe09e3560871480478e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:33:15 -0400 Subject: [PATCH 1018/2835] design --- doc/bugs/fat_support.mdwn | 3 --- doc/todo/object_dir_reorg_v2.mdwn | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 doc/todo/object_dir_reorg_v2.mdwn diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn index 8088e965a7..2c6c973856 100644 --- a/doc/bugs/fat_support.mdwn +++ b/doc/bugs/fat_support.mdwn @@ -8,8 +8,5 @@ be VFAT formatted: - Use of ":" in filenames of object files, also not supported. Could easily be fixed by reorganizing the object directory. - (If the object directory is reorganized, should consider adding hashing - at the same time.) - [[!tag wishlist]] diff --git a/doc/todo/object_dir_reorg_v2.mdwn b/doc/todo/object_dir_reorg_v2.mdwn new file mode 100644 index 0000000000..db18856995 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2.mdwn @@ -0,0 +1,19 @@ +Several things suggest now would be a good time to reorgaize the object +directory. This would be annex.version=2. It will be slightly painful for +all users, so this should be the *last* reorg in the forseeable future. + +1. Remove colons from filenames, for [[bugs/fat_support]] + +2. Add hashing, since some filesystems do suck (like er, fat at least :) + [[forum/hashing_objects_directories]] + +3. Add filesize metadata for [[bugs/free_space_checking]]. (Currently only + present in WORM, and in an ad-hoc way.) + +4. Perhaps use a generic format that will allow further metadata to be + added later. For example, + "bSHA1,s101111,kf3101c30bb23467deaec5d78c6daa71d395d1879" + + (Probably everything after ",k" should be part of the key, even if it + contains the "," separator character. Otherwise an escaping mechanism + would be needed.) From 61396e34b5c9cb8b94940e9e83baf4e1a9466205 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 17:00:38 +0000 Subject: [PATCH 1019/2835] --- doc/bugs/softlink_atime.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index 7ab0881df3..49e26013ff 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -19,3 +19,5 @@ Optionally, editing the meta-data should change the times in all annexes. >>> is also out of scope, and it's best to just use metastore if you need >>> it. Otherwise, git-annex would have to reimplement metastore, and is >>> unlikely to do it better. + +>>>> OK, thanks for the clarification. Would it be acceptable for you to put the timestamps into the metastore with vanilla git? If such an option existed, everyone would be able to benefit and not just me. -- RichiH From a4993f9321702d75d93c76227a8b8ee2fd863fb1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 17:06:30 +0000 Subject: [PATCH 1020/2835] --- doc/todo/add_a_git_backend.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index 25e780269f..34322d9811 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -10,3 +10,5 @@ use all of git-annex's features everywhere else. >> Might also be interesting to use `bup` in the git backend, to work >> around git's big file issues there. So git-annex would pull data out >> of the git backend using bup. --[[Joey]] + +>>> Very much so. Generally speaking, having one or more versioned storage back-ends with current data in the local annexes sounds incredibly useful. Still being able to get at old data in via the back-end and/or making offline backups of the full history are excellent use cases. -- RichiH From 96e074bb0333b6952fb9fbce2f0a810ebafd3d2c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 17:07:31 +0000 Subject: [PATCH 1021/2835] --- doc/todo/hidden_files.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn index 97eb71e7be..191e9c3286 100644 --- a/doc/todo/hidden_files.mdwn +++ b/doc/todo/hidden_files.mdwn @@ -26,3 +26,5 @@ TODO: Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH > Nah, `git annex unused/dropunused` already exist. --[[Joey]] + +>> OK, in that case forget what I said. No idea about your internal policy, but feel free to delete this part of the page, then. -- RichiH From bc5c54c987f548505a3877e8a0e460abe0b2a081 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 23:00:23 -0400 Subject: [PATCH 1022/2835] symlink touching fun When adding files to the annex, the symlinks pointing at the annexed content are made to have the same mtime as the original file. While git does not preserve that information, this allows a tool like metastore to be used with annexed files. --- .gitignore | 1 + Command/Add.hs | 7 ++++ Command/Find.hs | 1 - Command/PreCommit.hs | 1 - Makefile | 7 ++-- Touch.hsc | 70 ++++++++++++++++++++++++++++++++++++ debian/changelog | 4 +++ doc/bugs/softlink_atime.mdwn | 14 ++++++++ 8 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 Touch.hsc diff --git a/.gitignore b/.gitignore index 764a1af9d5..69d2c80709 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ doc/.ikiwiki html *.tix .hpc +Touch.hs diff --git a/Command/Add.hs b/Command/Add.hs index 26e7fa258d..09fff7cff7 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -18,6 +18,7 @@ import Types import Content import Messages import Utility +import Touch command :: [Command] command = [Command "add" paramPath seek "add files to annex"] @@ -53,5 +54,11 @@ cleanup file key = do link <- calcGitLink file key liftIO $ createSymbolicLink link file + + -- touch the symlink to have the same mtime as the file it points to + s <- liftIO $ getFileStatus file + let mtime = modificationTime s + _ <- liftIO $ touch file (TimeSpec mtime 0) False + Annex.queue "add" [Param "--"] file return True diff --git a/Command/Find.hs b/Command/Find.hs index 1ca6ff1e7c..3ed15c1537 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -12,7 +12,6 @@ import Control.Monad.State (liftIO) import Command import Content -import Messages command :: [Command] command = [Command "find" (paramOptional $ paramRepeating paramPath) seek diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 6f9adb79a5..1465ebc615 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -14,7 +14,6 @@ import qualified Annex import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix -import Messages import Utility command :: [Command] diff --git a/Makefile b/Makefile index c888fc2151..c381ae986d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,10 @@ SysConfig.hs: configure.hs TestConfig.hs $(GHCMAKE) configure ./configure -$(bins): SysConfig.hs +Touch.hs: Touch.hsc + hsc2hs $< + +$(bins): SysConfig.hs Touch.hs $(GHCMAKE) $@ git-annex.1: doc/git-annex.mdwn @@ -57,7 +60,7 @@ docs: $(mans) --exclude='news/.*' clean: - rm -rf build $(bins) $(mans) test configure SysConfig.hs *.tix .hpc + rm -rf build $(bins) $(mans) test configure Touch.hs SysConfig.hs *.tix .hpc rm -rf doc/.ikiwiki html find . \( -name \*.o -or -name \*.hi \) -exec rm {} \; diff --git a/Touch.hsc b/Touch.hsc new file mode 100644 index 0000000000..ad36761c4b --- /dev/null +++ b/Touch.hsc @@ -0,0 +1,70 @@ +{- More control over touching a file. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE ForeignFunctionInterface #-} + +module Touch ( + TimeSpec(..), + now, + omit, + touchBoth, + touch +) where + +import Foreign +import Foreign.C + +#include +#include +#include + +#let alignment t = "%lu", (unsigned long)offsetof(struct {char x__; t (y__); }, y__) + +data TimeSpec = TimeSpec CTime CLong + +instance Storable TimeSpec where + alignment _ = #{alignment struct timespec} + sizeOf _ = #{size struct timespec} + peek ptr = do + sec <- #{peek struct timespec, tv_sec} ptr + nsec <- #{peek struct timespec, tv_nsec} ptr + return $ TimeSpec sec nsec + poke ptr (TimeSpec sec nsec) = do + #{poke struct timespec, tv_sec} ptr sec + #{poke struct timespec, tv_nsec} ptr nsec + +{- special timespecs -} +omit :: TimeSpec +omit = TimeSpec 0 #const UTIME_OMIT +now :: TimeSpec +now = TimeSpec 0 #const UTIME_NOW + +{- While its interface is beastly, utimensat is in recent + POSIX standards, unlike futimes. -} +foreign import ccall "utimensat" + c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt + +{- Changes the access and/or modification times of a file. + Can follow symlinks, or not. -} +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO Bool +touchBoth file atime mtime follow = + allocaArray 2 $ \ptr -> + withCString file $ \f -> do + pokeArray ptr [atime, mtime] + r <- c_utimensat at_fdcwd f ptr flags + putStrLn $ "ret " ++ (show r) + return (r == 0) + where + at_fdcwd = #const AT_FDCWD + at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW + + flags = if follow + then 0 + else at_symlink_nofollow + +touch :: FilePath -> TimeSpec -> Bool -> IO Bool +touch file mtime follow = touchBoth file omit mtime follow diff --git a/debian/changelog b/debian/changelog index 6e70ac67e3..e7017a26d0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,10 @@ git-annex (0.24) UNRELEASED; urgency=low * Add Suggests on graphviz. Closes: #618039 + * When adding files to the annex, the symlinks pointing at the annexed + content are made to have the same mtime as the original file. + While git does not preserve that information, this allows a tool + like metastore to be used with annexed files. -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index 49e26013ff..ebb040dd1e 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -21,3 +21,17 @@ Optionally, editing the meta-data should change the times in all annexes. >>> unlikely to do it better. >>>> OK, thanks for the clarification. Would it be acceptable for you to put the timestamps into the metastore with vanilla git? If such an option existed, everyone would be able to benefit and not just me. -- RichiH + +>>>>> I've now committed to git changes to make git-annex add make +>>>>> symlinks that reflect the original file's mtime. (It's not possible +>>>>> to set the ctime of a symlink; nor would you want to as messing with +>>>>> ctimes can break backup software ... and atime doesn't much matter.) +>>>>> +>>>>> So all you have to do is make the pre-commit hook call +>>>>> [metastore](http://david.hardeman.nu/software.php). The hook +>>>>> would look like this: ---[[Joey]] [[!tag done]] + + #!/bin/sh + git annex pre-commit . + metastore --save + git add .metadata From 88ef64e566cac30f552982354642feb993d13823 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 01:16:27 -0400 Subject: [PATCH 1023/2835] calculate alignment in a way I can understand See http://therning.org/magnus/archives/315 --- Touch.hsc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Touch.hsc b/Touch.hsc index ad36761c4b..f85873125c 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -22,12 +22,14 @@ import Foreign.C #include #include -#let alignment t = "%lu", (unsigned long)offsetof(struct {char x__; t (y__); }, y__) - data TimeSpec = TimeSpec CTime CLong instance Storable TimeSpec where - alignment _ = #{alignment struct timespec} + -- use the larger alignment of the two types in the struct + alignment _ = max sec_alignment nsec_alignment + where + sec_alignment = alignment $ undefined::CTime + nsec_alignment = alignment $ undefined::CLong sizeOf _ = #{size struct timespec} peek ptr = do sec <- #{peek struct timespec, tv_sec} ptr From 199ffa6c36178ac7360ad957feccba188e0dad02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 02:31:47 -0400 Subject: [PATCH 1024/2835] remove debug --- Touch.hsc | 1 - 1 file changed, 1 deletion(-) diff --git a/Touch.hsc b/Touch.hsc index f85873125c..4e1052b94a 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -58,7 +58,6 @@ touchBoth file atime mtime follow = withCString file $ \f -> do pokeArray ptr [atime, mtime] r <- c_utimensat at_fdcwd f ptr flags - putStrLn $ "ret " ++ (show r) return (r == 0) where at_fdcwd = #const AT_FDCWD From bca68c263da765acedad5643a37f09f142555683 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 02:36:24 -0400 Subject: [PATCH 1025/2835] tweaks --- Touch.hsc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Touch.hsc b/Touch.hsc index 4e1052b94a..34b34be515 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -9,8 +9,8 @@ module Touch ( TimeSpec(..), - now, - omit, + nowTime, + omitTime, touchBoth, touch ) where @@ -28,8 +28,8 @@ instance Storable TimeSpec where -- use the larger alignment of the two types in the struct alignment _ = max sec_alignment nsec_alignment where - sec_alignment = alignment $ undefined::CTime - nsec_alignment = alignment $ undefined::CLong + sec_alignment = alignment (undefined::CTime) + nsec_alignment = alignment (undefined::CLong) sizeOf _ = #{size struct timespec} peek ptr = do sec <- #{peek struct timespec, tv_sec} ptr @@ -40,10 +40,10 @@ instance Storable TimeSpec where #{poke struct timespec, tv_nsec} ptr nsec {- special timespecs -} -omit :: TimeSpec -omit = TimeSpec 0 #const UTIME_OMIT -now :: TimeSpec -now = TimeSpec 0 #const UTIME_NOW +omitTime :: TimeSpec +omitTime = TimeSpec 0 #const UTIME_OMIT +nowTime :: TimeSpec +nowTime = TimeSpec 0 #const UTIME_NOW {- While its interface is beastly, utimensat is in recent POSIX standards, unlike futimes. -} @@ -68,4 +68,4 @@ touchBoth file atime mtime follow = else at_symlink_nofollow touch :: FilePath -> TimeSpec -> Bool -> IO Bool -touch file mtime follow = touchBoth file omit mtime follow +touch file mtime follow = touchBoth file omitTime mtime follow From 58b3e53ff6ee8af6e7a1d67d21c7ce7ae8c6cfda Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Tue, 15 Mar 2011 08:38:53 +0000 Subject: [PATCH 1026/2835] --- doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn diff --git a/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn b/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn new file mode 100644 index 0000000000..60014a7f53 --- /dev/null +++ b/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn @@ -0,0 +1 @@ +This is just a comment on git-annex building on haskell platform 2011.2.0.0 on archlinux. It just works. From 7b0c6177ff9e0d0f2c23f798b14f1e49128f3589 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Tue, 15 Mar 2011 10:49:10 +0000 Subject: [PATCH 1027/2835] --- doc/bugs/softlink_atime.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index ebb040dd1e..0d26d1ce6a 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -35,3 +35,5 @@ Optionally, editing the meta-data should change the times in all annexes. git annex pre-commit . metastore --save git add .metadata + +>>>>>> Thanks a lot. Doing this in a new git-annex repo from the start should at least ensure local consistency and I assume I can simply add a post-pull hook to restore the mtimes on all all other repositories? -- RichiH From 83a9bb624bcd7c5b4eee69bd91150d46c82146d8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 11:50:40 -0400 Subject: [PATCH 1028/2835] fix error throwing --- Command/Add.hs | 2 +- Touch.hsc | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Command/Add.hs b/Command/Add.hs index 09fff7cff7..a577203bf4 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -58,7 +58,7 @@ cleanup file key = do -- touch the symlink to have the same mtime as the file it points to s <- liftIO $ getFileStatus file let mtime = modificationTime s - _ <- liftIO $ touch file (TimeSpec mtime 0) False + liftIO $ touch file (TimeSpec mtime 0) False Annex.queue "add" [Param "--"] file return True diff --git a/Touch.hsc b/Touch.hsc index 34b34be515..689c58765f 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -50,15 +50,17 @@ nowTime = TimeSpec 0 #const UTIME_NOW foreign import ccall "utimensat" c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt -{- Changes the access and/or modification times of a file. - Can follow symlinks, or not. -} -touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO Bool +{- Changes the access and/or modification times of an existing file. + Can follow symlinks, or not. Throws IO error on failure. -} +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () touchBoth file atime mtime follow = allocaArray 2 $ \ptr -> withCString file $ \f -> do pokeArray ptr [atime, mtime] r <- c_utimensat at_fdcwd f ptr flags - return (r == 0) + if (r /= 0) + then throwErrno "touchBoth" + else return () where at_fdcwd = #const AT_FDCWD at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW @@ -67,5 +69,5 @@ touchBoth file atime mtime follow = then 0 else at_symlink_nofollow -touch :: FilePath -> TimeSpec -> Bool -> IO Bool +touch :: FilePath -> TimeSpec -> Bool -> IO () touch file mtime follow = touchBoth file omitTime mtime follow From 0e0f85e09d975a6062fb417f8bbae5fbadb6f79f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 17:47:00 -0400 Subject: [PATCH 1029/2835] add hash directory stuff, not used yet --- Locations.hs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Locations.hs b/Locations.hs index 908d5b74ed..91a61ddd7b 100644 --- a/Locations.hs +++ b/Locations.hs @@ -26,6 +26,9 @@ module Locations ( import System.FilePath import Data.String.Utils import Data.List +import Bits +import Word +import Data.Hash.MD5 import Types import qualified GitRepo as Git @@ -128,3 +131,29 @@ fileKey file = read $ prop_idempotent_fileKey :: String -> Bool prop_idempotent_fileKey s = k == fileKey (keyFile k) where k = read $ "test:" ++ s + +{- Given a filename, generates a short directory name to put it in, + - to do hashing to protect against filesystems that dislike having + - many items in a single directory. -} +hashDir :: FilePath -> FilePath +hashDir s = take 2 $ abcd_to_dir $ md5 (Str s) + +abcd_to_dir :: ABCD -> String +abcd_to_dir (ABCD (a,b,c,d)) = concat $ map display_32bits_as_dir [a,b,c,d] + +{- modified version of display_32bits_as_hex from Data.Hash.MD5 + - Copyright (C) 2001 Ian Lynagh + - License: Either BSD or GPL + -} +display_32bits_as_dir :: Word32 -> String +display_32bits_as_dir w = trim $ swap_pairs cs + where + -- Need 32 characters to use. To avoid inaverdently making + -- a real word, use the alphabet without vowels. + chars = ['0'..'9'] ++ "bcdfghjklnmpqrstvwxyzZ" + cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7] + getc n = chars !! (fromIntegral n) + swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs + swap_pairs _ = [] + -- Last 2 will always be 00, so omit. + trim s = take 6 s From fe09c2b7231485afced594cd27582bc6bd32f250 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 17:47:29 -0400 Subject: [PATCH 1030/2835] a new Key data type with metadata --- Key.hs | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ test.hs | 3 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 Key.hs diff --git a/Key.hs b/Key.hs new file mode 100644 index 0000000000..cc4effb11f --- /dev/null +++ b/Key.hs @@ -0,0 +1,88 @@ +{- git-annex Key data type + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Key where + +import Data.String.Utils +import Test.QuickCheck +import Data.Maybe +import Data.List + +{- A Key has a unique name, is associated with a backend, + - and may contain other metadata. -} +data Field = KeyName | KeyBackend | KeySize | KeyModTime + deriving (Eq, Ord, Show) +newtype Key = Key [(Field, String)] + deriving (Eq, Ord) + +{- Generates a Key given a name, a backend and a list of other metadata. -} +keyGen :: String -> String -> [(Field, String)] -> Key +keyGen name backend meta = Key $ (KeyName, name):(KeyBackend, backend):meta + +{- Gets the name of a Key. -} +keyName :: Key -> String +keyName key = fromJust $ keyField key KeyName + +{- Gets the backend associated with a Key. -} +keyBackend :: Key -> String +keyBackend key = fromJust $ keyField key KeyBackend + +{- Looks up a given Field of a Key's metadata. -} +keyField :: Key -> Field -> Maybe String +keyField (Key meta) field = + if null matches + then Nothing + else Just $ snd $ head matches + where + matches = filter match meta + match (f, _) = f == field + +fieldSep :: Char +fieldSep = ',' + +{- Keys show as strings that are suitable for use as filenames. + - The name field is always shown last, and is the only field + - allowed to contain the fieldSep. -} +instance Show Key where + show k@(Key meta) = join [fieldSep] $ map showp meta' ++ [name] + where + name = 'n':keyName k + meta' = sort $ (filter (\(f, _) -> f /= KeyName)) meta + showp (f, v) = (field f) : v + + field KeyBackend = 'b' + field KeySize = 's' + field KeyModTime = 'm' + field f = error $ "unknown key field" ++ show f + +instance Read Key where + readsPrec _ s = [(Key (meta s []), "")] + where + meta (c:r) m = findfield c r m + meta [] m = m + + findfield 'n' v m = (KeyName, v):m -- rest is name + findfield c v m = let (v', _:r) = span (/= fieldSep) v in + meta r (field c v' m) + + field 'b' v m = (KeyBackend, v):m + field 's' v m = (KeySize, v):m + field 'm' v m = (KeyModTime, v):m + field _ _ m = m -- just ignore unparseable fields + +-- for quickcheck +instance Arbitrary Key where + arbitrary = do + backendname <- arbitrary + value <- arbitrary + return $ keyGen value backendname [] + +prop_idempotent_key_read_show :: Key -> Bool +prop_idempotent_key_read_show k + -- backend names will never contain the fieldSep + | fieldSep `elem` (keyBackend k) = True + | otherwise = k == (read $ show k) diff --git a/test.hs b/test.hs index 31960bb2e2..bc849dadc5 100644 --- a/test.hs +++ b/test.hs @@ -38,6 +38,7 @@ import qualified Trust import qualified Remotes import qualified Content import qualified Command.DropUnused +import qualified Key main :: IO () main = do @@ -55,7 +56,7 @@ quickcheck :: Test quickcheck = TestLabel "quickcheck" $ TestList [ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode , qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey - , qctest "prop_idempotent_key_read_show" BackendTypes.prop_idempotent_key_read_show + , qctest "prop_idempotent_key_read_show" Key.prop_idempotent_key_read_show , qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics From 940c4e361dd5149d52c773a0e020150d1b5fed56 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 18:15:44 -0400 Subject: [PATCH 1031/2835] cleanup --- Key.hs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Key.hs b/Key.hs index cc4effb11f..cc089104e3 100644 --- a/Key.hs +++ b/Key.hs @@ -60,19 +60,19 @@ instance Show Key where field f = error $ "unknown key field" ++ show f instance Read Key where - readsPrec _ s = [(Key (meta s []), "")] + readsPrec _ s = [(Key (findfields s []), "")] where - meta (c:r) m = findfield c r m - meta [] m = m - - findfield 'n' v m = (KeyName, v):m -- rest is name - findfield c v m = let (v', _:r) = span (/= fieldSep) v in - meta r (field c v' m) + findfields ('n':v) m = (KeyName, v):m -- rest is name + findfields (c:v) m = + case span (/= fieldSep) v of + (v', _:r) -> findfields r (field c v' m) + _ -> m + findfields [] m = m field 'b' v m = (KeyBackend, v):m field 's' v m = (KeySize, v):m field 'm' v m = (KeyModTime, v):m - field _ _ m = m -- just ignore unparseable fields + field _ _ m = m -- for quickcheck instance Arbitrary Key where From 675ee89749ba2272d37b763078020b6e5f4cd380 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 19:11:21 -0400 Subject: [PATCH 1032/2835] redo using record syntax --- Key.hs | 100 ++++++++++++++++++++++++--------------------------------- 1 file changed, 42 insertions(+), 58 deletions(-) diff --git a/Key.hs b/Key.hs index cc089104e3..c542b46ed8 100644 --- a/Key.hs +++ b/Key.hs @@ -7,39 +7,17 @@ module Key where -import Data.String.Utils import Test.QuickCheck -import Data.Maybe -import Data.List +import Utility {- A Key has a unique name, is associated with a backend, - and may contain other metadata. -} -data Field = KeyName | KeyBackend | KeySize | KeyModTime - deriving (Eq, Ord, Show) -newtype Key = Key [(Field, String)] - deriving (Eq, Ord) - -{- Generates a Key given a name, a backend and a list of other metadata. -} -keyGen :: String -> String -> [(Field, String)] -> Key -keyGen name backend meta = Key $ (KeyName, name):(KeyBackend, backend):meta - -{- Gets the name of a Key. -} -keyName :: Key -> String -keyName key = fromJust $ keyField key KeyName - -{- Gets the backend associated with a Key. -} -keyBackend :: Key -> String -keyBackend key = fromJust $ keyField key KeyBackend - -{- Looks up a given Field of a Key's metadata. -} -keyField :: Key -> Field -> Maybe String -keyField (Key meta) field = - if null matches - then Nothing - else Just $ snd $ head matches - where - matches = filter match meta - match (f, _) = f == field +data Key = Key { + keyName :: String, + keyBackend :: String, + keySize :: Maybe Int, + keyMtime :: Maybe Int +} deriving (Eq, Ord) fieldSep :: Char fieldSep = ',' @@ -48,41 +26,47 @@ fieldSep = ',' - The name field is always shown last, and is the only field - allowed to contain the fieldSep. -} instance Show Key where - show k@(Key meta) = join [fieldSep] $ map showp meta' ++ [name] + show Key { keyBackend = b, keySize = s, keyMtime = m, keyName = n } = + ('b' : b) +++ ('s' ?: s) +++ ('m' ?: m) +++ ('n' : n) where - name = 'n':keyName k - meta' = sort $ (filter (\(f, _) -> f /= KeyName)) meta - showp (f, v) = (field f) : v + "" +++ y = y + x +++ "" = x + x +++ y = x ++ fieldSep:y + c ?: (Just v) = c:(show v) + _ ?: _ = "" - field KeyBackend = 'b' - field KeySize = 's' - field KeyModTime = 'm' - field f = error $ "unknown key field" ++ show f +readKey :: String -> Maybe Key +readKey s = if key == stub then Nothing else key + where + key = findfields s stub -instance Read Key where - readsPrec _ s = [(Key (findfields s []), "")] - where - findfields ('n':v) m = (KeyName, v):m -- rest is name - findfields (c:v) m = - case span (/= fieldSep) v of - (v', _:r) -> findfields r (field c v' m) - _ -> m - findfields [] m = m - - field 'b' v m = (KeyBackend, v):m - field 's' v m = (KeySize, v):m - field 'm' v m = (KeyModTime, v):m - field _ _ m = m + stub = Just Key { + keyName = "", + keyBackend = "", + keySize = Nothing, + keyMtime = Nothing + } + + findfields ('n':v) (Just k) = Just $ k { keyName = v } + findfields (c:v) (Just k) = + case span (/= fieldSep) v of + (v', _:r) -> findfields r $ addfield k c v' + _ -> Nothing + findfields _ v = v + + addfield k 'b' v = Just k { keyBackend = v } + addfield k 's' v = Just k { keySize = readMaybe v } + addfield k 'm' v = Just k { keyMtime = readMaybe v } + addfield _ _ _ = Nothing -- for quickcheck instance Arbitrary Key where arbitrary = do - backendname <- arbitrary - value <- arbitrary - return $ keyGen value backendname [] + n <- arbitrary + b <- elements ['A'..'Z'] + s <- arbitrary + m <- arbitrary + return $ Key { keyName = n, keyBackend = [b] , keySize = s, keyMtime = m } prop_idempotent_key_read_show :: Key -> Bool -prop_idempotent_key_read_show k - -- backend names will never contain the fieldSep - | fieldSep `elem` (keyBackend k) = True - | otherwise = k == (read $ show k) +prop_idempotent_key_read_show k = Just k == (readKey $ show k) From 2d5339d7cb887abb8e5baee8ba9244636492f023 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 16 Mar 2011 01:16:48 +0000 Subject: [PATCH 1033/2835] Added a comment --- .../comment_1_ba03333dc76ff49eccaba375e68cb525._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment b/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment new file mode 100644 index 0000000000..261c2a51f3 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-16T01:16:48Z" + content=""" +If you support generic meta-data, keep in mind that you will need to do conflict resolution. Timestamps may not be synched across all systems, so keeping a log of old metadata could be used, sorting by history and using the latest. Which leaves the situation of two incompatible changes. This would probably mean manual conflict resolution. You will probably have thought of this already, but I still wanted to make sure this is recorded. -- RichiH +"""]] From 7d316d0f0de9daced582eb957b0d2b9127a9c6dc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 16 Mar 2011 01:19:26 +0000 Subject: [PATCH 1034/2835] Added a comment --- .../comment_2_81276ac309959dc741bc90101c213ab7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment b/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment new file mode 100644 index 0000000000..9785f1989e --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-03-16T01:19:25Z" + content=""" +Hmm, I added quite a few comments at work, but they are stuck in moderation. Maybe I forgot to log in before adding them. I am surprised this one appeared immediately. -- RichiH +"""]] From 9d49fe2c172b135a1a3735827df014b5f45d99a2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 21:34:13 -0400 Subject: [PATCH 1035/2835] first pass at using new keys It compiles. It sorta works. Several subcommands are FIXME marked and broken, because things that used to accept separate --backend and --key params need to be changed to accept just a --key that encodes all the key info, now that there is metadata in keys. --- Backend.hs | 24 ++++++++++++----------- Backend/SHA.hs | 11 +++++++++-- Backend/WORM.hs | 40 ++++++++++++++++---------------------- BackendTypes.hs | 44 ++---------------------------------------- Command.hs | 13 +++++++++---- Command/DropKey.hs | 6 ++++-- Command/DropUnused.hs | 5 +++-- Command/FromKey.hs | 2 +- Command/InAnnex.hs | 9 +++++++-- Command/Move.hs | 5 +++-- Command/RecvKey.hs | 3 +++ Command/SendKey.hs | 3 +++ Command/Unused.hs | 2 +- Content.hs | 3 ++- Key.hs | 45 +++++++++++++++++++++++++------------------ Locations.hs | 9 +++++---- Makefile | 1 + Remotes.hs | 5 +++-- Types.hs | 6 ++---- Upgrade.hs | 3 ++- 20 files changed, 116 insertions(+), 123 deletions(-) diff --git a/Backend.hs b/Backend.hs index df23e80a33..94755e8d6b 100644 --- a/Backend.hs +++ b/Backend.hs @@ -39,6 +39,7 @@ import Locations import qualified GitRepo as Git import qualified Annex import Types +import Key import qualified BackendTypes as B import Messages @@ -135,18 +136,19 @@ lookupFile file = do getsymlink = do l <- readSymbolicLink file return $ takeFileName l - makekey bs l = do + makekey bs l = + case fileKey l of + Just k -> makeret k l bs + Nothing -> return Nothing + makeret k l bs = case maybeLookupBackendName bs bname of - Nothing -> do - unless (null kname || null bname || - not (isLinkToAnnex l)) $ - warning skip - return Nothing - Just backend -> return $ Just (k, backend) + Just backend -> return $ Just (k, backend) + Nothing -> do + when (isLinkToAnnex l) $ + warning skip + return Nothing where - k = fileKey l - bname = backendName k - kname = keyName k + bname = keyBackendName k skip = "skipping " ++ file ++ " (unknown backend " ++ bname ++ ")" @@ -164,4 +166,4 @@ chooseBackends fs = do keyBackend :: Key -> Annex (Backend Annex) keyBackend key = do bs <- Annex.getState Annex.supportedBackends - return $ lookupBackendName bs $ backendName key + return $ lookupBackendName bs $ keyBackendName key diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 4eea890ce4..3cdc3bf808 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -13,6 +13,7 @@ import System.Cmd.Utils import System.IO import System.Directory import Data.Maybe +import System.Posix.Files import qualified Backend.File import BackendTypes @@ -23,6 +24,7 @@ import Content import Types import Utility import qualified SysConfig +import Key type SHASize = Int @@ -63,11 +65,16 @@ shaN size file = do where command = "sha" ++ (show size) ++ "sum" --- A key is a checksum of its contents. +{- A key is a checksum of its contents. -} keyValue :: SHASize -> FilePath -> Annex (Maybe Key) keyValue size file = do s <- shaN size file - return $ Just $ Key (shaName size, s) + stat <- liftIO $ getFileStatus file + return $ Just $ stubKey { + keyName = s, + keyBackendName = shaName size, + keySize = Just $ fromIntegral $ fileSize stat + } -- A key's checksum is checked during fsck. checkKeyChecksum :: SHASize -> Key -> Annex Bool diff --git a/Backend/WORM.hs b/Backend/WORM.hs index a0d814aa08..324aee76be 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -10,9 +10,8 @@ module Backend.WORM (backends) where import Control.Monad.State import System.FilePath import System.Posix.Files -import System.Posix.Types import System.Directory -import Data.String.Utils +import Data.Maybe import qualified Backend.File import BackendTypes @@ -21,6 +20,7 @@ import qualified Annex import Content import Messages import Types +import Key backends :: [Backend Annex] backends = [backend] @@ -32,31 +32,25 @@ backend = Backend.File.backend { fsckKey = Backend.File.checkKey checkKeySize } --- The key is formed from the file size, modification time, and the --- basename of the filename. --- --- That allows multiple files with the same names to have different keys, --- while also allowing a file to be moved around while retaining the --- same key. +{- The key includes the file size, modification time, and the + - basename of the filename. + - + - That allows multiple files with the same names to have different keys, + - while also allowing a file to be moved around while retaining the + - same key. + -} keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do stat <- liftIO $ getFileStatus file - return $ Just $ Key (name backend, key stat) - where - key stat = uniqueid stat ++ sep ++ base - uniqueid stat = show (modificationTime stat) ++ sep ++ - show (fileSize stat) - base = takeFileName file - sep = ":" - -{- Extracts the file size from a key. -} -keySize :: Key -> FileOffset -keySize key = read $ section !! 1 - where - section = split ":" (keyName key) + return $ Just $ Key { + keyName = takeFileName file, + keyBackendName = name backend, + keySize = Just $ fromIntegral $ fileSize stat, + keyMtime = Just $ modificationTime stat + } {- The size of the data for a key is checked against the size encoded in - - the key. Note that the modification time is not checked. -} + - the key's metadata. -} checkKeySize :: Key -> Annex Bool checkKeySize key = do g <- Annex.gitRepo @@ -66,7 +60,7 @@ checkKeySize key = do then return True else do s <- liftIO $ getFileStatus file - if fileSize s == keySize key + if fromIntegral (fileSize s) == fromJust (keySize key) then return True else do dest <- moveBad key diff --git a/BackendTypes.hs b/BackendTypes.hs index c0705a550f..48b208a9b4 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -1,4 +1,4 @@ -{- git-annex key/value backend data types +{- git-annex key/value backend data type - - Most things should not need this, using Types instead - @@ -9,12 +9,7 @@ module BackendTypes where -import Data.String.Utils -import Test.QuickCheck - -type KeyName = String -type BackendName = String -newtype Key = Key (BackendName, KeyName) deriving (Eq, Ord) +import Key data Backend a = Backend { -- name of this backend @@ -42,38 +37,3 @@ instance Show (Backend a) where instance Eq (Backend a) where a == b = name a == name b - --- accessors for the parts of a key -keyName :: Key -> KeyName -keyName (Key (_,k)) = k -backendName :: Key -> BackendName -backendName (Key (b,_)) = b - --- constructs a key in a backend -genKey :: Backend a -> KeyName -> Key -genKey b f = Key (name b,f) - --- show a key to convert it to a string; the string includes the --- name of the backend to avoid collisions between key strings -instance Show Key where - show (Key (b, k)) = b ++ ":" ++ k - -instance Read Key where - readsPrec _ s = [(Key (b,k), "")] - where - l = split ":" s - b = if null l then "" else head l - k = join ":" $ drop 1 l - --- for quickcheck -instance Arbitrary Key where - arbitrary = do - backendname <- arbitrary - keyname <- arbitrary - return $ Key (backendname, keyname) - -prop_idempotent_key_read_show :: Key -> Bool -prop_idempotent_key_read_show k - -- backend names will never contain colons - | ':' `elem` (backendName k) = True - | otherwise = k == (read $ show k) diff --git a/Command.hs b/Command.hs index eba7f2cef5..38c63bd77e 100644 --- a/Command.hs +++ b/Command.hs @@ -17,11 +17,13 @@ import Data.List import Types import qualified Backend +import qualified BackendTypes import Messages import qualified Annex import qualified GitRepo as Git import Locations import Utility +import Key {- A command runs in four stages. - @@ -233,11 +235,14 @@ cmdlineKey :: Annex Key cmdlineKey = do k <- Annex.getState Annex.defaultkey backends <- Backend.list - return $ genKey (head backends) (keyname' k) + return $ stubKey { + keyName = kname k, + keyBackendName = BackendTypes.name $ head backends + } where - keyname' Nothing = badkey - keyname' (Just "") = badkey - keyname' (Just n) = n + kname Nothing = badkey + kname (Just "") = badkey + kname (Just n) = n badkey = error "please specify the key with --key" {- Given an original list of files, and an expanded list derived from it, diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 8c7566df84..f0450eea34 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -26,8 +26,10 @@ seek = [withKeys start] start :: CommandStartString start keyname = do backends <- Backend.list - let key = genKey (head backends) keyname - present <- inAnnex key + let key = error "fixme!!" + --let key = genKey (head backends) keyname --TODO FIXME + let present = error "fixme!!" + --present <- inAnnex key force <- Annex.getState Annex.force if not present then return Nothing diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 594564cb72..8ed61ba65b 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -11,6 +11,7 @@ import Control.Monad (when) import Control.Monad.State (liftIO) import qualified Data.Map as M import System.Directory +import Data.Maybe import Command import Types @@ -19,6 +20,7 @@ import Locations import qualified Annex import qualified Command.Drop import Backend +import Key command :: [Command] command = [Command "dropunused" (paramRepeating paramNumber) seek @@ -55,7 +57,6 @@ readUnusedLog = do return $ M.fromList $ map parse $ lines l else return $ M.empty where - parse line = (head ws, tokey $ unwords $ tail ws) + parse line = (head ws, fromJust $ readKey $ unwords $ tail ws) where ws = words line - tokey s = read s :: Key diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 717d528bc9..176d2cd54d 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -16,9 +16,9 @@ import Command import qualified Annex import Utility import qualified Backend -import Types import Content import Messages +import Key command :: [Command] command = [Command "fromkey" paramPath seek diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs index 68ac9a2c67..4a4102754a 100644 --- a/Command/InAnnex.hs +++ b/Command/InAnnex.hs @@ -11,9 +11,10 @@ import Control.Monad.State (liftIO) import System.Exit import Command -import Types import Content import qualified Backend +import qualified BackendTypes +import Key command :: [Command] command = [Command "inannex" (paramRepeating paramKey) seek @@ -25,7 +26,11 @@ seek = [withKeys start] start :: CommandStartString start keyname = do backends <- Backend.list - let key = genKey (head backends) keyname + let key = stubKey { + keyName = keyname, + keyBackendName = BackendTypes.name (head backends) + } + error "BROKEN. fixme!" present <- inAnnex key if present then return Nothing diff --git a/Command/Move.hs b/Command/Move.hs index 3774ccbe9d..1b14813089 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -20,7 +20,8 @@ import qualified Remotes import UUID import Messages import Utility - +import Key + command :: [Command] command = [Command "move" paramPath seek "move content of files to/from another repository"] @@ -136,7 +137,7 @@ fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup fromCleanup src True key = do ok <- Remotes.onRemote src (boolSystem, False) "dropkey" [ Params "--quiet --force" - , Param $ "--backend=" ++ backendName key + , Param $ "--backend=" ++ keyBackendName key , Param $ keyName key ] -- better safe than sorry: assume the src dropped the key diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 8a96730503..488bab62d6 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -27,6 +27,8 @@ seek = [withKeys start] start :: CommandStartString start keyname = do + error "BROKEN FIXME!" + {- backends <- Backend.list let key = genKey (head backends) keyname present <- inAnnex key @@ -41,3 +43,4 @@ start keyname = do _ <- shutdown liftIO exitSuccess else liftIO exitFailure + -} diff --git a/Command/SendKey.hs b/Command/SendKey.hs index cb883b53aa..ff269f21fe 100644 --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -28,6 +28,8 @@ seek = [withKeys start] start :: CommandStartString start keyname = do + error "BROKEN FIXME!" + {- backends <- Backend.list let key = genKey (head backends) keyname present <- inAnnex key @@ -36,3 +38,4 @@ start keyname = do when present $ liftIO $ rsyncServerSend file liftIO exitFailure + -} diff --git a/Command/Unused.hs b/Command/Unused.hs index a614ce5d94..52e483d870 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -126,4 +126,4 @@ tmpKeys = do contents <- liftIO $ getDirectoryContents tmp files <- liftIO $ filterM doesFileExist $ map (tmp ) contents - return $ map (fileKey . takeFileName) files + return $ catMaybes $ map (fileKey . takeFileName) files diff --git a/Content.hs b/Content.hs index dc675389f1..1a5a80a9f1 100644 --- a/Content.hs +++ b/Content.hs @@ -26,6 +26,7 @@ import System.Path import Control.Monad (when, unless, filterM) import System.Posix.Files import System.FilePath +import Data.Maybe import Types import Locations @@ -162,7 +163,7 @@ getKeysPresent' dir = do else do contents <- liftIO $ getDirectoryContents dir files <- liftIO $ filterM present contents - return $ map fileKey files + return $ catMaybes $ map fileKey files where present d = do result <- try $ diff --git a/Key.hs b/Key.hs index c542b46ed8..178f1ca69e 100644 --- a/Key.hs +++ b/Key.hs @@ -5,20 +5,35 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Key where +module Key ( + Key(..), + stubKey, + readKey, + + prop_idempotent_key_read_show +) where import Test.QuickCheck import Utility +import System.Posix.Types -{- A Key has a unique name, is associated with a backend, - - and may contain other metadata. -} +{- A Key has a unique name, is associated with a key/value backend, + - and may contain other optional metadata. -} data Key = Key { keyName :: String, - keyBackend :: String, - keySize :: Maybe Int, - keyMtime :: Maybe Int + keyBackendName :: String, + keySize :: Maybe Integer, + keyMtime :: Maybe EpochTime } deriving (Eq, Ord) +stubKey :: Key +stubKey = Key { + keyName = "", + keyBackendName = "", + keySize = Nothing, + keyMtime = Nothing +} + fieldSep :: Char fieldSep = ',' @@ -26,7 +41,7 @@ fieldSep = ',' - The name field is always shown last, and is the only field - allowed to contain the fieldSep. -} instance Show Key where - show Key { keyBackend = b, keySize = s, keyMtime = m, keyName = n } = + show Key { keyBackendName = b, keySize = s, keyMtime = m, keyName = n } = ('b' : b) +++ ('s' ?: s) +++ ('m' ?: m) +++ ('n' : n) where "" +++ y = y @@ -36,16 +51,9 @@ instance Show Key where _ ?: _ = "" readKey :: String -> Maybe Key -readKey s = if key == stub then Nothing else key +readKey s = if key == Just stubKey then Nothing else key where - key = findfields s stub - - stub = Just Key { - keyName = "", - keyBackend = "", - keySize = Nothing, - keyMtime = Nothing - } + key = findfields s $ Just stubKey findfields ('n':v) (Just k) = Just $ k { keyName = v } findfields (c:v) (Just k) = @@ -54,7 +62,7 @@ readKey s = if key == stub then Nothing else key _ -> Nothing findfields _ v = v - addfield k 'b' v = Just k { keyBackend = v } + addfield k 'b' v = Just k { keyBackendName = v } addfield k 's' v = Just k { keySize = readMaybe v } addfield k 'm' v = Just k { keyMtime = readMaybe v } addfield _ _ _ = Nothing @@ -65,8 +73,7 @@ instance Arbitrary Key where n <- arbitrary b <- elements ['A'..'Z'] s <- arbitrary - m <- arbitrary - return $ Key { keyName = n, keyBackend = [b] , keySize = s, keyMtime = m } + return $ Key { keyName = n, keyBackendName = [b] , keySize = s } prop_idempotent_key_read_show :: Key -> Bool prop_idempotent_key_read_show k = Just k == (readKey $ show k) diff --git a/Locations.hs b/Locations.hs index 91a61ddd7b..6cff910880 100644 --- a/Locations.hs +++ b/Locations.hs @@ -31,6 +31,7 @@ import Word import Data.Hash.MD5 import Types +import Key import qualified GitRepo as Git {- Conventions: @@ -123,14 +124,14 @@ keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} -fileKey :: FilePath -> Key -fileKey file = read $ +fileKey :: FilePath -> Maybe Key +fileKey file = readKey $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file {- for quickcheck -} prop_idempotent_fileKey :: String -> Bool -prop_idempotent_fileKey s = k == fileKey (keyFile k) - where k = read $ "test:" ++ s +prop_idempotent_fileKey s = Just k == fileKey (keyFile k) + where k = stubKey { keyName = s, keyBackendName = "test" } {- Given a filename, generates a short directory name to put it in, - to do hashing to protect against filesystems that dislike having diff --git a/Makefile b/Makefile index c381ae986d..c60e19b311 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ SysConfig.hs: configure.hs TestConfig.hs Touch.hs: Touch.hsc hsc2hs $< + perl -i -pe 's/^{-# INCLUDE.*//' $@ $(bins): SysConfig.hs Touch.hs $(GHCMAKE) $@ diff --git a/Remotes.hs b/Remotes.hs index 3c9db314c8..dd733e4545 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -27,6 +27,7 @@ import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) import Types +import Key import qualified GitRepo as Git import qualified Annex import LocationLog @@ -153,7 +154,7 @@ inAnnex r key = if Git.repoIsUrl r checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- onRemote r (boolSystem, False) "inannex" - [Param ("--backend=" ++ backendName key), Param (keyName key)] + [Param ("--backend=" ++ keyBackendName key), Param (keyName key)] return $ Right inannex {- Cost Ordered list of remotes. -} @@ -272,7 +273,7 @@ rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam] rsyncParams r sending key file = do Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") - [ Param $ "--backend=" ++ backendName key + [ Param $ "--backend=" ++ keyBackendName key , Param $ keyName key -- Command is terminated with "--", because -- rsync will tack on its own options afterwards, diff --git a/Types.hs b/Types.hs index 0890efd5e3..f48d4079be 100644 --- a/Types.hs +++ b/Types.hs @@ -8,11 +8,9 @@ module Types ( Annex, Backend, - Key, - genKey, - backendName, - keyName + Key ) where import BackendTypes import Annex +import Key diff --git a/Upgrade.hs b/Upgrade.hs index 3c16bcc862..7469d9ba75 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -13,6 +13,7 @@ import Control.Monad.State (liftIO) import Control.Monad (filterM, forM_) import System.Posix.Files import System.FilePath +import Data.Maybe import Content import Types @@ -74,7 +75,7 @@ getKeysPresent0' dir = do else do contents <- liftIO $ getDirectoryContents dir files <- liftIO $ filterM present contents - return $ map fileKey files + return $ catMaybes $ map fileKey files where present d = do result <- try $ From f27df5e6584ed6a09be4a9bb1030be132c2d8051 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 21:54:38 -0400 Subject: [PATCH 1036/2835] improve key filenames --- Key.hs | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Key.hs b/Key.hs index 178f1ca69e..3385148707 100644 --- a/Key.hs +++ b/Key.hs @@ -35,14 +35,14 @@ stubKey = Key { } fieldSep :: Char -fieldSep = ',' +fieldSep = '-' {- Keys show as strings that are suitable for use as filenames. - - The name field is always shown last, and is the only field - - allowed to contain the fieldSep. -} + - The name field is always shown last, separated by doubled fieldSeps, + - and is the only field allowed to contain the fieldSep. -} instance Show Key where show Key { keyBackendName = b, keySize = s, keyMtime = m, keyName = n } = - ('b' : b) +++ ('s' ?: s) +++ ('m' ?: m) +++ ('n' : n) + b +++ ('s' ?: s) +++ ('m' ?: m) +++ (fieldSep : n) where "" +++ y = y x +++ "" = x @@ -53,18 +53,22 @@ instance Show Key where readKey :: String -> Maybe Key readKey s = if key == Just stubKey then Nothing else key where - key = findfields s $ Just stubKey + key = startbackend stubKey s - findfields ('n':v) (Just k) = Just $ k { keyName = v } - findfields (c:v) (Just k) = - case span (/= fieldSep) v of - (v', _:r) -> findfields r $ addfield k c v' - _ -> Nothing - findfields _ v = v + startbackend k v = sepfield k v addbackend - addfield k 'b' v = Just k { keyBackendName = v } - addfield k 's' v = Just k { keySize = readMaybe v } - addfield k 'm' v = Just k { keyMtime = readMaybe v } + sepfield k v a = case span (/= fieldSep) v of + (v', _:r) -> findfields r $ a k v' + _ -> Nothing + + findfields (c:v) (Just k) + | c == fieldSep = Just $ k { keyName = v } + | otherwise = sepfield k v $ addfield c + findfields _ v = v + + addbackend k v = Just k { keyBackendName = v } + addfield 's' k v = Just k { keySize = readMaybe v } + addfield 'm' k v = Just k { keyMtime = readMaybe v } addfield _ _ _ = Nothing -- for quickcheck @@ -73,7 +77,12 @@ instance Arbitrary Key where n <- arbitrary b <- elements ['A'..'Z'] s <- arbitrary - return $ Key { keyName = n, keyBackendName = [b] , keySize = s } + return $ Key { + keyName = n, + keyBackendName = [b], + keySize = s, + keyMtime = Nothing + } prop_idempotent_key_read_show :: Key -> Bool prop_idempotent_key_read_show k = Just k == (readKey $ show k) From 4594bd51c1754bc7e1fdf03d83569aeea163e761 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:04:50 -0400 Subject: [PATCH 1037/2835] rename file --- Annex.hs | 10 +++++----- Backend.hs | 2 +- Backend/File.hs | 2 +- Backend/SHA.hs | 2 +- Backend/URL.hs | 2 +- Backend/WORM.hs | 2 +- BackendTypes.hs => BackendClass.hs | 2 +- Command.hs | 4 ++-- Command/InAnnex.hs | 4 ++-- Key.hs | 3 +-- Types.hs | 2 +- test.hs | 8 ++++---- 12 files changed, 21 insertions(+), 22 deletions(-) rename BackendTypes.hs => BackendClass.hs (97%) diff --git a/Annex.hs b/Annex.hs index dd3362b29d..f8cfd0ec92 100644 --- a/Annex.hs +++ b/Annex.hs @@ -25,7 +25,7 @@ import Data.Maybe import qualified GitRepo as Git import qualified GitQueue -import qualified BackendTypes +import qualified BackendClass import Utility -- git-annex's monad @@ -34,8 +34,8 @@ type Annex = StateT AnnexState IO -- internal state storage data AnnexState = AnnexState { repo :: Git.Repo - , backends :: [BackendTypes.Backend Annex] - , supportedBackends :: [BackendTypes.Backend Annex] + , backends :: [BackendClass.Backend Annex] + , supportedBackends :: [BackendClass.Backend Annex] , repoqueue :: GitQueue.Queue , quiet :: Bool , force :: Bool @@ -47,7 +47,7 @@ data AnnexState = AnnexState , remotesread :: Bool } deriving (Show) -newState :: Git.Repo -> [BackendTypes.Backend Annex] -> AnnexState +newState :: Git.Repo -> [BackendClass.Backend Annex] -> AnnexState newState gitrepo allbackends = AnnexState { repo = gitrepo , backends = [] @@ -64,7 +64,7 @@ newState gitrepo allbackends = AnnexState } {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> [BackendTypes.Backend Annex] -> IO AnnexState +new :: Git.Repo -> [BackendClass.Backend Annex] -> IO AnnexState new gitrepo allbackends = do gitrepo' <- liftIO $ Git.configRead gitrepo return $ newState gitrepo' allbackends diff --git a/Backend.hs b/Backend.hs index 94755e8d6b..e1f8f388b9 100644 --- a/Backend.hs +++ b/Backend.hs @@ -40,7 +40,7 @@ import qualified GitRepo as Git import qualified Annex import Types import Key -import qualified BackendTypes as B +import qualified BackendClass as B import Messages {- List of backends in the order to try them when storing a new key. -} diff --git a/Backend/File.hs b/Backend/File.hs index d76cd29391..a5e2431998 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -18,7 +18,7 @@ import Control.Monad.State import System.Directory import Data.List -import BackendTypes +import BackendClass import LocationLog import Locations import qualified Remotes diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 3cdc3bf808..0563851076 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -16,7 +16,7 @@ import Data.Maybe import System.Posix.Files import qualified Backend.File -import BackendTypes +import BackendClass import Messages import qualified Annex import Locations diff --git a/Backend/URL.hs b/Backend/URL.hs index 29dc8fefa7..b40ff39599 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import Data.String.Utils import Types -import BackendTypes +import BackendClass import Utility import Messages diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 324aee76be..a011995da3 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -14,7 +14,7 @@ import System.Directory import Data.Maybe import qualified Backend.File -import BackendTypes +import BackendClass import Locations import qualified Annex import Content diff --git a/BackendTypes.hs b/BackendClass.hs similarity index 97% rename from BackendTypes.hs rename to BackendClass.hs index 48b208a9b4..909ae8f96e 100644 --- a/BackendTypes.hs +++ b/BackendClass.hs @@ -7,7 +7,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module BackendTypes where +module BackendClass where import Key diff --git a/Command.hs b/Command.hs index 38c63bd77e..18b4a167aa 100644 --- a/Command.hs +++ b/Command.hs @@ -17,7 +17,7 @@ import Data.List import Types import qualified Backend -import qualified BackendTypes +import qualified BackendClass import Messages import qualified Annex import qualified GitRepo as Git @@ -237,7 +237,7 @@ cmdlineKey = do backends <- Backend.list return $ stubKey { keyName = kname k, - keyBackendName = BackendTypes.name $ head backends + keyBackendName = BackendClass.name $ head backends } where kname Nothing = badkey diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs index 4a4102754a..a2beda4a59 100644 --- a/Command/InAnnex.hs +++ b/Command/InAnnex.hs @@ -13,7 +13,7 @@ import System.Exit import Command import Content import qualified Backend -import qualified BackendTypes +import qualified BackendClass import Key command :: [Command] @@ -28,7 +28,7 @@ start keyname = do backends <- Backend.list let key = stubKey { keyName = keyname, - keyBackendName = BackendTypes.name (head backends) + keyBackendName = BackendClass.name (head backends) } error "BROKEN. fixme!" present <- inAnnex key diff --git a/Key.hs b/Key.hs index 3385148707..f52aea31b7 100644 --- a/Key.hs +++ b/Key.hs @@ -76,11 +76,10 @@ instance Arbitrary Key where arbitrary = do n <- arbitrary b <- elements ['A'..'Z'] - s <- arbitrary return $ Key { keyName = n, keyBackendName = [b], - keySize = s, + keySize = Nothing, keyMtime = Nothing } diff --git a/Types.hs b/Types.hs index f48d4079be..503e27d312 100644 --- a/Types.hs +++ b/Types.hs @@ -11,6 +11,6 @@ module Types ( Key ) where -import BackendTypes +import BackendClass import Annex import Key diff --git a/test.hs b/test.hs index bc849dadc5..2bc0c37a37 100644 --- a/test.hs +++ b/test.hs @@ -29,7 +29,7 @@ import qualified Backend import qualified GitRepo as Git import qualified Locations import qualified Utility -import qualified BackendTypes +import qualified BackendClass import qualified Types import qualified GitAnnex import qualified LocationLog @@ -120,8 +120,8 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup] test_setkey :: Test test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do writeFile tmp $ content sha1annexedfile - r <- annexeval $ BackendTypes.getKey backendSHA1 tmp - let sha1 = BackendTypes.keyName $ fromJust r + r <- annexeval $ BackendClass.getKey backendSHA1 tmp + let sha1 = Key.keyName $ fromJust r git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed" @@ -439,7 +439,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do checkunused [annexedfilekey, sha1annexedfilekey] -- good opportunity to test dropkey also - git_annex "dropkey" ["-q", "--force", BackendTypes.keyName annexedfilekey] + git_annex "dropkey" ["-q", "--force", Key.keyName annexedfilekey] @? "dropkey failed" checkunused [sha1annexedfilekey] From 4651688290086d70275e3d0b2976dbc57f8e4df1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:08:19 -0400 Subject: [PATCH 1038/2835] increase repo version --- Upgrade.hs | 6 ++++++ Version.hs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Upgrade.hs b/Upgrade.hs index 7469d9ba75..eba75bf587 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -31,10 +31,16 @@ upgrade = do version <- getVersion case version of Just "0" -> upgradeFrom0 + Just "1" -> upgradeFrom1 Nothing -> return True -- repo not initted yet, no version Just v | v == currentVersion -> return True Just _ -> error "this version of git-annex is too old for this git repository!" +upgradeFrom1 :: Annex Bool +upgradeFrom1 = do + showSideAction "Upgrading object directory layout..." + error "upgradeFrom1 TODO FIXME" + upgradeFrom0 :: Annex Bool upgradeFrom0 = do showSideAction "Upgrading object directory layout..." diff --git a/Version.hs b/Version.hs index 9e31d3c9eb..7fdbd1a49a 100644 --- a/Version.hs +++ b/Version.hs @@ -16,7 +16,7 @@ import qualified GitRepo as Git import Locations currentVersion :: String -currentVersion = "1" +currentVersion = "2" versionField :: String versionField = "annex.version" From 27472710c73320449ce96bc409fa82e0a46be1cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:19:44 -0400 Subject: [PATCH 1039/2835] initial pass at doc update --- debian/changelog | 3 +++ doc/git-annex.mdwn | 14 ++++---------- doc/walkthrough/modifying_annexed_files.mdwn | 2 +- .../moving_file_content_between_repositories.mdwn | 2 +- doc/walkthrough/unused_data.mdwn | 4 ++-- doc/walkthrough/using_ssh_remotes.mdwn | 2 +- doc/walkthrough/using_the_URL_backend.mdwn | 2 +- 7 files changed, 13 insertions(+), 16 deletions(-) diff --git a/debian/changelog b/debian/changelog index e7017a26d0..0d1832374d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,8 @@ git-annex (0.24) UNRELEASED; urgency=low + * Reorganized annexed object store. annex.version=2 + * The setkey, fromkey, and dropkey subcommands have changed how + the key is specified. --backend is no longer used with these. * Add Suggests on graphviz. Closes: #618039 * When adding files to the annex, the symlinks pointing at the annexed content are made to have the same mtime as the original file. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4998a64911..f2ec5fd15e 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -234,11 +234,11 @@ Many git-annex commands will stage changes for later `git commit` by you. This can be used to manually set up a file to link to a specified key in the key-value backend. How you determine an existing key in the backend - varies. For the URL backend, the key is just a URL to the content. + varies. For the URL backend, the key is based on an URL to the content. Example: - git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + git annex fromkey --key=URL--http://www.archive.org/somefile somefile * dropkey [key ...] @@ -248,24 +248,18 @@ Many git-annex commands will stage changes for later `git commit` by you. This can be used to drop content for arbitrary keys, which do not need to have a file in the git repository pointing at them. - A backend will typically need to be specified with --backend. If none - is specified, the first configured backend is used. - Example: - git annex dropkey --backend=SHA1 7da006579dd64330eb2456001fd01948430572f2 + git annex dropkey --key=SHA1-s10-7da006579dd64330eb2456001fd01948430572f2 * setkey file This plumbing-level command sets the annexed data for a key to the content of the specified file, and then removes the file. - A backend will typically need to be specified with --backend. If none - is specified, the first configured backend is used. - Example: - git annex setkey --backend=WORM --key=1287765018:3 /tmp/file + git annex setkey --key=WORM-s3-m1287765018--file /tmp/file # OPTIONS diff --git a/doc/walkthrough/modifying_annexed_files.mdwn b/doc/walkthrough/modifying_annexed_files.mdwn index 3ad4e82eab..f75b73a24c 100644 --- a/doc/walkthrough/modifying_annexed_files.mdwn +++ b/doc/walkthrough/modifying_annexed_files.mdwn @@ -27,7 +27,7 @@ and this symlink is what gets committed to git in the end. add my_cool_big_file ok [master 64cda67] changed an annexed file 2 files changed, 2 insertions(+), 1 deletions(-) - create mode 100644 .git-annex/WORM:1289672605:30:file.log + create mode 100644 .git-annex/WORM-s30-m1289672605--file.log There is one problem with using `git commit` like this: Git wants to first stage the entire contents of the file in its index. That can be slow for diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn index d7150f109a..6b3e3f4e80 100644 --- a/doc/walkthrough/moving_file_content_between_repositories.mdwn +++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn @@ -9,5 +9,5 @@ makes it very easy. move my_cool_big_file (moving to usbdrive...) ok # git annex move video/hackity_hack_and_kaxxt.mov --from fileserver move video/hackity_hack_and_kaxxt.mov (moving from fileserver...) - WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 + WORM-s86050597-m1274316523--hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 ok diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn index 69a581fe1e..9be32577c3 100644 --- a/doc/walkthrough/unused_data.mdwn +++ b/doc/walkthrough/unused_data.mdwn @@ -12,8 +12,8 @@ eliminate it to save space. unused (checking for unused data...) Some annexed data is no longer pointed to by any files in the repository. NUMBER KEY - 1 WORM:1289672605:3:file - 2 WORM:1289672605:14:file + 1 WORM-s3-m1289672605--file + 2 WORM-s14-m1289672605--file (To see where data was previously used, try: git log --stat -S'KEY') (To remove unwanted data: git-annex dropunused NUMBER) ok diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn index 6af9e1f477..4c2f830de8 100644 --- a/doc/walkthrough/using_ssh_remotes.mdwn +++ b/doc/walkthrough/using_ssh_remotes.mdwn @@ -13,7 +13,7 @@ Now you can get files and they will be transferred (using `rsync` via `ssh`): # git annex get my_cool_big_file get my_cool_big_file (getting UUID for origin...) (copying from origin...) - WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 + WORM-s2159-m1285650548--my_cool_big_file 100% 2159 2.1KB/s 00:00 ok When you drop files, git-annex will ssh over to the remote and make diff --git a/doc/walkthrough/using_the_URL_backend.mdwn b/doc/walkthrough/using_the_URL_backend.mdwn index fe79a6be2e..585fd0668a 100644 --- a/doc/walkthrough/using_the_URL_backend.mdwn +++ b/doc/walkthrough/using_the_URL_backend.mdwn @@ -5,7 +5,7 @@ Another handy backend is the URL backend, which can fetch file's content from remote URLs. Here's how to set up some files in your repository that use this backend: - # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + # git annex fromkey --key=URL--http://www.archive.org/somefile somefile fromkey somefile ok # git commit -m "added a file from the Internet Archive" From da504f647fdbec7aa3a3c08244520de2c00898ef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:28:18 -0400 Subject: [PATCH 1040/2835] fromkey, and url backend download work now --- Backend/URL.hs | 6 +++--- Command.hs | 19 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index b40ff39599..02ce3563cd 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -8,12 +8,12 @@ module Backend.URL (backends) where import Control.Monad.State (liftIO) -import Data.String.Utils import Types import BackendClass import Utility import Messages +import Key backends :: [Backend Annex] backends = [backend] @@ -52,8 +52,8 @@ dummyOk _ = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do - showNote "downloading" + showNote $ "downloading" showProgress -- make way for curl progress bar liftIO $ boolSystem "curl" [Params "-# -o", File file, File url] where - url = join ":" $ drop 1 $ split ":" $ show key + url = keyName key diff --git a/Command.hs b/Command.hs index 18b4a167aa..27598fff0a 100644 --- a/Command.hs +++ b/Command.hs @@ -17,7 +17,6 @@ import Data.List import Types import qualified Backend -import qualified BackendClass import Messages import qualified Annex import qualified GitRepo as Git @@ -230,20 +229,18 @@ paramName = "NAME" paramNothing :: String paramNothing = "" -{- The Key specified by the --key and --backend parameters. -} +{- The Key specified by the --key parameter. -} cmdlineKey :: Annex Key cmdlineKey = do k <- Annex.getState Annex.defaultkey - backends <- Backend.list - return $ stubKey { - keyName = kname k, - keyBackendName = BackendClass.name $ head backends - } + case k of + Nothing -> nokey + Just "" -> nokey + Just kstring -> case readKey kstring of + Nothing -> error "bad key" + Just key -> return key where - kname Nothing = badkey - kname (Just "") = badkey - kname (Just n) = n - badkey = error "please specify the key with --key" + nokey = error "please specify the key with --key" {- Given an original list of files, and an expanded list derived from it, - ensures that the original list's ordering is preserved. From 2e1cc2f8b98aaf9e01b620d557c42d5b1ae2aaa6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:42:34 -0400 Subject: [PATCH 1041/2835] fixed dropkey, setkey, and git-annex-shell subcommands key is now specified as the full key, no --backend needed --- Command.hs | 7 +++++-- Command/DropKey.hs | 14 +++++--------- Command/InAnnex.hs | 13 ++----------- Command/RecvKey.hs | 11 ++--------- Command/SendKey.hs | 11 ++--------- doc/git-annex.mdwn | 2 +- 6 files changed, 17 insertions(+), 41 deletions(-) diff --git a/Command.hs b/Command.hs index 27598fff0a..c3cb612ee0 100644 --- a/Command.hs +++ b/Command.hs @@ -14,6 +14,7 @@ import Control.Monad (filterM, liftM, when) import System.Path.WildMatch import Text.Regex.PCRE.Light.Char8 import Data.List +import Data.Maybe import Types import qualified Backend @@ -46,6 +47,8 @@ type CommandCleanup = Annex Bool - functions. -} type CommandSeekStrings = CommandStartString -> CommandSeek type CommandStartString = String -> CommandStart +type CommandSeekKeys = CommandStartKey -> CommandSeek +type CommandStartKey = Key -> CommandStart type BackendFile = (FilePath, Maybe (Backend Annex)) type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek type CommandStartBackendFile = BackendFile -> CommandStart @@ -167,8 +170,8 @@ withFilesUnlocked' typechanged a params = do map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' -withKeys :: CommandSeekStrings -withKeys a params = return $ map a params +withKeys :: CommandSeekKeys +withKeys a params = return $ map a $ catMaybes $ map readKey params withTempFile :: CommandSeekStrings withTempFile a params = return $ map a params withNothing :: CommandSeekNothing diff --git a/Command/DropKey.hs b/Command/DropKey.hs index f0450eea34..419d9caa43 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -14,6 +14,7 @@ import LocationLog import Types import Content import Messages +import Key command :: [Command] command = [Command "dropkey" (paramRepeating paramKey) seek @@ -22,21 +23,16 @@ command = [Command "dropkey" (paramRepeating paramKey) seek seek :: [CommandSeek] seek = [withKeys start] -{- Drops cached content for a key. -} -start :: CommandStartString -start keyname = do - backends <- Backend.list - let key = error "fixme!!" - --let key = genKey (head backends) keyname --TODO FIXME - let present = error "fixme!!" - --present <- inAnnex key +start :: CommandStartKey +start key = do + present <- inAnnex key force <- Annex.getState Annex.force if not present then return Nothing else if not force then error "dropkey is can cause data loss; use --force if you're sure you want to do this" else do - showStart "dropkey" keyname + showStart "dropkey" (show key) return $ Just $ perform key perform :: Key -> CommandPerform diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs index a2beda4a59..fa81fc9a4c 100644 --- a/Command/InAnnex.hs +++ b/Command/InAnnex.hs @@ -12,9 +12,6 @@ import System.Exit import Command import Content -import qualified Backend -import qualified BackendClass -import Key command :: [Command] command = [Command "inannex" (paramRepeating paramKey) seek @@ -23,14 +20,8 @@ command = [Command "inannex" (paramRepeating paramKey) seek seek :: [CommandSeek] seek = [withKeys start] -start :: CommandStartString -start keyname = do - backends <- Backend.list - let key = stubKey { - keyName = keyname, - keyBackendName = BackendClass.name (head backends) - } - error "BROKEN. fixme!" +start :: CommandStartKey +start key = do present <- inAnnex key if present then return Nothing diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 488bab62d6..c7c37d1e31 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -12,10 +12,8 @@ import Control.Monad.State (liftIO) import System.Exit import Command -import Types import CmdLine import Content -import qualified Backend import RsyncFile command :: [Command] @@ -25,12 +23,8 @@ command = [Command "recvkey" paramKey seek seek :: [CommandSeek] seek = [withKeys start] -start :: CommandStartString -start keyname = do - error "BROKEN FIXME!" - {- - backends <- Backend.list - let key = genKey (head backends) keyname +start :: CommandStartKey +start key = do present <- inAnnex key when present $ error "key is already present in annex" @@ -43,4 +37,3 @@ start keyname = do _ <- shutdown liftIO exitSuccess else liftIO exitFailure - -} diff --git a/Command/SendKey.hs b/Command/SendKey.hs index ff269f21fe..56974bda96 100644 --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -14,9 +14,7 @@ import System.Exit import Locations import qualified Annex import Command -import Types import Content -import qualified Backend import RsyncFile command :: [Command] @@ -26,16 +24,11 @@ command = [Command "sendkey" paramKey seek seek :: [CommandSeek] seek = [withKeys start] -start :: CommandStartString -start keyname = do - error "BROKEN FIXME!" - {- - backends <- Backend.list - let key = genKey (head backends) keyname +start :: CommandStartKey +start key = do present <- inAnnex key g <- Annex.gitRepo let file = gitAnnexLocation g key when present $ liftIO $ rsyncServerSend file liftIO exitFailure - -} diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index f2ec5fd15e..bfec527d9a 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -250,7 +250,7 @@ Many git-annex commands will stage changes for later `git commit` by you. Example: - git annex dropkey --key=SHA1-s10-7da006579dd64330eb2456001fd01948430572f2 + git annex dropkey SHA1-s10-7da006579dd64330eb2456001fd01948430572f2 * setkey file From 9d24cc7bdb011d66e41229a3b96401808be47268 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:46:47 -0400 Subject: [PATCH 1042/2835] make commands that take a key as a parameter error if it's bad --- Command.hs | 7 +++++-- Command/DropKey.hs | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Command.hs b/Command.hs index c3cb612ee0..41ad884a93 100644 --- a/Command.hs +++ b/Command.hs @@ -14,7 +14,6 @@ import Control.Monad (filterM, liftM, when) import System.Path.WildMatch import Text.Regex.PCRE.Light.Char8 import Data.List -import Data.Maybe import Types import qualified Backend @@ -171,7 +170,11 @@ withFilesUnlocked' typechanged a params = do unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' withKeys :: CommandSeekKeys -withKeys a params = return $ map a $ catMaybes $ map readKey params +withKeys a params = return $ map a $ map parse params + where + parse p = case readKey p of + Just k -> k + Nothing -> error "bad key" withTempFile :: CommandSeekStrings withTempFile a params = return $ map a params withNothing :: CommandSeekNothing diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 419d9caa43..b3cc60961c 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -9,12 +9,10 @@ module Command.DropKey where import Command import qualified Annex -import qualified Backend import LocationLog import Types import Content import Messages -import Key command :: [Command] command = [Command "dropkey" (paramRepeating paramKey) seek From 49b7f5918341c30140779ea1f376b4d9f81d8a30 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:53:14 -0400 Subject: [PATCH 1043/2835] test suite passes again doesn't test remote functionality.. but that may be working too now --- Command/Move.hs | 4 +--- Remotes.hs | 6 ++---- doc/git-annex.mdwn | 5 ++++- test.hs | 8 ++++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index 1b14813089..2d6c973fe0 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -20,7 +20,6 @@ import qualified Remotes import UUID import Messages import Utility -import Key command :: [Command] command = [Command "move" paramPath seek @@ -137,8 +136,7 @@ fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup fromCleanup src True key = do ok <- Remotes.onRemote src (boolSystem, False) "dropkey" [ Params "--quiet --force" - , Param $ "--backend=" ++ keyBackendName key - , Param $ keyName key + , Param $ show key ] -- better safe than sorry: assume the src dropped the key -- even if it seemed to fail; the failure could have occurred diff --git a/Remotes.hs b/Remotes.hs index dd733e4545..8b760ac957 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -27,7 +27,6 @@ import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) import Types -import Key import qualified GitRepo as Git import qualified Annex import LocationLog @@ -154,7 +153,7 @@ inAnnex r key = if Git.repoIsUrl r checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- onRemote r (boolSystem, False) "inannex" - [Param ("--backend=" ++ keyBackendName key), Param (keyName key)] + [Param (show key)] return $ Right inannex {- Cost Ordered list of remotes. -} @@ -273,8 +272,7 @@ rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam] rsyncParams r sending key file = do Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") - [ Param $ "--backend=" ++ keyBackendName key - , Param $ keyName key + [ Param $ show key -- Command is terminated with "--", because -- rsync will tack on its own options afterwards, -- and they need to be ignored. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index bfec527d9a..e559e8cba6 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -296,7 +296,10 @@ Many git-annex commands will stage changes for later `git commit` by you. * --backend=name - Specifies which key-value backend to use. + Specifies which key-value backend to use. This can be used when + adding a file to the annex, or migrating a file. Once files + are in the annex, their backend is known and this option is not + necessary. * --key=name diff --git a/test.hs b/test.hs index 2bc0c37a37..49f7f2ab99 100644 --- a/test.hs +++ b/test.hs @@ -121,9 +121,9 @@ test_setkey :: Test test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do writeFile tmp $ content sha1annexedfile r <- annexeval $ BackendClass.getKey backendSHA1 tmp - let sha1 = Key.keyName $ fromJust r - git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" - git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" + let key = show $ fromJust r + git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed" + git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed" Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed" annexed_present sha1annexedfile where @@ -439,7 +439,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do checkunused [annexedfilekey, sha1annexedfilekey] -- good opportunity to test dropkey also - git_annex "dropkey" ["-q", "--force", Key.keyName annexedfilekey] + git_annex "dropkey" ["-q", "--force", show annexedfilekey] @? "dropkey failed" checkunused [sha1annexedfilekey] From fe4e482a9633e977beb1c132c3705b5e103846fa Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 02:58:44 +0000 Subject: [PATCH 1044/2835] Comment moderation --- ..._a868e805be43c5a7c19c41f1af8e41e6._comment | 10 +++++++ ..._7101d07400ad5935f880dc00d89bf90e._comment | 27 +++++++++++++++++++ ..._008554306dd082d7f543baf283510e92._comment | 19 +++++++++++++ ..._504c96959c779176f991f4125ea22009._comment | 14 ++++++++++ ..._79bdf9c51dec9f52372ce95b53233bb2._comment | 12 +++++++++ 5 files changed, 82 insertions(+) create mode 100644 doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment create mode 100644 doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment create mode 100644 doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment create mode 100644 doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment diff --git a/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment b/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment new file mode 100644 index 0000000000..954433deb4 --- /dev/null +++ b/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-15T14:11:27Z" + content=""" +Keep in mind that lots of small files may have significant overhead, so a warning that it's not possible to make sure there's enough space would make sense for certain corner cases. Actually finding out the exact overhead is beyond git-annex' scope and, given transparent compression etc, ability, but a warning, optionally with a \"do you want to continue\" prompt can't hurt. + +-- RichiH +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment b/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment new file mode 100644 index 0000000000..7d50c58d1b --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="praet" + ip="81.240.159.215" + subject="Use variable symlinks, relative to the repo's root ?" + date="2011-03-10T16:50:28Z" + content=""" +It all boils down to the fact that the path to a relative symlink's target is determined relative to the symlink itself. + +Now, if we define the symlink's target relative to the git repo's root (eg. using the $GIT_DIR environment variable, which can be a relative or absolute path itself), this unfortunately results in an absolute symlink, which would -for obvious reasons- only be usable locally: + + user@host:~$ mkdir -p tmp/{.git/annex,somefolder} + user@host:~$ export GIT_DIR=~/tmp + user@host:~$ touch $GIT_DIR/.git/annex/realfile + user@host:~$ ln -s $GIT_DIR/.git/annex/realfile $GIT_DIR/somefolder/file + user@host:~$ ls -al $GIT_DIR/somefolder/ + total 12 + drwxr-x--- 2 user group 4096 2011-03-10 16:54 . + drwxr-x--- 4 user group 4096 2011-03-10 16:53 .. + lrwxrwxrwx 1 user group 33 2011-03-10 16:54 file -> /home/user/tmp/.git/annex/realfile + user@host:~$ + +So, what we need is the ability to record the actual variable name (instead of it's value) in our symlinks. + +It *is* possible, using [variable/variant symlinks](http://en.wikipedia.org/wiki/Symbolic_link#Variable_symbolic_links), yet I'm unsure as to whether or not this is available on Linux systems, and even if it is, it would introduce compatibility issues in multi-OS environments. + +Thoughts on this? +"""]] diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment new file mode 100644 index 0000000000..ab114bb1c8 --- /dev/null +++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://dieter-be.myopenid.com/" + nickname="dieter" + subject="comment 2" + date="2011-02-16T21:32:04Z" + content=""" +thanks Joey, + +is it possible to run some git annex command that tells me, for a specific directory, which files are available in an other remote? (and which remote, and which filenames?) +I guess I could run that, do my own policy thingie, and run `git annex get` for the files I want. + +For your podcast use case (and some of my use cases) don't you think git [annex] might actually be overkill? For example your podcasts use case, what value does git annex give over a simple rsync/rm script? +such a script wouldn't even need a data store to store its state, unlike git. it seems simpler and cleaner to me. + +for the mpd thing, check http://alip.github.com/mpdcron/ (bad project name, it's a plugin based \"event handler\") +you should be able to write a simple plugin for mpdcron that does what you want (or even interface with mpd yourself from perl/python/.. to use its idle mode to get events) + +Dieter +"""]] diff --git a/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment b/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment new file mode 100644 index 0000000000..64f1e16b50 --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-03-15T13:52:16Z" + content=""" +Can't you just use an underscore instead of a colon? + +Would it be feasible to split directories dynamically? I.e. start with SHA1_123456789abcdef0123456789abcdef012345678/SHA1_123456789abcdef0123456789abcdef012345678 and, at a certain cut-off point, switch to shorter directory names? This could even be done per subdirectory and based purely on a locally-configured number. Different annexes on different file systems or with different file subsets might even have different thresholds. This would ensure scale while not forcing you to segment from the start. Also, while segmenting with longer directory names means a flatter tree, segments longer than four characters might not make too much sense. Segmenting too often could lead to some directories becoming too populated, bringing us back to the dynamic segmentation. + +All of the above would make merging annexes by hand a _lot_ harder, but I don't know if this is a valid use case. And if all else fails, one could merge everything with the unsegemented directory names and start again from there. + +-- RichiH +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment b/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment new file mode 100644 index 0000000000..886941be72 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-15T14:08:41Z" + content=""" +What is the potential time-frame for this change? As I am not using git-annex for production yet, I can see myself waiting to avoid any potential hassle. + +Supporting generic metadata seems like a great idea. Though if you are going this path, wouldn't it make sense to avoid metastore for mtime etc and support this natively without outside dependencies? + +-- RichiH +"""]] From 795502624dd8bdbcd59767232e9e3d4d8906ca9a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:01:17 +0000 Subject: [PATCH 1045/2835] Added a comment --- ...mment_3_4c69097fe2ee81359655e59a03a9bb8d._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment new file mode 100644 index 0000000000..5cdd6aa0c6 --- /dev/null +++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T03:01:17Z" + content=""" +Whups, the comment above got stuck in moderation queue for 27 days. I will try to check that more frequently. + +In the meantime, I've implemented \"git annex whereis\" -- enjoy! + +I find keeping my podcasts in the annex useful because it allows me to download individual episodes or poscasts easily when low bandwidth is available (ie, dialup), or over sneakernet. And generally keeps everything organised. +"""]] From 9741c2026898ef453716d0368091bd17dbebc13a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:03:19 +0000 Subject: [PATCH 1046/2835] Added a comment --- .../comment_3_57010bcaca42089b451ad8659a1e018e._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment b/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment new file mode 100644 index 0000000000..534723254a --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T03:03:19Z" + content=""" +Interesting, I had not heard of variable symlinks before. AFAIK linux does not have them. +"""]] From cf5e002daf481fa4d209ac10d3a991a09e55e13a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:04:50 +0000 Subject: [PATCH 1047/2835] Added a comment --- .../comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment diff --git a/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment b/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment new file mode 100644 index 0000000000..9a43fe3f27 --- /dev/null +++ b/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-03-16T03:04:50Z" + content=""" +Right. You probably don't want git-annex to fill up your entire drive anyway, so if it tries to reseve 10 mb or 1% or whatever (probably configurable) for overhead, that should be good enough. +"""]] From 682db07d308118064725e57ee90e1aed4323b08c Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:13:39 +0000 Subject: [PATCH 1048/2835] Added a comment --- ...mment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment diff --git a/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment b/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment new file mode 100644 index 0000000000..51deb2f959 --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T03:13:39Z" + content=""" +It is unfortunatly not possible to do system-dependant hashing, so long as git-annex stores symlinks to the content in git. + +It might be possible to start without hashing, and add hashing for new files after a cutoff point. It would add complexity. + +I'm currently looking at a 2 character hash directory segment, based on an md5sum of the key, which splits it into 1024 buckets. git uses just 256 buckets for its object directory, but then its objects tend to get packed away. I sorta hope that one level is enough, but guess I could go to 2 levels (objects/ab/cd/key), which would provide 1048576 buckets, probably plenty, as if you are storing more than a million files, you are probably using a modern enough system to have a filesystem that doesn't need hashing. +"""]] From 0ead8b59c247c6208a7223fe5ab013ae79b6a1e2 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:22:46 +0000 Subject: [PATCH 1049/2835] Added a comment --- ...mment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment b/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment new file mode 100644 index 0000000000..475359abbf --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-03-16T03:22:45Z" + content=""" +Well, I spent a few hours playing this evening in the 'reorg' branch in git. It seems to be shaping up pretty well; type-based refactoring in haskell makes these kind of big systematic changes a matter of editing until it compiles. And it compiles and test suite passes. But, so far I've only covered 1. 3. and 4. on the list, and have yet to deal with upgrades. + +I'd recommend you not wait before using git-annex. I am committed to provide upgradability between annexes created with all versions of git-annex, going forward. This is important because we can have offline archival drives that sit unused for years. Git-annex will upgrade a repository to current standard the first time it sees it, and I hope the upgrade will be pretty smooth. It was not bad for the annex.version 0 to 1 upgrade earlier. The only annoyance with upgrades is that it will result in some big commits to git, as every symlink in the repo gets changed, and log files get moved to new names. + +(The metadata being stored with keys is data that a particular backend can use, and is static to a given key, so there are no merge issues (and it won't be used to preserve mtimes, etc).) +"""]] From 955bda7803b5d8dcbe103d986d53a35186b34ab0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 23:25:12 -0400 Subject: [PATCH 1050/2835] update --- doc/todo/object_dir_reorg_v2.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/object_dir_reorg_v2.mdwn b/doc/todo/object_dir_reorg_v2.mdwn index db18856995..1c2d2f21b7 100644 --- a/doc/todo/object_dir_reorg_v2.mdwn +++ b/doc/todo/object_dir_reorg_v2.mdwn @@ -6,6 +6,8 @@ all users, so this should be the *last* reorg in the forseeable future. 2. Add hashing, since some filesystems do suck (like er, fat at least :) [[forum/hashing_objects_directories]] + (Also, may as well hash .git-annex/* while at it -- that's what + really gets big.) 3. Add filesize metadata for [[bugs/free_space_checking]]. (Currently only present in WORM, and in an ad-hoc way.) From fd2f04694f8ba52d9b67e35c95f1bddc33bbe292 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 23:25:16 -0400 Subject: [PATCH 1051/2835] add comments page --- doc/comments.mdwn | 9 +++++++++ doc/index.mdwn | 1 + 2 files changed, 10 insertions(+) create mode 100644 doc/comments.mdwn diff --git a/doc/comments.mdwn b/doc/comments.mdwn new file mode 100644 index 0000000000..e19962b92a --- /dev/null +++ b/doc/comments.mdwn @@ -0,0 +1,9 @@ +[[!sidebar content=""" +[[!inline pages="comment_pending(*)" feedfile=pendingmoderation +description="comments pending moderation" show=-1]] +Comments in the [[!commentmoderation desc="moderation queue"]]: +[[!pagecount pages="comment_pending(*)"]] +"""]] + +Recent comments posted to this site: +[[!inline pages="comment(*)" template="comment"]] diff --git a/doc/index.mdwn b/doc/index.mdwn index 47682349ff..4f967b71f5 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -10,6 +10,7 @@ To get a feel for it, see the [[walkthrough]]. * [[bugs]] * [[todo]] * [[forum]] +* [[comments]] * [[contact]] [[News]]: From 6c412fb9f55155b0b7bf58d578e51640514ec562 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 23:39:04 -0400 Subject: [PATCH 1052/2835] escape colons in key files --- Locations.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Locations.hs b/Locations.hs index 6cff910880..9ffcd9f42b 100644 --- a/Locations.hs +++ b/Locations.hs @@ -118,15 +118,19 @@ isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s - a slash - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping - is one to one. + - ":" is escaped to "&c", because despite it being 2011, people still care + - about FAT. - -} keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key +keyFile key = replace "/" "%" $ replace ":" "&c" $ + replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Maybe Key fileKey file = readKey $ - replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + replace "&a" "&" $ replace "&s" "%" $ + replace "&c" ":" $ replace "%" "/" file {- for quickcheck -} prop_idempotent_fileKey :: String -> Bool From dd5448eb075c3774aa173cb9f2e4344ce62b3e13 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 23:58:27 -0400 Subject: [PATCH 1053/2835] added 2 level hashing This means there can be 1024 subdirs, each with up to 1024 sub-subdirs. So with hundreds of millions of annexed objects, each leaf directory will have only a few files on average. --- LocationLog.hs | 5 ----- Locations.hs | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index f778df3864..a939af825d 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -123,11 +123,6 @@ logNow s u = do now <- getPOSIXTime return $ LogLine now s u -{- Returns the filename of the log file for a given key. -} -logFile :: Git.Repo -> Key -> String -logFile repo key = - gitStateDir repo ++ keyFile key ++ ".log" - {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} keyLocations :: Git.Repo -> Key -> IO [UUID] diff --git a/Locations.hs b/Locations.hs index 9ffcd9f42b..b2d31a1bf8 100644 --- a/Locations.hs +++ b/Locations.hs @@ -19,6 +19,7 @@ module Locations ( gitAnnexBadDir, gitAnnexUnusedLog, isLinkToAnnex, + logFile, prop_idempotent_fileKey ) where @@ -66,7 +67,7 @@ objectDir = addTrailingPathSeparator $ annexDir "objects" {- Annexed file's location relative to the .git directory. -} annexLocation :: Key -> FilePath -annexLocation key = objectDir f f +annexLocation key = objectDir hashDir key f f where f = keyFile key @@ -109,6 +110,11 @@ gitAnnexUnusedLog r = gitAnnexDir r "unused" isLinkToAnnex :: FilePath -> Bool isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s +{- The filename of the log file for a given key. -} +logFile :: Git.Repo -> Key -> String +logFile repo key = + gitStateDir repo ++ hashDir key ++ keyFile key ++ ".log" + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid @@ -137,11 +143,13 @@ prop_idempotent_fileKey :: String -> Bool prop_idempotent_fileKey s = Just k == fileKey (keyFile k) where k = stubKey { keyName = s, keyBackendName = "test" } -{- Given a filename, generates a short directory name to put it in, +{- Given a key, generates a short directory name to put it in, - to do hashing to protect against filesystems that dislike having - many items in a single directory. -} -hashDir :: FilePath -> FilePath -hashDir s = take 2 $ abcd_to_dir $ md5 (Str s) +hashDir :: Key -> FilePath +hashDir k = addTrailingPathSeparator $ take 2 dir drop 2 dir + where + dir = take 4 $ abcd_to_dir $ md5 $ Str $ show k abcd_to_dir :: ABCD -> String abcd_to_dir (ABCD (a,b,c,d)) = concat $ map display_32bits_as_dir [a,b,c,d] From 2e26caa8568001b33a969efc46f6911278686e0e Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 04:06:19 +0000 Subject: [PATCH 1054/2835] Added a comment --- ...mment_4_0de9170e429cbfea66f5afa8980d45ac._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment diff --git a/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment b/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment new file mode 100644 index 0000000000..b29eea1b2b --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-03-16T04:06:19Z" + content=""" +The .git-annex/ directory is what really needs hashing. + +Consider that when git looks for changes in there, it has to scan every file in the directory. With hashing, it should be able to more quickly identify just the subdirectories that contained changed files, by the directory mtimes. + +And the real kicker is that when committing there, git has to create a tree object containing every single file, even if only 1 file changed. That will be a lot of extra work; with hashed subdirs it will instead create just 2 or 3 small tree objects leading down to the changed file. (Probably these trees both pack down to similar size pack files, not sure.) +"""]] From 09a7689bc30faaf938a0b32a417d38ac093a6f7a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 00:08:02 -0400 Subject: [PATCH 1055/2835] update and bug closures for v2 layout --- debian/changelog | 7 +++++++ doc/bugs/fat_support.mdwn | 3 +++ doc/forum/hashing_objects_directories.mdwn | 8 ++++++++ doc/internals.mdwn | 10 +++++++--- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0d1832374d..ac7c854ff5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,13 @@ git-annex (0.24) UNRELEASED; urgency=low * Reorganized annexed object store. annex.version=2 + * Colons are now avoided in filenames, so bare clones of git repos + can be put on USB thumb drives formatted with vFAT or similar + filesystems. + * Added two levels of hashing to object directory and .git-annex logs, + to improve scalability with enormous numbers of annexed + objects. (With one hundred million annexed objects, each + directory would contain fewer than 1024 files.) * The setkey, fromkey, and dropkey subcommands have changed how the key is specified. --backend is no longer used with these. * Add Suggests on graphviz. Closes: #618039 diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn index 2c6c973856..60633c29bf 100644 --- a/doc/bugs/fat_support.mdwn +++ b/doc/bugs/fat_support.mdwn @@ -10,3 +10,6 @@ be VFAT formatted: [[!tag wishlist]] +[[Done]]; in annex.version 2 repos, colons are entirely avoided in +filenames. So a bare git clone can be put on VFAT, and git-annex +used to move stuff --to and --from it, for sneakernet. diff --git a/doc/forum/hashing_objects_directories.mdwn b/doc/forum/hashing_objects_directories.mdwn index 715e972ca4..5b7708fb58 100644 --- a/doc/forum/hashing_objects_directories.mdwn +++ b/doc/forum/hashing_objects_directories.mdwn @@ -17,3 +17,11 @@ or anything in between to a paranoid Also the use of a colon specifically breaks FAT32 ([[bugs/fat_support]]), must it be a colon or could an extra directory be used? i.e. `.git/annex/objects/SHA1/*/...` `git annex init` could also create all but the last level directory on initialization. I'm thinking `SHA1/1/1, SHA1/1/2, ..., SHA256/f/f, ..., URL/f/f, ..., WORM/f/f` + +> This is done now with a 2-level hash. It also hashes .git-annex/ log +> files which were the worse problem really. Scales to hundreds of millions +> of files with each dir having 1024 or fewer contents. Example: +> +> `me -> .git/annex/objects/71/9t/WORM-s3-m1300247299--me/WORM-s3-m1300247299--me` +> +> --[[Joey]] diff --git a/doc/internals.mdwn b/doc/internals.mdwn index 3f680dd8f2..a133320b4b 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -2,12 +2,15 @@ In the world of git, we're not scared about internal implementation details, and sometimes we like to dive in and tweak things by hand. Here's some documentation to that end. -## `.git/annex/objects/*/*` +## `.git/annex/objects/aa/bb/*/*` This is where locally available file contents are actually stored. Files added to the annex get a symlink checked into git that points to the file content. +First there are two levels of directories used for hashing, to prevent +too many things ending up in any one directory. + Each subdirectory has the name of a key in one of the [[key-value_backends|backends]]. The file inside also has the name of the key. This two-level structure is used because it allows the write bit to be removed @@ -41,10 +44,11 @@ Example: e605dca6-446a-11e0-8b2a-002170d25c55 1 26339d22-446b-11e0-9101-002170d25c55 ? -## `.git-annex/*.log` +## `.git-annex/aa/bb/*.log` The remainder of the log files record [[location_tracking]] information -for file contents. The name of the key is the filename, and the content +for file contents. Again these are placed in two levels of subdirectories +for hashing. The name of the key is the filename, and the content consists of a timestamp, either 1 (present) or 0 (not present), and the UUID of the repository that has or lacks the file content. From f1e010f42e373bd0658c3b9c6ab67cd84715ad60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 00:32:15 -0400 Subject: [PATCH 1056/2835] upgrade thoughts long comments :) --- Upgrade.hs | 47 +++++++++++++++++++++++++++++++++++++++++++++-- debian/changelog | 1 + 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/Upgrade.hs b/Upgrade.hs index eba75bf587..d63397ce0f 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -38,12 +38,52 @@ upgrade = do upgradeFrom1 :: Annex Bool upgradeFrom1 = do - showSideAction "Upgrading object directory layout..." + showSideAction "Upgrading object directory layout v1 to v2..." error "upgradeFrom1 TODO FIXME" + -- v2 adds hashing of filenames of content and location log files. + -- + -- Key information is encoded in filenames differently. + -- + -- When upgrading a v1 key to v2, file size metadata needs to be + -- added to the key (unless it is a WORM key, which encoded + -- mtime:size in v1). This can only be done when the file content + -- is present. + -- + -- So there are two approaches -- either upgrade + -- everything, leaving out file size information for files not + -- present in the current repo; or upgrade peicemeil, only + -- upgrading keys whose content is present. + -- + -- The latter approach would mean that, until every clone of an + -- annex is upgraded, git annex would refuse to operate on annexed + -- files that had not yet been committed. Unless it were taught to + -- work with both v1 and v2 keys in the same repo. + -- + -- Another problem with the latter approach might involve content + -- being moved between repos while the conversion is still + -- incomplete. If repo A has already upgraded, and B has not, and B + -- has K, moving K from B -> A would result in it lurking + -- unconverted on A. Unless A upgraded it in passing. But that's + -- getting really complex, and would mean a constant trickle of + -- upgrade commits, which users would find annoying. + -- + -- So, the former option it is! Note that file size metadata + -- will only be used for detecting situations where git-annex + -- would run out of disk space, so if some keys don't have it, + -- the impact is small. At least initially. It could be used in the + -- future by smart auto-repo balancing code, etc. + -- + -- Anyway, since v2 plans ahead for other metadata being included + -- in keys, there should probably be a way to update a key. + -- Something similar to the migrate subcommand could be used, + -- and users could then run that at their leisure. Or, this upgrade + -- could to that key update for all keys that have been converted + -- and have content in the repo. + upgradeFrom0 :: Annex Bool upgradeFrom0 = do - showSideAction "Upgrading object directory layout..." + showSideAction "Upgrading object directory layout v0 to v1..." g <- Annex.gitRepo -- do the reorganisation of the files @@ -56,6 +96,9 @@ upgradeFrom0 = do fixlinks files Annex.queueRun + -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 + upgradeFrom1 + setVersion return True diff --git a/debian/changelog b/debian/changelog index ac7c854ff5..738faf916e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,6 @@ git-annex (0.24) UNRELEASED; urgency=low + * TODO: upgrade v1 -> v2 * Reorganized annexed object store. annex.version=2 * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar From e227c210ec817eca6b4409cb4cc893f791d51c00 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 01:23:20 -0400 Subject: [PATCH 1057/2835] upgrade groundwork pulled in old versions of functions for working with keys Wrote a parser from old key filenames to new keys. --- Backend.hs | 3 +- Upgrade.hs | 118 ++------------------------------------ Upgrade/V0.hs | 80 ++++++++++++++++++++++++++ Upgrade/V1.hs | 155 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 241 insertions(+), 115 deletions(-) create mode 100644 Upgrade/V0.hs create mode 100644 Upgrade/V1.hs diff --git a/Backend.hs b/Backend.hs index e1f8f388b9..cd14ce50e1 100644 --- a/Backend.hs +++ b/Backend.hs @@ -27,7 +27,8 @@ module Backend ( lookupFile, chooseBackends, keyBackend, - lookupBackendName + lookupBackendName, + maybeLookupBackendName ) where import Control.Monad.State diff --git a/Upgrade.hs b/Upgrade.hs index d63397ce0f..a152582043 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -7,128 +7,18 @@ module Upgrade where -import System.IO.Error (try) -import System.Directory -import Control.Monad.State (liftIO) -import Control.Monad (filterM, forM_) -import System.Posix.Files -import System.FilePath -import Data.Maybe - -import Content import Types -import Locations -import qualified GitRepo as Git -import qualified Annex -import qualified Backend -import Messages import Version -import Utility +import qualified Upgrade.V0 +import qualified Upgrade.V1 {- Uses the annex.version git config setting to automate upgrades. -} upgrade :: Annex Bool upgrade = do version <- getVersion case version of - Just "0" -> upgradeFrom0 - Just "1" -> upgradeFrom1 + Just "0" -> Upgrade.V0.upgrade + Just "1" -> Upgrade.V1.upgrade Nothing -> return True -- repo not initted yet, no version Just v | v == currentVersion -> return True Just _ -> error "this version of git-annex is too old for this git repository!" - -upgradeFrom1 :: Annex Bool -upgradeFrom1 = do - showSideAction "Upgrading object directory layout v1 to v2..." - error "upgradeFrom1 TODO FIXME" - - -- v2 adds hashing of filenames of content and location log files. - -- - -- Key information is encoded in filenames differently. - -- - -- When upgrading a v1 key to v2, file size metadata needs to be - -- added to the key (unless it is a WORM key, which encoded - -- mtime:size in v1). This can only be done when the file content - -- is present. - -- - -- So there are two approaches -- either upgrade - -- everything, leaving out file size information for files not - -- present in the current repo; or upgrade peicemeil, only - -- upgrading keys whose content is present. - -- - -- The latter approach would mean that, until every clone of an - -- annex is upgraded, git annex would refuse to operate on annexed - -- files that had not yet been committed. Unless it were taught to - -- work with both v1 and v2 keys in the same repo. - -- - -- Another problem with the latter approach might involve content - -- being moved between repos while the conversion is still - -- incomplete. If repo A has already upgraded, and B has not, and B - -- has K, moving K from B -> A would result in it lurking - -- unconverted on A. Unless A upgraded it in passing. But that's - -- getting really complex, and would mean a constant trickle of - -- upgrade commits, which users would find annoying. - -- - -- So, the former option it is! Note that file size metadata - -- will only be used for detecting situations where git-annex - -- would run out of disk space, so if some keys don't have it, - -- the impact is small. At least initially. It could be used in the - -- future by smart auto-repo balancing code, etc. - -- - -- Anyway, since v2 plans ahead for other metadata being included - -- in keys, there should probably be a way to update a key. - -- Something similar to the migrate subcommand could be used, - -- and users could then run that at their leisure. Or, this upgrade - -- could to that key update for all keys that have been converted - -- and have content in the repo. - -upgradeFrom0 :: Annex Bool -upgradeFrom0 = do - showSideAction "Upgrading object directory layout v0 to v1..." - g <- Annex.gitRepo - - -- do the reorganisation of the files - let olddir = gitAnnexDir g - keys <- getKeysPresent0' olddir - forM_ keys $ \k -> moveAnnex k $ olddir keyFile k - - -- update the symlinks to the files - files <- liftIO $ Git.inRepo g [Git.workTree g] - fixlinks files - Annex.queueRun - - -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 - upgradeFrom1 - - setVersion - - return True - - where - fixlinks [] = return () - fixlinks (f:fs) = do - r <- Backend.lookupFile f - case r of - Nothing -> return () - Just (k, _) -> do - link <- calcGitLink f k - liftIO $ removeFile f - liftIO $ createSymbolicLink link f - Annex.queue "add" [Param "--"] f - fixlinks fs - -getKeysPresent0' :: FilePath -> Annex [Key] -getKeysPresent0' dir = do - exists <- liftIO $ doesDirectoryExist dir - if (not exists) - then return [] - else do - contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM present contents - return $ catMaybes $ map fileKey files - where - present d = do - result <- try $ - getFileStatus $ dir ++ "/" ++ takeFileName d - case result of - Right s -> return $ isRegularFile s - Left _ -> return False diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs new file mode 100644 index 0000000000..25b6f27635 --- /dev/null +++ b/Upgrade/V0.hs @@ -0,0 +1,80 @@ +{- git-annex v0 -> v1 upgrade support + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V0 where + +import System.IO.Error (try) +import System.Directory +import Control.Monad.State (liftIO) +import Control.Monad (filterM, forM_) +import System.Posix.Files +import System.FilePath + +import Content +import Types +import Locations +import qualified GitRepo as Git +import qualified Annex +import Messages +import Utility +import qualified Upgrade.V1 + +upgrade :: Annex Bool +upgrade = do + showSideAction "Upgrading object directory layout v0 to v1..." + g <- Annex.gitRepo + + -- do the reorganisation of the key files + let olddir = gitAnnexDir g + keys <- getKeysPresent0 olddir + forM_ keys $ \k -> moveAnnex k $ olddir keyFile0 k + + -- update the symlinks to the key files + files <- liftIO $ Git.inRepo g [Git.workTree g] + fixlinks files + Annex.queueRun + + -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 + Upgrade.V1.upgrade + + where + fixlinks [] = return () + fixlinks (f:fs) = do + r <- lookupFile0 f + case r of + Nothing -> return () + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" [Param "--"] f + fixlinks fs + +-- these stayed unchanged between v0 and v1 +keyFile0 :: Key -> FilePath +keyFile0 = Upgrade.V1.keyFile1 +fileKey0 :: FilePath -> Key +fileKey0 = Upgrade.V1.fileKey1 +lookupFile0 :: FilePath -> Annex (Maybe (Key, Backend Annex)) +lookupFile0 = Upgrade.V1.lookupFile1 + +getKeysPresent0 :: FilePath -> Annex [Key] +getKeysPresent0 dir = do + exists <- liftIO $ doesDirectoryExist dir + if (not exists) + then return [] + else do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey0 files + where + present d = do + result <- try $ + getFileStatus $ dir ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs new file mode 100644 index 0000000000..dd51206b30 --- /dev/null +++ b/Upgrade/V1.hs @@ -0,0 +1,155 @@ +{- git-annex v1 -> v2 upgrade support + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V1 where + +import System.IO.Error (try) +import System.Directory +import Control.Monad.State (liftIO) +import Control.Monad (filterM, forM_, unless) +import System.Posix.Files +import System.FilePath +import Data.String.Utils +import Key +import System.Posix.Types + +import Content +import Types +import Locations +import qualified Annex +import Backend +import Messages +import Version + +upgrade :: Annex Bool +upgrade = do + showSideAction "Upgrading object directory layout v1 to v2..." + error "upgradeFrom1 TODO FIXME" + + -- v2 adds hashing of filenames of content and location log files. + -- + -- Key information is encoded in filenames differently. + -- + -- When upgrading a v1 key to v2, file size metadata needs to be + -- added to the key (unless it is a WORM key, which encoded + -- mtime:size in v1). This can only be done when the file content + -- is present. + -- + -- So there are two approaches -- either upgrade + -- everything, leaving out file size information for files not + -- present in the current repo; or upgrade peicemeil, only + -- upgrading keys whose content is present. + -- + -- The latter approach would mean that, until every clone of an + -- annex is upgraded, git annex would refuse to operate on annexed + -- files that had not yet been committed. Unless it were taught to + -- work with both v1 and v2 keys in the same repo. + -- + -- Another problem with the latter approach might involve content + -- being moved between repos while the conversion is still + -- incomplete. If repo A has already upgraded, and B has not, and B + -- has K, moving K from B -> A would result in it lurking + -- unconverted on A. Unless A upgraded it in passing. But that's + -- getting really complex, and would mean a constant trickle of + -- upgrade commits, which users would find annoying. + -- + -- So, the former option it is! Note that file size metadata + -- will only be used for detecting situations where git-annex + -- would run out of disk space, so if some keys don't have it, + -- the impact is small. At least initially. It could be used in the + -- future by smart auto-repo balancing code, etc. + -- + -- Anyway, since v2 plans ahead for other metadata being included + -- in keys, there should probably be a way to update a key. + -- Something similar to the migrate subcommand could be used, + -- and users could then run that at their leisure. Or, this upgrade + -- could to that key update for all keys that have been converted + -- and have content in the repo. + + -- do the reorganisation of the log files + + -- do the reorganisation of the key files + g <- Annex.gitRepo + let olddir = gitAnnexDir g + keys <- getKeysPresent1 + forM_ keys $ \k -> moveAnnex k $ olddir keyFile1 k + + -- update the symlinks to the key files + + Annex.queueRun + + setVersion + + return True + +keyFile1 :: Key -> FilePath +keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key + +fileKey1 :: FilePath -> Key +fileKey1 file = readKey1 $ + replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + +readKey1 :: String -> Key +readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } + where + bits = split ":" v + b = head bits + n = join ":" $ drop (if wormy then 3 else 1) bits + t = if wormy + then Just (read (bits !! 1) :: EpochTime) + else Nothing + s = if wormy + then Just (read (bits !! 2) :: Integer) + else Nothing + wormy = b == "WORM" + +lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex)) +lookupFile1 file = do + bs <- Annex.getState Annex.supportedBackends + tl <- liftIO $ try getsymlink + case tl of + Left _ -> return Nothing + Right l -> makekey bs l + where + getsymlink = do + l <- readSymbolicLink file + return $ takeFileName l + makekey bs l = do + case maybeLookupBackendName bs bname of + Nothing -> do + unless (null kname || null bname || + not (isLinkToAnnex l)) $ + warning skip + return Nothing + Just backend -> return $ Just (k, backend) + where + k = fileKey1 l + bname = keyBackendName k + kname = keyName k + skip = "skipping " ++ file ++ + " (unknown backend " ++ bname ++ ")" + +getKeysPresent1 :: Annex [Key] +getKeysPresent1 = do + g <- Annex.gitRepo + getKeysPresent1' $ gitAnnexObjectDir g +getKeysPresent1' :: FilePath -> Annex [Key] +getKeysPresent1' dir = do + exists <- liftIO $ doesDirectoryExist dir + if (not exists) + then return [] + else do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey1 files + where + present d = do + result <- try $ + getFileStatus $ dir ++ "/" ++ d ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False From a4d0250298a503c7d9093e6dd1618ff5b07b19e5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 02:15:54 -0400 Subject: [PATCH 1058/2835] slways set current version in new repos detect v1 repos that don't have a version set --- Upgrade.hs | 9 ++++----- Version.hs | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Upgrade.hs b/Upgrade.hs index a152582043..76dd156f83 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -17,8 +17,7 @@ upgrade :: Annex Bool upgrade = do version <- getVersion case version of - Just "0" -> Upgrade.V0.upgrade - Just "1" -> Upgrade.V1.upgrade - Nothing -> return True -- repo not initted yet, no version - Just v | v == currentVersion -> return True - Just _ -> error "this version of git-annex is too old for this git repository!" + "0" -> Upgrade.V0.upgrade + "1" -> Upgrade.V1.upgrade + v | v == currentVersion -> return True + _ -> error "this version of git-annex is too old for this git repository!" diff --git a/Version.hs b/Version.hs index 7fdbd1a49a..5f414e93b8 100644 --- a/Version.hs +++ b/Version.hs @@ -21,21 +21,28 @@ currentVersion = "2" versionField :: String versionField = "annex.version" -getVersion :: Annex (Maybe String) +getVersion :: Annex String getVersion = do g <- Annex.gitRepo let v = Git.configGet g versionField "" if not $ null v - then return $ Just v + then return v else do -- version 0 was not recorded in .git/config; -- such a repo should have an gitAnnexDir but no - -- gitAnnexObjectDir + -- gitAnnexObjectDir. + -- + -- version 1 may not be recorded if the user + -- forgot to init. Such a repo should have a + -- gitAnnexObjectDir already. d <- liftIO $ doesDirectoryExist $ gitAnnexDir g o <- liftIO $ doesDirectoryExist $ gitAnnexObjectDir g - if d && not o - then return $ Just "0" - else return Nothing -- no version yet + case (d, o) of + (True, False) -> return "0" + (True, True) -> return "1" + _ -> do + setVersion + return currentVersion setVersion :: Annex () setVersion = Annex.setConfig versionField currentVersion From 500c4e44c560a04aaa30e165b70d4d8491ad9c32 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 02:35:48 -0400 Subject: [PATCH 1059/2835] v1 -> v2 upgrade partially working still need to move location log files, and auto-commit --- Upgrade/V0.hs | 19 +------ Upgrade/V1.hs | 136 +++++++++++++++++++++++++++-------------------- debian/changelog | 2 +- 3 files changed, 81 insertions(+), 76 deletions(-) diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs index 25b6f27635..5ba305817b 100644 --- a/Upgrade/V0.hs +++ b/Upgrade/V0.hs @@ -17,10 +17,8 @@ import System.FilePath import Content import Types import Locations -import qualified GitRepo as Git import qualified Annex import Messages -import Utility import qualified Upgrade.V1 upgrade :: Annex Bool @@ -34,26 +32,11 @@ upgrade = do forM_ keys $ \k -> moveAnnex k $ olddir keyFile0 k -- update the symlinks to the key files - files <- liftIO $ Git.inRepo g [Git.workTree g] - fixlinks files - Annex.queueRun + -- No longer needed here; V1.upgrade does the same thing -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 Upgrade.V1.upgrade - where - fixlinks [] = return () - fixlinks (f:fs) = do - r <- lookupFile0 f - case r of - Nothing -> return () - Just (k, _) -> do - link <- calcGitLink f k - liftIO $ removeFile f - liftIO $ createSymbolicLink link f - Annex.queue "add" [Param "--"] f - fixlinks fs - -- these stayed unchanged between v0 and v1 keyFile0 :: Key -> FilePath keyFile0 = Upgrade.V1.keyFile1 diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index dd51206b30..850080436f 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -21,78 +21,83 @@ import Content import Types import Locations import qualified Annex +import qualified GitRepo as Git import Backend import Messages import Version +import Utility + +-- v2 adds hashing of filenames of content and location log files. +-- Key information is encoded in filenames differently, so +-- both content and location log files move around, and symlinks +-- to content need to be changed. +-- +-- When upgrading a v1 key to v2, file size metadata ought to be +-- added to the key (unless it is a WORM key, which encoded +-- mtime:size in v1). This can only be done when the file content +-- is present. Since upgrades need to happen consistently, +-- (so that two repos get changed the same way by the upgrade, and +-- will merge), that metadata cannot be added on upgrade. +-- +-- Note that file size metadata +-- will only be used for detecting situations where git-annex +-- would run out of disk space, so if some keys don't have it, +-- the impact is minor. At least initially. It could be used in the +-- future by smart auto-repo balancing code, etc. +-- +-- Anyway, since v2 plans ahead for other metadata being included +-- in keys, there should probably be a way to update a key. +-- Something similar to the migrate subcommand could be used, +-- and users could then run that at their leisure. upgrade :: Annex Bool upgrade = do showSideAction "Upgrading object directory layout v1 to v2..." - error "upgradeFrom1 TODO FIXME" - -- v2 adds hashing of filenames of content and location log files. - -- - -- Key information is encoded in filenames differently. - -- - -- When upgrading a v1 key to v2, file size metadata needs to be - -- added to the key (unless it is a WORM key, which encoded - -- mtime:size in v1). This can only be done when the file content - -- is present. - -- - -- So there are two approaches -- either upgrade - -- everything, leaving out file size information for files not - -- present in the current repo; or upgrade peicemeil, only - -- upgrading keys whose content is present. - -- - -- The latter approach would mean that, until every clone of an - -- annex is upgraded, git annex would refuse to operate on annexed - -- files that had not yet been committed. Unless it were taught to - -- work with both v1 and v2 keys in the same repo. - -- - -- Another problem with the latter approach might involve content - -- being moved between repos while the conversion is still - -- incomplete. If repo A has already upgraded, and B has not, and B - -- has K, moving K from B -> A would result in it lurking - -- unconverted on A. Unless A upgraded it in passing. But that's - -- getting really complex, and would mean a constant trickle of - -- upgrade commits, which users would find annoying. - -- - -- So, the former option it is! Note that file size metadata - -- will only be used for detecting situations where git-annex - -- would run out of disk space, so if some keys don't have it, - -- the impact is small. At least initially. It could be used in the - -- future by smart auto-repo balancing code, etc. - -- - -- Anyway, since v2 plans ahead for other metadata being included - -- in keys, there should probably be a way to update a key. - -- Something similar to the migrate subcommand could be used, - -- and users could then run that at their leisure. Or, this upgrade - -- could to that key update for all keys that have been converted - -- and have content in the repo. - - -- do the reorganisation of the log files - - -- do the reorganisation of the key files - g <- Annex.gitRepo - let olddir = gitAnnexDir g - keys <- getKeysPresent1 - forM_ keys $ \k -> moveAnnex k $ olddir keyFile1 k - - -- update the symlinks to the key files + moveContent + updateSymlinks + moveLocationLogs Annex.queueRun - setVersion - return True -keyFile1 :: Key -> FilePath -keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key +moveContent :: Annex () +moveContent = do + keys <- getKeysPresent1 + forM_ keys move + where + move k = do + g <- Annex.gitRepo + let f = gitAnnexObjectDir g keyFile1 k keyFile1 k + let d = parentDir f + liftIO $ allowWrite d + liftIO $ allowWrite f + moveAnnex k f + liftIO $ removeDirectory d -fileKey1 :: FilePath -> Key -fileKey1 file = readKey1 $ - replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file +updateSymlinks :: Annex () +updateSymlinks = do + g <- Annex.gitRepo + files <- liftIO $ Git.inRepo g [Git.workTree g] + forM_ files $ fixlink + where + fixlink f = do + r <- lookupFile1 f + case r of + Nothing -> return () + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" [Param "--"] f +moveLocationLogs :: Annex () +moveLocationLogs = do + warning "TODO location log move" + +-- WORM backend keys: "WORM:mtime:size:filename" +-- all the rest: "backend:key" readKey1 :: String -> Key readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } where @@ -107,6 +112,23 @@ readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } else Nothing wormy = b == "WORM" +showKey1 :: Key -> String +showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } = + join ":" $ filter (not . null) [b, showifhere t, showifhere s, n] + where + showifhere Nothing = "" + showifhere (Just v) = show v + +keyFile1 :: Key -> FilePath +keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ showKey1 key + +fileKey1 :: FilePath -> Key +fileKey1 file = readKey1 $ + replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + +logFile1 :: Git.Repo -> Key -> String +logFile1 repo key = gitStateDir repo ++ keyFile1 key ++ ".log" + lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex)) lookupFile1 file = do bs <- Annex.getState Annex.supportedBackends diff --git a/debian/changelog b/debian/changelog index 738faf916e..6cd8a9326b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.24) UNRELEASED; urgency=low +git-annex (0.20110316) UNRELEASED; urgency=low * TODO: upgrade v1 -> v2 * Reorganized annexed object store. annex.version=2 From 137257ded1acb1b15c0413a8998af1e999cd1a53 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 02:50:13 -0400 Subject: [PATCH 1060/2835] better letter choice for hashing --- Locations.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Locations.hs b/Locations.hs index b2d31a1bf8..3cce4c2611 100644 --- a/Locations.hs +++ b/Locations.hs @@ -162,8 +162,8 @@ display_32bits_as_dir :: Word32 -> String display_32bits_as_dir w = trim $ swap_pairs cs where -- Need 32 characters to use. To avoid inaverdently making - -- a real word, use the alphabet without vowels. - chars = ['0'..'9'] ++ "bcdfghjklnmpqrstvwxyzZ" + -- a real word, use letters that appear less frequently. + chars = ['0'..'9'] ++ "zqjxkmvwgpfZQJXKMVWGPF" cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7] getc n = chars !! (fromIntegral n) swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs From ea81da347f952bedb8d793c7fa190c229bce04ee Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 16 Mar 2011 14:27:45 +0000 Subject: [PATCH 1061/2835] --- ...has_problems_on_non-linux_based_systems.mdwn | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn new file mode 100644 index 0000000000..20c16e3cfa --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn @@ -0,0 +1,17 @@ +It seems that commit bc5c54c987f548505a3877e8a0e460abe0b2a081 introduced some linux specific things... + +

+hsc2hs Touch.hsc
+Touch.hsc: In function ‘main’:
+Touch.hsc:46: error: ‘UTIME_OMIT’ undeclared (first use in this function)
+Touch.hsc:46: error: (Each undeclared identifier is reported only once
+Touch.hsc:46: error: for each function it appears in.)
+Touch.hsc:48: error: ‘UTIME_NOW’ undeclared (first use in this function)
+Touch.hsc:67: error: ‘AT_FDCWD’ undeclared (first use in this function)
+Touch.hsc:68: error: ‘AT_SYMLINK_NOFOLLOW’ undeclared (first use in this function)
+compiling Touch_hsc_make.c failed
+command was: /usr/bin/gcc -c -m32 -I/Library/Frameworks/GHC.framework/Versions/612/usr/lib/ghc-6.12.3/include/ Touch_hsc_make.c -o Touch_hsc_make.o
+make: *** [Touch.hs] Error 1
+
+ +I dug around the OSX documentation and fcntl.h header file and it seems that UTIME_OMIT, UTIME_NOW, AT_FDCWD and AT_SYMLINK_NOFOLLOW aren't defined (at least on OSX). I suspect the BSD's in general will have problems compiling git-annex. From a0807999001017bd6897bad82e747c16e18af6bc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 10:56:59 -0400 Subject: [PATCH 1062/2835] upgrades seem to fully work --- Upgrade/V1.hs | 34 ++++++++++++++++++++++++++++++++-- debian/changelog | 9 +++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 850080436f..602ba41c59 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -14,9 +14,10 @@ import Control.Monad (filterM, forM_, unless) import System.Posix.Files import System.FilePath import Data.String.Utils -import Key import System.Posix.Types +import Data.Maybe +import Key import Content import Types import Locations @@ -94,7 +95,36 @@ updateSymlinks = do moveLocationLogs :: Annex () moveLocationLogs = do - warning "TODO location log move" + logkeys <- oldlocationlogs + forM_ logkeys move + where + oldlocationlogs = do + g <- Annex.gitRepo + let dir = gitStateDir g + contents <- liftIO $ getDirectoryContents dir + return $ catMaybes $ map oldlog2key contents + move (l, k) = do + g <- Annex.gitRepo + let dest = logFile g k + let dir = gitStateDir g + let f = dir l + liftIO $ createDirectoryIfMissing True (parentDir dest) + -- could just git mv, but this way deals with + -- log files that are not checked into git + liftIO $ copyFile f dest + Annex.queue "add" [Param "--"] dest + Annex.queue "add" [Param "--"] f + Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f + +oldlog2key :: FilePath -> Maybe (FilePath, Key) +oldlog2key l = + let len = length l - 4 in + if drop len l == ".log" + then let k = readKey1 (take len l) in + if null (keyName k) || null (keyBackendName k) + then Nothing + else Just (l, k) + else Nothing -- WORM backend keys: "WORM:mtime:size:filename" -- all the rest: "backend:key" diff --git a/debian/changelog b/debian/changelog index 6cd8a9326b..c40385bebe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,12 @@ git-annex (0.20110316) UNRELEASED; urgency=low - * TODO: upgrade v1 -> v2 - * Reorganized annexed object store. annex.version=2 + * Reorganized .git/annex/objects and .git-annex/; annex.version=2 + * The first time git-annex is run in an old format repository, it + will automatically upgrade it to the new format, staging all + necessary changes to git. + * Note that remotes must be running this version of git-annex, + and must also have been upgraded, in order for git-annex to + communicate with them. * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar filesystems. From 744638197f51811fca13a37c7bbc51dfb626793b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 11:27:29 -0400 Subject: [PATCH 1063/2835] fix getKeyspresent to work with hashed dirs --- Content.hs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Content.hs b/Content.hs index 1a5a80a9f1..a59484b5ab 100644 --- a/Content.hs +++ b/Content.hs @@ -161,13 +161,21 @@ getKeysPresent' dir = do if (not exists) then return [] else do - contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM present contents - return $ catMaybes $ map fileKey files + -- 2 levels of hashing + levela <- liftIO $ subdirContent dir + levelb <- liftIO $ mapM subdirContent levela + contents <- liftIO $ mapM subdirContent (concat levelb) + files <- liftIO $ filterM present (concat contents) + return $ catMaybes $ map (fileKey . takeFileName) files where present d = do result <- try $ - getFileStatus $ dir ++ "/" ++ d ++ "/" ++ takeFileName d + getFileStatus $ d takeFileName d + liftIO $ putStrLn $ "trying " ++ (d takeFileName d) case result of Right s -> return $ isRegularFile s Left _ -> return False + subdirContent d = do + c <- getDirectoryContents d + return $ map (d ) $ filter notcruft c + notcruft f = f /= "." && f /= ".." From 5eb76d2b033436973e0732215fa9d0227a2187bf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 11:53:46 -0400 Subject: [PATCH 1064/2835] improve upgrade --- Command/Init.hs | 17 +++++++++++------ Command/Uninit.hs | 2 +- LocationLog.hs | 2 ++ Upgrade/V1.hs | 18 +++++++++++++++--- debian/changelog | 5 +---- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Command/Init.hs b/Command/Init.hs index 6618351699..d9ea394a33 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -8,7 +8,7 @@ module Command.Init where import Control.Monad.State (liftIO) -import Control.Monad (when) +import Control.Monad (when, unless) import System.Directory import System.FilePath @@ -74,12 +74,14 @@ gitAttributesWrite repo = do exists <- doesFileExist attributes if not exists then do - safeWriteFile attributes $ attrLine ++ "\n" + safeWriteFile attributes $ unlines attrLines commit else do content <- readFile attributes - when (all (/= attrLine) (lines content)) $ do - appendFile attributes $ attrLine ++ "\n" + let present = lines content + let missing = filter (\l -> not $ l `elem` present) attrLines + unless (null missing) $ do + appendFile attributes $ unlines missing commit where attributes = Git.attributes repo @@ -91,8 +93,11 @@ gitAttributesWrite repo = do , Param attributes ] -attrLine :: String -attrLine = stateDir "*.log merge=union" +attrLines :: [String] +attrLines = + [ stateDir "*.log merge=union" + , stateDir "*/*/*.log merge=union" + ] {- set up a git pre-commit hook, if one is not already present -} gitPreCommitHookWrite :: Git.Repo -> Annex () diff --git a/Command/Uninit.hs b/Command/Uninit.hs index e9406ce3af..e8ac1bbd5e 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -60,4 +60,4 @@ gitAttributesUnWrite repo = do when attrexists $ do c <- readFileStrict attributes safeWriteFile attributes $ unlines $ - filter (/= Command.Init.attrLine) $ lines c + filter (\l -> not $ l `elem` Command.Init.attrLines) $ lines c diff --git a/LocationLog.hs b/LocationLog.hs index a939af825d..f1e54432ca 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -24,6 +24,8 @@ module LocationLog ( LogStatus(..), logChange, logFile, + readLog, + writeLog, keyLocations ) where diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 602ba41c59..797bdee0dc 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -21,13 +21,15 @@ import Key import Content import Types import Locations +import LocationLog import qualified Annex import qualified GitRepo as Git import Backend import Messages import Version import Utility - +import qualified Command.Init + -- v2 adds hashing of filenames of content and location log files. -- Key information is encoded in filenames differently, so -- both content and location log files move around, and symlinks @@ -61,6 +63,12 @@ upgrade = do Annex.queueRun setVersion + + -- add new line to auto-merge hashed location logs + -- this commits, so has to come after the upgrade + g <- Annex.gitRepo + liftIO $ Command.Init.gitAttributesWrite g + return True moveContent :: Annex () @@ -110,8 +118,12 @@ moveLocationLogs = do let f = dir l liftIO $ createDirectoryIfMissing True (parentDir dest) -- could just git mv, but this way deals with - -- log files that are not checked into git - liftIO $ copyFile f dest + -- log files that are not checked into git, + -- as well as merging with already upgraded + -- logs that have been pulled from elsewhere + old <- liftIO $ readLog f + new <- liftIO $ readLog dest + liftIO $ writeLog dest (old++new) Annex.queue "add" [Param "--"] dest Annex.queue "add" [Param "--"] f Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f diff --git a/debian/changelog b/debian/changelog index c40385bebe..68afb12933 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,12 +1,9 @@ git-annex (0.20110316) UNRELEASED; urgency=low - * Reorganized .git/annex/objects and .git-annex/; annex.version=2 + * New repository format, annex.version=2. * The first time git-annex is run in an old format repository, it will automatically upgrade it to the new format, staging all necessary changes to git. - * Note that remotes must be running this version of git-annex, - and must also have been upgraded, in order for git-annex to - communicate with them. * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar filesystems. From c93cd81a3372f50fb5b5c8b8668536c125dce463 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 12:06:32 -0400 Subject: [PATCH 1065/2835] define feature test macro --- Touch.hsc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Touch.hsc b/Touch.hsc index 689c58765f..f0c4debb33 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -22,6 +22,10 @@ import Foreign.C #include #include +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE >= 200809L +#endif + data TimeSpec = TimeSpec CTime CLong instance Storable TimeSpec where From 0512c7dcfa9e4e95044b9717fc864380b4c08214 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 16:07:26 +0000 Subject: [PATCH 1066/2835] Added a comment --- ...comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment new file mode 100644 index 0000000000..f26239c3e9 --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-16T16:07:26Z" + content=""" +Hmm.. is utimensat available at all? + +I've committed an update that may convince at least some compilers to expose this newer POSIX stuff. I don't know if it will help, please let me know. +"""]] From e044d40cbda1a2740fca373922f29f2d57174de8 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 16:29:33 +0000 Subject: [PATCH 1067/2835] Comment moderation --- ...mment_3_0fc6ff79a357b1619d13018ccacc7c10._comment | 8 ++++++++ ...mment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment | 12 ++++++++++++ ...mment_5_821c382987f105da72a50e0a5ce61fdc._comment | 12 ++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment create mode 100644 doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment create mode 100644 doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment diff --git a/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment b/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment new file mode 100644 index 0000000000..ea4fb6c23e --- /dev/null +++ b/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 3" + date="2011-03-16T15:40:56Z" + content=""" +Sometimes, I might want to fill up the disk as much as possible. Thus, a warning is preferable to erroring out too early, imo -- Richard +"""]] diff --git a/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment b/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment new file mode 100644 index 0000000000..c558ee65ee --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 5" + date="2011-03-16T15:47:17Z" + content=""" +If you can't segment the names retroactively, it's better to start with segmenting, imo. + +As subdirectories are cheap, going with ab/cd/rest or even ab/cd/ef/rest by default wouldn't hurt. + +Your point about git not needing to create as many tree objects is a kicker indeed. If I were you, I would default to segmentation. +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment b/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment new file mode 100644 index 0000000000..2032bce3c0 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 5" + date="2011-03-16T15:51:30Z" + content=""" +Hashing & segmenting seems to be around the corner, which is nice :) + +Is there a chance that you will optionally add mtime to your native metadata store? If yes, I'd rather wait for v2 to start with the native system from the start. If not, I will probably set it up tonight. + +PS: While posting from work, my comments are held for moderation once again. I am somewhat confused as to why this happens when I can just submit directly from home. And yes, I am using the same auth provider and user in both cases. +"""]] From ba584005b04a33d2defcfec391f4ea3d94eec1ab Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 16:32:52 +0000 Subject: [PATCH 1068/2835] Added a comment --- ...comment_6_8834c3a3f1258c4349d23aff8549bf35._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment b/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment new file mode 100644 index 0000000000..ff86e3970b --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 6" + date="2011-03-16T16:32:52Z" + content=""" +The mtime cannot be stored for all keys. Consider a SHA1 key. The mtime is irrelevant; 2 files with different mtimes, when added to the SHA1 backend, should get the same key. + +Probably our spam filter doesn't like your work IP. +"""]] From f655cecfedfa17d6d59415da0e787d5c63629f34 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 16 Mar 2011 16:49:20 +0000 Subject: [PATCH 1069/2835] Added a comment --- .../comment_2_bf112edd075fbebe4fc959a387946eb9._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment new file mode 100644 index 0000000000..0222e645b9 --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2011-03-16T16:49:18Z" + content=""" +Just pulled the changes, it still fails to build. utimensat doesn't seem to exist on OSX 10.6.6. +"""]] From 1443fcfe022028c2c074fc555d1e0d595fd4db95 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 12:52:30 -0400 Subject: [PATCH 1070/2835] don't use queue when upgrading In a large repo, just queuing the things to do used a lot of ram. --- Upgrade/V1.hs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 797bdee0dc..ffb774f7de 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -61,7 +61,6 @@ upgrade = do updateSymlinks moveLocationLogs - Annex.queueRun setVersion -- add new line to auto-merge hashed location logs @@ -89,17 +88,18 @@ updateSymlinks :: Annex () updateSymlinks = do g <- Annex.gitRepo files <- liftIO $ Git.inRepo g [Git.workTree g] - forM_ files $ fixlink + forM_ files $ (fixlink g) where - fixlink f = do + fixlink g f = do r <- lookupFile1 f case r of Nothing -> return () Just (k, _) -> do link <- calcGitLink f k - liftIO $ removeFile f - liftIO $ createSymbolicLink link f - Annex.queue "add" [Param "--"] f + liftIO $ do + removeFile f + createSymbolicLink link f + Git.run g "add" [Param "--", File f] moveLocationLogs :: Annex () moveLocationLogs = do @@ -123,10 +123,11 @@ moveLocationLogs = do -- logs that have been pulled from elsewhere old <- liftIO $ readLog f new <- liftIO $ readLog dest - liftIO $ writeLog dest (old++new) - Annex.queue "add" [Param "--"] dest - Annex.queue "add" [Param "--"] f - Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f + liftIO $ do + writeLog dest (old++new) + Git.run g "add" [Param "--", File dest] + Git.run g "add" [Param "--", File f] + Git.run g "rm" [Param "--quiet", Param "-f", Param "--", File f] oldlog2key :: FilePath -> Maybe (FilePath, Key) oldlog2key l = From a4bc3d6f38e45d982fc20c7c7e84207b5140b24f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 13:16:52 -0400 Subject: [PATCH 1071/2835] bare repo upgrade support --- Upgrade/V1.hs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index ffb774f7de..1bf3cc0e85 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -57,17 +57,21 @@ upgrade :: Annex Bool upgrade = do showSideAction "Upgrading object directory layout v1 to v2..." - moveContent - updateSymlinks - moveLocationLogs + g <- Annex.gitRepo + if Git.repoIsLocalBare g + then do + moveContent + else do + moveContent + updateSymlinks + moveLocationLogs + + -- add new line to auto-merge hashed location logs + -- this commits, so has to come after the upgrade + g <- Annex.gitRepo + liftIO $ Command.Init.gitAttributesWrite g setVersion - - -- add new line to auto-merge hashed location logs - -- this commits, so has to come after the upgrade - g <- Annex.gitRepo - liftIO $ Command.Init.gitAttributesWrite g - return True moveContent :: Annex () From 35cbd107d5107eb96f52860098b1036c73281715 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 13:46:08 -0400 Subject: [PATCH 1072/2835] detect systems w/o utmensat and ifdef out code that needs it --- Touch.hsc | 37 +++++++++++++++++++++++++------------ debian/changelog | 1 + 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Touch.hsc b/Touch.hsc index f0c4debb33..456175182e 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -28,6 +28,25 @@ import Foreign.C data TimeSpec = TimeSpec CTime CLong +touch :: FilePath -> TimeSpec -> Bool -> IO () +touch file mtime follow = touchBoth file omitTime mtime follow + +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () + +omitTime :: TimeSpec +nowTime :: TimeSpec + +#if (defined UTIME_OMIT && defined UTIME_NOW && defined AT_FDCWD && defined AT_SYMLINK_NOFOLLOW) + +at_fdcwd :: CInt +at_fdcwd = #const AT_FDCWD + +at_symlink_nofollow :: CInt +at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW + +omitTime = TimeSpec 0 #const UTIME_OMIT +nowTime = TimeSpec 0 #const UTIME_NOW + instance Storable TimeSpec where -- use the larger alignment of the two types in the struct alignment _ = max sec_alignment nsec_alignment @@ -43,12 +62,6 @@ instance Storable TimeSpec where #{poke struct timespec, tv_sec} ptr sec #{poke struct timespec, tv_nsec} ptr nsec -{- special timespecs -} -omitTime :: TimeSpec -omitTime = TimeSpec 0 #const UTIME_OMIT -nowTime :: TimeSpec -nowTime = TimeSpec 0 #const UTIME_NOW - {- While its interface is beastly, utimensat is in recent POSIX standards, unlike futimes. -} foreign import ccall "utimensat" @@ -56,7 +69,6 @@ foreign import ccall "utimensat" {- Changes the access and/or modification times of an existing file. Can follow symlinks, or not. Throws IO error on failure. -} -touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () touchBoth file atime mtime follow = allocaArray 2 $ \ptr -> withCString file $ \f -> do @@ -66,12 +78,13 @@ touchBoth file atime mtime follow = then throwErrno "touchBoth" else return () where - at_fdcwd = #const AT_FDCWD - at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW - flags = if follow then 0 else at_symlink_nofollow -touch :: FilePath -> TimeSpec -> Bool -> IO () -touch file mtime follow = touchBoth file omitTime mtime follow +#else +#warning "utimensat not available; building without symlink timestamp preservation support" +omitTime = TimeSpec 0 (-1) +nowTime = TimeSpec 0 (-2) +touchBoth _ _ _ _ = return () +#endif diff --git a/debian/changelog b/debian/changelog index e7017a26d0..110fc2b4c0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ git-annex (0.24) UNRELEASED; urgency=low content are made to have the same mtime as the original file. While git does not preserve that information, this allows a tool like metastore to be used with annexed files. + (Currently this is only done on systems supporting POSIX 200809.) -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 From a9111ff7e04161afcdf2e335a03d528d5afc780b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 17:46:40 +0000 Subject: [PATCH 1073/2835] Added a comment --- .../comment_3_a46080fbe82adf0986c5dc045e382501._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment new file mode 100644 index 0000000000..7e79dea881 --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T17:46:40Z" + content=""" +Alright, I've added #idefs and the symlink timestamp mirroring feature will be unavailable on OSX until I get a version that works there. +"""]] From bc21502b9a640e798dc6bbbb255aa9742a1c6187 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 15:10:15 -0400 Subject: [PATCH 1074/2835] use queue when upgrading, flushing every so often Added a cheap way to query the size of a queue. runQueueAt is not the default yet only because there may be some code that expects to be able to queue some suff, do something else, and run the whole queue at the end. 10240 is an arbitrary size for the queue. If we assume annexed filenames are between 10 and 255 characters long, then the queue will build up between 100kb and 2550kb long commands. The max command line length on linux is somewhere above 20k, so this is a fairly good balance -- the queue will buffer only a few megabytes of stuff and a minimal number of commands will be run by xargs. Also, insert queue items strictly, this should save memory. --- Annex.hs | 8 ++++++++ CmdLine.hs | 2 +- GitQueue.hs | 17 ++++++++++++----- Upgrade/V1.hs | 25 +++++++++++++------------ 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/Annex.hs b/Annex.hs index f8cfd0ec92..608151d824 100644 --- a/Annex.hs +++ b/Annex.hs @@ -16,6 +16,7 @@ module Annex ( gitRepo, queue, queueRun, + queueRunAt, setConfig, repoConfig ) where @@ -109,6 +110,13 @@ queueRun = do liftIO $ GitQueue.run g q put state { repoqueue = GitQueue.empty } +{- Runs the queue if the specified number of items have been queued. -} +queueRunAt :: Integer -> Annex () +queueRunAt n = do + state <- get + let q = repoqueue state + when (GitQueue.size q >= n) queueRun + {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () setConfig k value = do diff --git a/CmdLine.hs b/CmdLine.hs index b8fd6af7ce..0698f2f5ea 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -99,7 +99,7 @@ startup = do shutdown :: Annex Bool shutdown = do q <- Annex.getState Annex.repoqueue - unless (q == GitQueue.empty) $ do + unless (0 == GitQueue.size q) $ do showSideAction "Recording state in git..." Annex.queueRun diff --git a/GitQueue.hs b/GitQueue.hs index 07cf9f62fc..097516c195 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -9,6 +9,7 @@ module GitQueue ( Queue, empty, add, + size, run ) where @@ -31,22 +32,28 @@ data Action = Action { {- A queue of actions to perform (in any order) on a git repository, - with lists of files to perform them on. This allows coalescing - similar git commands. -} -type Queue = M.Map Action [FilePath] +data Queue = Queue Integer (M.Map Action [FilePath]) + deriving (Show, Eq) {- Constructor for empty queue. -} empty :: Queue -empty = M.empty +empty = Queue 0 M.empty {- Adds an action to a queue. -} add :: Queue -> String -> [CommandParam] -> FilePath -> Queue -add queue subcommand params file = M.insertWith (++) action [file] queue +add (Queue n m) subcommand params file = Queue (n + 1) m' where action = Action subcommand params + m' = M.insertWith' (++) action [file] m + +{- Number of items in a queue. -} +size :: Queue -> Integer +size (Queue n _) = n {- Runs a queue on a git repository. -} run :: Git.Repo -> Queue -> IO () -run repo queue = do - forM_ (M.toList queue) $ uncurry $ runAction repo +run repo (Queue _ m) = do + forM_ (M.toList m) $ uncurry $ runAction repo return () {- Runs an Action on a list of files in a git repository. diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 1bf3cc0e85..64ca298eb4 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -66,9 +66,10 @@ upgrade = do updateSymlinks moveLocationLogs + Annex.queueRun + -- add new line to auto-merge hashed location logs -- this commits, so has to come after the upgrade - g <- Annex.gitRepo liftIO $ Command.Init.gitAttributesWrite g setVersion @@ -92,18 +93,18 @@ updateSymlinks :: Annex () updateSymlinks = do g <- Annex.gitRepo files <- liftIO $ Git.inRepo g [Git.workTree g] - forM_ files $ (fixlink g) + forM_ files $ fixlink where - fixlink g f = do + fixlink f = do r <- lookupFile1 f case r of Nothing -> return () Just (k, _) -> do link <- calcGitLink f k - liftIO $ do - removeFile f - createSymbolicLink link f - Git.run g "add" [Param "--", File f] + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" [Param "--"] f + Annex.queueRunAt 1024 moveLocationLogs :: Annex () moveLocationLogs = do @@ -127,11 +128,11 @@ moveLocationLogs = do -- logs that have been pulled from elsewhere old <- liftIO $ readLog f new <- liftIO $ readLog dest - liftIO $ do - writeLog dest (old++new) - Git.run g "add" [Param "--", File dest] - Git.run g "add" [Param "--", File f] - Git.run g "rm" [Param "--quiet", Param "-f", Param "--", File f] + liftIO $ writeLog dest (old++new) + Annex.queue "add" [Param "--"] dest + Annex.queue "add" [Param "--"] f + Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f + Annex.queueRunAt 1024 oldlog2key :: FilePath -> Maybe (FilePath, Key) oldlog2key l = From d7ef5fd2941fa66aa7f9c998fe4acfda60e63295 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 15:48:26 -0400 Subject: [PATCH 1075/2835] add explicit upgrade command --- Command/Upgrade.hs | 22 ++++++++++++++++++++++ GitAnnex.hs | 2 ++ Upgrade/V1.hs | 4 ++-- debian/changelog | 3 ++- doc/git-annex.mdwn | 6 ++++++ 5 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 Command/Upgrade.hs diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs new file mode 100644 index 0000000000..3c9fa3eebf --- /dev/null +++ b/Command/Upgrade.hs @@ -0,0 +1,22 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Upgrade where + +import Command + +command :: [Command] +command = [Command "upgrade" paramNothing seek "upgrade repository layout"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStartNothing +start = do + -- The actual upgrading is handled by just running any command, + -- so nothing extra needs to be done. + return $ Just $ return $ Just $ return True diff --git a/GitAnnex.hs b/GitAnnex.hs index da91f6e74e..b9c22bdfb4 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -41,6 +41,7 @@ import qualified Command.Trust import qualified Command.Untrust import qualified Command.Semitrust import qualified Command.Map +import qualified Command.Upgrade cmds :: [Command] cmds = concat @@ -70,6 +71,7 @@ cmds = concat , Command.Whereis.command , Command.Migrate.command , Command.Map.command + , Command.Upgrade.command ] options :: [Option] diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 64ca298eb4..f1c3e6143c 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -104,7 +104,7 @@ updateSymlinks = do liftIO $ removeFile f liftIO $ createSymbolicLink link f Annex.queue "add" [Param "--"] f - Annex.queueRunAt 1024 + Annex.queueRunAt 10240 moveLocationLogs :: Annex () moveLocationLogs = do @@ -132,7 +132,7 @@ moveLocationLogs = do Annex.queue "add" [Param "--"] dest Annex.queue "add" [Param "--"] f Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f - Annex.queueRunAt 1024 + Annex.queueRunAt 10240 oldlog2key :: FilePath -> Maybe (FilePath, Key) oldlog2key l = diff --git a/debian/changelog b/debian/changelog index e9fdceee6f..a9b9249c1b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,8 @@ git-annex (0.20110316) UNRELEASED; urgency=low * New repository format, annex.version=2. * The first time git-annex is run in an old format repository, it will automatically upgrade it to the new format, staging all - necessary changes to git. + necessary changes to git. See + for details. * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar filesystems. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e559e8cba6..ee40190682 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -261,6 +261,12 @@ Many git-annex commands will stage changes for later `git commit` by you. git annex setkey --key=WORM-s3-m1287765018--file /tmp/file +* upgrade + + Upgrades the repository to current layout. Upgrades are done automatically + whenever a newer git annex encounters an old repository; this command + allows explcitly starting an upgrade. + # OPTIONS * --force From 49da5d1a7b6d86b6795399322ce7ff6be921855f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 15:51:42 -0400 Subject: [PATCH 1076/2835] upgrade documentation --- debian/NEWS | 11 ++++++++ doc/upgrades.mdwn | 67 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 debian/NEWS create mode 100644 doc/upgrades.mdwn diff --git a/debian/NEWS b/debian/NEWS new file mode 100644 index 0000000000..19c277eb0c --- /dev/null +++ b/debian/NEWS @@ -0,0 +1,11 @@ +git-annex (0.20110316) unstable; urgency=low + + This version reorganises the layout of git-annex's files in your repository. + There is an upgrade process to convert a repository from the old git-annex + to this version. While git-annex will attempt to transparently handle + upgrades, you may want to drive the upgrade process by hand. + + See or + /usr/share/doc/git-annex/html/upgrades.html + + -- Joey Hess Wed, 16 Mar 2011 15:49:15 -0400 diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn new file mode 100644 index 0000000000..245212124a --- /dev/null +++ b/doc/upgrades.mdwn @@ -0,0 +1,67 @@ +Occasionally improvments are made to how git-annex stores its data, +that require an upgrade process to convert repositories made with an older +version to be used by a newer version. It's annoying, it should happen +rarely, but sometimes, it's worth it. + +There's a committment that git-annex will always support upgrades from all +past versions. After all, you may have offline drives from an earlier +git-annex, and might want to use them with a newer git-annex. + +## Upgrade process + +git-annex will automatically notice if it is run in a repository that +needs an upgrade, and perform the upgrade before running whatever it +was asked to do. Or you can use the "git annex upgrade" command to +explicitly do an upgrade. The upgrade can tend to take a while, +if you have a lot of files. + +Each clone of a repository should be individually upgraded. +Until a repository's remotes have been upgraded, git-annex +may refuse to communicate with them. + +Generally, start by upgrading one repository, and then you can commit +the changes git-annex staged during upgrade, and push them out to other +repositories. And then upgrade those other repositories. Doing it this +way avoids git-annex doing some duplicate work during the upgrade. + +The upgrade process is guaranteed to be conflict-free. Unless you +already have git conflicts in your repository or between repositories. +Upgrading a repository with conflicts is not recommended; resolve the +conflicts first before upgrading git-annex. + +Example upgrade process: + + cd localrepo + git pull + git annex upgrade + (Upgrading object directory layout v1 to v2...) + git commit -a -m "upgrade v1 to v2" + git push + + ssh remote + cd remoterepo + git pull + git annex upgrade + ... + +## Upgrade events, so far + +### v1 -> v2 (git-annex version 0.23 to version 0.20110316) + +Involved adding hashing to .git/annex/ and changing the names of all keys. +Symlinks changed. + +Also, hashing was added to location log files in .git-annex/. +And .gitattributes needed to have another line added to it. + +Handled transparently. + +### v0 -> v1 (git-annex version 0.03 to version 0.04) + +Involved a reogranisation of the layout of .git/annex/. Symlinks changed. + +Handled more or less transparently, although git-annex was just 2 weeks +old at the time, and had few users other than Joey. + +This upgrade is belived to still be supported, but has not been tested +lately. From 8ad3fd26570e8d6a8066f35353b535e4af81b7ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 16:07:28 -0400 Subject: [PATCH 1077/2835] remove debugging --- Content.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Content.hs b/Content.hs index a59484b5ab..4bd8265c2d 100644 --- a/Content.hs +++ b/Content.hs @@ -171,7 +171,6 @@ getKeysPresent' dir = do present d = do result <- try $ getFileStatus $ d takeFileName d - liftIO $ putStrLn $ "trying " ++ (d takeFileName d) case result of Right s -> return $ isRegularFile s Left _ -> return False From b7a49283fb94a9fc1d5b4d66f0f992e329ef1ee7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 16:07:33 -0400 Subject: [PATCH 1078/2835] set version before running operation that can commit --- Upgrade/V1.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index f1c3e6143c..270de5f74b 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -61,18 +61,19 @@ upgrade = do if Git.repoIsLocalBare g then do moveContent + setVersion else do moveContent updateSymlinks moveLocationLogs Annex.queueRun + setVersion -- add new line to auto-merge hashed location logs -- this commits, so has to come after the upgrade liftIO $ Command.Init.gitAttributesWrite g - setVersion return True moveContent :: Annex () From 6255865c6c0a2fee75ff120cd30211674044c1fb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 16:14:50 -0400 Subject: [PATCH 1079/2835] update --- doc/bugs/free_space_checking.mdwn | 10 ++++++++++ doc/todo/object_dir_reorg_v2.mdwn | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/doc/bugs/free_space_checking.mdwn b/doc/bugs/free_space_checking.mdwn index 34528a7b35..eaa3294d60 100644 --- a/doc/bugs/free_space_checking.mdwn +++ b/doc/bugs/free_space_checking.mdwn @@ -6,3 +6,13 @@ file around. * And, need a way to tell the size of a file before copying it from a remote, to check local disk space. + + As of annex.version 2, this metadata can be available for any type + of backend. Newly added files will always have file size metadata, + while files that used a SHA backend and were added before the upgrade + won't. + + So, need a migration process from eg SHA1 to SHA1+filesize. It will + find files that lack size info, and rename their keys to add the size + info. Users with old repos can run this on them, to get the missing + info recorded. diff --git a/doc/todo/object_dir_reorg_v2.mdwn b/doc/todo/object_dir_reorg_v2.mdwn index 1c2d2f21b7..49666ddc79 100644 --- a/doc/todo/object_dir_reorg_v2.mdwn +++ b/doc/todo/object_dir_reorg_v2.mdwn @@ -19,3 +19,7 @@ all users, so this should be the *last* reorg in the forseeable future. (Probably everything after ",k" should be part of the key, even if it contains the "," separator character. Otherwise an escaping mechanism would be needed.) + +[[done]] now! + +Although [[bugs/free_space_checking]] is not quite there --[[Joey]] From 00eb8ae82981b36d1f96740a703d92ed3c913877 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 16:25:20 -0400 Subject: [PATCH 1080/2835] prepping experimental release --- debian/NEWS | 2 +- debian/changelog | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/NEWS b/debian/NEWS index 19c277eb0c..df8518cef9 100644 --- a/debian/NEWS +++ b/debian/NEWS @@ -1,4 +1,4 @@ -git-annex (0.20110316) unstable; urgency=low +git-annex (0.20110316) experimental; urgency=low This version reorganises the layout of git-annex's files in your repository. There is an upgrade process to convert a repository from the old git-annex diff --git a/debian/changelog b/debian/changelog index a9b9249c1b..c572a4e768 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110316) UNRELEASED; urgency=low +git-annex (0.20110316) experimental; urgency=low * New repository format, annex.version=2. * The first time git-annex is run in an old format repository, it @@ -21,7 +21,7 @@ git-annex (0.20110316) UNRELEASED; urgency=low like metastore to be used with annexed files. (Currently this is only done on systems supporting POSIX 200809.) - -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 + -- Joey Hess Wed, 16 Mar 2011 16:20:23 -0400 git-annex (0.23) unstable; urgency=low From b630705981ba4be8d54896e9418ead43bc4c4630 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 16 Mar 2011 20:32:03 +0000 Subject: [PATCH 1081/2835] Added a comment --- .../comment_4_760437bf3ba972a775bb190fb4b38202._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment new file mode 100644 index 0000000000..6b1e03b026 --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2011-03-16T20:32:01Z" + content=""" +Just tried it out on my mac and it's working again. I guess this issue could be closed for now. +"""]] From a5b48c7b66b16aee4d624325d4cae7e2b473224a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 16 Mar 2011 21:05:39 +0000 Subject: [PATCH 1082/2835] Added a comment --- ...mment_7_42501404c82ca07147e2cce0cff59474._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment b/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment new file mode 100644 index 0000000000..fc866c57a6 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 7" + date="2011-03-16T21:05:38Z" + content=""" +Ah, OK. I assumed the metadata would be attached to a key, not part of the key. This seems to make upgrades/extensions down the line harder than they need to be, but you are right that this way, merges are not, and never will be, an issue. + +Though with the SHA1 backend, changing files can be tracked. This means that tracking changes in mtime or other is possible. It also means that there are potential merge issues. But I won't argue the point endlessly. I can accept design decisions :) + +The prefix at work is from a university netblock so yes, it might be on a few hundred proxy lists etc. +"""]] From 67be8f13fbd36d70083d56d2f038af7c085a9b89 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 17:37:56 -0400 Subject: [PATCH 1083/2835] tweak --- doc/upgrades.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn index 245212124a..1371dc0330 100644 --- a/doc/upgrades.mdwn +++ b/doc/upgrades.mdwn @@ -35,7 +35,7 @@ Example upgrade process: git pull git annex upgrade (Upgrading object directory layout v1 to v2...) - git commit -a -m "upgrade v1 to v2" + git commit -m "upgrade v1 to v2" git push ssh remote From 63360f776783d2527af8c8ed9a4dd79d3159b9a1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 18:33:28 -0400 Subject: [PATCH 1084/2835] update --- debian/changelog | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index c572a4e768..c71c5ee25a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,8 +3,7 @@ git-annex (0.20110316) experimental; urgency=low * New repository format, annex.version=2. * The first time git-annex is run in an old format repository, it will automatically upgrade it to the new format, staging all - necessary changes to git. See - for details. + necessary changes to git. Also added a "git annex upgrade" command. * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar filesystems. From 1079ade2083a7294cb2ce266d8f32bf70821cf1b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 18:41:02 -0400 Subject: [PATCH 1085/2835] releasing version 0.24 --- debian/changelog | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 110fc2b4c0..f5fc4eebe4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,8 @@ -git-annex (0.24) UNRELEASED; urgency=low +git-annex (0.24) unstable; urgency=low + + Branched the 0.24 series, which will be maintained for a while to + support v1 git-annex repos, while main development moves to the 0.2011 + series, with v2 git-annex repos. * Add Suggests on graphviz. Closes: #618039 * When adding files to the annex, the symlinks pointing at the annexed @@ -7,7 +11,7 @@ git-annex (0.24) UNRELEASED; urgency=low like metastore to be used with annexed files. (Currently this is only done on systems supporting POSIX 200809.) - -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 + -- Joey Hess Wed, 16 Mar 2011 18:35:13 -0400 git-annex (0.23) unstable; urgency=low From 7e99ac0222b3b3e69cce06d7de720a78be11b209 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 18:41:32 -0400 Subject: [PATCH 1086/2835] add news item for git-annex 0.24 --- doc/news/version_0.19.mdwn | 17 ----------------- doc/news/version_0.24.mdwn | 11 +++++++++++ 2 files changed, 11 insertions(+), 17 deletions(-) delete mode 100644 doc/news/version_0.19.mdwn create mode 100644 doc/news/version_0.24.mdwn diff --git a/doc/news/version_0.19.mdwn b/doc/news/version_0.19.mdwn deleted file mode 100644 index 5d6ab47be0..0000000000 --- a/doc/news/version_0.19.mdwn +++ /dev/null @@ -1,17 +0,0 @@ -git-annex 0.19 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * configure: Support using the uuidgen command if the uuid command is - not available. - * Allow --exclude to be specified more than once. - * There are now three levels of repository trust. - * untrust: Now marks the current repository as untrusted. - * semitrust: Now restores the default trust level. (What untrust used to do.) - * fsck, drop: Take untrusted repositories into account. - * Bugfix: Files were copied from trusted remotes first even if their - annex.cost was higher than other remotes. - * Improved temp file handling. Transfers of content can now be resumed - from temp files later; the resume does not have to be the immediate - next git-annex run. - * unused: Include partially transferred content in the list. - * Bugfix: Running a second git-annex while a first has a transfer in - progress no longer deletes the first processes's temp file."""]] \ No newline at end of file diff --git a/doc/news/version_0.24.mdwn b/doc/news/version_0.24.mdwn new file mode 100644 index 0000000000..b240820408 --- /dev/null +++ b/doc/news/version_0.24.mdwn @@ -0,0 +1,11 @@ +git-annex 0.24 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + Branched the 0.24 series, which will be maintained for a while to + support v1 git-annex repos, while main development moves to the 0.2011 + series, with v2 git-annex repos. + * Add Suggests on graphviz. Closes: #[618039](http://bugs.debian.org/618039) + * When adding files to the annex, the symlinks pointing at the annexed + content are made to have the same mtime as the original file. + While git does not preserve that information, this allows a tool + like metastore to be used with annexed files. + (Currently this is only done on systems supporting POSIX 200809.)"""]] \ No newline at end of file From 40652e0eaf019ce076fcee24c25fb1f8c950ac5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 18:43:32 -0400 Subject: [PATCH 1087/2835] format --- doc/news/version_0.24.mdwn | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/doc/news/version_0.24.mdwn b/doc/news/version_0.24.mdwn index b240820408..2d94a0e9bb 100644 --- a/doc/news/version_0.24.mdwn +++ b/doc/news/version_0.24.mdwn @@ -1,11 +1,12 @@ +Branched the 0.24 series, which will be maintained for a while to +support v1 git-annex repos, while main development moves to the 0.2011 +series, with v2 git-annex repos. + git-annex 0.24 released with [[!toggle text="these changes"]] [[!toggleable text=""" - Branched the 0.24 series, which will be maintained for a while to - support v1 git-annex repos, while main development moves to the 0.2011 - series, with v2 git-annex repos. - * Add Suggests on graphviz. Closes: #[618039](http://bugs.debian.org/618039) - * When adding files to the annex, the symlinks pointing at the annexed - content are made to have the same mtime as the original file. - While git does not preserve that information, this allows a tool - like metastore to be used with annexed files. - (Currently this is only done on systems supporting POSIX 200809.)"""]] \ No newline at end of file +* Add Suggests on graphviz. Closes: #[618039](http://bugs.debian.org/618039) +* When adding files to the annex, the symlinks pointing at the annexed + content are made to have the same mtime as the original file. + While git does not preserve that information, this allows a tool + like metastore to be used with annexed files. + (Currently this is only done on systems supporting POSIX 200809.)"""]] From 2f515fb57f1316bd5732faa4ce03d30ed099b74a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 17 Mar 2011 00:04:38 +0000 Subject: [PATCH 1088/2835] --- doc/bugs/check_for_curl_in_configure.hs.mdwn | 39 ++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 doc/bugs/check_for_curl_in_configure.hs.mdwn diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn new file mode 100644 index 0000000000..ee24769ce7 --- /dev/null +++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn @@ -0,0 +1,39 @@ +I thought this might be useful, since curl is being used for the URL backend, it might be worth checking for it's existence. + +
+diff --git a/configure.hs b/configure.hs
+index 772ba54..1a563e0 100644
+--- a/configure.hs
++++ b/configure.hs
+@@ -13,6 +13,7 @@ tests = [
+        , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"]
+        , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null"
++       , TestCase "curl" $ requireCmd "curl" "curl --version >/dev/null"
+        , TestCase "unicode FilePath support" $ unicodeFilePath
+        ] ++ shaTestCases [1, 256, 512, 224, 384]
+
+ +also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not) + +
+diff --git a/Backend/URL.hs b/Backend/URL.hs
+index 29dc8fe..4afcf86 100644
+--- a/Backend/URL.hs
++++ b/Backend/URL.hs
+@@ -50,10 +50,13 @@ dummyFsck _ _ _ = return True
+ dummyOk :: Key -> Annex Bool
+ dummyOk _ = return True
+ 
++curl :: [CommandParam] -> IO Bool
++curl = boolSystem "curl"
++
+ downloadUrl :: Key -> FilePath -> Annex Bool
+ downloadUrl key file = do
+        showNote "downloading"
+        showProgress -- make way for curl progress bar
+-       liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
++       liftIO $ curl [Params "-# -o", File file, File url]
+        where
+                url = join ":" $ drop 1 $ split ":" $ show key 
+
From 14d7049886314cc73148c61921c2193be92c835f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 21:04:25 -0400 Subject: [PATCH 1089/2835] releasing version 0.20110316 --- doc/news/version_0.24.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/news/version_0.24.mdwn b/doc/news/version_0.24.mdwn index 2d94a0e9bb..81b013a26f 100644 --- a/doc/news/version_0.24.mdwn +++ b/doc/news/version_0.24.mdwn @@ -1,6 +1,6 @@ -Branched the 0.24 series, which will be maintained for a while to -support v1 git-annex repos, while main development moves to the 0.2011 -series, with v2 git-annex repos. +Branched the 0.24 series, which will be maintained for a while (in the +stable branch in git) to support v1 git-annex repos, while main development +moves to the 0.2011 series, with v2 git-annex repos. git-annex 0.24 released with [[!toggle text="these changes"]] [[!toggleable text=""" From 455721597b4ed8d9e2379e66ccc47a4c9459867c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 21:05:00 -0400 Subject: [PATCH 1090/2835] add news item for git-annex 0.20110316 --- doc/news/version_0.20.mdwn | 12 ------------ doc/news/version_0.20110316.mdwn | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 12 deletions(-) delete mode 100644 doc/news/version_0.20.mdwn create mode 100644 doc/news/version_0.20110316.mdwn diff --git a/doc/news/version_0.20.mdwn b/doc/news/version_0.20.mdwn deleted file mode 100644 index 9b95b652e5..0000000000 --- a/doc/news/version_0.20.mdwn +++ /dev/null @@ -1,12 +0,0 @@ -git-annex 0.20 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Preserve specified file ordering when instructed to act on multiple - files or directories. For example, "git annex get a b" will now always - get "a" before "b". Previously it could operate in either order. - * unannex: Commit staged changes at end, to avoid some confusing behavior - with the pre-commit hook, which would see some types of commits after - an unannex as checking in of an unlocked file. - * map: New subcommand that uses graphviz to display a nice map of - the git repository network. - * Deal with the mtl/monads-fd conflict. - * configure: Check for sha1sum."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110316.mdwn b/doc/news/version_0.20110316.mdwn new file mode 100644 index 0000000000..5654c15bcf --- /dev/null +++ b/doc/news/version_0.20110316.mdwn @@ -0,0 +1,24 @@ +News for git-annex 0.20110316: + + This version reorganises the layout of git-annex's files in your repository. + There is an upgrade process to convert a repository from the old git-annex + to this version. While git-annex will attempt to transparently handle + upgrades, you may want to drive the upgrade process by hand. + See <http://git-annex.branchable.com/upgrades/> or + /usr/share/doc/git-annex/html/upgrades.html + +git-annex 0.20110316 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * New repository format, annex.version=2. + * The first time git-annex is run in an old format repository, it + will automatically upgrade it to the new format, staging all + necessary changes to git. Also added a "git annex upgrade" command. + * Colons are now avoided in filenames, so bare clones of git repos + can be put on USB thumb drives formatted with vFAT or similar + filesystems. + * Added two levels of hashing to object directory and .git-annex logs, + to improve scalability with enormous numbers of annexed + objects. (With one hundred million annexed objects, each + directory would contain fewer than 1024 files.) + * The setkey, fromkey, and dropkey subcommands have changed how + the key is specified. --backend is no longer used with these."""]] \ No newline at end of file From cb1c68de5c26fcaa83ccdbc3d6ef79e400a3e297 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 17 Mar 2011 00:47:03 -0400 Subject: [PATCH 1091/2835] format --- doc/news/version_0.20110316.mdwn | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/news/version_0.20110316.mdwn b/doc/news/version_0.20110316.mdwn index 5654c15bcf..968722c0e6 100644 --- a/doc/news/version_0.20110316.mdwn +++ b/doc/news/version_0.20110316.mdwn @@ -1,11 +1,8 @@ -News for git-annex 0.20110316: - - This version reorganises the layout of git-annex's files in your repository. - There is an upgrade process to convert a repository from the old git-annex - to this version. While git-annex will attempt to transparently handle - upgrades, you may want to drive the upgrade process by hand. - See <http://git-annex.branchable.com/upgrades/> or - /usr/share/doc/git-annex/html/upgrades.html +This version reorganises the layout of git-annex's files in your repository. +There is an upgrade process to convert a repository from the old git-annex +to this version. While git-annex will attempt to transparently handle +upgrades, you may want to drive the upgrade process by hand. +See [[upgrades]] for details. git-annex 0.20110316 released with [[!toggle text="these changes"]] [[!toggleable text=""" @@ -21,4 +18,4 @@ git-annex 0.20110316 released with [[!toggle text="these changes"]] objects. (With one hundred million annexed objects, each directory would contain fewer than 1024 files.) * The setkey, fromkey, and dropkey subcommands have changed how - the key is specified. --backend is no longer used with these."""]] \ No newline at end of file + the key is specified. --backend is no longer used with these."""]] From a24a04ff7e417f588bb627af8a8a139c46834fc6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 17 Mar 2011 09:59:05 +0000 Subject: [PATCH 1092/2835] --- doc/install.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/install.mdwn b/doc/install.mdwn index 9eb8bbacf7..a531015782 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -16,6 +16,7 @@ To build and use git-annex, you will need: (or uuidgen from util-linux) * `xargs`: * `rsync`: +* `curl` : (optional, but recommended) * `sha1sum`: (optional, but recommended) * Then just [[download]] git-annex and run: `make; make install` From ba91851632a5756c96e47d6036713fdb417e9ce9 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 17 Mar 2011 10:14:19 +0000 Subject: [PATCH 1093/2835] --- doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn diff --git a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn new file mode 100644 index 0000000000..8ce7e84ad8 --- /dev/null +++ b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn @@ -0,0 +1,11 @@ +I was trying out the example with the walkthrough [[walkthrough/using_the_URL_backend]]. I tried dropping files that I had after doing an "git annex get ." which have the URL backend associated with the files it fails with + + +
+[jtang@lenny gc]$ git annex drop -v curl-7.21.4.tar.gz
+drop curl-7.21.4.tar.gz
+failed
+git-annex: 1 failed
+
+ +At first I thought it was just my OSX machine not having the coreutils stuff load up before the BSD utils, but I then tried the same thing on my archlinux machine and it showed the same behaviour, that is I could not drop a file with the URL backend as shown in the walkthrough. From 9aada06745d138c953d28929ac561449ddc8fabe Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 17 Mar 2011 15:44:27 +0000 Subject: [PATCH 1094/2835] --- ...magic_upgrade_of_the_object_directory_safe__63__.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn diff --git a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn new file mode 100644 index 0000000000..5643f6b7a0 --- /dev/null +++ b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn @@ -0,0 +1,9 @@ +Consider the following two use cases: + +* I have a git-annex repo on a portable medium and carry it around between several machines. I use it on a non-important system with the most current git-annex installed, automagic upgrade happens. I am now forced to upgrade git-annex on all other machines. Bonus points if this happens in the background and I don't even notice it until it's too late. + +* My system crashes and I use a rescue CD to access local data, including git-annex. The rescue CD includes a newer version of git-annex and once my system is restored, I am forced to upgrade git-annex locally. + +My suggestion would be not to upgrade automatically, but to either ask the user if this is OK or to error out and request that they run git annex update by hand. + +Optionally, this could be done via a local config variable which should default to error or ask, not upgrade. From 7b5b1276085be3d2c12c8c28b7be1aceccae44f9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 17 Mar 2011 11:49:21 -0400 Subject: [PATCH 1095/2835] Fix dropping of files using the URL backend. --- Backend/URL.hs | 2 +- debian/changelog | 6 ++++++ doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index 02ce3563cd..210c7c5b48 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -42,7 +42,7 @@ dummyStore :: FilePath -> Key -> Annex Bool dummyStore _ _ = return False dummyRemove :: Key -> Maybe a -> Annex Bool -dummyRemove _ _ = return False +dummyRemove _ _ = return True dummyFsck :: Key -> Maybe FilePath -> Maybe a -> Annex Bool dummyFsck _ _ _ = return True diff --git a/debian/changelog b/debian/changelog index 47a9148124..81257a24be 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.20110317) UNRELEASED; urgency=low + + * Fix dropping of files using the URL backend. + + -- Joey Hess Thu, 17 Mar 2011 11:46:53 -0400 + git-annex (0.20110316) experimental; urgency=low * New repository format, annex.version=2. diff --git a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn index 8ce7e84ad8..e88bf07f4d 100644 --- a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn +++ b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn @@ -9,3 +9,5 @@ git-annex: 1 failed At first I thought it was just my OSX machine not having the coreutils stuff load up before the BSD utils, but I then tried the same thing on my archlinux machine and it showed the same behaviour, that is I could not drop a file with the URL backend as shown in the walkthrough. + +> Whoops, got some logic backwards. [[fixed|done]]! --[[Joey]] From 8543d9add4c717987ce4b8bbd44b285b4963c125 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 17 Mar 2011 11:57:03 -0400 Subject: [PATCH 1096/2835] check for curl in configure, thanks Jimmy --- configure.hs | 1 + doc/bugs/check_for_curl_in_configure.hs.mdwn | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/configure.hs b/configure.hs index 772ba54899..f5c2956488 100644 --- a/configure.hs +++ b/configure.hs @@ -13,6 +13,7 @@ tests = [ , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" + , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath ] ++ shaTestCases [1, 256, 512, 224, 384] diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn index ee24769ce7..18edabe5e0 100644 --- a/doc/bugs/check_for_curl_in_configure.hs.mdwn +++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn @@ -14,8 +14,21 @@ index 772ba54..1a563e0 100644 ] ++ shaTestCases [1, 256, 512, 224, 384] +> Well, curl is an optional extra, so requireCmd is too strong. Changed +> to testCmd and applied, thank you! +> +> I thought about actually *using* the resulting SysConfig.curl +> to disable the URL backend if False.. but probably it's better +> to just let it fail if curl is not available. Although, if we wanted +> to add a check for wget or something and use it when curl was not +> available, that might be worth doing. --[[Joey]] + also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not) +> It's correct, typewise, but I don't see any real reason to bother +> with the change. But I do appreciate patches, which have been rare +> so far, probaby because of Haskell.. :) --[[Joey]] + 
 diff --git a/Backend/URL.hs b/Backend/URL.hs
 index 29dc8fe..4afcf86 100644

From c196875932d7b51083cc835ce0518d7a3d961198 Mon Sep 17 00:00:00 2001
From: Richard Hartmann 
Date: Thu, 17 Mar 2011 12:20:55 +0100
Subject: [PATCH 1097/2835] Fix typos

---
 doc/upgrades.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 1371dc0330..1bd97c46e2 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -58,10 +58,10 @@ Handled transparently.
 
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 
-Involved a reogranisation of the layout of .git/annex/. Symlinks changed.
+Involved a reorganisation of the layout of .git/annex/. Symlinks changed.
 
 Handled more or less transparently, although git-annex was just 2 weeks
 old at the time, and had few users other than Joey.
 
-This upgrade is belived to still be supported, but has not been tested
+This upgrade is believed to still be supported, but has not been tested
 lately.

From 98364839ca5367bdf21bc2a2cbfab6e8189aafaf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Thu, 17 Mar 2011 19:11:21 +0000
Subject: [PATCH 1098/2835]

---
 doc/bugs/No_version_information_from_cli.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/No_version_information_from_cli.mdwn

diff --git a/doc/bugs/No_version_information_from_cli.mdwn b/doc/bugs/No_version_information_from_cli.mdwn
new file mode 100644
index 0000000000..c432cf99b7
--- /dev/null
+++ b/doc/bugs/No_version_information_from_cli.mdwn
@@ -0,0 +1,16 @@
+git-annex does not listen to -v, --version or version.
+
+At the very least, it should return both the version of the binary and the version of the object store it supports.
+If it supports several annex versions, they should be listed in a comma-separated fashion.
+If git-annex is called from within an annex, it should print the version of the local object store.
+
+Sample:
+
+    % git annex version
+    git-annex version               : 0.24
+    default object store version    : 3
+    supported object store versions : 2,3
+    local object store version      : 2
+    % 
+
+The above might look like overkill, but it's in a form that will, most likely, never need to be extended.

From e31be22c12a0c69e17042138663cc6e9fa539db2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 17 Mar 2011 20:27:54 +0000
Subject: [PATCH 1099/2835]

---
 doc/bugs/check_for_curl_in_configure.hs.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn
index 18edabe5e0..bba2b81fcb 100644
--- a/doc/bugs/check_for_curl_in_configure.hs.mdwn
+++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn
@@ -23,12 +23,23 @@ index 772ba54..1a563e0 100644
 > to add a check for wget or something and use it when curl was not
 > available, that might be worth doing. --[[Joey]] 
 
+>> I was thinking that is it worth doing a generic "stat", "delete", "get" 
+>> and "put" options, I do like the idea of having the possibility of 
+>> being about to use completely arbitrary storage systems or arbitrary 
+>> transfer systems. If there was the capability of doing so it would be 
+>> interesting to see possibilities of using aria2 for using something 
+>> like bittorrent as backend, or using something like irods or some 
+>> grid storage system as the storage archive. It's just an idea as 
+>> I have seen it implemented quite well in irods.
+
 also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not)
 
 > It's correct, typewise, but I don't see any real reason to bother
 > with the change. But I do appreciate patches, which have been rare
 > so far, probaby because of Haskell.. :) --[[Joey]] 
 
+>> heh agreed
+
 
 diff --git a/Backend/URL.hs b/Backend/URL.hs
 index 29dc8fe..4afcf86 100644

From be27fa41fa3aff7fb0c455d51c7d67d7b4114b69 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 17 Mar 2011 20:36:25 -0400
Subject: [PATCH 1100/2835] response

---
 doc/bugs/check_for_curl_in_configure.hs.mdwn | 21 ++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn
index bba2b81fcb..2a5227491a 100644
--- a/doc/bugs/check_for_curl_in_configure.hs.mdwn
+++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn
@@ -1,3 +1,9 @@
+[[!meta title="arbitrary/configurable backends"]]
+
+(Retitling as this has drifted..)
+
+---
+
 I thought this might be useful, since curl is being used for the URL backend, it might be worth checking for it's existence.
 
 
@@ -32,6 +38,21 @@ index 772ba54..1a563e0 100644
 >> grid storage system as the storage archive. It's just an idea as 
 >> I have seen it implemented quite well in irods.
 
+>>> I'm unsure about the idea of having a backend where that is
+>>> parameterized. It would mean that one annex's GENERIC-foo key
+>>> might be entirely different from another's key with the same backend
+>>> and details. And a misconfiguration could get data the wrong
+>>> way and get the wrong data, etc.
+>>>
+>>> I mostly look at the URL backend as an example that can be modified to
+>>> make this kind of custom backend. You already probably know enough to
+>>> make a TORRENT backend where keys are the urls to torrents to download
+>>> with `aria2c --follow-torrent=mem`.
+>>>
+>>> I am also interested in doing backends that use eg, cloud storage.
+>>> A S3 backend that could upload files to S3 in addition to downloading
+>>> them, for example, would be handy. --[[Joey]]
+
 also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not)
 
 > It's correct, typewise, but I don't see any real reason to bother

From d235677926109bad7567bda101364087fd121223 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 18 Mar 2011 00:38:51 +0000
Subject: [PATCH 1101/2835] Added a comment

---
 ...comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment

diff --git a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
new file mode 100644
index 0000000000..6a34becd37
--- /dev/null
+++ b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-18T00:38:51Z"
+ content="""
+These are good examples; I think you've convinced me at least for upgrades going forward after v2. I'm not sure we have enough users and outdated git-annex installations to worry about it for v1.
+
+(Hoping such upgrades are rare anyway.. Part of the point of changes made in v2 was to allow lots of changes to be made later w/o needing a v3.)
+"""]]

From 028eb96f408cddab03a4b77709fb6a24806a5e57 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 09:39:27 -0400
Subject: [PATCH 1102/2835] design

---
 doc/todo/S3.mdwn | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index ec2d403ced..cb5186b095 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -1 +1,38 @@
 Support Amazon S3 as a file storage backend.
+
+There's a haskell library that looks good. Not yet in Debian.
+
+Multiple ways of using S3 are possible. Current plan is to have a S3BUCKET
+backend, that is derived from Backend.File, so it caches files locally and
+can transfer files between systems too, without involving S3.
+
+get will try to get it from S3 or from a remote. A annex.s3.cost can
+configure the cost of S3 vs the cost of other remotes.
+
+add will always upload a copy to S3.
+
+Each file in the S3 bucket is assumed to be in the annex. So unused
+will show files in the bucket that nothing points to, and dropunused remove
+them.
+
+For numcopies counting, S3 will count as 1 copy (or maybe more?), so if
+numcopies=2, then you don't fully trust S3 and request git-annex assure
+one other copy.
+
+drop will remove a file locally, but keep it in S3. drop --force *might*
+remove it from S3. TBD.
+
+annex.s3.bucket would configure the bucket the use. (And an env var or
+something configure the password.) Although the bucket
+would also be encoded in the keys. So, the configured bucket would be used
+when adding new files. A system could move from one bucket to another over
+time while still having legacy files in an earlier one; 
+perhaps you move to Europe and want new files to be put in that region.
+
+And git annex `migrate --backend=S3BUCKET --force` could move files
+between datacenters!
+
+Problem: Then the only way for unused to know what buckets are in use
+is to see what keys point to them -- but if the last file from a bucket is
+deleted, it would then not be able to say that the files in that bucket are
+all unused. Need cached list of recently seen S3 buckets?

From 70a8a3ab711fb266657653177ec58f8234bb716c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 09:47:12 -0400
Subject: [PATCH 1103/2835] update

---
 doc/todo/S3.mdwn | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index cb5186b095..3d18527d48 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -36,3 +36,17 @@ Problem: Then the only way for unused to know what buckets are in use
 is to see what keys point to them -- but if the last file from a bucket is
 deleted, it would then not be able to say that the files in that bucket are
 all unused. Need cached list of recently seen S3 buckets?
+
+-----
+
+One problem with this is what key metadata to include. Should it be like
+WORM? Or like SHA1? Or just a new unique identifier for each file? It might
+be worth having S3 variants of *all* the Backend.File derived backends.
+
+More blue-sky, it might be nice to be able to union or stack together
+multiple backends, so S3BUCKET+SHA1 or S3BUCKET+WORM. That would likely
+be hard to get right.
+
+Less blue-sky, if the S3 capability were added directly to Backend.File,
+and bucket name was configured by annex.s3.bucket, then any existing
+annexed file could be upgraded to also store on S3.

From 8fed2539a3056bafb92dc55b1bd393be03fef758 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 17:54:32 +0000
Subject: [PATCH 1104/2835] new: bugs/fsck output

---
 doc/bugs/fsck_output.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/fsck_output.mdwn

diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
new file mode 100644
index 0000000000..81fecceb3f
--- /dev/null
+++ b/doc/bugs/fsck_output.mdwn
@@ -0,0 +1,16 @@
+When you check several files and the fsck fails, you get confusing output:
+
+
+fsck test1 (checksum...) 
+  Only 1 of 2 trustworthy copies of test1 exist.
+  Back it up with git-annex copy.
+
+failed
+fsck test2 (checksum...) 
+  Only 1 of 2 trustworthy copies of test2 exist.
+  Back it up with git-annex copy.
+
+failed
+

+
+The newline is in the wrong place and confuses the user. It should be printed _after_ "failed".

From 12a2f366c235dc7b7280411622c8462f0afaae3c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 18:07:52 +0000
Subject: [PATCH 1105/2835]

---
 doc/walkthrough/adding_a_remote.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn
index be8e8e7fe5..013ac51100 100644
--- a/doc/walkthrough/adding_a_remote.mdwn
+++ b/doc/walkthrough/adding_a_remote.mdwn
@@ -17,3 +17,5 @@ of the new repository created on the USB drive.
 Notice that both repos are set up as remotes of one another. This lets
 either get annexed files from the other. You'll want to do that even
 if you are using git in a more centralized fashion.
+
+Note that you need to pull from the new remote at least once to make your local annex aware of the remote annex.

From bc72c0bab6bb2d17293f7549aa031b4d439ea954 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 18:37:05 +0000
Subject: [PATCH 1106/2835]

---
 doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn

diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
new file mode 100644
index 0000000000..8399ff9844
--- /dev/null
+++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
@@ -0,0 +1 @@
+I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. The name scheme for an annex should be the same as for git repos themselves.

From 482782ad05d0a8a8ce4375c437e092d40aa574cb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 18:38:30 +0000
Subject: [PATCH 1107/2835]

---
 ...bout_remotes_with_dots_in_their_names.mdwn | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn

diff --git a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
new file mode 100644
index 0000000000..e6dee3e0b7
--- /dev/null
+++ b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
@@ -0,0 +1,22 @@
+For test.com//test, I get this:
+
+    % git annex copy . --to test.com//test
+    (getting UUID for test...) git-annex: there is no git remote named "test.com//test"
+
+And my .git/config changes from
+
+    [remote "test.com//test"]
+    	url = richih@test.com:/test
+    	fetch = +refs/heads/*:refs/remotes/test.com//test/*
+
+to
+
+    [remote "test.com//test"]
+    	url = richih@test.com:/test
+    	fetch = +refs/heads/*:refs/remotes/test.com//test/*
+    	annex-uuid = xyz
+    [remote "test"]
+    	annex-uuid = xyz
+
+
+Unless I am misunderstanding something, git annex gets confused about what the name of the remote it supposed to be, truncates at the dot for some operations and uses the full name for others.

From ca48255495e1b8ef4bda5f7f019c482d2a59b431 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 19:24:10 +0000
Subject: [PATCH 1108/2835]

---
 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn

diff --git a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
new file mode 100644
index 0000000000..06e940a104
--- /dev/null
+++ b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
@@ -0,0 +1 @@
+* I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. Name schemes for an annex should be the same as for git repos themselves.

From ba7970c644a2583d0cd50242f170770c4a0ff1b7 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 19:24:40 +0000
Subject: [PATCH 1109/2835]

---
 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
index 06e940a104..17b3c65b16 100644
--- a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
+++ b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
@@ -1 +1 @@
-* I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. Name schemes for an annex should be the same as for git repos themselves.
+* I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. Name schemes for an annex should be the same as for git remotes themselves.

From b7ed29f3c07c7025c203813e4ecc5b6c6908279f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 19:24:56 +0000
Subject: [PATCH 1110/2835]

---
 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
index 17b3c65b16..7862c44e93 100644
--- a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
+++ b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
@@ -1 +1 @@
-* I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. Name schemes for an annex should be the same as for git remotes themselves.
+I can create an annex remote named 'test:/test'. git itself does not allow colons in remote names, though. Name schemes for an annex should be the same as for git remotes themselves.

From efd555848d042450d867761799ebd1b23459ea4e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:13:54 -0400
Subject: [PATCH 1111/2835] remove bad information

You do not need to pull from a remote to make git-annex aware of it.
Pulling only makes git know about remote/master branches, but git-annex
does not care about that. When git-annex encounters a remote in .git/config
that has no recorded annex.uuid, it connects to it to get the uuid
automatically, and will immediately start using it.

A later part of this walkthrough shows how to pull from a remote
in order to get its location tracking info, so that git-annex can retrieve
files from it. That's the right place to document the need to pull from
a remote periodically.
---
 doc/walkthrough/adding_a_remote.mdwn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn
index 013ac51100..be8e8e7fe5 100644
--- a/doc/walkthrough/adding_a_remote.mdwn
+++ b/doc/walkthrough/adding_a_remote.mdwn
@@ -17,5 +17,3 @@ of the new repository created on the USB drive.
 Notice that both repos are set up as remotes of one another. This lets
 either get annexed files from the other. You'll want to do that even
 if you are using git in a more centralized fashion.
-
-Note that you need to pull from the new remote at least once to make your local annex aware of the remote annex.

From 0cc05e0c19d7e0a6112167440a9ba48629407625 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:29:14 -0400
Subject: [PATCH 1112/2835] response

---
 doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
index 8399ff9844..aa658819b4 100644
--- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
+++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
@@ -1 +1,6 @@
 I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. The name scheme for an annex should be the same as for git repos themselves.
+
+> What do you mean by "an annex remote"? git-annex uses the same
+> remotes configuration as does git. If you put invalid
+> stuff in .git/config it might handle it slightly different than 
+> git, I don't know. Examples needed. --[[Joey]] 

From 0663f14cf7cb49189352a9c35ce649f3ad10de8b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:29:42 -0400
Subject: [PATCH 1113/2835] Fix support for remotes with '.' in their names.

---
 GitRepo.hs       | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index a62d765961..34a59a10d4 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -404,7 +404,7 @@ configRemotes repo = mapM construct remotepairs
 		remotepairs = Map.toList $ filterremotes $ config repo
 		filterremotes = Map.filterWithKey (\k _ -> isremote k)
 		isremote k = startswith "remote." k && endswith ".url" k
-		remotename k = split "." k !! 1
+		remotename k = join "." $ reverse $ drop 1 $ reverse $ drop 1 $ split "." k
 		construct (k,v) = do
 			r <- gen v
 			return $ r { remoteName = Just $ remotename k }
diff --git a/debian/changelog b/debian/changelog
index 81257a24be..e1c0576e8b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (0.20110317) UNRELEASED; urgency=low
 
   * Fix dropping of files using the URL backend.
+  * Fix support for remotes with '.' in their names.
 
  -- Joey Hess   Thu, 17 Mar 2011 11:46:53 -0400
 

From 69841b67c9dfc58d33735b6e15d81099b5f502e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:32:31 -0400
Subject: [PATCH 1114/2835] done

---
 ...ets_confused_about_remotes_with_dots_in_their_names.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
index e6dee3e0b7..6f35cef8d1 100644
--- a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
+++ b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
@@ -20,3 +20,9 @@ to
 
 
 Unless I am misunderstanding something, git annex gets confused about what the name of the remote it supposed to be, truncates at the dot for some operations and uses the full name for others.
+
+> I've fixed this bug. [[done]]
+> 
+> However, using "/" in a remote name seems likely to me to confuse 
+> git's own remote branch handling. Although I've never tried it.
+> --[[Joey]] 

From 75ad0f22d84603fa609acb5a62f4cc4dcd86ee72 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:47:20 -0400
Subject: [PATCH 1115/2835] analysis

---
 doc/bugs/fsck_output.mdwn | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 81fecceb3f..413336b313 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -1,16 +1,28 @@
 When you check several files and the fsck fails, you get confusing output:
 
 
-fsck test1 (checksum...) 
-  Only 1 of 2 trustworthy copies of test1 exist.
-  Back it up with git-annex copy.
-
-failed
-fsck test2 (checksum...) 
-  Only 1 of 2 trustworthy copies of test2 exist.
-  Back it up with git-annex copy.
-
-failed
+O fsck test1 (checksum...) 
+E  Only 1 of 2 trustworthy copies of test1 exist.
+E  Back it up with git-annex copy.
+O
+O failed
+O fsck test2 (checksum...) 
+E  Only 1 of 2 trustworthy copies of test2 exist.
+E  Back it up with git-annex copy.
+O 
+O failed
 

 
 The newline is in the wrong place and confuses the user. It should be printed _after_ "failed".
+
+> This is a consequence of part of the output being printed to stderr, and
+> part to stdout. I've marked the lines above with E and O.
+> 
+> Normally a "failed" is preceeded by a message output to stdout desribing
+> the problem; such a message will not be "\n" terminated, so a newline
+> is always displayed before "failed". In this case, since the message
+> is sent to stderr, it is newline terminated.
+> 
+> Fixing this properly would involve storing state, or rethinking 
+> when git-annex displays newlines (and I rather like its behavior
+> otherwise). --[[Joey]]

From 885215508ce8e07676fd92cd8bbb4ac68bd5c1b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 17:05:03 -0400
Subject: [PATCH 1116/2835] add

---
 doc/bugs/fsck_output.mdwn | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 413336b313..3ded1b409e 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -25,4 +25,12 @@ The newline is in the wrong place and confuses the user. It should be printed _a
 > 
 > Fixing this properly would involve storing state, or rethinking 
 > when git-annex displays newlines (and I rather like its behavior
-> otherwise). --[[Joey]]
+> otherwise).
+> 
+> A related problem occurs if an error message is unexpetedly printed.
+> Dummying up an example:
+> 
+> O get test1 (copying from foo...) E git-annex: failed to run ssh
+> failed
+> 
+> --[[Joey]] 

From 3dbff3356d14eadc3463693ae44921b98d7e03f4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 19 Mar 2011 01:09:22 +0000
Subject: [PATCH 1117/2835]

---
 ...ets_confused_about_remotes_with_dots_in_their_names.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
index 6f35cef8d1..d35282e750 100644
--- a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
+++ b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
@@ -26,3 +26,9 @@ Unless I am misunderstanding something, git annex gets confused about what the n
 > However, using "/" in a remote name seems likely to me to confuse 
 > git's own remote branch handling. Although I've never tried it.
 > --[[Joey]] 
+
+>> From what I can see, git handles / just fine, but would get upset about : which is why it's not allowed in a remote's name.
+>> My naming scheme is host//path/to/annex. It sorts nicely and gives all important information left to right with the most specific parts at the beginning and end.
+>> If you have any other ideas or scheme, I am all ears :)
+>> Either way, thanks for fixing this so quickly.
+>> -- RichiH

From debc7b27445da764561abd83044e7865024b39f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 19 Mar 2011 01:10:26 +0000
Subject: [PATCH 1118/2835] removed

---
 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn

diff --git a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
deleted file mode 100644
index 7862c44e93..0000000000
--- a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
+++ /dev/null
@@ -1 +0,0 @@
-I can create an annex remote named 'test:/test'. git itself does not allow colons in remote names, though. Name schemes for an annex should be the same as for git remotes themselves.

From 9e8d5373365281f67b9539e0443572e92bde5b60 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 19 Mar 2011 01:16:09 +0000
Subject: [PATCH 1119/2835]

---
 ...me_scheme_does_not_follow_git__39__s_rules.mdwn | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
index aa658819b4..5971888891 100644
--- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
+++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
@@ -4,3 +4,17 @@ I can create an annex remote named 'test:/test'. git itself does not allow colon
 > remotes configuration as does git. If you put invalid
 > stuff in .git/config it might handle it slightly different than 
 > git, I don't know. Examples needed. --[[Joey]] 
+
+>> What I mean is this:
+
+    % cd 1
+    % git init
+    % git annex init "my:colon"
+    % [...]
+    % cd ../2
+    % git init
+    % git annex init "second"
+    % git remote add "my:colon" ../1
+    fatal: 'my:colon' is not a valid remote name
+
+>> -- RichiH

From 5e76dab6febf727a37421ab124d6f380114f5deb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 19 Mar 2011 01:18:49 +0000
Subject: [PATCH 1120/2835] Added a comment

---
 .../comment_1_0a59355bd33a796aec97173607e6adc9._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment

diff --git a/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment b/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment
new file mode 100644
index 0000000000..4b0b9c0fd2
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-19T01:18:49Z"
+ content="""
+After doing the above with two required copy per file, `git annex fsck` complained that I had only one copy per file even though I had created my clone, already. Once I `git pull`ed from the second repo, not getting any changes for obvious reasons, `git annex fsck` was happy. So I am not sure how my addition was incorrect. -- RichiH
+"""]]

From a3ee0da1e0706834d3e8da8d2abf1f134e26f625 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 19 Mar 2011 11:31:23 +0000
Subject: [PATCH 1121/2835]

---
 ...askell-platform_statically_links_things.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn

diff --git a/doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn b/doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn
new file mode 100644
index 0000000000..f656427589
--- /dev/null
+++ b/doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn
@@ -0,0 +1,17 @@
+This isn't really a bug of git-annex, but a problem with haskell-platform/ghc6.12.x so this post might need to be moved to a better place (maybe tips).
+
+OSX's haskell-platform doesn't have the dynamic libraries available, as far as I know it just isn't supported therefore git-annex will always be statically built on OSX, so wrappers like  or [[!google dsocks]] for preloading connect() calls won't work. 
+
+
+jtang@x00:~/annex $ tsocks git annex get .
+dyld: could not load inserted library: /opt/local/lib/libtsocks.dylib
+
+error: git-annex died of signal 5
+

+
+The side effect of this means that users who are behind restrictive firewalls that allow only ssh via a socks proxy, they will need to configure ssh to use something like "http://bent.latency.net/bent/git/goto-san-connect-1.85/src/connect.html".
+
+
+host remotemyhost
+        ProxyCommand connect -S proxy.mydomain:1080 -R local %h %p
+


From 4395429ab1d8165240c6c76dc90e22fd0c023532 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 19 Mar 2011 11:39:28 +0000
Subject: [PATCH 1122/2835]

---
 ...default_sshd_behaviour_has_limited_paths_set.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn

diff --git a/doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn b/doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
new file mode 100644
index 0000000000..99b7092c7f
--- /dev/null
+++ b/doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
@@ -0,0 +1,12 @@
+This is a tip for users who wish to use remotes which are based on OSX systems and have used macports to install some of the required utilities for git-annex to work.
+
+The default behaviour of OSX's sshd is to have a "highly restricted" restricted environment. The defaults that it allows is
+
+    jtang@x00:~ $ ssh x00 echo \$PATH
+    /usr/bin:/bin:/usr/sbin:/sbin
+
+One solution is to enable *PermitUserEnvironment yes* in `/etc/sshd_config` and then in your own `~/.ssh/environment` file you could add something like (the below is an example)
+
+    PATH=/Users/jtang/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/X11/bin:/usr/X11/bin:/Users/jtang/.cabal/bin:/opt/local/libexec/gnubin
+
+If the above is not done, cloning from the OSX host will fail if git is not installed isn't /usr/bin (which it probably won't be).

From 065a3e4d2bfa2340b57830baf4e3899a8bc25217 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 11:31:08 -0400
Subject: [PATCH 1123/2835] move OSX tips to forum and link to from install/OSX

---
 ...__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn | 0
 .../OSX__39__s_haskell-platform_statically_links_things.mdwn | 0
 doc/install/OSX.mdwn                                         | 5 +++++
 3 files changed, 5 insertions(+)
 rename doc/{bugs => forum}/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn (100%)
 rename doc/{bugs => forum}/OSX__39__s_haskell-platform_statically_links_things.mdwn (100%)

diff --git a/doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
similarity index 100%
rename from doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
rename to doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
diff --git a/doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn b/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
similarity index 100%
rename from doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn
rename to doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 3c29fc101c..c8c381486b 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -18,3 +18,8 @@ sudo make install
 

 
 Originally posted by Jon at  --[[Joey]]
+
+See also:
+
+* [[forum/OSX__39__s_haskell-platform_statically_links_things]]
+* [[forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set]]

From 6783c31ba36cbebfb6c6f256d0562e7a0d0195da Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 11:33:04 -0400
Subject: [PATCH 1124/2835] notabug

---
 doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
index 5971888891..f90eb5ae3c 100644
--- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
+++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
@@ -18,3 +18,9 @@ I can create an annex remote named 'test:/test'. git itself does not allow colon
     fatal: 'my:colon' is not a valid remote name
 
 >> -- RichiH
+
+>>> I see.. Git annex init does not specifiy a remote's name, it specifies
+>>> an arbitrary human-readable description of the repository, which will
+>>> be displayed when there is no configured remote corresponding to the
+>>> repository. So this is not a bug unless some documentation of that is
+>>> unclear. --[[Joey]] 

From 9d96ef4ddada7c015264db72fb3fceca0ad87c3d Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 19 Mar 2011 15:35:38 +0000
Subject: [PATCH 1125/2835] Added a comment

---
 .../comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment

diff --git a/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment b/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment
new file mode 100644
index 0000000000..015417a4f7
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-19T15:35:38Z"
+ content="""
+Yes, you have to pull down location tracking information in order for fsck to be satisfied in that situation. But since this is a walkthrough, and neither fsck or numcopies settings are mentioned until later, it's ok for this pull to be described a few steps along in [[getting file content]].
+
+"""]]

From f3f57f0c5ea50ab0da7c102d0041f1f056a6408c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 19 Mar 2011 15:37:22 +0000
Subject: [PATCH 1126/2835] Added a comment

---
 .../comment_2_af4f8b52526d8bea2904c95406fd2796._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment

diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment
new file mode 100644
index 0000000000..ca599b2857
--- /dev/null
+++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-19T15:37:22Z"
+ content="""
+Now it's fully supported, so long as you put a bare git repo on your key.
+"""]]

From d6e1d0680421152541e74d6372bfcc2b46bebfed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 12:49:37 -0400
Subject: [PATCH 1127/2835] alt approach

---
 doc/todo/S3.mdwn | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 3d18527d48..56023e71e1 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -50,3 +50,24 @@ be hard to get right.
 Less blue-sky, if the S3 capability were added directly to Backend.File,
 and bucket name was configured by annex.s3.bucket, then any existing
 annexed file could be upgraded to also store on S3.
+
+## alternate approach
+
+The above assumes S3 should be a separate backend somehow. What if,
+instead a S3 bucket is treated as a separate **remote**.
+
+* Could "git annex add" while offline, and "git annex push --to S3" when
+  online.
+* No need to choose whether a file goes to S3 at add time; no need to
+  migrate to move files there.
+* numcopies counting Just Works
+* Could have multiple S3 buckets as desired.
+
+The bucket name could 1:1 map with its annex.uuid, so not much
+configuration would be needed when cloning a repo to get it using S3 --
+just configure the S3 access token(s) to use for various UUIDs.
+
+Implementing this might not be as conceptually nice as making S3 a separate
+backend. It would need some changes to the remotes code, perhaps lifting
+some of it into backend-specific hooks. Then the S3 backend could be
+implicitly stacked in front of a backend like WORM.

From c0eabcc19165d5ba7c03f53002fb1b186f9f43c5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 19 Mar 2011 17:33:21 +0000
Subject: [PATCH 1128/2835] link to page

---
 .../OSX__39__s_haskell-platform_statically_links_things.mdwn    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn b/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
index f656427589..537c85d019 100644
--- a/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
+++ b/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
@@ -9,7 +9,7 @@ dyld: could not load inserted library: /opt/local/lib/libtsocks.dylib
 error: git-annex died of signal 5
 

 
-The side effect of this means that users who are behind restrictive firewalls that allow only ssh via a socks proxy, they will need to configure ssh to use something like "http://bent.latency.net/bent/git/goto-san-connect-1.85/src/connect.html".
+The side effect of this means that users who are behind restrictive firewalls that allow only ssh via a socks proxy, they will need to configure ssh to use something like .
 
 
 host remotemyhost

From 942480c47f69e13cf053b8f50c98c2ce4eaa256e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 19 Mar 2011 17:33:32 +0000
Subject: [PATCH 1129/2835] fix typo

---
 ...OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
index 99b7092c7f..5e417b14c9 100644
--- a/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
+++ b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
@@ -9,4 +9,4 @@ One solution is to enable *PermitUserEnvironment yes* in `/etc/sshd_config` and
 
     PATH=/Users/jtang/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/X11/bin:/usr/X11/bin:/Users/jtang/.cabal/bin:/opt/local/libexec/gnubin
 
-If the above is not done, cloning from the OSX host will fail if git is not installed isn't /usr/bin (which it probably won't be).
+If the above is not done, cloning from the OSX host will fail if git is not installed in /usr/bin (which it probably won't be).

From 828a84ba3341d4b7a84292d8b9002a8095dd2382 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 14:33:24 -0400
Subject: [PATCH 1130/2835] Add version command to show git-annex version as
 well as repository version information.

---
 Command/Version.hs | 34 ++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |  2 ++
 Upgrade.hs         |  5 ++++-
 Version.hs         | 15 ++++++++++-----
 configure.hs       | 13 ++++++++++++-
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  4 ++++
 7 files changed, 68 insertions(+), 7 deletions(-)
 create mode 100644 Command/Version.hs

diff --git a/Command/Version.hs b/Command/Version.hs
new file mode 100644
index 0000000000..480f2166b1
--- /dev/null
+++ b/Command/Version.hs
@@ -0,0 +1,34 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Version where
+
+import Control.Monad.State (liftIO)
+import Data.String.Utils
+
+import Command
+import qualified SysConfig
+import Version
+import Upgrade
+
+command :: [Command]
+command = [Command "version" paramNothing seek "show versions"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStartNothing
+start = do
+	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion
+	v <- getVersion
+	liftIO $ putStrLn $ "local repository version: " ++ v
+	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion
+	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions
+	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
+	return Nothing
+	where
+		vs l = join " " l
diff --git a/GitAnnex.hs b/GitAnnex.hs
index b9c22bdfb4..adf07e5b3e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -42,6 +42,7 @@ import qualified Command.Untrust
 import qualified Command.Semitrust
 import qualified Command.Map
 import qualified Command.Upgrade
+import qualified Command.Version
 
 cmds :: [Command]
 cmds = concat
@@ -72,6 +73,7 @@ cmds = concat
 	, Command.Migrate.command
 	, Command.Map.command
 	, Command.Upgrade.command
+	, Command.Version.command
 	]
 
 options :: [Option]
diff --git a/Upgrade.hs b/Upgrade.hs
index 76dd156f83..d201cc73ed 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -12,6 +12,9 @@ import Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
 
+upgradableVersions :: [Version]
+upgradableVersions = ["0", "1"]
+
 {- Uses the annex.version git config setting to automate upgrades. -}
 upgrade :: Annex Bool
 upgrade = do
@@ -19,5 +22,5 @@ upgrade = do
 	case version of
 		"0" -> Upgrade.V0.upgrade
 		"1" -> Upgrade.V1.upgrade
-		v | v == currentVersion -> return True
+		v | v `elem` supportedVersions -> return True
 		_ -> error "this version of git-annex is too old for this git repository!"
diff --git a/Version.hs b/Version.hs
index 5f414e93b8..d4a58d77a2 100644
--- a/Version.hs
+++ b/Version.hs
@@ -15,13 +15,18 @@ import qualified Annex
 import qualified GitRepo as Git
 import Locations
 
-currentVersion :: String
-currentVersion = "2"
+type Version = String
+
+defaultVersion :: Version
+defaultVersion = "2"
+
+supportedVersions :: [Version]
+supportedVersions = [defaultVersion]
 
 versionField :: String
 versionField = "annex.version"
 
-getVersion :: Annex String
+getVersion :: Annex Version
 getVersion = do
 	g <- Annex.gitRepo
 	let v = Git.configGet g versionField ""
@@ -42,7 +47,7 @@ getVersion = do
 				(True, True) -> return "1"
 				_ -> do
 					setVersion
-					return currentVersion
+					return defaultVersion
 
 setVersion :: Annex ()
-setVersion = Annex.setConfig versionField currentVersion
+setVersion = Annex.setConfig versionField defaultVersion
diff --git a/configure.hs b/configure.hs
index f5c2956488..f8cd577e99 100644
--- a/configure.hs
+++ b/configure.hs
@@ -7,7 +7,8 @@ import TestConfig
 
 tests :: [TestCase]
 tests = [
-	  testCp "cp_a" "-a"
+	  TestCase "version" $ getVersion
+	, testCp "cp_a" "-a"
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
 	, TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"]
@@ -49,6 +50,16 @@ unicodeFilePath = do
 	let file = head $ filter (isInfixOf "unicode-test") fs
 	return $ Config "unicodefilepath" (BoolConfig $ isInfixOf "ü" file)
 
+{- Pulls package version out of the changelog. -}
+getVersion :: Test
+getVersion = do
+	changelog <- readFile "debian/changelog"
+	let verline = head $ lines changelog
+	let version = middle (words verline !! 1)
+	return $ Config "packageversion" (StringConfig version)
+	where
+		middle s = drop 1 $ take (length s - 1) s
+
 setup :: IO ()
 setup = do
 	createDirectoryIfMissing True tmpDir
diff --git a/debian/changelog b/debian/changelog
index e1c0576e8b..751fcaff96 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (0.20110317) UNRELEASED; urgency=low
 
   * Fix dropping of files using the URL backend.
   * Fix support for remotes with '.' in their names.
+  * Add version command to show git-annex version as well as repository
+    version information.
 
  -- Joey Hess   Thu, 17 Mar 2011 11:46:53 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index ee40190682..1e4af022f1 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -267,6 +267,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
   whenever a newer git annex encounters an old repository; this command
   allows explcitly starting an upgrade.
 
+* version
+
+  Shows the version of git-annex, as well as repository version information.
+
 # OPTIONS
 
 * --force

From 5022a69e45a073046a2b14b6a4e798910c920ee9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 14:39:03 -0400
Subject: [PATCH 1131/2835] implemented

---
 doc/bugs/No_version_information_from_cli.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/No_version_information_from_cli.mdwn b/doc/bugs/No_version_information_from_cli.mdwn
index c432cf99b7..a0d30db414 100644
--- a/doc/bugs/No_version_information_from_cli.mdwn
+++ b/doc/bugs/No_version_information_from_cli.mdwn
@@ -14,3 +14,5 @@ Sample:
     % 
 
 The above might look like overkill, but it's in a form that will, most likely, never need to be extended.
+
+> Great idea, [[done]] --[[Joey]] 

From 43b3675d4298ffe8cfaefed0d70f0ef1fce25e37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 14:46:44 -0400
Subject: [PATCH 1132/2835] don't crash on upgrade if .git-annex DNE

---
 Upgrade/V1.hs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 270de5f74b..43f279ad08 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -115,8 +115,12 @@ moveLocationLogs = do
 			oldlocationlogs = do
 				g <- Annex.gitRepo
 				let dir = gitStateDir g
-				contents <- liftIO $ getDirectoryContents dir
-				return $ catMaybes $ map oldlog2key contents
+				exists <- liftIO $ doesDirectoryExist dir
+				if exists
+					then do
+						contents <- liftIO $ getDirectoryContents dir
+						return $ catMaybes $ map oldlog2key contents
+					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
 				let dest = logFile g k

From 6a2a17658c1d508cec022132768e9bc1957098d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 18:33:39 -0400
Subject: [PATCH 1133/2835] No longer auto-upgrade to repository format 2, to
 avoid accidental upgrades, etc. Use git-annex upgrade when you're ready to
 run this version.

---
 CmdLine.hs                                       |  5 +++--
 Command/Version.hs                               |  1 -
 Upgrade.hs                                       |  6 +-----
 Version.hs                                       | 16 ++++++++++++++++
 debian/NEWS                                      |  5 +----
 debian/changelog                                 |  3 +++
 ...t_1_c25900b9d2d62cc0b8c77150bcfebadf._comment |  3 +++
 doc/upgrades.mdwn                                | 13 +++++--------
 8 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 0698f2f5ea..6b4fd0f368 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,7 +22,7 @@ import qualified GitQueue
 import Types
 import Command
 import BackendList
-import Upgrade
+import Version
 import Options
 import Messages
 import UUID
@@ -33,7 +33,7 @@ dispatch gitrepo args cmds options header = do
 	setupConsole
 	state <- Annex.new gitrepo allBackends
 	(actions, state') <- Annex.run state $ parseCmd args header cmds options
-	tryRun state' $ [startup, upgrade] ++ actions ++ [shutdown]
+	tryRun state' $ [startup] ++ actions ++ [shutdown]
 
 {- Parses command line, stores configure flags, and returns a 
  - list of actions to be run in the Annex monad. -}
@@ -93,6 +93,7 @@ tryRun' _ errnum [] = do
 startup :: Annex Bool
 startup = do
 	prepUUID
+	checkVersion
 	return True
 
 {- Cleanup actions. -}
diff --git a/Command/Version.hs b/Command/Version.hs
index 480f2166b1..ac8fdd48c9 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -13,7 +13,6 @@ import Data.String.Utils
 import Command
 import qualified SysConfig
 import Version
-import Upgrade
 
 command :: [Command]
 command = [Command "version" paramNothing seek "show versions"]
diff --git a/Upgrade.hs b/Upgrade.hs
index d201cc73ed..08481755f3 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -12,9 +12,6 @@ import Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
 
-upgradableVersions :: [Version]
-upgradableVersions = ["0", "1"]
-
 {- Uses the annex.version git config setting to automate upgrades. -}
 upgrade :: Annex Bool
 upgrade = do
@@ -22,5 +19,4 @@ upgrade = do
 	case version of
 		"0" -> Upgrade.V0.upgrade
 		"1" -> Upgrade.V1.upgrade
-		v | v `elem` supportedVersions -> return True
-		_ -> error "this version of git-annex is too old for this git repository!"
+		_ -> return True
diff --git a/Version.hs b/Version.hs
index d4a58d77a2..d061a2eab2 100644
--- a/Version.hs
+++ b/Version.hs
@@ -8,6 +8,7 @@
 module Version where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (unless)
 import System.Directory
 
 import Types
@@ -23,6 +24,9 @@ defaultVersion = "2"
 supportedVersions :: [Version]
 supportedVersions = [defaultVersion]
 
+upgradableVersions :: [Version]
+upgradableVersions = ["0", "1"]
+
 versionField :: String
 versionField = "annex.version"
 
@@ -51,3 +55,15 @@ getVersion = do
 
 setVersion :: Annex ()
 setVersion = Annex.setConfig versionField defaultVersion
+
+checkVersion :: Annex ()
+checkVersion = do
+	v <- getVersion
+	unless (v `elem` supportedVersions) $ do
+		error $ "Repository version " ++ v ++ 
+			" is not supported. " ++
+			msg v
+	where
+		msg v
+			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"
+			| otherwise = "Upgrade git-annex."
diff --git a/debian/NEWS b/debian/NEWS
index df8518cef9..a8b258bfe4 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -2,10 +2,7 @@ git-annex (0.20110316) experimental; urgency=low
 
   This version reorganises the layout of git-annex's files in your repository.
   There is an upgrade process to convert a repository from the old git-annex
-  to this version. While git-annex will attempt to transparently handle
-  upgrades, you may want to drive the upgrade process by hand.
-
-  See  or
+  to this version. See  or
   /usr/share/doc/git-annex/html/upgrades.html
 
  -- Joey Hess   Wed, 16 Mar 2011 15:49:15 -0400
diff --git a/debian/changelog b/debian/changelog
index 751fcaff96..032dedfb9e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,9 @@ git-annex (0.20110317) UNRELEASED; urgency=low
   * Fix support for remotes with '.' in their names.
   * Add version command to show git-annex version as well as repository
     version information.
+  * No longer auto-upgrade to repository format 2, to avoid accidental
+    upgrades, etc. Use git-annex upgrade when you're ready to run this
+    version.
 
  -- Joey Hess   Thu, 17 Mar 2011 11:46:53 -0400
 
diff --git a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
index 6a34becd37..8420d7bb3a 100644
--- a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
+++ b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
@@ -7,4 +7,7 @@
 These are good examples; I think you've convinced me at least for upgrades going forward after v2. I'm not sure we have enough users and outdated git-annex installations to worry about it for v1.
 
 (Hoping such upgrades are rare anyway.. Part of the point of changes made in v2 was to allow lots of changes to be made later w/o needing a v3.)
+
+Update: Upgrades from v1 to v2 will no longer be handled automatically
+now.
 """]]
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 1bd97c46e2..5c2187c891 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -9,15 +9,14 @@ git-annex, and might want to use them with a newer git-annex.
 
 ## Upgrade process
 
-git-annex will automatically notice if it is run in a repository that
-needs an upgrade, and perform the upgrade before running whatever it
-was asked to do. Or you can use the "git annex upgrade" command to
-explicitly do an upgrade. The upgrade can tend to take a while,
-if you have a lot of files.
+git-annex will notice if it is run in a repository that
+needs an upgrade, and refuse to do anything. To upgrade,
+use the "git annex upgrade" command. The upgrade can tend
+to take a while, if you have a lot of files.
 
 Each clone of a repository should be individually upgraded.
 Until a repository's remotes have been upgraded, git-annex
-may refuse to communicate with them.
+will refuse to communicate with them.
 
 Generally, start by upgrading one repository, and then you can commit
 the changes git-annex staged during upgrade, and push them out to other
@@ -54,8 +53,6 @@ Symlinks changed.
 Also, hashing was added to location log files in .git-annex/.
 And .gitattributes needed to have another line added to it.
 
-Handled transparently.
-
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 
 Involved a reorganisation of the layout of .git/annex/. Symlinks changed.

From 91d1d935dd034850c152455bfab8f9df227a682f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 18:56:36 -0400
Subject: [PATCH 1134/2835] git-annex setup

---
 .gitattributes | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitattributes b/.gitattributes
index 5d425843f2..b83edcae09 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1 +1,3 @@
 debian/changelog merge=dpkg-mergechangelogs
+.git-annex/*.log merge=union
+.git-annex/*/*/*.log merge=union

From 54c39d09ba0185cea398876e609e59d3059b18e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 18:58:10 -0400
Subject: [PATCH 1135/2835] need to explicitly run upgrade now

---
 Command/Upgrade.hs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 3c9fa3eebf..94398c70a1 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -8,15 +8,18 @@
 module Command.Upgrade where
 
 import Command
+import Upgrade
+import Version
 
 command :: [Command]
-command = [Command "upgrade" paramNothing seek "upgrade repository layout"]
+command = [standaloneCommand "upgrade" paramNothing seek
+	"upgrade repository layout"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
 
 start :: CommandStartNothing
 start = do
-	-- The actual upgrading is handled by just running any command,
-	-- so nothing extra needs to be done.
-	return $ Just $ return $ Just $ return True
+	r <- upgrade
+	checkVersion
+	return $ Just $ return $ Just $ return r

From 140a351fc535235ae5714122f0c98174cbdb19ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 18:58:49 -0400
Subject: [PATCH 1136/2835] avoid version check before running version and
 upgrade commands

There are two types of commands; those that access the repository and those
that don't. Sorted.
---
 CmdLine.hs            |  5 +++--
 Command.hs            | 13 ++++++++++---
 Command/Add.hs        |  2 +-
 Command/ConfigList.hs |  2 +-
 Command/Copy.hs       |  2 +-
 Command/Describe.hs   |  2 +-
 Command/Drop.hs       |  2 +-
 Command/DropKey.hs    |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  2 +-
 Command/FromKey.hs    |  2 +-
 Command/Fsck.hs       |  2 +-
 Command/Get.hs        |  2 +-
 Command/InAnnex.hs    |  2 +-
 Command/Init.hs       |  2 +-
 Command/Lock.hs       |  2 +-
 Command/Map.hs        |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/Move.hs       |  2 +-
 Command/PreCommit.hs  |  2 +-
 Command/RecvKey.hs    |  2 +-
 Command/Semitrust.hs  |  2 +-
 Command/SendKey.hs    |  2 +-
 Command/SetKey.hs     |  2 +-
 Command/Trust.hs      |  2 +-
 Command/Unannex.hs    |  2 +-
 Command/Uninit.hs     |  2 +-
 Command/Unlock.hs     |  4 ++--
 Command/Untrust.hs    |  2 +-
 Command/Unused.hs     |  3 ++-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  2 +-
 33 files changed, 46 insertions(+), 37 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 6b4fd0f368..de03d96ed4 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -45,7 +45,9 @@ parseCmd argv header cmds options = do
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			prepCmd command (drop 1 params)
+			when (cmdusesrepo command) $
+				checkVersion
+			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
 		getopt = case getOpt Permute options argv of
@@ -93,7 +95,6 @@ tryRun' _ errnum [] = do
 startup :: Annex Bool
 startup = do
 	prepUUID
-	checkVersion
 	return True
 
 {- Cleanup actions. -}
diff --git a/Command.hs b/Command.hs
index 41ad884a93..1449d7eedf 100644
--- a/Command.hs
+++ b/Command.hs
@@ -61,13 +61,20 @@ data Command = Command {
 	cmdname :: String,
 	cmdparams :: String,
 	cmdseek :: [CommandSeek],
-	cmddesc :: String
+	cmddesc :: String,
+	cmdusesrepo :: Bool
 }
 
+repoCommand :: String -> String -> [CommandSeek] -> String -> Command
+repoCommand n p s d = Command n p s d True
+
+standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command
+standaloneCommand n p s d = Command n p s d False
+
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
-prepCmd :: Command -> [String] -> Annex [Annex Bool]
-prepCmd Command { cmdseek = seek } params = do
+prepCommand :: Command -> [String] -> Annex [Annex Bool]
+prepCommand Command { cmdseek = seek } params = do
 	lists <- mapM (\s -> s params) seek
 	return $ map doCommand $ concat lists
 
diff --git a/Command/Add.hs b/Command/Add.hs
index a577203bf4..f6ccf0fb88 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -21,7 +21,7 @@ import Utility
 import Touch
 
 command :: [Command]
-command = [Command "add" paramPath seek "add files to annex"]
+command = [repoCommand "add" paramPath seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index b91c2a9160..476d73cfbb 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -14,7 +14,7 @@ import Command
 import UUID
 
 command :: [Command]
-command = [Command "configlist" paramNothing seek
+command = [standaloneCommand "configlist" paramNothing seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 93342e11bb..46d49dd058 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -11,7 +11,7 @@ import Command
 import qualified Command.Move
 
 command :: [Command]
-command = [Command "copy" paramPath seek
+command = [repoCommand "copy" paramPath seek
 	"copy content of files to/from another repository"]
 
 -- A copy is just a move that does not delete the source file.
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 643ca04718..32aef4f245 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -16,7 +16,7 @@ import Messages
 import qualified Command.Init
 
 command :: [Command]
-command = [Command "describe" (paramPair paramRemote paramDesc) seek
+command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index fdc55969f0..52b724e62b 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -18,7 +18,7 @@ import Messages
 import Utility
 
 command :: [Command]
-command = [Command "drop" paramPath seek
+command = [repoCommand "drop" paramPath seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index b3cc60961c..4c6f1ab2e1 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -15,7 +15,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [Command "dropkey" (paramRepeating paramKey) seek
+command = [repoCommand "dropkey" (paramRepeating paramKey) seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 8ed61ba65b..c6a28663ec 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -23,7 +23,7 @@ import Backend
 import Key
 
 command :: [Command]
-command = [Command "dropunused" (paramRepeating paramNumber) seek
+command = [repoCommand "dropunused" (paramRepeating paramNumber) seek
 	"drop unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Find.hs b/Command/Find.hs
index 3ed15c1537..6a6ae29787 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -14,7 +14,7 @@ import Command
 import Content
 
 command :: [Command]
-command = [Command "find" (paramOptional $ paramRepeating paramPath) seek
+command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
 	"lists available files"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 0047548715..513e07a310 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -18,7 +18,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [Command "fix" paramPath seek
+command = [repoCommand "fix" paramPath seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 176d2cd54d..8c1a1028fe 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -21,7 +21,7 @@ import Messages
 import Key
 
 command :: [Command]
-command = [Command "fromkey" paramPath seek
+command = [repoCommand "fromkey" paramPath seek
 	"adds a file using a specific key"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 76d0e38b4b..216c87493b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -20,7 +20,7 @@ import Content
 import LocationLog
 
 command :: [Command]
-command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek
+command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
 	"check for problems"]
 
 seek :: [CommandSeek]
diff --git a/Command/Get.hs b/Command/Get.hs
index 2aa3c0c150..0463dccb05 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -14,7 +14,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [Command "get" paramPath seek
+command = [repoCommand "get" paramPath seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index fa81fc9a4c..a7e2ecff60 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -14,7 +14,7 @@ import Command
 import Content
 
 command :: [Command]
-command = [Command "inannex" (paramRepeating paramKey) seek
+command = [repoCommand "inannex" (paramRepeating paramKey) seek
 		"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Init.hs b/Command/Init.hs
index d9ea394a33..cca2e8faef 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -23,7 +23,7 @@ import Types
 import Utility
 	
 command :: [Command]
-command = [Command "init" paramDesc seek
+command = [repoCommand "init" paramDesc seek
 		"initialize git-annex with repository description"]
 
 seek :: [CommandSeek]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index a3a39a9078..cdbc560194 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -17,7 +17,7 @@ import qualified GitRepo as Git
 import Utility
 	
 command :: [Command]
-command = [Command "lock" paramPath seek "undo unlock command"]
+command = [repoCommand "lock" paramPath seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start]
diff --git a/Command/Map.hs b/Command/Map.hs
index 6c3e0b3df5..4ae947b15e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -29,7 +29,7 @@ import qualified Dot
 data Link = Link Git.Repo Git.Repo
 
 command :: [Command]
-command = [Command "map" paramNothing seek "generate map of repositories"]
+command = [repoCommand "map" paramNothing seek "generate map of repositories"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index c0e80c5b47..584f6e34e1 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -21,7 +21,7 @@ import Messages
 import qualified Command.Add
 
 command :: [Command]
-command = [Command "migrate" paramPath seek "switch data to different backend"]
+command = [repoCommand "migrate" paramPath seek "switch data to different backend"]
 
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
diff --git a/Command/Move.hs b/Command/Move.hs
index 2d6c973fe0..8056e95dbe 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -22,7 +22,7 @@ import Messages
 import Utility
 
 command :: [Command]
-command = [Command "move" paramPath seek
+command = [repoCommand "move" paramPath seek
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 1465ebc615..727a637285 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -17,7 +17,7 @@ import qualified Command.Fix
 import Utility
 
 command :: [Command]
-command = [Command "pre-commit" paramPath seek "run by git pre-commit hook"]
+command = [repoCommand "pre-commit" paramPath seek "run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index c7c37d1e31..126608f614 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -17,7 +17,7 @@ import Content
 import RsyncFile
 
 command :: [Command]
-command = [Command "recvkey" paramKey seek
+command = [repoCommand "recvkey" paramKey seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 13c6847e17..351336b899 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -15,7 +15,7 @@ import Trust
 import Messages
 
 command :: [Command]
-command = [Command "semitrust" (paramRepeating paramRemote) seek
+command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 56974bda96..871a530af7 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -18,7 +18,7 @@ import Content
 import RsyncFile
 
 command :: [Command]
-command = [Command "sendkey" paramKey seek
+command = [repoCommand "sendkey" paramKey seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index fdda1c3bee..af46fe06e4 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -16,7 +16,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [Command "setkey" (paramRepeating paramKey) seek
+command = [repoCommand "setkey" (paramRepeating paramKey) seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index ea661da2a6..f7dba56485 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -15,7 +15,7 @@ import UUID
 import Messages
 
 command :: [Command]
-command = [Command "trust" (paramRepeating paramRemote) seek
+command = [repoCommand "trust" (paramRepeating paramRemote) seek
 	"trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 42dc1fb0ab..b0ce31ceed 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -22,7 +22,7 @@ import qualified GitRepo as Git
 import Messages
 
 command :: [Command]
-command = [Command "unannex" paramPath seek "undo accidential add command"]
+command = [repoCommand "unannex" paramPath seek "undo accidential add command"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index e8ac1bbd5e..ee0cbde6b3 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -21,7 +21,7 @@ import qualified Command.Unannex
 import qualified Command.Init
 
 command :: [Command]
-command = [Command "uninit" paramPath seek 
+command = [repoCommand "uninit" paramPath seek 
         "de-initialize git-annex and clean out repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index bd1021cc3c..ac7b22ac72 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -22,8 +22,8 @@ import CopyFile
 
 command :: [Command]
 command =
-	[ Command "unlock" paramPath seek "unlock files for modification"
-	, Command "edit" paramPath seek "same as unlock"
+	[ repoCommand "unlock" paramPath seek "unlock files for modification"
+	, repoCommand "edit" paramPath seek "same as unlock"
 	]
 
 seek :: [CommandSeek]
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index fdf9a83dec..9c11efe465 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -15,7 +15,7 @@ import Trust
 import Messages
 
 command :: [Command]
-command = [Command "untrust" (paramRepeating paramRemote) seek
+command = [repoCommand "untrust" (paramRepeating paramRemote) seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 52e483d870..a1c4ee03c9 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -25,7 +25,8 @@ import qualified GitRepo as Git
 import qualified Backend
 
 command :: [Command]
-command = [Command "unused" paramNothing seek "look for unused file content"]
+command = [repoCommand "unused" paramNothing seek
+	"look for unused file content"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Version.hs b/Command/Version.hs
index ac8fdd48c9..2b294c80be 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -15,7 +15,7 @@ import qualified SysConfig
 import Version
 
 command :: [Command]
-command = [Command "version" paramNothing seek "show versions"]
+command = [standaloneCommand "version" paramNothing seek "show version info"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 5b0bcbbd26..599df44676 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -17,7 +17,7 @@ import UUID
 import Types
 
 command :: [Command]
-command = [Command "whereis" (paramOptional $ paramRepeating paramPath) seek
+command = [repoCommand "whereis" (paramOptional $ paramRepeating paramPath) seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]

From 6634b6a6b84a924f6f6059b5bea61f449d056eee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Mar 2011 14:04:39 -0400
Subject: [PATCH 1137/2835] imcomplete attempt at supporting lutimes(3) for BSD
 compat

---
 Command/Add.hs |  2 +-
 Touch.hsc      | 84 ++++++++++++++++++++++++++++++++++----------------
 2 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index f6ccf0fb88..da98bffa4f 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -58,7 +58,7 @@ cleanup file key = do
 	-- touch the symlink to have the same mtime as the file it points to
 	s <- liftIO $ getFileStatus file
 	let mtime = modificationTime s
-	liftIO $ touch file (TimeSpec mtime 0) False
+	liftIO $ touch file (TimeSpec mtime) False
 
 	Annex.queue "add" [Param "--"] file
 	return True
diff --git a/Touch.hsc b/Touch.hsc
index 456175182e..8a3e31b571 100644
--- a/Touch.hsc
+++ b/Touch.hsc
@@ -9,8 +9,6 @@
 
 module Touch (
 	TimeSpec(..),
-	nowTime,
-	omitTime,
 	touchBoth,
 	touch
 ) where
@@ -18,24 +16,24 @@ module Touch (
 import Foreign
 import Foreign.C
 
+data TimeSpec = TimeSpec CTime
+
+{- Changes the access and modification times of an existing file.
+   Can follow symlinks, or not. Throws IO error on failure. -}
+touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO ()
+
+touch :: FilePath -> TimeSpec -> Bool -> IO ()
+touch file mtime follow = touchBoth file mtime mtime follow
+
 #include 
 #include 
 #include 
+#include 
 
-#ifndef _POSIX_C_SOURCE
-#define _POSIX_C_SOURCE >= 200809L
+#ifndef _BSD_SOURCE
+#define _BSD_SOURCE
 #endif
 
-data TimeSpec = TimeSpec CTime CLong
-
-touch :: FilePath -> TimeSpec -> Bool -> IO ()
-touch file mtime follow = touchBoth file omitTime mtime follow
-
-touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO ()
-
-omitTime :: TimeSpec
-nowTime :: TimeSpec
-
 #if (defined UTIME_OMIT && defined UTIME_NOW && defined AT_FDCWD && defined AT_SYMLINK_NOFOLLOW)
 
 at_fdcwd :: CInt
@@ -44,9 +42,6 @@ at_fdcwd = #const AT_FDCWD
 at_symlink_nofollow :: CInt
 at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW
 
-omitTime = TimeSpec 0 #const UTIME_OMIT
-nowTime = TimeSpec 0 #const UTIME_NOW
-
 instance Storable TimeSpec where
 	-- use the larger alignment of the two types in the struct
 	alignment _ = max sec_alignment nsec_alignment
@@ -56,19 +51,16 @@ instance Storable TimeSpec where
 	sizeOf _ = #{size struct timespec}
 	peek ptr = do
 		sec <- #{peek struct timespec, tv_sec} ptr
-		nsec <- #{peek struct timespec, tv_nsec} ptr
-		return $ TimeSpec sec nsec
-	poke ptr (TimeSpec sec nsec) = do
+		return $ TimeSpec sec
+	poke ptr (TimeSpec sec) = do
 		#{poke struct timespec, tv_sec} ptr sec
-		#{poke struct timespec, tv_nsec} ptr nsec
+		#{poke struct timespec, tv_nsec} ptr (0 :: CLong)
 
 {- While its interface is beastly, utimensat is in recent
-   POSIX standards, unlike futimes. -}
+   POSIX standards, unlike lutimes. -}
 foreign import ccall "utimensat" 
 	c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt
 
-{- Changes the access and/or modification times of an existing file.
-   Can follow symlinks, or not. Throws IO error on failure. -}
 touchBoth file atime mtime follow = 
 	allocaArray 2 $ \ptr ->
 	withCString file $ \f -> do
@@ -83,8 +75,46 @@ touchBoth file atime mtime follow =
 			else at_symlink_nofollow 
 
 #else
-#warning "utimensat not available; building without symlink timestamp preservation support"
-omitTime = TimeSpec 0 (-1)
-nowTime = TimeSpec 0 (-2)
+#if 0
+{- Using lutimes is needed for BSD.
+ - 
+ - TODO: test if lutimes is available. May have to do it in configure.
+ - TODO: TimeSpec uses a CTime, while tv_sec is a CLong. It is implementation
+ - dependent whether these are the same; need to find a cast that works.
+ - (The cast below fails.. without the cast it works on linux i386, but
+ - maybe not elsewhere.)
+ -}
+
+instance Storable TimeSpec where
+	alignment _ = alignment (undefined::CLong)
+	sizeOf _ = #{size struct timeval}
+	peek ptr = do
+		sec <- #{peek struct timeval, tv_sec} ptr
+		return $ TimeSpec sec
+	poke ptr (TimeSpec sec) = do
+		#{poke struct timeval, tv_sec} ptr (sec :: CLong)
+		#{poke struct timeval, tv_usec} ptr (0 :: CLong) 
+
+foreign import ccall "utimes" 
+	c_utimes :: CString -> Ptr TimeSpec -> IO CInt
+foreign import ccall "lutimes" 
+	c_lutimes :: CString -> Ptr TimeSpec -> IO CInt
+
+touchBoth file atime mtime follow = 
+	allocaArray 2 $ \ptr ->
+	withCString file $ \f -> do
+		pokeArray ptr [atime, mtime]
+		r <- syscall f ptr
+		if (r /= 0)
+			then throwErrno "touchBoth"
+			else return ()
+	where
+		syscall = if follow
+			then c_lutimes
+			else c_utimes
+
+#else
+warning "utimensat and lutimes not available; building without symlink timestamp preservation support"
 touchBoth _ _ _ _ = return ()
 #endif
+#endif

From 05ae9cb1bb55c95ff4436d15f66903eb757cac2a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 20 Mar 2011 18:12:59 +0000
Subject: [PATCH 1138/2835] Added a comment

---
 ...comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment
new file mode 100644
index 0000000000..aeb576be37
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-03-20T18:12:59Z"
+ content="""
+I'm leaving this bug open because this feature, however minor is not available on OSX and BSD. 
+
+I have added a partial implementation using lutimes(3), which should be available on the BSDs. However, it's ifdefed out due to a casting problem: The TimeSpec uses a CTime, while lutimes uses a CLong. These data types may be internally the same on some or all platforms, so if you want this feature you can try changing the \"ifdef 0\" in Touch.hsc to 1 and try it, see if \"git annex add\" mirrors file modification time in created symlinks, and let me know.
+"""]]

From 23cf9dac8633f110e900e7b79d0e1c12ce9d3b5f Mon Sep 17 00:00:00 2001
From: praet 
Date: Sun, 20 Mar 2011 20:11:28 +0000
Subject: [PATCH 1139/2835] Added a comment: Brainfart

---
 ..._79d96599f757757f34d7b784e6c0e81c._comment | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment b/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment
new file mode 100644
index 0000000000..c265b58995
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="praet"
+ ip="81.240.27.89"
+ subject="Brainfart"
+ date="2011-03-20T20:11:27Z"
+ content="""
+Haven't given these any serious thought (which will become apparent in a moment) but hoping they will give birth to some less retarded ideas:
+
+---
+
+### Bait'n'switch
+
+- pre-commit: Replace all staged symlinks (when pointing to annexed files) with plaintext files containing the key of their respective annexed content, re-stage, and add their paths (relative to repo root) to .gitignore.
+- post-commit: Replace the plaintext files with (git annex fix'ed) symlinks.
+
+In doing so, the blobs to be committed can remain unaltered, irrespective of their related files' depth in the directory hierarchy.
+
+To prevent git from reporting ALL annexed files as unstaged changes after running post-commit hook, their paths would need to be added to .gitignore.
+
+This wouldn't cause any issues when adding files, very little when modifying files (would need some alterations to \"git annex unlock\"), BUT would make git totally oblivious to removals...
+
+---
+
+### Manifest-based (re)population
+- Keep a manifest of all annexed files (key + relative path)
+- DON'T track the symlinks (.gitignore)
+- Populate/update the directory structure using a post-commit hook.
+
+... thus circumventing the issue entirely, yet diffstats (et al.) would be rather uninformative.
+
+---
+
+***Wide open to suggestions, criticism, mocking laughter and finger-pointing :)***
+"""]]

From cb4ed36f13b05ca9baa64820f64bc84448a7af46 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 20 Mar 2011 20:48:42 +0000
Subject: [PATCH 1140/2835] Added a comment

---
 ..._548303d6ffb21a9370b6904f41ff49c1._comment | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment
new file mode 100644
index 0000000000..cd116c232d
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 6"
+ date="2011-03-20T20:48:41Z"
+ content="""
+ok, pulling the latest master and building on OSX now does this...
+
+
+ghc -O2 -Wall -ignore-package monads-fd --make git-annex
+[ 1 of 63] Compiling Touch            ( Touch.hs, Touch.o )
+
+Touch.hsc:24:0:
+    The type signature for `touchBoth' lacks an accompanying binding
+
+Touch.hsc:27:26: Not in scope: `touchBoth'
+make: *** [git-annex] Error 1
+

+
+changing the #if 0 to 1 gives this...
+
+
+ghc -O2 -Wall -ignore-package monads-fd --make git-annex
+[ 1 of 63] Compiling Touch            ( Touch.hs, Touch.o )
+
+Touch.hsc:95:43:
+    Couldn't match expected type `CLong' against inferred type `CTime'
+    In the second argument of `(\ hsc_ptr
+                                    -> pokeByteOff hsc_ptr 0)', namely
+        `(sec :: CLong)'
+    In a stmt of a 'do' expression:
+        (\ hsc_ptr -> pokeByteOff hsc_ptr 0) ptr (sec :: CLong)
+    In the expression:
+        do { (\ hsc_ptr -> pokeByteOff hsc_ptr 0) ptr (sec :: CLong);
+             (\ hsc_ptr -> pokeByteOff hsc_ptr 4) ptr (0 :: CLong) }
+make: *** [git-annex] Error 1
+

+
+
+it seems that commit 6634b6a6b84a924f6f6059b5bea61f449d056eee has broken support for OSX.
+
+"""]]

From 08d1c13628c1f927be58f8454058f458cca22817 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Mar 2011 18:06:06 -0400
Subject: [PATCH 1141/2835] fix typo

---
 Touch.hsc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Touch.hsc b/Touch.hsc
index 8a3e31b571..fd3500f869 100644
--- a/Touch.hsc
+++ b/Touch.hsc
@@ -81,7 +81,7 @@ touchBoth file atime mtime follow =
  - TODO: test if lutimes is available. May have to do it in configure.
  - TODO: TimeSpec uses a CTime, while tv_sec is a CLong. It is implementation
  - dependent whether these are the same; need to find a cast that works.
- - (The cast below fails.. without the cast it works on linux i386, but
+ - (Without the cast it works on linux i386, but
  - maybe not elsewhere.)
  -}
 
@@ -92,7 +92,7 @@ instance Storable TimeSpec where
 		sec <- #{peek struct timeval, tv_sec} ptr
 		return $ TimeSpec sec
 	poke ptr (TimeSpec sec) = do
-		#{poke struct timeval, tv_sec} ptr (sec :: CLong)
+		#{poke struct timeval, tv_sec} ptr sec
 		#{poke struct timeval, tv_usec} ptr (0 :: CLong) 
 
 foreign import ccall "utimes" 
@@ -114,7 +114,7 @@ touchBoth file atime mtime follow =
 			else c_utimes
 
 #else
-warning "utimensat and lutimes not available; building without symlink timestamp preservation support"
+#warning "utimensat and lutimes not available; building without symlink timestamp preservation support"
 touchBoth _ _ _ _ = return ()
 #endif
 #endif

From fccd30cacde588ca36dd066b1df5ec586a4647cb Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 20 Mar 2011 22:06:26 +0000
Subject: [PATCH 1142/2835] Added a comment

---
 .../comment_7_7ca00527ab5db058aadec4fe813e51fd._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment
new file mode 100644
index 0000000000..e35dc8a827
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-03-20T22:06:25Z"
+ content="""
+Fixed that, and removed the impossible cast so it can be built with #if 1
+"""]]

From 09b16afe02882485f60953f01726ea65d38af920 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Mar 2011 18:11:00 -0400
Subject: [PATCH 1143/2835] releasing version 0.20110320

---
 debian/changelog           | 4 ++--
 doc/news/version_0.25.mdwn | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)
 create mode 100644 doc/news/version_0.25.mdwn

diff --git a/debian/changelog b/debian/changelog
index 032dedfb9e..eb8d735047 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20110317) UNRELEASED; urgency=low
+git-annex (0.20110320) experimental; urgency=low
 
   * Fix dropping of files using the URL backend.
   * Fix support for remotes with '.' in their names.
@@ -8,7 +8,7 @@ git-annex (0.20110317) UNRELEASED; urgency=low
     upgrades, etc. Use git-annex upgrade when you're ready to run this
     version.
 
- -- Joey Hess   Thu, 17 Mar 2011 11:46:53 -0400
+ -- Joey Hess   Sun, 20 Mar 2011 16:36:33 -0400
 
 git-annex (0.20110316) experimental; urgency=low
 
diff --git a/doc/news/version_0.25.mdwn b/doc/news/version_0.25.mdwn
new file mode 100644
index 0000000000..13abd66cd5
--- /dev/null
+++ b/doc/news/version_0.25.mdwn
@@ -0,0 +1,6 @@
+git-annex 0.25 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix dropping of files using the URL backend.
+   * Fix support for remotes with '.' in their names.
+   * Add version command to show git-annex version as well as repository
+     version information."""]]
\ No newline at end of file

From 162047b8da7ede5da44d031d4265890669ed8e78 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Mar 2011 18:11:24 -0400
Subject: [PATCH 1144/2835] add news item for git-annex 0.20110320

---
 doc/news/version_0.20110320.mdwn |  9 +++++++++
 doc/news/version_0.21.mdwn       |  7 -------
 doc/news/version_0.22.mdwn       | 24 ------------------------
 3 files changed, 9 insertions(+), 31 deletions(-)
 create mode 100644 doc/news/version_0.20110320.mdwn
 delete mode 100644 doc/news/version_0.21.mdwn
 delete mode 100644 doc/news/version_0.22.mdwn

diff --git a/doc/news/version_0.20110320.mdwn b/doc/news/version_0.20110320.mdwn
new file mode 100644
index 0000000000..84475829c3
--- /dev/null
+++ b/doc/news/version_0.20110320.mdwn
@@ -0,0 +1,9 @@
+git-annex 0.20110320 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix dropping of files using the URL backend.
+   * Fix support for remotes with '.' in their names.
+   * Add version command to show git-annex version as well as repository
+     version information.
+   * No longer auto-upgrade to repository format 2, to avoid accidental
+     upgrades, etc. Use git-annex upgrade when you're ready to run this
+     version."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.21.mdwn b/doc/news/version_0.21.mdwn
deleted file mode 100644
index 996474910e..0000000000
--- a/doc/news/version_0.21.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 0.21 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * test: Don't rely on chmod -R working.
-   * unannex: Fix recently introduced bug when attempting to unannex more
-     than one file at a time.
-   * test: Set git user name and email in case git can't guess values.
-   * Fix display of unicode filenames."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.22.mdwn b/doc/news/version_0.22.mdwn
deleted file mode 100644
index 4848e861e1..0000000000
--- a/doc/news/version_0.22.mdwn
+++ /dev/null
@@ -1,24 +0,0 @@
-git-annex 0.22 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Git annexes can now be attached to bare git repositories.
-     (Both the local and remote host must have this version of git-annex
-     installed for it to work.)
-   * Support filenames that start with a dash; when such a file is passed
-     to a utility it will be escaped to avoid it being interpreted as an
-     option. (I went a little overboard and got the type checker involved
-     in this, so such files are rather comprehensively supported now.)
-   * New backends: SHA512 SHA384 SHA256 SHA224
-     (Supported on systems where corresponding shaNsum commands are available.)
-   * describe: New subcommand that can set or change the description of
-     a repository.
-   * Fix test suite to reap zombies.
-     (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang
-     for his help eliminating the infestation... for now.)
-   * Make test suite not rely on a working cp -pr.
-     (The Unix wars are still ON!)
-   * Look for dir.git directories the same as git does.
-   * Support remote urls specified as relative paths.
-   * Support non-ssh remote paths that contain tilde expansions.
-   * fsck: Check for and repair location log damage.
-   * Bugfix: When fsck detected and moved away corrupt file content, it did
-     not update the location log."""]]
\ No newline at end of file

From d8dfd2c103606e29cd16fc33680f53650144836c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmSbJHbvlxbCjtPXk_Io3qP3MFqJr3pUgQ"
 
Date: Mon, 21 Mar 2011 04:27:46 +0000
Subject: [PATCH 1145/2835]

---
 .../Unfortunate_interaction_with_Calibre.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/Unfortunate_interaction_with_Calibre.mdwn

diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn b/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn
new file mode 100644
index 0000000000..d00a6720cd
--- /dev/null
+++ b/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn
@@ -0,0 +1,21 @@
+# Calibre 
+
+Calibre is a somewhat popular eBook management package that's also free software.    
+
+Install via
+    # apt-get install calibre
+
+There is a somewhat unfortunate interaction between Calibre and git-annex...
+
+* git-annex makes its files become read-only.  By the way, that's not quite obvious from the documentation; I suggest making that more prominent.
+* Calibre modifies files (not quite sure of semantics, how, or why) when doing various operations, notably such as when copying a book from one's library to one's portable reading device.
+
+These don't play well together, sadly.
+
+I'd expect most of the issue to sit on the Calibre side, and have reported it as a bug.
+[Calibre bug #739045](https://bugs.launchpad.net/calibre/+bug/739045)
+Preliminary indication is that they're treating it as a functionality change they'll decline to fix.  Which isn't entirely unreasonable - I anticipated as much, and I don't want to treat that as a bad/wrong decision.
+
+However, I think it's:
+* Unfortunate, as fitting Calibre together with git-annex seems like a neat idea.
+* Useful to make sure that this kind of "doesn't play well together" condition is documented, even if only as a bug report.

From 6fe02b24fa8c5f9a11e4deb319acad671285f1a5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 21 Mar 2011 08:52:19 +0000
Subject: [PATCH 1146/2835] Added a comment

---
 .../comment_8_881aecb9ae671689453f6d5d780d844b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment
new file mode 100644
index 0000000000..56a7eb360e
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 8"
+ date="2011-03-21T08:52:18Z"
+ content="""
+Just tried building both of the code paths, and they seem to build and somewhat function on OSX. I have yet to confirm the functionality is working correctly, but so far it's looking good. (I somewhat care less about the utimes/mtimes of my files since I care more about the content :) )
+"""]]

From e1147b4454109dddff0b6e79decda710fd71cbb2 Mon Sep 17 00:00:00 2001
From: praet 
Date: Mon, 21 Mar 2011 19:58:36 +0000
Subject: [PATCH 1147/2835] Added a comment

---
 ...mment_5_d61f5693d947b9736b29fca1dbc7ad76._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment b/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment
new file mode 100644
index 0000000000..93db97e704
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="praet"
+ ip="81.242.56.203"
+ subject="comment 5"
+ date="2011-03-21T19:58:34Z"
+ content="""
+In the meantime, would it be acceptable to split the pre-commit hook
+into two discrete parts?
+
+This would allow to (if preferred) defer \"git annex fix\" until
+post-commit while still keeping the safety net for unlocked files.
+"""]]

From c048905dc4c2ce155dbb03d7e60014568faa9553 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 21 Mar 2011 20:46:43 -0400
Subject: [PATCH 1148/2835] upgrade messages

---
 Command/Upgrade.hs | 2 ++
 Upgrade/V0.hs      | 2 +-
 Upgrade/V1.hs      | 5 ++++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 94398c70a1..880a5324f7 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -10,6 +10,7 @@ module Command.Upgrade where
 import Command
 import Upgrade
 import Version
+import Messages
 
 command :: [Command]
 command = [standaloneCommand "upgrade" paramNothing seek
@@ -20,6 +21,7 @@ seek = [withNothing start]
 
 start :: CommandStartNothing
 start = do
+	showStart "upgrade" ""
 	r <- upgrade
 	checkVersion
 	return $ Just $ return $ Just $ return r
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index 5ba305817b..eabd030098 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -23,7 +23,7 @@ import qualified Upgrade.V1
 
 upgrade :: Annex Bool
 upgrade = do
-	showSideAction "Upgrading object directory layout v0 to v1..."
+	showNote "v0 to v1..."
 	g <- Annex.gitRepo
 
 	-- do the reorganisation of the key files
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 43f279ad08..f18a76c875 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -55,7 +55,7 @@ import qualified Command.Init
 
 upgrade :: Annex Bool
 upgrade = do
-	showSideAction "Upgrading object directory layout v1 to v2..."
+	showNote "v1 to v2"
 
 	g <- Annex.gitRepo
 	if Git.repoIsLocalBare g
@@ -78,6 +78,7 @@ upgrade = do
 
 moveContent :: Annex ()
 moveContent = do
+	showNote "moving content..."
 	keys <- getKeysPresent1
 	forM_ keys move
 	where
@@ -92,6 +93,7 @@ moveContent = do
 
 updateSymlinks :: Annex ()
 updateSymlinks = do
+	showNote "updating symlinks content..."
 	g <- Annex.gitRepo
 	files <- liftIO $ Git.inRepo g [Git.workTree g]
 	forM_ files $ fixlink
@@ -109,6 +111,7 @@ updateSymlinks = do
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
+	showNote "moving location logs..."
 	logkeys <- oldlocationlogs
 	forM_ logkeys move
 		where

From c94261020f29887b46fffc5652f700a00fefb1a2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 21 Mar 2011 23:37:46 -0400
Subject: [PATCH 1149/2835] typo

---
 Upgrade/V1.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index f18a76c875..c0470a3bc8 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -93,7 +93,7 @@ moveContent = do
 
 updateSymlinks :: Annex ()
 updateSymlinks = do
-	showNote "updating symlinks content..."
+	showNote "updating symlinks..."
 	g <- Annex.gitRepo
 	files <- liftIO $ Git.inRepo g [Git.workTree g]
 	forM_ files $ fixlink

From e6dfcbf32b051547b78d68f289d5a505341e9d85 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 22 Mar 2011 13:06:59 +0000
Subject: [PATCH 1150/2835] Added a comment: Please provide stable tarballs or
 zipfiles

---
 .../comment_1_fbd8b6d39e9d3c71791551358c863966._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment

diff --git a/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment b/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment
new file mode 100644
index 0000000000..488e005278
--- /dev/null
+++ b/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="84.189.2.244"
+ subject="Please provide stable tarballs or zipfiles"
+ date="2011-03-22T13:06:58Z"
+ content="""
+I'm trying to package git annex for ArchLinux and NixOS. That task would be a *lot* easier, if there were proper release archives available for download. The Gitweb site offers to create snapshot tarballs on the fly, but those tarballs have a different SHA hash every time they're generated, so they cannot be used for the purposes of a distribution. A simple solution for this problem would be to enable snapshots in zip format (because zip files look the same every time they're generated).
+"""]]

From 90a15fc214d913d9e8aa3a937ca214af1d8d66e6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 22 Mar 2011 14:01:38 +0000
Subject: [PATCH 1151/2835] Added a comment

---
 .../comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment

diff --git a/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment b/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment
new file mode 100644
index 0000000000..5441c3e4ce
--- /dev/null
+++ b/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-22T14:01:37Z"
+ content="""
+maybe snag tarballs from  ? 
+"""]]

From 6815a9974febe2b872523672772aeb1c3fea9689 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 22 Mar 2011 18:09:21 +0000
Subject: [PATCH 1152/2835] Added a comment

---
 ...comment_3_cf6044ebe99f71158034e21197228abd._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment

diff --git a/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment b/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment
new file mode 100644
index 0000000000..b72b848f80
--- /dev/null
+++ b/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-22T18:09:21Z"
+ content="""
+The tarballs produced by gitweb are actually stable. They are wrapped in a gz file with a varying timestamp however. It might be nice if gitweb passed --no-name to gzip to avoid that inconsistency.
+
+git-annex also has a [pristine-tar](http://kitenet.net/~joey/code/pristine-tar/) branch in git that can be used to recreate the tarballs I upload to Debian.
+"""]]

From 25605d18fd0a9404dba92e0c7d9f438fbd70fcd4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 15:39:36 -0400
Subject: [PATCH 1153/2835] generalize

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c60e19b311..08e2f59fb0 100644
--- a/Makefile
+++ b/Makefile
@@ -11,7 +11,7 @@ SysConfig.hs: configure.hs TestConfig.hs
 	$(GHCMAKE) configure
 	./configure
 
-Touch.hs: Touch.hsc
+%.hs: %.hsc
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 

From 8ede2e255ffd2397f0714df066d078bc45d839ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 15:48:14 -0400
Subject: [PATCH 1154/2835] add StatFS.hsc, copied from xmobar

---
 .gitignore       |   1 +
 StatFS.hsc       | 109 +++++++++++++++++++++++++++++++++++++++++++++++
 debian/copyright |   4 ++
 3 files changed, 114 insertions(+)
 create mode 100644 StatFS.hsc

diff --git a/.gitignore b/.gitignore
index 69d2c80709..aa677c1335 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ html
 *.tix
 .hpc
 Touch.hs
+StatFS.hs
diff --git a/StatFS.hsc b/StatFS.hsc
new file mode 100644
index 0000000000..f91a6912ce
--- /dev/null
+++ b/StatFS.hsc
@@ -0,0 +1,109 @@
+-----------------------------------------------------------------------------
+-- |
+--
+-- (This code comes from xmobar)
+-- 
+-- Module      :  StatFS
+-- Copyright   :  (c) Jose A Ortega Ruiz
+-- License     :  BSD-3-clause
+--
+-- All rights reserved.
+-- 
+-- Redistribution and use in source and binary forms, with or without
+-- modification, are permitted provided that the following conditions
+-- are met:
+-- 
+-- 1. Redistributions of source code must retain the above copyright
+--    notice, this list of conditions and the following disclaimer.
+-- 2. Redistributions in binary form must reproduce the above copyright
+--    notice, this list of conditions and the following disclaimer in the
+--    documentation and/or other materials provided with the distribution.
+-- 3. Neither the name of the author nor the names of his contributors
+--    may be used to endorse or promote products derived from this software
+--    without specific prior written permission.
+-- 
+-- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+-- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+-- ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+-- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+-- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+-- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+-- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+-- SUCH DAMAGE.
+--
+-- Maintainer  :  Jose A Ortega Ruiz 
+-- Stability   :  unstable
+-- Portability :  unportable
+--
+--  A binding to C's statvfs(2)
+--
+-----------------------------------------------------------------------------
+
+{-# LANGUAGE CPP, ForeignFunctionInterface, EmptyDataDecls #-}
+
+
+module StatFS ( FileSystemStats(..), getFileSystemStats ) where
+
+import Foreign
+import Foreign.C.Types
+import Foreign.C.String
+import Data.ByteString (useAsCString)
+import Data.ByteString.Char8 (pack)
+
+#if defined (__FreeBSD__)
+# include 
+# include 
+#else
+#include 
+#endif
+
+data FileSystemStats = FileSystemStats {
+  fsStatBlockSize :: Integer
+  -- ^ Optimal transfer block size.
+  , fsStatBlockCount :: Integer
+  -- ^ Total data blocks in file system.
+  , fsStatByteCount :: Integer
+  -- ^ Total bytes in file system.
+  , fsStatBytesFree :: Integer
+  -- ^ Free bytes in file system.
+  , fsStatBytesAvailable :: Integer
+  -- ^ Free bytes available to non-superusers.
+  , fsStatBytesUsed :: Integer
+  -- ^ Bytes used.
+  } deriving (Show, Eq)
+
+data CStatfs
+
+#if defined(__FreeBSD__)
+foreign import ccall unsafe "sys/mount.h statfs"
+#else
+foreign import ccall unsafe "sys/vfs.h statfs64"
+#endif
+  c_statfs :: CString -> Ptr CStatfs -> IO CInt
+
+toI :: CLong -> Integer
+toI = toInteger
+
+getFileSystemStats :: String -> IO (Maybe FileSystemStats)
+getFileSystemStats path =
+  allocaBytes (#size struct statfs) $ \vfs ->
+  useAsCString (pack path) $ \cpath -> do
+    res <- c_statfs cpath vfs
+    if res == -1 then return Nothing
+      else do
+        bsize <- (#peek struct statfs, f_bsize) vfs
+        bcount <- (#peek struct statfs, f_blocks) vfs
+        bfree <- (#peek struct statfs, f_bfree) vfs
+        bavail <- (#peek struct statfs, f_bavail) vfs
+        let bpb = toI bsize
+        return $ Just FileSystemStats
+                       { fsStatBlockSize = bpb
+                       , fsStatBlockCount = toI bcount
+                       , fsStatByteCount = toI bcount * bpb
+                       , fsStatBytesFree = toI bfree * bpb
+                       , fsStatBytesAvailable = toI bavail * bpb
+                       , fsStatBytesUsed = toI (bcount - bfree) * bpb
+                       }
diff --git a/debian/copyright b/debian/copyright
index 90e26b7524..2144501e11 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -7,3 +7,7 @@ License: GPL-3+
  The full text of version 3 of the GPL is distributed as doc/GPL in
  this package's source, or in /usr/share/common-licenses/GPL-3 on
  Debian systems.
+
+Files: StatFS.hsc
+Copyright: Jose A Ortega Ruiz 
+License: BSD-3-clause

From aa1bc31e0aede63a1e68d2ec3e2653a7f5be0ae7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 15:58:47 -0400
Subject: [PATCH 1155/2835] be a no-op on non-linux, non-freebsd systems

Todo later: use POSIX statvfs

Note: Re OSX, see http://code.google.com/p/xmobar/issues/detail?id=28
Apparently xmobar's code will work on OSX, probably __FreeBSD__ is defined
there.
---
 StatFS.hsc | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index f91a6912ce..8b453dc199 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -1,7 +1,7 @@
 -----------------------------------------------------------------------------
 -- |
 --
--- (This code comes from xmobar)
+-- (This code originally comes from xmobar)
 -- 
 -- Module      :  StatFS
 -- Copyright   :  (c) Jose A Ortega Ruiz
@@ -57,7 +57,11 @@ import Data.ByteString.Char8 (pack)
 # include 
 # include 
 #else
+#if defined (__linux__)
 #include 
+#else
+#define UNKNOWN
+#endif
 #endif
 
 data FileSystemStats = FileSystemStats {
@@ -77,18 +81,25 @@ data FileSystemStats = FileSystemStats {
 
 data CStatfs
 
+#ifdef UNKNOWN
+#warning free space checking code not available for this OS
+#else
 #if defined(__FreeBSD__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"
 #endif
   c_statfs :: CString -> Ptr CStatfs -> IO CInt
+#endif
 
 toI :: CLong -> Integer
 toI = toInteger
 
 getFileSystemStats :: String -> IO (Maybe FileSystemStats)
 getFileSystemStats path =
+#ifdef UNKNOWN
+  return Nothing
+#else
   allocaBytes (#size struct statfs) $ \vfs ->
   useAsCString (pack path) $ \cpath -> do
     res <- c_statfs cpath vfs
@@ -107,3 +118,4 @@ getFileSystemStats path =
                        , fsStatBytesAvailable = toI bavail * bpb
                        , fsStatBytesUsed = toI (bcount - bfree) * bpb
                        }
+#endif

From 66f1d7dc5b63d1f017e557eb354407f1d8454207 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 22 Mar 2011 21:25:29 +0000
Subject: [PATCH 1156/2835]

---
 doc/bugs/softlink_atime.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn
index 0d26d1ce6a..c48d3a4de4 100644
--- a/doc/bugs/softlink_atime.mdwn
+++ b/doc/bugs/softlink_atime.mdwn
@@ -37,3 +37,13 @@ Optionally, editing the meta-data should change the times in all annexes.
 	git add .metadata
 
 >>>>>> Thanks a lot. Doing this in a new git-annex repo from the start should at least ensure local consistency and I assume I can simply add a post-pull hook to restore the mtimes on all all other repositories? -- RichiH
+
+>>>>>>> This is even better:
+
+    #!/bin/sh
+    git annex pre-commit .
+    which metastore || echo "$0: metastore is not installed; exiting" ; exit 99
+    metastore --save
+    git add .metadata
+
+>>>>>>> -- RichiH

From 0fe3ff8e1404f9893f4340281fedb70de2622de7 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 22 Mar 2011 21:25:47 +0000
Subject: [PATCH 1157/2835]

---
 doc/bugs/softlink_atime.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn
index c48d3a4de4..69d6f6600e 100644
--- a/doc/bugs/softlink_atime.mdwn
+++ b/doc/bugs/softlink_atime.mdwn
@@ -42,7 +42,7 @@ Optionally, editing the meta-data should change the times in all annexes.
 
     #!/bin/sh
     git annex pre-commit .
-    which metastore || echo "$0: metastore is not installed; exiting" ; exit 99
+    which metastore || echo "$0: metastore is not installed; exiting"; exit 99
     metastore --save
     git add .metadata
 

From aa2d8e33df3fc6ba204e28001ab0d1d231c9c58e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 17:27:04 -0400
Subject: [PATCH 1158/2835] free space checking

Free space checking is now done, for transfers of data for keys that have free space metadata.
(Notably, not for SHA* keys generated with git-annex 0.24 or earlier.)

The code is believed to work on Linux, FreeBSD, and OSX; check compile-time
messages to see if it is not enabled for your OS.
---
 Command/Migrate.hs                |  2 +-
 Command/SetKey.hs                 |  3 +-
 Command/Unlock.hs                 |  2 ++
 Content.hs                        | 51 +++++++++++++++++++++++++++++++
 debian/changelog                  | 10 ++++++
 doc/bugs/free_space_checking.mdwn |  3 ++
 6 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 584f6e34e1..56147113b9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -55,7 +55,7 @@ perform file oldkey newbackend = do
 	case stored of
 		Nothing -> return Nothing
 		Just (newkey, _) -> do
-			ok <- getViaTmp newkey $ \t -> do
+			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
 				-- cached key, to avoid wasting disk space.
 				liftIO $ createLink src t
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index af46fe06e4..6f6078e4ba 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -32,7 +32,8 @@ perform :: FilePath -> CommandPerform
 perform file = do
 	key <- cmdlineKey
 	-- the file might be on a different filesystem, so mv is used
-	-- rather than simply calling moveToObjectDir
+	-- rather than simply calling moveToObjectDir; disk space is also
+	-- checked this way.
 	ok <- getViaTmp key $ \dest -> do
 		if dest /= file
 			then liftIO $
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index ac7b22ac72..bf593e1e99 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -41,6 +41,8 @@ perform dest key = do
 	inbackend <- Backend.hasKey key
 	when (not inbackend) $
 		error "content not present"
+	
+	checkDiskSpace key
 
 	g <- Annex.gitRepo
 	let src = gitAnnexLocation g key
diff --git a/Content.hs b/Content.hs
index 4bd8265c2d..596274ad09 100644
--- a/Content.hs
+++ b/Content.hs
@@ -10,6 +10,8 @@ module Content (
 	calcGitLink,
 	logStatus,
 	getViaTmp,
+	getViaTmpUnchecked,
+	checkDiskSpace,
 	preventWrite,
 	allowWrite,
 	moveAnnex,
@@ -35,6 +37,8 @@ import UUID
 import qualified GitRepo as Git
 import qualified Annex
 import Utility
+import StatFS
+import Key
 
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
@@ -75,6 +79,27 @@ getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
 	g <- Annex.gitRepo
 	let tmp = gitAnnexTmpLocation g key
+
+	-- Check that there is enough free disk space.
+	-- When the temp file already exists, count the space
+	-- it is using as free.
+	e <- liftIO $ doesFileExist tmp
+	if e
+		then do
+			stat <- liftIO $ getFileStatus tmp
+			checkDiskSpace' (fromIntegral $ fileSize stat) key
+		else checkDiskSpace key
+
+	getViaTmpUnchecked key action
+
+{- Like getViaTmp, but does not check that there is enough disk space
+ - for the incoming key. For use when the key content is already on disk
+ - and not being copied into place. -}
+getViaTmpUnchecked :: Key -> (FilePath -> Annex Bool) -> Annex Bool
+getViaTmpUnchecked key action = do
+	g <- Annex.gitRepo
+	let tmp = gitAnnexTmpLocation g key
+
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	success <- action tmp
 	if success
@@ -87,6 +112,32 @@ getViaTmp key action = do
 			-- to resume its transfer
 			return False
 
+{- Checks that there is disk space available to store a given key,
+ - throwing an error if not. -}
+checkDiskSpace :: Key -> Annex ()
+checkDiskSpace = checkDiskSpace' 0
+
+checkDiskSpace' :: Integer -> Key -> Annex ()
+checkDiskSpace' adjustment key = do
+	liftIO $ putStrLn $ "adjust " ++ show adjustment
+	g <- Annex.gitRepo
+	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
+	case (stats, keySize key) of
+		(Nothing, _) -> return ()
+		(_, Nothing) -> return ()
+		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
+			if (need + overhead >= have + adjustment)
+				then error $ "not enough free space (have " ++ 
+					showsize (have + adjustment) ++ "; need " ++
+					showsize (need + overhead) ++ ")"
+				else return ()
+	where
+		showsize i = show i
+		-- Adding a file to the annex requires some overhead beyond
+		-- just the file size; the git index must be updated, etc. 
+		-- This is an arbitrary value.
+		overhead = 1024 * 1024 -- 1 mb
+
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
 preventWrite f = unsetFileMode f writebits
diff --git a/debian/changelog b/debian/changelog
index eb8d735047..a5830884ab 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,13 @@
+git-annex (0.20110321) UNRELEASED; urgency=low
+
+  * Free space checking is now done, for transfers of data for keys
+    that have free space metadata. (Notably, not for SHA* keys generated
+    with git-annex 0.24 or earlier.) The code is believed to work on
+    Linux, FreeBSD, and OSX; check compile-time messages to see if it
+    is not enabled for your OS.
+
+ -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
+
 git-annex (0.20110320) experimental; urgency=low
 
   * Fix dropping of files using the URL backend.
diff --git a/doc/bugs/free_space_checking.mdwn b/doc/bugs/free_space_checking.mdwn
index eaa3294d60..92e8be40d1 100644
--- a/doc/bugs/free_space_checking.mdwn
+++ b/doc/bugs/free_space_checking.mdwn
@@ -16,3 +16,6 @@ file around.
   find files that lack size info, and rename their keys to add the size
   info. Users with old repos can run this on them, to get the missing
   info recorded.
+
+> [[done]]; no migtation process for old SHA1 keys from v1 repo though.
+> --[[Joey]] 

From c21998722cb6a65993a3b72e66b225443cfce48b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 17:41:06 -0400
Subject: [PATCH 1159/2835] fast mode

Add --fast flag, that can enable less expensive, but also less thurough versions of some commands.

* Add --fast flag, that can enable less expensive, but also less thurough
  versions of some commands.
* fsck: In fast mode, avoid checking checksums.
* unused: In fast mode, just show all existing temp files as unused,
  and avoid expensive scan for other unused content.
---
 Annex.hs           |  2 ++
 Backend/SHA.hs     |  3 ++-
 Command/Unused.hs  | 27 ++++++++++++++++++---------
 Options.hs         |  3 +++
 debian/changelog   |  5 +++++
 doc/git-annex.mdwn |  6 ++++++
 6 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 608151d824..f45415a72f 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -40,6 +40,7 @@ data AnnexState = AnnexState
 	, repoqueue :: GitQueue.Queue
 	, quiet :: Bool
 	, force :: Bool
+	, fast :: Bool
 	, defaultbackend :: Maybe String
 	, defaultkey :: Maybe String
 	, toremote :: Maybe String
@@ -56,6 +57,7 @@ newState gitrepo allbackends = AnnexState
 	, repoqueue = GitQueue.empty
 	, quiet = False
 	, force = False
+	, fast = False
 	, defaultbackend = Nothing
 	, defaultkey = Nothing
 	, toremote = Nothing
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 0563851076..0ec555ce3f 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -80,9 +80,10 @@ keyValue size file = do
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
 	g <- Annex.gitRepo
+	fast <- Annex.getState Annex.fast
 	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
-	if not present
+	if (not present || fast)
 		then return True
 		else do
 			s <- shaN size file
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a1c4ee03c9..518e986561 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -44,7 +44,6 @@ perform = do
 
 checkUnused :: Annex Bool
 checkUnused = do
-	showNote "checking for unused data..."
 	(unused, staletmp) <- unusedKeys
 	let unusedlist = number 0 unused
 	let staletmplist = number (length unused) staletmp
@@ -81,17 +80,27 @@ number n (x:xs) = (n+1, x):(number (n+1) xs)
 unusedKeys :: Annex ([Key], [Key])
 unusedKeys = do
 	g <- Annex.gitRepo
-	present <- getKeysPresent
-	referenced <- getKeysReferenced
-	tmps <- tmpKeys
 	
-	let (unused, staletmp, duptmp) = calcUnusedKeys present referenced tmps
+	fast <- Annex.getState Annex.fast
+	if fast
+		then do
+			showNote "fast mode enabled; assuming all temporary files are unused"
+			tmps <- tmpKeys
+			return ([], tmps)
+		else do
+			showNote "checking for unused data..."
+			present <- getKeysPresent
+			referenced <- getKeysReferenced
+			tmps <- tmpKeys
+	
+			let (unused, staletmp, duptmp) = calcUnusedKeys present referenced tmps
 
-	-- Tmp files that are dups of content already present can simply
-	-- be removed.
-	liftIO $ forM_ duptmp $ \t -> removeFile $ gitAnnexTmpLocation g t
+			-- Tmp files that are dups of content already present
+			-- can simply be removed.
+			liftIO $ forM_ duptmp $ \t -> removeFile $
+				gitAnnexTmpLocation g t
 
-	return (unused, staletmp)
+			return (unused, staletmp)
 
 calcUnusedKeys :: [Key] -> [Key] -> [Key] -> ([Key], [Key], [Key])
 calcUnusedKeys present referenced tmps = (unused, staletmp, duptmp)
diff --git a/Options.hs b/Options.hs
index 4cd62c2222..10c3714e41 100644
--- a/Options.hs
+++ b/Options.hs
@@ -22,6 +22,8 @@ commonOptions :: [Option]
 commonOptions =
 	[ Option ['f'] ["force"] (NoArg (setforce True))
 		"allow actions that may lose annexed data"
+	, Option ['F'] ["fast"] (NoArg (setfast True))
+		"avoid slow operations"
 	, Option ['q'] ["quiet"] (NoArg (setquiet True))
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setquiet False))
@@ -31,5 +33,6 @@ commonOptions =
 	]
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
+		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
 		setdefaultbackend v = Annex.changeState $ \s -> s { Annex.defaultbackend = Just v }
diff --git a/debian/changelog b/debian/changelog
index a5830884ab..e0927817a7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,11 @@ git-annex (0.20110321) UNRELEASED; urgency=low
     with git-annex 0.24 or earlier.) The code is believed to work on
     Linux, FreeBSD, and OSX; check compile-time messages to see if it
     is not enabled for your OS.
+  * Add --fast flag, that can enable less expensive, but also less thurough
+    versions of some commands.
+  * fsck: In fast mode, avoid checking checksums.
+  * unused: In fast mode, just show all existing temp files as unused,
+    and avoid expensive scan for other unused content.
 
  -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 1e4af022f1..6168ebae2f 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -278,6 +278,12 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Force unsafe actions, such as dropping a file's content when no other
   source of it can be verified to still exist. Use with care.
 
+* --fast
+
+  Enables less expensive, but also less thorough versions of some commands.
+  What is avoided depends on the command. A fast fsck avoids calculating
+  checksums; a fast unused only shows temp files and not other unused files.
+
 * --quiet
 
   Avoid the default verbose logging of what is done; only show errors

From 368e20eb84fac8224a2ab33616cdd31f2c4d5ff1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 17:53:40 -0400
Subject: [PATCH 1160/2835] diskreserve setting

Add annex.diskreserve config setting, to control how much free space to
reserve for other purposes and avoid using (defaults to 1 mb).
---
 Content.hs         | 13 ++++++-------
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  8 ++++++++
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/Content.hs b/Content.hs
index 596274ad09..39a3addccf 100644
--- a/Content.hs
+++ b/Content.hs
@@ -119,24 +119,23 @@ checkDiskSpace = checkDiskSpace' 0
 
 checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
-	liftIO $ putStrLn $ "adjust " ++ show adjustment
 	g <- Annex.gitRepo
+	r <- Annex.repoConfig g "diskreserve" ""
+	let reserve = if null r then megabyte else (read r :: Integer)
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
 		(_, Nothing) -> return ()
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
-			if (need + overhead >= have + adjustment)
+			if (need + reserve > have + adjustment)
 				then error $ "not enough free space (have " ++ 
 					showsize (have + adjustment) ++ "; need " ++
-					showsize (need + overhead) ++ ")"
+					showsize (need + reserve) ++ ")"
 				else return ()
 	where
 		showsize i = show i
-		-- Adding a file to the annex requires some overhead beyond
-		-- just the file size; the git index must be updated, etc. 
-		-- This is an arbitrary value.
-		overhead = 1024 * 1024 -- 1 mb
+		megabyte :: Integer
+		megabyte = 1024 * 1024
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
diff --git a/debian/changelog b/debian/changelog
index e0927817a7..88e0986a94 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (0.20110321) UNRELEASED; urgency=low
     with git-annex 0.24 or earlier.) The code is believed to work on
     Linux, FreeBSD, and OSX; check compile-time messages to see if it
     is not enabled for your OS.
+  * Add annex.diskreserve config setting, to control how much free space
+    to reserve for other purposes and avoid using (defaults to 1 mb).
   * Add --fast flag, that can enable less expensive, but also less thurough
     versions of some commands.
   * fsck: In fast mode, avoid checking checksums.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 6168ebae2f..3cf408939e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -377,6 +377,14 @@ Here are all the supported configuration settings.
   Default ssh and rsync options to use if a remote does not have
   specific options.
 
+* `annex.diskreserve`
+
+  Amount of disk space to reserve. Disk space is checked when transferring
+  content to avoid running out, and additional free space can be reserved
+  via this option, to make space for more important content (such as git
+  commit logs). The units are bytes.
+  The default reserve is 1048576 (1 megabyte).
+
 * `annex.version`
 
   Automatically maintained, and used to automate upgrades between versions.

From 4ca3a3a9b5d38f1ac239b441c75ccb3a461e35cb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 22 Mar 2011 22:25:27 +0000
Subject: [PATCH 1161/2835]

---
 .../Makefile_is_missing_dependancies.mdwn     | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies.mdwn

diff --git a/doc/bugs/Makefile_is_missing_dependancies.mdwn b/doc/bugs/Makefile_is_missing_dependancies.mdwn
new file mode 100644
index 0000000000..5a04ba49f6
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies.mdwn
@@ -0,0 +1,31 @@
+
+
+From e45c73e66fc18d27bdf5797876fbeb07786a4af1 Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Tue, 22 Mar 2011 22:24:07 +0000
+Subject: [PATCH] Touch up Makefile to depend on StatFS.hs
+
+---
+ Makefile |    2 +-
+ 1 files changed, 1 insertions(+), 1 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 08e2f59..4ae8392 100644
+--- a/Makefile
++++ b/Makefile
+@@ -15,7 +15,7 @@ SysConfig.hs: configure.hs TestConfig.hs
+        hsc2hs $<
+        perl -i -pe 's/^{-# INCLUDE.*//' $@
+ 
+-$(bins): SysConfig.hs Touch.hs
++$(bins): SysConfig.hs Touch.hs StatFS.hs
+        $(GHCMAKE) $@
+ 
+ git-annex.1: doc/git-annex.mdwn
+-- 
+1.7.4.1
+
+

+
+
+StatFS.hs never gets depended on and compiled, the makefile was just missing something

From c44c318eafbfcd55a568c2761721520a95dfe530 Mon Sep 17 00:00:00 2001
From: Jimmy Tang 
Date: Tue, 22 Mar 2011 22:24:07 +0000
Subject: [PATCH 1162/2835] Touch up Makefile to depend on StatFS.hs

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 08e2f59fb0..4ae8392c71 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@ SysConfig.hs: configure.hs TestConfig.hs
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 
-$(bins): SysConfig.hs Touch.hs
+$(bins): SysConfig.hs Touch.hs StatFS.hs
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn

From fea20d260c7c095775b4667a95501f4e9cb7b741 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:45:11 -0400
Subject: [PATCH 1163/2835] applied

---
 doc/bugs/Makefile_is_missing_dependancies.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/Makefile_is_missing_dependancies.mdwn b/doc/bugs/Makefile_is_missing_dependancies.mdwn
index 5a04ba49f6..4bda91f17d 100644
--- a/doc/bugs/Makefile_is_missing_dependancies.mdwn
+++ b/doc/bugs/Makefile_is_missing_dependancies.mdwn
@@ -29,3 +29,6 @@ index 08e2f59..4ae8392 100644
 
 
 StatFS.hs never gets depended on and compiled, the makefile was just missing something
+
+> Thanks, [done]]! Interested to hear if StatFS.hs works on OSX (no warning) or
+> is a no-op (with warning). --[[Joey]] 

From ee6fabb7cac48f0f2eaec999164b5f40186ca92b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:50:36 -0400
Subject: [PATCH 1164/2835] update

---
 doc/upgrades.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 5c2187c891..dbea5e9c4a 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -53,6 +53,13 @@ Symlinks changed.
 Also, hashing was added to location log files in .git-annex/.
 And .gitattributes needed to have another line added to it.
 
+Previously, files added to the SHA [[backends]] did not have their file
+size tracked, while files added to the WORM backend did. Files added to
+the SHA backends after the conversion will have their file size tracked,
+and that information will be used by git-annex for disk space checking.
+There is not yet a way to add file size tracking information to old files
+in the SHA backend.
+
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 
 Involved a reorganisation of the layout of .git/annex/. Symlinks changed.

From 9f9325aa3da7df99e09b8b1d393814e4590b9358 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:51:14 -0400
Subject: [PATCH 1165/2835] clean too

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 4ae8392c71..e1aaf8ec37 100644
--- a/Makefile
+++ b/Makefile
@@ -61,7 +61,8 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure Touch.hs SysConfig.hs *.tix .hpc
+	rm -rf build $(bins) $(mans) test configure \
+		StatFS.hs Touch.hs SysConfig.hs *.tix .hpc
 	rm -rf doc/.ikiwiki html
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 

From 1ea373e2466e8a0f665a42392c71cb2db46f23e3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:52:52 -0400
Subject: [PATCH 1166/2835] clarify

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3cf408939e..d7b57675d0 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -133,7 +133,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
-  and warns about any problems found.
+  and warns about or fixes any problems found.
 
   With parameters, only the specified files are checked.
 

From 3a419237895f1d8c5c598134330830b10d0176e6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:53:16 -0400
Subject: [PATCH 1167/2835] clarify wording

temp files are always assumed to be unused, in either mode
---
 Command/Unused.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 518e986561..83d8757cff 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -84,7 +84,7 @@ unusedKeys = do
 	fast <- Annex.getState Annex.fast
 	if fast
 		then do
-			showNote "fast mode enabled; assuming all temporary files are unused"
+			showNote "fast mode enabled; only finding temporary files"
 			tmps <- tmpKeys
 			return ([], tmps)
 		else do

From 5d759195618a96ce745b8ee559b439c86426a0f3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:55:29 -0400
Subject: [PATCH 1168/2835] update

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 88e0986a94..3c5c91b1b0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,7 +7,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
     is not enabled for your OS.
   * Add annex.diskreserve config setting, to control how much free space
     to reserve for other purposes and avoid using (defaults to 1 mb).
-  * Add --fast flag, that can enable less expensive, but also less thurough
+  * Add --fast flag, that can enable less expensive, but also less thorough
     versions of some commands.
   * fsck: In fast mode, avoid checking checksums.
   * unused: In fast mode, just show all existing temp files as unused,

From d2a8e511776ac84793ab70fcd33e7498bafe83cf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 22 Mar 2011 22:59:55 +0000
Subject: [PATCH 1169/2835]

---
 doc/bugs/softlink_atime.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn
index 69d6f6600e..c62610be02 100644
--- a/doc/bugs/softlink_atime.mdwn
+++ b/doc/bugs/softlink_atime.mdwn
@@ -41,8 +41,8 @@ Optionally, editing the meta-data should change the times in all annexes.
 >>>>>>> This is even better:
 
     #!/bin/sh
+    if ! type metastore >/dev/null; then echo "$0: metastore is not installed; exiting"; exit 1; fi
     git annex pre-commit .
-    which metastore || echo "$0: metastore is not installed; exiting"; exit 99
     metastore --save
     git add .metadata
 

From 686b12ae303a5ddc69b51beb467113f0e8e6d1b5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 22 Mar 2011 23:41:53 +0000
Subject: [PATCH 1170/2835] Added a comment

---
 ..._4c30ade91fc7113a95960aa3bd1d5427._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment

diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment
new file mode 100644
index 0000000000..b3dc8fe7a2
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-22T23:41:51Z"
+ content="""
+I may be missing something obvious, but when I copy to a remote repository, the object files are created, but no softlinks are created. When I pull everything from the remote, it pulls only files the local repo knows about already.
+
+        A
+       / \
+      B   C
+
+Moving from B to A creates no symlinks in A but the object files are moved to A. Copying back from A to B restores the object files in B and keeps them in A.
+
+Copying from A to an empty C does not create any object files nor symlinks. Copying from C to A creates no symlinks in A but the object files are copied to A.
+
+-- RichiH
+
+"""]]

From c1dc4079419cff94cca72441d5e67a866110ec7e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 20:31:22 -0400
Subject: [PATCH 1171/2835] Fix space leak in fsck and drop commands.

The space leak was somehow caused by this line:

	absfiles <- mapM absPath files

I confess, I don't quite understand why this caused bad buffering,
but apparently the whole pipeline from git-ls-files backed up at that
point.

Happily, rewriting the code to only get the cwd once and use a pure
function to calculate absfiles clears it up, and should be a little more
efficient in syscalls too.
---
 Command.hs       | 3 +--
 GitRepo.hs       | 4 ++--
 Utility.hs       | 9 ++++++++-
 debian/changelog | 1 +
 4 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index 1449d7eedf..446b1b55fe 100644
--- a/Command.hs
+++ b/Command.hs
@@ -132,8 +132,7 @@ withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
-	files' <- filterFiles files
-	liftM (map a) $ liftIO $ Git.checkAttr repo attr files'
+	liftM (map a) $ liftIO $ Git.checkAttr repo attr files
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
diff --git a/GitRepo.hs b/GitRepo.hs
index 34a59a10d4..4e4a063e82 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -449,9 +449,9 @@ checkAttr repo attr files = do
 	-- top of the repo). But we're passed files relative to the current
 	-- directory. Convert to absolute, and then convert the filenames
 	-- in its output back to relative.
-	absfiles <- mapM absPath files
-	(_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles
 	cwd <- getCurrentDirectory
+	let absfiles = map  (absPathFrom cwd) files
+	(_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles
 	return $ map (topair $ cwd++"/") $ lines s
 	where
 		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]
diff --git a/Utility.hs b/Utility.hs
index e63fa1f6be..8312335f86 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -12,6 +12,7 @@ module Utility (
 	readFileStrict,
 	parentDir,
 	absPath,
+	absPathFrom,
 	relPathCwdToDir,
 	relPathDirToDir,
 	boolSystem,
@@ -165,8 +166,14 @@ dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
 absPath :: FilePath -> IO FilePath
 absPath file = do
 	cwd <- getCurrentDirectory
+	return $ absPathFrom cwd file
+
+{- Converts a filename into a normalized, absolute path
+ - from the specified cwd. -}
+absPathFrom :: FilePath -> FilePath -> FilePath
+absPathFrom cwd file =
 	case absNormPath cwd file of
-		Just f -> return f
+		Just f -> f
 		Nothing -> error $ "unable to normalize " ++ file
 
 {- Constructs a relative path from the CWD to a directory.
diff --git a/debian/changelog b/debian/changelog
index 3c5c91b1b0..3738c77589 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * fsck: In fast mode, avoid checking checksums.
   * unused: In fast mode, just show all existing temp files as unused,
     and avoid expensive scan for other unused content.
+  * Fix space leak in fsck and drop commands.
 
  -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
 

From 0e8f304dec2a0c2c1c20ce4c1cfd95b936d2e494 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 00:38:10 +0000
Subject: [PATCH 1172/2835] Added a comment

---
 ...comment_2_7d90e1e150e7524ba31687108fcc38d6._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment

diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment
new file mode 100644
index 0000000000..a6f8e9cf97
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-23T00:38:10Z"
+ content="""
+`git annex move` only moves content. All symlink management is handled by git, so you have to keep repositories in sync using git as you would any other repo. When you `git pull B` in A, it will get whatever symlinks were added to B.
+
+(It can be useful to use a central bare repo and avoid needing to git pull from one repo to another, then you can just always push commits to the central repo, and pull down all changes from other repos.)
+"""]]

From 7051763b5b9ef4c230169a432384488d66078cee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 21:00:18 -0400
Subject: [PATCH 1173/2835] tweak

---
 GitRepo.hs       | 2 +-
 debian/changelog | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 4e4a063e82..ad58b28a00 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -450,7 +450,7 @@ checkAttr repo attr files = do
 	-- directory. Convert to absolute, and then convert the filenames
 	-- in its output back to relative.
 	cwd <- getCurrentDirectory
-	let absfiles = map  (absPathFrom cwd) files
+	let absfiles = map (absPathFrom cwd) files
 	(_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles
 	return $ map (topair $ cwd++"/") $ lines s
 	where
diff --git a/debian/changelog b/debian/changelog
index 3738c77589..f124296cc8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,7 +2,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
 
   * Free space checking is now done, for transfers of data for keys
     that have free space metadata. (Notably, not for SHA* keys generated
-    with git-annex 0.24 or earlier.) The code is believed to work on
+    with git-annex 0.2x or earlier.) The code is believed to work on
     Linux, FreeBSD, and OSX; check compile-time messages to see if it
     is not enabled for your OS.
   * Add annex.diskreserve config setting, to control how much free space

From 376fc65d3e5a16e7d3b5dc8bf2dd35d72a28c2c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 21:23:35 -0400
Subject: [PATCH 1174/2835] add license text

---
 debian/copyright | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/debian/copyright b/debian/copyright
index 2144501e11..112ad54aaa 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -11,3 +11,29 @@ License: GPL-3+
 Files: StatFS.hsc
 Copyright: Jose A Ortega Ruiz 
 License: BSD-3-clause
+ -- All rights reserved.
+ -- 
+ -- Redistribution and use in source and binary forms, with or without
+ -- modification, are permitted provided that the following conditions
+ -- are met:
+ -- 
+ -- 1. Redistributions of source code must retain the above copyright
+ --    notice, this list of conditions and the following disclaimer.
+ -- 2. Redistributions in binary form must reproduce the above copyright
+ --    notice, this list of conditions and the following disclaimer in the
+ --    documentation and/or other materials provided with the distribution.
+ -- 3. Neither the name of the author nor the names of his contributors
+ --    may be used to endorse or promote products derived from this software
+ --    without specific prior written permission.
+ -- 
+ -- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ -- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ -- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ -- ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ -- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ -- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ -- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ -- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ -- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ -- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ -- SUCH DAMAGE.

From 745f48297d46ada2242c8a698fa1e133d6d7d3f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 23 Mar 2011 02:07:49 +0000
Subject: [PATCH 1175/2835] Added a comment

---
 ...mment_3_558d80384434207b9cfc033763863de3._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment

diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment
new file mode 100644
index 0000000000..9a128f1ed6
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-23T02:07:49Z"
+ content="""
+Ah yes, I feel kinda stupid in hindsight.
+
+As the central server is most likely a common use case, would you object if I added that to the walkthrough? If you have any best practices on how to automate a push with every copy to a bare remote? AFAIK, git does not store information about bare/non-bare remotes, but this could easily be put into .git/config by git annex.
+
+-- RichiH
+"""]]

From 1161fa8bdffbf2c83351024fdd653beefe050f82 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 01:06:14 -0400
Subject: [PATCH 1176/2835] add units to disk size check message

---
 Content.hs   |  6 ++---
 DataUnits.hs | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 3 deletions(-)
 create mode 100644 DataUnits.hs

diff --git a/Content.hs b/Content.hs
index 39a3addccf..2a3106c108 100644
--- a/Content.hs
+++ b/Content.hs
@@ -39,6 +39,7 @@ import qualified Annex
 import Utility
 import StatFS
 import Key
+import DataUnits
 
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
@@ -129,11 +130,10 @@ checkDiskSpace' adjustment key = do
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
 			if (need + reserve > have + adjustment)
 				then error $ "not enough free space (have " ++ 
-					showsize (have + adjustment) ++ "; need " ++
-					showsize (need + reserve) ++ ")"
+					roughSize True (have + adjustment) ++ "; need " ++
+					roughSize True (need + reserve) ++ ")"
 				else return ()
 	where
-		showsize i = show i
 		megabyte :: Integer
 		megabyte = 1024 * 1024
 
diff --git a/DataUnits.hs b/DataUnits.hs
new file mode 100644
index 0000000000..3404e0ab18
--- /dev/null
+++ b/DataUnits.hs
@@ -0,0 +1,66 @@
+{- data size display
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module DataUnits (roughSize) where
+
+{- And now a rant: 
+ -
+ - In the beginning, we had powers of two, and they were good.
+ -
+ - Disk drive manufacturers noticed that some powers of two were
+ - sorta close to some powers of ten, and that rounding down to the nearest
+ - power of ten allowed them to advertise their drives were bigger. This
+ - was sorta annoying.
+ -
+ - Then drives got big. Really, really big. This was good.
+ -
+ - Except that the small rounding error perpretrated by the drive
+ - manufacturers suffered the fate of a small error, and became a large
+ - error. This was bad.
+ -
+ - So, a committee was formed. And it arrived at a committee-like decision,
+ - which satisfied noone, confused everyone, and made the world an uglier
+ - place. As with all committees, this was meh.
+ -
+ - And the drive manufacturers happily continued selling drives that are
+ - increasingly smaller than you'd expect, if you don't count on your
+ - fingers. But that are increasingly bigger.
+ -
+ - Thus, I use units here that I loathe. Because if I didn't, people would
+ - be confused that their drives seem the wrong size, and other people would
+ - complain at me for not being standards compliant. And we call this
+ - progress?
+ -}
+
+{- approximate display of a particular number of bytes -}
+roughSize :: Bool -> Integer -> String
+roughSize short i
+	| i < 0 = "-" ++ roughSize short (negate i)
+	| i >= at 8 = units 8 "yottabyte" "YB"
+	| i >= at 7 = units 7 "zettabyte" "ZB"
+	| i >= at 6 = units 6 "exabyte" "EB"
+	| i >= at 5 = units 5 "petabyte" "PB"
+	| i >= at 4 = units 4 "terabyte" "TB"
+	| i >= at 3 = units 3 "gigabyte" "GB"
+	| i >= at 2 = units 2 "megabyte" "MB"
+	| i >= at 1 = units 1 "kilobyte" "kB"
+	| otherwise = units 0 "byte" "B"
+	where
+		at :: Integer -> Integer
+		at n = 1000^n
+
+		chop :: Integer -> Integer
+		chop d = round ((fromInteger i :: Double) / fromInteger (at d))
+
+		units :: Integer -> String -> String -> String
+		units d u u' = let num = chop d in
+			show num ++ " " ++
+			(if short then u' else plural num u)
+
+		plural n u
+			| n == 1 = u
+			| otherwise = u ++ "s"

From 12cdc96216e0c516ceeee922f9ca1568e9d4d592 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 01:13:13 -0400
Subject: [PATCH 1177/2835] tweak

---
 DataUnits.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/DataUnits.hs b/DataUnits.hs
index 3404e0ab18..329bf7e580 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -54,9 +54,8 @@ roughSize short i
 		at n = 1000^n
 
 		chop :: Integer -> Integer
-		chop d = round ((fromInteger i :: Double) / fromInteger (at d))
+		chop d = round $ (fromInteger i :: Double) / fromInteger (at d)
 
-		units :: Integer -> String -> String -> String
 		units d u u' = let num = chop d in
 			show num ++ " " ++
 			(if short then u' else plural num u)

From c43e3b5c787050664089fa4498e660a475addcd9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 02:10:59 -0400
Subject: [PATCH 1178/2835] check key size when available, no matter the
 backend

Now that SHA and other backends can have size info, fsck should check it
whenever available.
---
 Backend.hs      | 40 ++++++++++++++++++++++++++++++++++++++--
 Backend/File.hs |  5 ++++-
 Backend/WORM.hs | 27 +--------------------------
 3 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index cd14ce50e1..d7334f144e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -35,6 +35,7 @@ import Control.Monad.State
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
+import System.Directory
 
 import Locations
 import qualified GitRepo as Git
@@ -43,6 +44,8 @@ import Types
 import Key
 import qualified BackendClass as B
 import Messages
+import Content
+import DataUnits
 
 {- List of backends in the order to try them when storing a new key. -}
 list :: Annex [Backend Annex]
@@ -120,9 +123,12 @@ hasKey key = do
 	backend <- keyBackend key
 	(B.hasKey backend) key
 
-{- Checks a key's backend for problems. -}
+{- Checks a key for problems. -}
 fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-fsckKey backend key file numcopies = (B.fsckKey backend) key file numcopies
+fsckKey backend key file numcopies = do
+	size_ok <- checkKeySize key
+	backend_ok <-(B.fsckKey backend) key file numcopies
+	return $ size_ok && backend_ok
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
@@ -168,3 +174,33 @@ keyBackend :: Key -> Annex (Backend Annex)
 keyBackend key = do
 	bs <- Annex.getState Annex.supportedBackends
 	return $ lookupBackendName bs $ keyBackendName key
+
+{- The size of the data for a key is checked against the size encoded in
+ - the key's metadata, if available. -}
+checkKeySize :: Key -> Annex Bool
+checkKeySize key = do
+	g <- Annex.gitRepo
+	let file = gitAnnexLocation g key
+	present <- liftIO $ doesFileExist file
+	case (present, keySize key) of
+		(_, Nothing) -> return True
+		(False, _) -> return True
+		(True, Just size) -> do
+			stat <- liftIO $ getFileStatus file
+			let size' = fromIntegral (fileSize stat)
+			if size == size'
+				then return True
+				else do
+					dest <- moveBad key
+					warning $ badsizeNote dest size size'
+					return False
+
+badsizeNote :: FilePath -> Integer -> Integer -> String
+badsizeNote dest expected got = "Bad file size (" ++ aside ++ "); moved to " ++ dest
+	where
+		expected' = roughSize True expected
+		got' = roughSize True got
+		aside = 
+			if expected' == got'
+				then show expected ++ " not " ++ show got
+				else expected' ++ " not " ++ got'
diff --git a/Backend/File.hs b/Backend/File.hs
index a5e2431998..a6d42eabde 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -38,7 +38,7 @@ backend = Backend {
 	retrieveKeyFile = copyKeyFile,
 	removeKey = checkRemoveKey,
 	hasKey = inAnnex,
-	fsckKey = mustProvide
+	fsckKey = checkKeyOnly
 }
 
 mustProvide :: a
@@ -172,6 +172,9 @@ checkKey a key file numcopies = do
 	copies_ok <- checkKeyNumCopies key file numcopies
 	return $ a_ok && copies_ok
 
+checkKeyOnly :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKeyOnly = checkKey (\_ -> return True)
+
 checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index a011995da3..b33c607632 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -10,15 +10,9 @@ module Backend.WORM (backends) where
 import Control.Monad.State
 import System.FilePath
 import System.Posix.Files
-import System.Directory
-import Data.Maybe
 
 import qualified Backend.File
 import BackendClass
-import Locations
-import qualified Annex
-import Content
-import Messages
 import Types
 import Key
 
@@ -28,8 +22,7 @@ backends = [backend]
 backend :: Backend Annex
 backend = Backend.File.backend {
 	name = "WORM",
-	getKey = keyValue,
-	fsckKey = Backend.File.checkKey checkKeySize
+	getKey = keyValue
 }
 
 {- The key includes the file size, modification time, and the
@@ -48,21 +41,3 @@ keyValue file = do
 		keySize = Just $ fromIntegral $ fileSize stat,
 		keyMtime = Just $ modificationTime stat
 	}
-
-{- The size of the data for a key is checked against the size encoded in
- - the key's metadata. -}
-checkKeySize :: Key -> Annex Bool
-checkKeySize key = do
-	g <- Annex.gitRepo
-	let file = gitAnnexLocation g key
-	present <- liftIO $ doesFileExist file
-	if not present
-		then return True
-		else do
-			s <- liftIO $ getFileStatus file
-			if fromIntegral (fileSize s) == fromJust (keySize key)
-				then return True
-				else do
-					dest <- moveBad key
-					warning $ "Bad file size; moved to " ++ dest
-					return False

From cd1cb526522029cbc9c6c8fe396da1685bfb603b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 02:15:26 -0400
Subject: [PATCH 1179/2835] tweak

---
 Backend.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index d7334f144e..fe61739e83 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -202,5 +202,5 @@ badsizeNote dest expected got = "Bad file size (" ++ aside ++ "); moved to " ++
 		got' = roughSize True got
 		aside = 
 			if expected' == got'
-				then show expected ++ " not " ++ show got
-				else expected' ++ " not " ++ got'
+				then show got ++ " not " ++ show expected
+				else got' ++ " not " ++ expected'

From 04539d16718265441c607da08d3e27d959f749c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 02:42:14 -0400
Subject: [PATCH 1180/2835] improve size change display

---
 Backend.hs   | 14 +++-----------
 DataUnits.hs |  8 +++++++-
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index fe61739e83..0ee56d2623 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -192,15 +192,7 @@ checkKeySize key = do
 				then return True
 				else do
 					dest <- moveBad key
-					warning $ badsizeNote dest size size'
+					warning $ "Bad file size (" ++
+						compareSizes True size size' ++ 
+						"); moved to " ++ dest
 					return False
-
-badsizeNote :: FilePath -> Integer -> Integer -> String
-badsizeNote dest expected got = "Bad file size (" ++ aside ++ "); moved to " ++ dest
-	where
-		expected' = roughSize True expected
-		got' = roughSize True got
-		aside = 
-			if expected' == got'
-				then show got ++ " not " ++ show expected
-				else got' ++ " not " ++ expected'
diff --git a/DataUnits.hs b/DataUnits.hs
index 329bf7e580..c2845affea 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module DataUnits (roughSize) where
+module DataUnits (roughSize, compareSizes) where
 
 {- And now a rant: 
  -
@@ -63,3 +63,9 @@ roughSize short i
 		plural n u
 			| n == 1 = u
 			| otherwise = u ++ "s"
+
+compareSizes :: Bool -> Integer -> Integer -> String
+compareSizes short old new
+	| old > new = roughSize short (old - new) ++ " smaller"
+	| old < new = roughSize short (new - old) ++ " larger"
+	| otherwise = "same"

From ba064654a3707ac9407cf17ea3c0a7e7fb573382 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 07:27:14 +0000
Subject: [PATCH 1181/2835]

---
 doc/bugs/Makefile_is_missing_dependancies.mdwn | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/Makefile_is_missing_dependancies.mdwn b/doc/bugs/Makefile_is_missing_dependancies.mdwn
index 4bda91f17d..3e9d6e903c 100644
--- a/doc/bugs/Makefile_is_missing_dependancies.mdwn
+++ b/doc/bugs/Makefile_is_missing_dependancies.mdwn
@@ -1,4 +1,3 @@
-
 
 From e45c73e66fc18d27bdf5797876fbeb07786a4af1 Mon Sep 17 00:00:00 2001
 From: Jimmy Tang 
@@ -30,5 +29,19 @@ index 08e2f59..4ae8392 100644
 
 StatFS.hs never gets depended on and compiled, the makefile was just missing something
 
-> Thanks, [done]]! Interested to hear if StatFS.hs works on OSX (no warning) or
+> Thanks, [[done]]! Interested to hear if StatFS.hs works on OSX (no warning) or
 > is a no-op (with warning). --[[Joey]] 
+
+>> 
+>> for now it gives a warning, it looks like it should be easy enough to add OSX
+>> support, I guess it's a case of just digging around documentation to find the equivalent
+>> calls/headers. I'll give it a go at making this feature work on OSX and get back to you.
+>> 
+
+
+jtang@exia:~/develop/git-annex $ make
+hsc2hs StatFS.hsc
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+


From 847629ee69e891185484704bd20b137c7ad06a25 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 08:21:32 +0000
Subject: [PATCH 1182/2835] Added a comment

---
 ..._5a3da5f79c8563c7a450aa29728abe7c._comment | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment
new file mode 100644
index 0000000000..ab8493a7a8
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-03-23T08:21:30Z"
+ content="""
+Just did some minor digging around and checking, this seems to satisfy the compilers etc... I have yet to confirm that it *really* is working as expected. Also it might be better to check for a darwin operating system instead of apple I think, though I don't know of any one really using a pure darwin OS. But for now it works (I think)
+
+
+From fbfe27c2e19906ac02e3673b91bffa920f6dae5d Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Wed, 23 Mar 2011 08:15:39 +0000
+Subject: [PATCH] Define (__APPLE__) in StatFS
+
+At least on OSX 10.6.6 it appears to have the same defintions as
+FreeBSD. The build process doesn't complain and the code is enabled,
+this needs to be tested and checked more.
+---
+ StatFS.hsc |    4 ++--
+ 1 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/StatFS.hsc b/StatFS.hsc
+index 8b453dc..45fd7e4 100644
+--- a/StatFS.hsc
++++ b/StatFS.hsc
+@@ -53,7 +53,7 @@ import Foreign.C.String
+ import Data.ByteString (useAsCString)
+ import Data.ByteString.Char8 (pack)
+ 
+-#if defined (__FreeBSD__)
++#if defined (__FreeBSD__) || defined(__APPLE__)
+ # include 
+ # include 
+ #else
+@@ -84,7 +84,7 @@ data CStatfs
+ #ifdef UNKNOWN
+ #warning free space checking code not available for this OS
+ #else
+-#if defined(__FreeBSD__)
++#if defined(__FreeBSD__) || defined(__APPLE__)
+ foreign import ccall unsafe \"sys/mount.h statfs\"
+ #else
+ foreign import ccall unsafe \"sys/vfs.h statfs64\"
+-- 
+1.7.4.1
+

+"""]]

From 002db413a53ec83c93f389b09fe9926cebfaa0ec Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 23 Mar 2011 09:53:16 +0000
Subject: [PATCH 1183/2835]

---
 doc/bugs/softlink_atime.mdwn | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn
index c62610be02..fe09ade38b 100644
--- a/doc/bugs/softlink_atime.mdwn
+++ b/doc/bugs/softlink_atime.mdwn
@@ -29,7 +29,7 @@ Optionally, editing the meta-data should change the times in all annexes.
 >>>>> 
 >>>>> So all you have to do is make the pre-commit hook call
 >>>>> [metastore](http://david.hardeman.nu/software.php). The hook
->>>>> would look like this: ---[[Joey]] [[!tag done]]
+>>>>> would look like this: ---[[Joey]]
 
 	#!/bin/sh
 	git annex pre-commit .
@@ -47,3 +47,5 @@ Optionally, editing the meta-data should change the times in all annexes.
     git add .metadata
 
 >>>>>>> -- RichiH
+
+>>>>>>>> After getting to actually play with this from different machines with a bare git as central instance for several distributed repos, the metastore trick does not work. The .metadata is causing merge conflicts for every pull. I removed the "done" tag from this issue. -- RichiH

From 6d857b540aab2e80394984572ebd26be31ececff Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 23 Mar 2011 11:31:06 +0000
Subject: [PATCH 1184/2835] Added a comment: Why isn't this package built with
 Cabal?

---
 .../comment_1_d9f7b851567445c7aa7ebbb440781819._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment

diff --git a/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment b/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
new file mode 100644
index 0000000000..616b3c4dd5
--- /dev/null
+++ b/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="84.189.1.247"
+ subject="Why isn't this package built with Cabal?"
+ date="2011-03-23T11:31:06Z"
+ content="""
+It would be a lot easier to compile this package, if it had a Cabal file to describe the build; especially the build-time dependencies. Why isn't Cabal used?
+"""]]

From b0eccb003ae70792e89eda54d0c0bea1f38847ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 10:52:40 -0400
Subject: [PATCH 1185/2835] improve free space needed display

---
 Content.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Content.hs b/Content.hs
index 2a3106c108..bc43e4b443 100644
--- a/Content.hs
+++ b/Content.hs
@@ -129,9 +129,9 @@ checkDiskSpace' adjustment key = do
 		(_, Nothing) -> return ()
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
 			if (need + reserve > have + adjustment)
-				then error $ "not enough free space (have " ++ 
-					roughSize True (have + adjustment) ++ "; need " ++
-					roughSize True (need + reserve) ++ ")"
+				then error $ "not enough free space, need " ++ 
+					roughSize True (need + reserve - have - adjustment) ++
+					" more"
 				else return ()
 	where
 		megabyte :: Integer

From 5b4ae90cdb27b0024f5caf129ffa6f05488869a4 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:05:12 +0000
Subject: [PATCH 1186/2835] Added a comment

---
 .../comment_2_416f12dbd0c2b841fac8164645b81df5._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment
new file mode 100644
index 0000000000..d355514a31
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-23T15:05:12Z"
+ content="""
+There's a simple test -- just configure annex.diskreserve to be say, 10 megabytes less than the total free space on your disk. Then try to git annex get a 11 mb file, and a 9 mb file. :)
+"""]]

From 4440ecf4a74b85341d5ecc1ecb1a9349b6fc5d3b Mon Sep 17 00:00:00 2001
From: Jimmy Tang 
Date: Wed, 23 Mar 2011 08:15:39 +0000
Subject: [PATCH 1187/2835] Define (__APPLE__) in StatFS

At least on OSX 10.6.6 it appears to have the same defintions as
FreeBSD. The build process doesn't complain and the code is enabled,
this needs to be tested and checked more.
---
 StatFS.hsc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index 8b453dc199..45fd7e4db9 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -53,7 +53,7 @@ import Foreign.C.String
 import Data.ByteString (useAsCString)
 import Data.ByteString.Char8 (pack)
 
-#if defined (__FreeBSD__)
+#if defined (__FreeBSD__) || defined(__APPLE__)
 # include 
 # include 
 #else
@@ -84,7 +84,7 @@ data CStatfs
 #ifdef UNKNOWN
 #warning free space checking code not available for this OS
 #else
-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined(__APPLE__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"

From cf70075c4be00781e87b5b9a4774c2bdb95d949e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:13:33 +0000
Subject: [PATCH 1188/2835] Added a comment

---
 ..._c38b6f4abc9b9ad413c3b83ca04386c3._comment | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment
new file mode 100644
index 0000000000..6b4cf5789c
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-23T15:13:33Z"
+ content="""
+Alternatively, you can just load it up in ghci and see if it reports numbers that make sense:
+
+
+joey@gnu:~/src/git-annex>make StatFS.hs
+hsc2hs StatFS.hsc
+perl -i -pe 's/^{-# INCLUDE.*//' StatFS.hs
+joey@gnu:~/src/git-annex>ghci StatFS.hs
+GHCi, version 6.12.1: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.5 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 7427989, fsStatByteCount = 30425042944, fsStatBytesFree = 2528489472, fsStatBytesAvailable = 2219384832, fsStatBytesUsed = 27896553472})
+

+"""]]

From 7808de3d471dffff3ee2360f3bbf1627c492cb0f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:18:29 +0000
Subject: [PATCH 1189/2835] Added a comment

---
 ...mment_2_cf0f829536744098d6846500db998b6a._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment

diff --git a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
new file mode 100644
index 0000000000..134024908e
--- /dev/null
+++ b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-23T15:18:29Z"
+ content="""
+Because I haven't learned Cabal yet.
+
+But also because I've had bad experiences with both a) tying a particular program to a particular language's pet build system and then having to add ugliness when I later need to do something in the build that has nothing to do with that language and b) as a user, needing to deal with the pet build systems of languages when I just need to make some small change to the build process that is trivial in a Makefile.
+
+With that said, I do have a configure program written in Haskell, so at least it doesn't use autotools. :)
+"""]]

From d0a7513bbbe8b13c99ed3af986aef82e5d0f0248 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:28:00 +0000
Subject: [PATCH 1190/2835] Added a comment

---
 .../comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment

diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment
new file mode 100644
index 0000000000..8b4d9a0538
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-03-23T15:28:00Z"
+ content="""
+I would not mind if the walkthrough documented the central git repo case. But I don't want to complicate it unduely (it's long enough), and it's important that the fully distributed case be shown to work, and I assume that people already have basic git knowledge, so documenting the details of set up of a bare git repo is sorta out of scope. (There are also a lot of way to do it, using github, or gitosis, or raw git, etc.)
+"""]]

From da198e87fb87430a8d5325d3f793ceeb8f746677 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:31:54 +0000
Subject: [PATCH 1191/2835] bit about bare repos

---
 doc/walkthrough/getting_file_content.mdwn | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index a863303cef..5c899ee3c5 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -13,4 +13,7 @@ USB drive.
 
 Notice that you had to git pull from laptop first, this lets git-annex know
 what has changed in laptop, and so it knows about the files present there and
-can get them.
+can get them. The alternate approach is to set up a central bare repository,
+and always push changes to it after committing them, then in the above,
+you can just pull from the central repository to get synced up to all
+repositories.

From 328b023c5381513cb0b9fb00f832cb8251799b4a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 16:02:35 +0000
Subject: [PATCH 1192/2835] Added a comment

---
 ..._cc13873175edf191047282700315beee._comment | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment
new file mode 100644
index 0000000000..c3ad2dafd6
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2011-03-23T16:02:34Z"
+ content="""
+Ok, well it looks like it isn't doing anything useful at all.
+
+
+jtang@x00:~/develop/git-annex $ make StatFS.hs                                                                                                                                    
+hsc2hs StatFS.hsc
+perl -i -pe 's/^{-# INCLUDE.*//' StatFS.hs
+jtang@x00:~/develop/git-annex $ ghci StatFS.hs                                                                                                                                    
+GHCi, version 6.12.3: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+Loading package ffi-1.0 ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.7 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 0, fsStatBlockCount = 1048576, fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0, fsStatBytesUsed = 0})
+*StatFS> s <- getFileSystemStats \"/\"
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 0, fsStatBlockCount = 1048576, fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0, fsStatBytesUsed = 0})
+*StatFS> 
+

+"""]]

From 48a8f811fa85f94912bd5009728d054cf6e8ac48 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 16:14:22 +0000
Subject: [PATCH 1193/2835] Added a comment

---
 ..._0a1c52e2c96d19b9c3eb7e99b8c2434f._comment | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment
new file mode 100644
index 0000000000..149aeeb75a
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment
@@ -0,0 +1,59 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2011-03-23T16:14:22Z"
+ content="""
+Actually I may have just been stupid and should have read the man page on statfs...
+
+
+jtang@x00:~/develop/git-annex $ git diff
+diff --git a/StatFS.hsc b/StatFS.hsc
+index 8b453dc..e10b2dd 100644
+--- a/StatFS.hsc
++++ b/StatFS.hsc
+@@ -53,7 +53,7 @@ import Foreign.C.String
+ import Data.ByteString (useAsCString)
+ import Data.ByteString.Char8 (pack)
+ 
+-#if defined (__FreeBSD__)
++#if defined (__FreeBSD__) || defined (__APPLE__)
+ # include 
+ # include 
+ #else
+@@ -84,8 +84,8 @@ data CStatfs
+ #ifdef UNKNOWN
+ #warning free space checking code not available for this OS
+ #else
+-#if defined(__FreeBSD__)
+-foreign import ccall unsafe \"sys/mount.h statfs\"
++#if defined(__FreeBSD__) || defined (__APPLE__)
++foreign import ccall unsafe \"sys/mount.h statfs64\"
+ #else
+ foreign import ccall unsafe \"sys/vfs.h statfs64\"
+ #endif
+

+
+yields this...
+
+
+jtang@x00:~/develop/git-annex $ ghci StatFS.hs                                                                                                                                    
+GHCi, version 6.12.3: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+Loading package ffi-1.0 ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.7 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 244106668, fsStatByteCount = 999860912128, fsStatBytesFree = 423097798656, fsStatBytesAvailable = 422835654656, fsStatBytesUsed = 576763113472})
+*StatFS> 
+

+
+
+we could just stick another if defined (__APPLE__) instead of what I previously had and it looks like it will do the right thing on OSX.
+
+
+"""]]

From 0dac2b4f6ce0a616711a24e964c138055607e35e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 12:23:13 -0400
Subject: [PATCH 1194/2835] Revert "Define (__APPLE__) in StatFS"

This reverts commit 4440ecf4a74b85341d5ecc1ecb1a9349b6fc5d3b.

Turns out that it is reporting a block size of 0 and so all bogus.
---
 StatFS.hsc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index 45fd7e4db9..8b453dc199 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -53,7 +53,7 @@ import Foreign.C.String
 import Data.ByteString (useAsCString)
 import Data.ByteString.Char8 (pack)
 
-#if defined (__FreeBSD__) || defined(__APPLE__)
+#if defined (__FreeBSD__)
 # include 
 # include 
 #else
@@ -84,7 +84,7 @@ data CStatfs
 #ifdef UNKNOWN
 #warning free space checking code not available for this OS
 #else
-#if defined(__FreeBSD__) || defined(__APPLE__)
+#if defined(__FreeBSD__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"

From 7ffc3a72ae05dde951cee7bdf972545ac461a184 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 16:23:57 +0000
Subject: [PATCH 1195/2835] Added a comment

---
 ...comment_6_24119fc5d5963ce9dd669f7dcf006859._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment
new file mode 100644
index 0000000000..714459fbe8
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 6"
+ date="2011-03-23T16:23:56Z"
+ content="""
+I forgot to mention that the statfs64 stuff in OSX seems to be deprecated, see http://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man2/statfs64.2.html
+
+on a slightly different note, is anonymous pushing to the \"wiki\" over git allowed? I'd prefer to be able to edit stuff inline for updating some of my own comments if I can :P
+"""]]

From 68d90b3328f7b8e7c9aa0bd997bef20724d7cafc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 12:45:34 -0400
Subject: [PATCH 1196/2835] allow force overriding the disk space check

---
 Content.hs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/Content.hs b/Content.hs
index bc43e4b443..633d827e6f 100644
--- a/Content.hs
+++ b/Content.hs
@@ -129,13 +129,17 @@ checkDiskSpace' adjustment key = do
 		(_, Nothing) -> return ()
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
 			if (need + reserve > have + adjustment)
-				then error $ "not enough free space, need " ++ 
-					roughSize True (need + reserve - have - adjustment) ++
-					" more"
+				then needmorespace (need + reserve - have - adjustment)
 				else return ()
 	where
 		megabyte :: Integer
 		megabyte = 1024 * 1024
+		needmorespace n = do
+			force <- Annex.getState Annex.force
+			unless force $ 
+				error $ "not enough free space, need " ++ 
+					roughSize True n ++
+					" more (use --force to override this check or adjust annex.diskreserve)"
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()

From 7400c8318aea1cc5f43352ae97454fd34e1126b1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 12:46:51 -0400
Subject: [PATCH 1197/2835] correct

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index f124296cc8..eef1d7e29f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,7 +3,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * Free space checking is now done, for transfers of data for keys
     that have free space metadata. (Notably, not for SHA* keys generated
     with git-annex 0.2x or earlier.) The code is believed to work on
-    Linux, FreeBSD, and OSX; check compile-time messages to see if it
+    Linux and FreeBSD; check compile-time messages to see if it
     is not enabled for your OS.
   * Add annex.diskreserve config setting, to control how much free space
     to reserve for other purposes and avoid using (defaults to 1 mb).

From b9c4c006bdc8f0317d0598f05b45e4fdc1102a6c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 12:53:57 -0400
Subject: [PATCH 1198/2835] use statfs64 on apple, while retaining statfs on
 freebsd

http://git-annex.branchable.com/bugs/Makefile_is_missing_dependancies/#comment-3196b43b7d745ab206435d0a69686815
indicates statfs64 works on apple. Probably on freebsd too, but I
have not tested it and so will stick with the old code there.
---
 StatFS.hsc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index 8b453dc199..dd0c90bebd 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -53,7 +53,7 @@ import Foreign.C.String
 import Data.ByteString (useAsCString)
 import Data.ByteString.Char8 (pack)
 
-#if defined (__FreeBSD__)
+#if defined (__FreeBSD__) || defined (__APPLE__)
 # include 
 # include 
 #else
@@ -84,10 +84,14 @@ data CStatfs
 #ifdef UNKNOWN
 #warning free space checking code not available for this OS
 #else
+#if defined(__APPLE__)
+foreign import ccall unsafe "sys/mount.h statfs64"
+#else
 #if defined(__FreeBSD__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"
+#endif
 #endif
   c_statfs :: CString -> Ptr CStatfs -> IO CInt
 #endif

From 4727ebab53d52e5126238e64ca03d03cf309d599 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 16:57:56 +0000
Subject: [PATCH 1199/2835] Added a comment

---
 ...mment_7_96fd4725df4b54e670077a18d3ac4943._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment
new file mode 100644
index 0000000000..8ba8e8d1f6
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-03-23T16:57:56Z"
+ content="""
+Try the changes I've pushed to use statfs64 on apple.
+
+There is actually a standardized statvfs that I'd rather use, but after the last time that I tried going with the POSIX option first only to find it was not broadly implemented, I was happy to find some already existing code that worked for some OSs.
+
+(While ikiwiki supports anonymous git push, it's a feature we have not rolled out on Branchable.com yet, and anyway, ikiwiki disallows editing existing comments that way. I would, however, be happy to git pull changes from somewhere.)
+"""]]

From 443bf2ade72a2117fc5ca77f7b9ee2dda2bb4c45 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 17:03:53 +0000
Subject: [PATCH 1200/2835] Added a comment

---
 .../comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment
new file mode 100644
index 0000000000..63d188bcce
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 8"
+ date="2011-03-23T17:03:51Z"
+ content="""
+The latest change looks good, it seems to be returning sensible numbers for me. Just tried it out on a few different mount points and it appears to be working.
+"""]]

From 0cd70cb5c0c33ce5db2a68fb0c76341f9d71f400 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 13:10:20 -0400
Subject: [PATCH 1201/2835] kFreeBSD support

Tested on Debian kfreebsd-amd64. The BSD #includes worked. Both statfs64
and statfs worked. Using statfs to keep the same as general freebsd, and
because I didn't try it on 32 bit.
---
 StatFS.hsc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index dd0c90bebd..0828f1378c 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -53,7 +53,7 @@ import Foreign.C.String
 import Data.ByteString (useAsCString)
 import Data.ByteString.Char8 (pack)
 
-#if defined (__FreeBSD__) || defined (__APPLE__)
+#if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__)
 # include 
 # include 
 #else
@@ -87,7 +87,7 @@ data CStatfs
 #if defined(__APPLE__)
 foreign import ccall unsafe "sys/mount.h statfs64"
 #else
-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined (__FreeBSD_kernel__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"

From af45a62980b6b231225e4456e602eec4dfc4e04b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 13:13:51 -0400
Subject: [PATCH 1202/2835] update

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index eef1d7e29f..f124296cc8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,7 +3,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * Free space checking is now done, for transfers of data for keys
     that have free space metadata. (Notably, not for SHA* keys generated
     with git-annex 0.2x or earlier.) The code is believed to work on
-    Linux and FreeBSD; check compile-time messages to see if it
+    Linux, FreeBSD, and OSX; check compile-time messages to see if it
     is not enabled for your OS.
   * Add annex.diskreserve config setting, to control how much free space
     to reserve for other purposes and avoid using (defaults to 1 mb).

From 8beb72e20604b0d2cc6359e75ed4eac45fcd1081 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 17:25:28 -0400
Subject: [PATCH 1203/2835] migrate: Bugfix for case when migrating a file
 results in a key that is already present in .git/annex/objects.

For example, this could happen if using SHA1 and a file with content
"foo" were added to that backend. Then a file with "content" foo were
migrated from the WORM backend.

Assume that, if a backend assigned the same key, the already annexed
content must be the same. So, the "old" content can be reused.
---
 Command/Migrate.hs | 4 +++-
 debian/changelog   | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 56147113b9..38dfd06b23 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,6 +8,7 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (unless)
 import System.Posix.Files
 import System.Directory
 
@@ -58,7 +59,8 @@ perform file oldkey newbackend = do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
 				-- cached key, to avoid wasting disk space.
-				liftIO $ createLink src t
+				exists <- liftIO $ doesFileExist t
+				unless exists $ liftIO $ createLink src t
 				return True
 			if ok
 				then do
diff --git a/debian/changelog b/debian/changelog
index f124296cc8..9a3ffbafaf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,8 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * unused: In fast mode, just show all existing temp files as unused,
     and avoid expensive scan for other unused content.
   * Fix space leak in fsck and drop commands.
+  * migrate: Bugfix for case when migrating a file results in a key that
+    is already present in .git/annex/objects.
 
  -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
 

From ad08273ac5118f1faac539b53f1fa63908dc5656 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 17:29:54 -0400
Subject: [PATCH 1204/2835] refactor

---
 Command/Migrate.hs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 38dfd06b23..a7014b9bcf 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -31,7 +31,8 @@ start :: CommandStartBackendFile
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
-	if (newbackend /= oldbackend) && exists
+	force <- Annex.getState Annex.force
+	if (newbackend /= oldbackend || force) && exists
 		then do
 			showStart "migrate" file
 			return $ Just $ perform file key newbackend
@@ -59,8 +60,9 @@ perform file oldkey newbackend = do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
 				-- cached key, to avoid wasting disk space.
-				exists <- liftIO $ doesFileExist t
-				unless exists $ liftIO $ createLink src t
+				liftIO $ do
+					exists <- doesFileExist t
+					unless exists $ createLink src t
 				return True
 			if ok
 				then do

From 6246b807f7df32877a87d906cfbe1ae26c51dd8e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 17:57:10 -0400
Subject: [PATCH 1205/2835] migrate: Support migrating v1 SHA keys to v2 SHA
 keys with size information that can be used for free space checking.

---
 Backend.hs                 |  5 +++++
 Backend/File.hs            | 10 +++++++++-
 Backend/URL.hs             |  4 +++-
 BackendClass.hs            |  4 +++-
 Command/Migrate.hs         |  4 ++--
 debian/changelog           |  2 ++
 doc/git-annex.mdwn         |  9 +++++++--
 doc/upgrades.mdwn          |  6 +++---
 doc/upgrades/SHA_size.mdwn | 20 ++++++++++++++++++++
 9 files changed, 54 insertions(+), 10 deletions(-)
 create mode 100644 doc/upgrades/SHA_size.mdwn

diff --git a/Backend.hs b/Backend.hs
index 0ee56d2623..4b0095214c 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -24,6 +24,7 @@ module Backend (
 	removeKey,
 	hasKey,
 	fsckKey,
+	upgradableKey,
 	lookupFile,
 	chooseBackends,
 	keyBackend,
@@ -130,6 +131,10 @@ fsckKey backend key file numcopies = do
 	backend_ok <-(B.fsckKey backend) key file numcopies
 	return $ size_ok && backend_ok
 
+{- Checks if a key is upgradable to a newer representation. -}
+upgradableKey :: Backend Annex -> Key -> Annex Bool
+upgradableKey backend key = (B.upgradableKey backend) key
+
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex))
diff --git a/Backend/File.hs b/Backend/File.hs
index a6d42eabde..fb8a052553 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -29,6 +29,7 @@ import Types
 import UUID
 import Messages
 import Trust
+import Key
 
 backend :: Backend Annex
 backend = Backend {
@@ -38,7 +39,8 @@ backend = Backend {
 	retrieveKeyFile = copyKeyFile,
 	removeKey = checkRemoveKey,
 	hasKey = inAnnex,
-	fsckKey = checkKeyOnly
+	fsckKey = checkKeyOnly,
+	upgradableKey = checkUpgradableKey
 }
 
 mustProvide :: a
@@ -159,6 +161,12 @@ getNumCopies Nothing = do
 	where
 		config = "annex.numcopies"
 
+{- Ideally, all keys have file size metadata. Old keys may not. -}
+checkUpgradableKey :: Key -> Annex Bool
+checkUpgradableKey key
+	| keySize key == Nothing = return True
+	| otherwise = return False
+
 {- This is used to check that numcopies is satisfied for the key on fsck.
  - This trusts data in the the location log, and so can check all keys, even
  - those with data not present in the current annex.
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 210c7c5b48..3068c30270 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -30,7 +30,9 @@ backend = Backend {
 	-- similarly, keys are always assumed to be out there on the web
 	hasKey = dummyOk,
 	-- and nothing needed to fsck
-	fsckKey = dummyFsck
+	fsckKey = dummyFsck,
+	-- and key upgrade not needed
+	upgradableKey = \_ -> return False
 }
 
 -- cannot generate url from filename
diff --git a/BackendClass.hs b/BackendClass.hs
index 909ae8f96e..b2d8879c2f 100644
--- a/BackendClass.hs
+++ b/BackendClass.hs
@@ -29,7 +29,9 @@ data Backend a = Backend {
 	-- (second parameter may be the filename associated with it)
 	-- (third parameter may be the number of copies that there should
 	-- be of the key)
-	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool
+	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool,
+	-- Is a newer repesentation possible for a key?
+	upgradableKey :: Key -> a Bool
 }
 
 instance Show (Backend a) where
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index a7014b9bcf..0d21fcbdf9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -31,8 +31,8 @@ start :: CommandStartBackendFile
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
-	force <- Annex.getState Annex.force
-	if (newbackend /= oldbackend || force) && exists
+	upgradable <- Backend.upgradableKey oldbackend key
+	if (newbackend /= oldbackend || upgradable) && exists
 		then do
 			showStart "migrate" file
 			return $ Just $ perform file key newbackend
diff --git a/debian/changelog b/debian/changelog
index 9a3ffbafaf..7bb5bd8d21 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,8 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * fsck: In fast mode, avoid checking checksums.
   * unused: In fast mode, just show all existing temp files as unused,
     and avoid expensive scan for other unused content.
+  * migrate: Support migrating v1 SHA keys to v2 SHA keys with
+    size information that can be used for free space checking.
   * Fix space leak in fsck and drop commands.
   * migrate: Bugfix for case when migrating a file results in a key that
     is already present in .git/annex/objects.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d7b57675d0..81cea04cd7 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -162,10 +162,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * migrate [path ...]
 
   Changes the specified annexed files to store their content in the
-  default backend (or the one specified with --backend).
+  default backend (or the one specified with --backend). Only files whose
+  content is currently available are migrated.
 
   Note that the content is not removed from the backend it was previously in.
-  Use `git annex unused` to find and remove such content. 
+  Use `git annex unused` to find and remove such content.
+
+  Normally, nothing will be done to files already in the backend.
+  However, if a backend changes the information it uses to construct a key,
+  this can also be used to migrate files to use the new key format.
 
 * map
 
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index dbea5e9c4a..0a07ef7aa8 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -56,9 +56,9 @@ And .gitattributes needed to have another line added to it.
 Previously, files added to the SHA [[backends]] did not have their file
 size tracked, while files added to the WORM backend did. Files added to
 the SHA backends after the conversion will have their file size tracked,
-and that information will be used by git-annex for disk space checking.
-There is not yet a way to add file size tracking information to old files
-in the SHA backend.
+and that information will be used by git-annex for disk free space checking.
+To ensure that information is available for all your annexed files, see
+[[upgrades/SHA_size]].
 
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 
diff --git a/doc/upgrades/SHA_size.mdwn b/doc/upgrades/SHA_size.mdwn
new file mode 100644
index 0000000000..319b91108a
--- /dev/null
+++ b/doc/upgrades/SHA_size.mdwn
@@ -0,0 +1,20 @@
+Before version 2 of the git-annex repository, files added to the SHA
+[[backends]] did not have their file size tracked, while files added to the
+WORM backend did. The file size information is used for disk free space
+checking.
+
+Files added to the SHA backends after the conversion will have their file
+size tracked automatically. This disk free space checking is an optional
+feature and since you're more likely to be using more recently added files,
+you're unlikely to see any bad effect if you do nothing.
+
+That said, if you have old files added to SHA backends that lack file size
+tracking info, here's how you can add that info. After [[upgrading|upgrades]]
+to repository version 2, in each repository run:
+
+	git annex migrate
+	git commit -m 'migrated keys for v2'
+
+The usual caveats about [[walkthrough/migrating_data_to_a_new_backend]]
+apply; you will end up with unused keys that you can later clean up with
+`git annex unused`.

From 03fdd0d56e554c65946b9eadd32c5be5d6d0c806 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 23:47:02 -0400
Subject: [PATCH 1206/2835] dropunused: Significantly sped up; only read unused
 log file once.

---
 Command/DropUnused.hs | 12 ++++++++----
 debian/changelog      |  1 +
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index c6a28663ec..932a8b8635 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -27,12 +27,16 @@ command = [repoCommand "dropunused" (paramRepeating paramNumber) seek
 	"drop unused file content"]
 
 seek :: [CommandSeek]
-seek = [withStrings start]
+seek = [withUnusedMap]
 
-{- Drops unused content by number. -} 
-start :: CommandStartString
-start s = notBareRepo $ do
+{- Read unusedlog once, and pass the map to each start action. -}
+withUnusedMap :: CommandSeek
+withUnusedMap params = do
 	m <- readUnusedLog
+	return $ map (start m) params
+
+start :: M.Map String Key -> CommandStartString
+start m s = notBareRepo $ do
 	case M.lookup s m of
 		Nothing -> return Nothing
 		Just key -> do
diff --git a/debian/changelog b/debian/changelog
index 7bb5bd8d21..9235c3ca33 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * Fix space leak in fsck and drop commands.
   * migrate: Bugfix for case when migrating a file results in a key that
     is already present in .git/annex/objects.
+  * dropunused: Significantly sped up; only read unused log file once.
 
  -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
 

From 31bf31583be78b589c4b78a0dc8705f54fd9946e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 24 Mar 2011 14:32:22 +0000
Subject: [PATCH 1207/2835] Did a fresh install of haskell-platform on a rhel5
 32bit host and iinstalling HP, pcre-light, missingh wasnt enough to build
 git-annex

---
 doc/install.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index a531015782..0501663231 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -12,6 +12,7 @@ To build and use git-annex, you will need:
 * The Haskell Platform: 
 * MissingH: 
 * pcre-light: 
+* utf8-string: 
 * `uuid`: 
   (or uuidgen from util-linux)
 * `xargs`: 

From e691727f24e8d42f2bd402f08e89d5f29af96cb0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 24 Mar 2011 16:25:44 +0000
Subject: [PATCH 1208/2835]

---
 doc/forum/Behaviour_of_fsck.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck.mdwn

diff --git a/doc/forum/Behaviour_of_fsck.mdwn b/doc/forum/Behaviour_of_fsck.mdwn
new file mode 100644
index 0000000000..29aa3ec645
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck.mdwn
@@ -0,0 +1,13 @@
+The current behaviour of 'fsck' is a bit verbose. I have an annex'd directory of tarballs for my own build system for "science" applications, there's about ~600 or blobs in my repo, I do occassionally like to run fsck across all my data to see what files don't meet the min num copies requirement that I have set.
+
+Would it be better for the default behaviour of fsck when it has not been given a path to only output errors and not bother to show that a file is ok for every single file in a repo. i.e.
+
+    git annex fsck
+
+should show only 'errors' and maybe a simple indicator showing the status (show a spinner or dots?) and when 
+
+    git annex fsck PATH/FILE 
+
+it should have the current behaviour? 
+
+Right now the current fsck behaviour might get annoying for anyone who would want to run fsck with repos with lots of big files.

From 0ed90ae99246c44a36c8956124c1c645cb845809 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 24 Mar 2011 17:45:08 +0000
Subject: [PATCH 1209/2835] Added a comment

---
 .../comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment b/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment
new file mode 100644
index 0000000000..dc48e2f943
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-24T17:45:08Z"
+ content="""
+I tend to agree that the default output of fsck is not quite right. I often use git annex fsck -q. A progress spinner display is a good idea.
+"""]]

From bc5f5428d39459228f61ec2e164bcbd00e110593 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 24 Mar 2011 18:38:38 +0000
Subject: [PATCH 1210/2835]

---
 doc/forum/Behaviour_of_fsck.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/Behaviour_of_fsck.mdwn b/doc/forum/Behaviour_of_fsck.mdwn
index 29aa3ec645..cd27d49f76 100644
--- a/doc/forum/Behaviour_of_fsck.mdwn
+++ b/doc/forum/Behaviour_of_fsck.mdwn
@@ -1,4 +1,4 @@
-The current behaviour of 'fsck' is a bit verbose. I have an annex'd directory of tarballs for my own build system for "science" applications, there's about ~600 or blobs in my repo, I do occassionally like to run fsck across all my data to see what files don't meet the min num copies requirement that I have set.
+The current behaviour of 'fsck' is a bit verbose. I have an annex'd directory of tarballs for my own build system for "science" applications, there's about ~600 or so blobs in my repo, I do occassionally like to run fsck across all my data to see what files don't meet the min num copies requirement that I have set.
 
 Would it be better for the default behaviour of fsck when it has not been given a path to only output errors and not bother to show that a file is ok for every single file in a repo. i.e.
 

From bc80ace96b2251262d1836c312f3924fe32e7cac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 25 Mar 2011 00:51:12 -0400
Subject: [PATCH 1211/2835] releasing version 0.20110325

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 9235c3ca33..c664566a9e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20110321) UNRELEASED; urgency=low
+git-annex (0.20110325) experimental; urgency=low
 
   * Free space checking is now done, for transfers of data for keys
     that have free space metadata. (Notably, not for SHA* keys generated
@@ -19,7 +19,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
     is already present in .git/annex/objects.
   * dropunused: Significantly sped up; only read unused log file once.
 
- -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
+ -- Joey Hess   Fri, 25 Mar 2011 00:47:37 -0400
 
 git-annex (0.20110320) experimental; urgency=low
 

From 25f842f58fbdf7e52b93e6e0a79cf3fed77dc78b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 25 Mar 2011 00:52:46 -0400
Subject: [PATCH 1212/2835] add news item for git-annex 0.20110325

---
 doc/news/version_0.20110325.mdwn | 20 ++++++++++++++++++++
 doc/news/version_0.23.mdwn       |  7 -------
 2 files changed, 20 insertions(+), 7 deletions(-)
 create mode 100644 doc/news/version_0.20110325.mdwn
 delete mode 100644 doc/news/version_0.23.mdwn

diff --git a/doc/news/version_0.20110325.mdwn b/doc/news/version_0.20110325.mdwn
new file mode 100644
index 0000000000..d5a59afe65
--- /dev/null
+++ b/doc/news/version_0.20110325.mdwn
@@ -0,0 +1,20 @@
+git-annex 0.20110325 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Free space checking is now done, for transfers of data for keys
+     that have free space metadata. (Notably, not for SHA* keys generated
+     with git-annex 0.2x or earlier.) The code is believed to work on
+     Linux, FreeBSD, and OSX; check compile-time messages to see if it
+     is not enabled for your OS.
+   * Add annex.diskreserve config setting, to control how much free space
+     to reserve for other purposes and avoid using (defaults to 1 mb).
+   * Add --fast flag, that can enable less expensive, but also less thorough
+     versions of some commands.
+   * fsck: In fast mode, avoid checking checksums.
+   * unused: In fast mode, just show all existing temp files as unused,
+     and avoid expensive scan for other unused content.
+   * migrate: Support migrating v1 SHA keys to v2 SHA keys with
+     size information that can be used for free space checking.
+   * Fix space leak in fsck and drop commands.
+   * migrate: Bugfix for case when migrating a file results in a key that
+     is already present in .git/annex/objects.
+   * dropunused: Significantly sped up; only read unused log file once."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.23.mdwn b/doc/news/version_0.23.mdwn
deleted file mode 100644
index e045352ad4..0000000000
--- a/doc/news/version_0.23.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 0.23 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Support ssh remotes with a port specified.
-   * whereis: New subcommand to show where a file's content has gotten to.
-   * Rethink filename encoding handling for display. Since filename encoding
-     may or may not match locale settings, any attempt to decode filenames
-     will fail for some files. So instead, do all output in binary mode."""]]
\ No newline at end of file

From 1bae07641eab449f74a411fe6f134dda83eaa42c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 26 Mar 2011 10:57:44 +0000
Subject: [PATCH 1213/2835] Added a comment

---
 ..._ead36a23c3e6efa1c41e4555f93e014e._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment b/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment
new file mode 100644
index 0000000000..357b48a234
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-26T10:57:41Z"
+ content="""
+After some thought, perhaps the default fsck output should be at least machine readable and copy and pasteable i.e.
+
+
+$ git annex fsck
+Files with errors
+
+    file1
+    file2
+
+

+
+so I can then copy the list of borked files and then just paste it into a for loop in my shell to recover the files. it's just an idea.
+"""]]

From ceb9593a9cbd39b00daf57ce52724eb40d85f1e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:24:29 -0400
Subject: [PATCH 1214/2835] added dat unit parsing

Also added all 3 existing kinds of data units. And even more of my opinions
to this opinionated piece of code.
---
 DataUnits.hs | 132 +++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 106 insertions(+), 26 deletions(-)

diff --git a/DataUnits.hs b/DataUnits.hs
index c2845affea..d8c8d31491 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -1,11 +1,27 @@
-{- data size display
+{- data size display and parsing
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module DataUnits (roughSize, compareSizes) where
+module DataUnits (
+	dataUnits,
+	storageUnits,
+	memoryUnits,
+	oldSchoolUnits,
+	roughSize,
+	compareSizes
+) where
+
+import Data.List
+import Data.Char
+
+type ByteSize = Integer
+type Name = String
+type Abbrev = String
+data Unit = Unit ByteSize Abbrev Name
+	deriving (Ord, Show, Eq)
 
 {- And now a rant: 
  -
@@ -28,7 +44,8 @@ module DataUnits (roughSize, compareSizes) where
  -
  - And the drive manufacturers happily continued selling drives that are
  - increasingly smaller than you'd expect, if you don't count on your
- - fingers. But that are increasingly bigger.
+ - fingers. But that are increasingly too big for anyone to much notice.
+ - This caused me to need git-annex.
  -
  - Thus, I use units here that I loathe. Because if I didn't, people would
  - be confused that their drives seem the wrong size, and other people would
@@ -36,36 +53,99 @@ module DataUnits (roughSize, compareSizes) where
  - progress?
  -}
 
-{- approximate display of a particular number of bytes -}
-roughSize :: Bool -> Integer -> String
-roughSize short i
-	| i < 0 = "-" ++ roughSize short (negate i)
-	| i >= at 8 = units 8 "yottabyte" "YB"
-	| i >= at 7 = units 7 "zettabyte" "ZB"
-	| i >= at 6 = units 6 "exabyte" "EB"
-	| i >= at 5 = units 5 "petabyte" "PB"
-	| i >= at 4 = units 4 "terabyte" "TB"
-	| i >= at 3 = units 3 "gigabyte" "GB"
-	| i >= at 2 = units 2 "megabyte" "MB"
-	| i >= at 1 = units 1 "kilobyte" "kB"
-	| otherwise = units 0 "byte" "B"
+dataUnits = storageUnits ++ memoryUnits
+
+{- Storage units are (stupidly) powers of ten. -}
+storageUnits :: [Unit]
+storageUnits =
+	[ Unit (p 8) "YB" "yottabyte"
+	, Unit (p 7) "ZB" "zettabyte"
+	, Unit (p 6) "EB" "exabyte"
+	, Unit (p 5) "PB" "petabyte"
+	, Unit (p 4) "TB" "terabyte"
+	, Unit (p 3) "GB" "gigabyte"
+	, Unit (p 2) "MB" "megabyte"
+	, Unit (p 1) "kB" "kilobyte" -- weird capitalization thanks to committe
+	, Unit (p 0) "B" "byte"
+	]
 	where
-		at :: Integer -> Integer
-		at n = 1000^n
+		p n = 1000^n
 
-		chop :: Integer -> Integer
-		chop d = round $ (fromInteger i :: Double) / fromInteger (at d)
+{- Memory units are (stupidly named) powers of 2. -}
+memoryUnits :: [Unit]
+memoryUnits =
+	[ Unit (p 8) "YiB" "yobibyte"
+	, Unit (p 7) "ZiB" "zebibyte"
+	, Unit (p 6) "EiB" "exbibyte"
+	, Unit (p 5) "PiB" "pebibyte"
+	, Unit (p 4) "TiB" "tebibyte"
+	, Unit (p 3) "GiB" "gigabyte"
+	, Unit (p 2) "MiB" "mebibyte"
+	, Unit (p 1) "kiB" "kibibyte"
+	, Unit (p 0) "B" "byte"
+	]
+	where
+		p n = 2^(n*10)
 
-		units d u u' = let num = chop d in
+{- Do you yearn for the days when men were men and megabytes were megabytes? -}
+oldSchoolUnits = map mingle $ zip storageUnits memoryUnits
+	where
+		mingle (Unit s a n, Unit s' a' n') = Unit s' a n
+
+{- approximate display of a particular number of bytes -}
+roughSize :: [Unit] -> Bool -> ByteSize -> String
+roughSize units abbrev i
+	| i < 0 = "-" ++ findUnit units' (negate i)
+	| otherwise = findUnit units' i
+	where
+		units' = reverse $ sort units -- largest first
+
+		findUnit (u@(Unit s _ _):us) i'
+			| i' >= s = showUnit i' u
+			| otherwise = findUnit us i'
+		findUnit [] i' = showUnit i' (last units') -- bytes
+
+		showUnit i' (Unit s a n) = let num = chop i' s in
 			show num ++ " " ++
-			(if short then u' else plural num u)
+			(if abbrev then a else plural num n)
+
+		chop :: Integer -> Integer -> Integer
+		chop i' d = round $ (fromInteger i' :: Double) / fromInteger d
 
 		plural n u
 			| n == 1 = u
 			| otherwise = u ++ "s"
 
-compareSizes :: Bool -> Integer -> Integer -> String
-compareSizes short old new
-	| old > new = roughSize short (old - new) ++ " smaller"
-	| old < new = roughSize short (new - old) ++ " larger"
+{- displays comparison of two sizes -}
+compareSizes :: [Unit] -> Bool -> ByteSize -> ByteSize -> String
+compareSizes units abbrev old new
+	| old > new = roughSize units abbrev (old - new) ++ " smaller"
+	| old < new = roughSize units abbrev (new - old) ++ " larger"
 	| otherwise = "same"
+
+{- Parses strings like "10 kilobytes" or "0.5tb". -}
+readSize :: String -> [Unit] -> Maybe ByteSize
+readSize s units
+	| null parsednum = Nothing
+	| null parsedunit = Nothing
+	| otherwise = Just $ round $ number * (fromIntegral multiplier)
+	where
+		(number, rest) = head parsednum
+		multiplier = head $ parsedunit
+
+		parsednum = reads s :: [(Double, String)]
+		parsedunit = lookupUnit units unit
+
+		unit = takeWhile isAlpha $ dropWhile isSpace rest
+
+		lookupUnit _ [] = [1] -- no unit given, assume bytes
+		lookupUnit [] _ = []
+		lookupUnit (u@(Unit s a n):us) v
+			| a ~~ v || n ~~ v = [s]
+			| plural n ~~ v || a ~~ byteabbrev v = [s]
+			| otherwise = lookupUnit us v
+		
+		a ~~ b = map toLower a == map toLower b
+		
+		plural n = n ++ "s"
+		byteabbrev a = a ++ "b"

From 8bcdf42b99675d507813205f097ab7b64b30f514 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:37:39 -0400
Subject: [PATCH 1215/2835] annex.diskreserve can be given in arbitrary units
 (ie "0.5 gigabytes")

---
 Backend.hs         |  2 +-
 Content.hs         |  8 +++++---
 DataUnits.hs       | 17 +++++++++++------
 debian/changelog   |  6 ++++++
 doc/git-annex.mdwn |  6 ++++--
 5 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 4b0095214c..c0e93acc23 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -198,6 +198,6 @@ checkKeySize key = do
 				else do
 					dest <- moveBad key
 					warning $ "Bad file size (" ++
-						compareSizes True size size' ++ 
+						compareSizes storageUnits True size size' ++ 
 						"); moved to " ++ dest
 					return False
diff --git a/Content.hs b/Content.hs
index 633d827e6f..7aa30f7ffc 100644
--- a/Content.hs
+++ b/Content.hs
@@ -122,7 +122,9 @@ checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- Annex.gitRepo
 	r <- Annex.repoConfig g "diskreserve" ""
-	let reserve = if null r then megabyte else (read r :: Integer)
+	let reserve = case readSize dataUnits r of
+		Nothing -> megabyte
+		Just v -> v
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
@@ -133,12 +135,12 @@ checkDiskSpace' adjustment key = do
 				else return ()
 	where
 		megabyte :: Integer
-		megabyte = 1024 * 1024
+		megabyte = 1000000
 		needmorespace n = do
 			force <- Annex.getState Annex.force
 			unless force $ 
 				error $ "not enough free space, need " ++ 
-					roughSize True n ++
+					roughSize storageUnits True n ++
 					" more (use --force to override this check or adjust annex.diskreserve)"
 
 {- Removes the write bits from a file. -}
diff --git a/DataUnits.hs b/DataUnits.hs
index d8c8d31491..37b0fa4295 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -11,7 +11,8 @@ module DataUnits (
 	memoryUnits,
 	oldSchoolUnits,
 	roughSize,
-	compareSizes
+	compareSizes,
+	readSize
 ) where
 
 import Data.List
@@ -53,6 +54,7 @@ data Unit = Unit ByteSize Abbrev Name
  - progress?
  -}
 
+dataUnits :: [Unit]
 dataUnits = storageUnits ++ memoryUnits
 
 {- Storage units are (stupidly) powers of ten. -}
@@ -69,6 +71,7 @@ storageUnits =
 	, Unit (p 0) "B" "byte"
 	]
 	where
+		p :: Integer -> Integer
 		p n = 1000^n
 
 {- Memory units are (stupidly named) powers of 2. -}
@@ -85,12 +88,14 @@ memoryUnits =
 	, Unit (p 0) "B" "byte"
 	]
 	where
+		p :: Integer -> Integer
 		p n = 2^(n*10)
 
 {- Do you yearn for the days when men were men and megabytes were megabytes? -}
+oldSchoolUnits :: [Unit]
 oldSchoolUnits = map mingle $ zip storageUnits memoryUnits
 	where
-		mingle (Unit s a n, Unit s' a' n') = Unit s' a n
+		mingle (Unit _ a n, Unit s' _ _) = Unit s' a n
 
 {- approximate display of a particular number of bytes -}
 roughSize :: [Unit] -> Bool -> ByteSize -> String
@@ -124,8 +129,8 @@ compareSizes units abbrev old new
 	| otherwise = "same"
 
 {- Parses strings like "10 kilobytes" or "0.5tb". -}
-readSize :: String -> [Unit] -> Maybe ByteSize
-readSize s units
+readSize :: [Unit] -> String -> Maybe ByteSize
+readSize units input
 	| null parsednum = Nothing
 	| null parsedunit = Nothing
 	| otherwise = Just $ round $ number * (fromIntegral multiplier)
@@ -133,14 +138,14 @@ readSize s units
 		(number, rest) = head parsednum
 		multiplier = head $ parsedunit
 
-		parsednum = reads s :: [(Double, String)]
+		parsednum = reads input :: [(Double, String)]
 		parsedunit = lookupUnit units unit
 
 		unit = takeWhile isAlpha $ dropWhile isSpace rest
 
 		lookupUnit _ [] = [1] -- no unit given, assume bytes
 		lookupUnit [] _ = []
-		lookupUnit (u@(Unit s a n):us) v
+		lookupUnit (Unit s a n:us) v
 			| a ~~ v || n ~~ v = [s]
 			| plural n ~~ v || a ~~ byteabbrev v = [s]
 			| otherwise = lookupUnit us v
diff --git a/debian/changelog b/debian/changelog
index c664566a9e..e9bc896bf0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.20110326) UNRELEASED; urgency=low
+
+  * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
+
+ -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
+
 git-annex (0.20110325) experimental; urgency=low
 
   * Free space checking is now done, for transfers of data for keys
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 81cea04cd7..32f190e75d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -387,8 +387,10 @@ Here are all the supported configuration settings.
   Amount of disk space to reserve. Disk space is checked when transferring
   content to avoid running out, and additional free space can be reserved
   via this option, to make space for more important content (such as git
-  commit logs). The units are bytes.
-  The default reserve is 1048576 (1 megabyte).
+  commit logs). Can be specified with any commonly used units, for example,
+  "0.5 gb" or "100 KiloBytes"
+
+  The default reserve is 1 megabyte.
 
 * `annex.version`
 

From a5c1c2770e967ec16d64d015fff9ab3eecb314df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:47:55 -0400
Subject: [PATCH 1216/2835] add more pointless opinion

---
 DataUnits.hs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/DataUnits.hs b/DataUnits.hs
index 37b0fa4295..fb8ce3d89c 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -91,6 +91,10 @@ memoryUnits =
 		p :: Integer -> Integer
 		p n = 2^(n*10)
 
+{- Bandwidth units are only measured in bits if you're some crazy telco. -}
+bandwidthUnits :: [Unit]
+bandwidthUnits = error "stop trying to rip people off"
+
 {- Do you yearn for the days when men were men and megabytes were megabytes? -}
 oldSchoolUnits :: [Unit]
 oldSchoolUnits = map mingle $ zip storageUnits memoryUnits

From 70fe1ee70b53fe5df766abcf38646f70dcea6c75 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:49:23 -0400
Subject: [PATCH 1217/2835] update

---
 DataUnits.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/DataUnits.hs b/DataUnits.hs
index fb8ce3d89c..fa706ef068 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -9,6 +9,7 @@ module DataUnits (
 	dataUnits,
 	storageUnits,
 	memoryUnits,
+	bandwidthUnits,
 	oldSchoolUnits,
 	roughSize,
 	compareSizes,

From fd8cba164c180149127656070ac4a68bbe13b9cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:54:11 -0400
Subject: [PATCH 1218/2835] thanks liw

---
 DataUnits.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DataUnits.hs b/DataUnits.hs
index fa706ef068..a175b5f6aa 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -85,7 +85,7 @@ memoryUnits =
 	, Unit (p 4) "TiB" "tebibyte"
 	, Unit (p 3) "GiB" "gigabyte"
 	, Unit (p 2) "MiB" "mebibyte"
-	, Unit (p 1) "kiB" "kibibyte"
+	, Unit (p 1) "KiB" "kibibyte"
 	, Unit (p 0) "B" "byte"
 	]
 	where

From 849c183f23a68d6204d5aacd0b9750374a5f59b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 15:10:12 -0400
Subject: [PATCH 1219/2835] tweak

---
 DataUnits.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/DataUnits.hs b/DataUnits.hs
index a175b5f6aa..c81c6e42e5 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -11,6 +11,7 @@ module DataUnits (
 	memoryUnits,
 	bandwidthUnits,
 	oldSchoolUnits,
+
 	roughSize,
 	compareSizes,
 	readSize
@@ -142,11 +143,10 @@ readSize units input
 	where
 		(number, rest) = head parsednum
 		multiplier = head $ parsedunit
+		unitname = takeWhile isAlpha $ dropWhile isSpace rest
 
 		parsednum = reads input :: [(Double, String)]
-		parsedunit = lookupUnit units unit
-
-		unit = takeWhile isAlpha $ dropWhile isSpace rest
+		parsedunit = lookupUnit units unitname
 
 		lookupUnit _ [] = [1] -- no unit given, assume bytes
 		lookupUnit [] _ = []

From 433c1780cff5673591be94deae02e775f804ab7c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 18:26:36 -0400
Subject: [PATCH 1220/2835] more thoughts

---
 Remotes.hs       |  1 -
 doc/todo/S3.mdwn | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/Remotes.hs b/Remotes.hs
index 8b760ac957..5a65e4fc7d 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -7,7 +7,6 @@
 
 module Remotes (
 	list,
-	tryGitConfigRead,
 	readConfigs,
 	keyPossibilities,
 	inAnnex,
diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 56023e71e1..946fa68170 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -71,3 +71,21 @@ Implementing this might not be as conceptually nice as making S3 a separate
 backend. It would need some changes to the remotes code, perhaps lifting
 some of it into backend-specific hooks. Then the S3 backend could be
 implicitly stacked in front of a backend like WORM.
+
+---
+
+Maybe the right way to look at this is that a list of Stores
+should be a property of the Backend. Backend.File is a Backend, that
+uses various Stores, which can be of different types (the local
+git repo, remote git repos, S3, etc). Backend.URL is a backend that uses
+other Stores (the local git repo, and the web).
+
+Operations on Stores are:
+
+* uuid -- each store has a unique uuid value
+* cost -- each store has a use cost value
+* getConfig -- attempts to look up values (uuid, possibly cost)
+* copyToStore -- store a file's contents to a key
+* copyFromStore -- retrieve a key's contents to a file
+* removeFromStore -- removes a key's contents from the store
+* hasKey -- checks if the key's content is available

From 2f32b7afc0ba361a4b92b74357115284634af529 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 10:21:18 +0000
Subject: [PATCH 1221/2835]

---
 ...has_issues_with_git_when_staging__47__commiting_logs.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
new file mode 100644
index 0000000000..89b6f9bee2
--- /dev/null
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -0,0 +1,5 @@
+After a series of pretty convoluted copying files around between annex'd repos and pulling changes around between repos. I noticed that occassionally when git-annex tries to stage files (the `.git-annex/*/*/*logs`) git some times gets wedged and doing a "git commit -a" doesn't seem to work or files might not get added thus leaving a bunch of untracked files or modified files that aren't staged for a commit.
+
+I tried running a *git add -u .git-annex/* and also just the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.
+
+For now it's just a bit of extra work for me when it does occur but it does not seem to occur often.

From 07d15ab2eae88059f2a90e1b389e5ce34deeafe6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 10:33:08 +0000
Subject: [PATCH 1222/2835]

---
 ...ex_has_issues_with_git_when_staging__47__commiting_logs.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 89b6f9bee2..d4e2635f56 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -1,5 +1,5 @@
 After a series of pretty convoluted copying files around between annex'd repos and pulling changes around between repos. I noticed that occassionally when git-annex tries to stage files (the `.git-annex/*/*/*logs`) git some times gets wedged and doing a "git commit -a" doesn't seem to work or files might not get added thus leaving a bunch of untracked files or modified files that aren't staged for a commit.
 
-I tried running a *git add -u .git-annex/* and also just the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.
+I tried running a *`git rm --cached -f -r *`* then *git add -u .git-annex/* or the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.
 
 For now it's just a bit of extra work for me when it does occur but it does not seem to occur often.

From ca5a6f4f42237136550347539f7d51376beef997 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 12:58:15 +0000
Subject: [PATCH 1223/2835]

---
 ...git-annex_directory_hashing_problems_on_osx.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
new file mode 100644
index 0000000000..9320b30a99
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -0,0 +1,13 @@
+Currently the hashed directories in .git-annex allow for upper and lower case directory names... on linux (or any case sensitive filesystem) the directory names such as 'Gg' and 'GG' are different and unique. However on systems like OSX (and probably windows if it is ever supported) the directory names 'Gg' is the same as 'GG'
+
+In one of the annex'd repos that I have this has occured...
+
+
+$ git add -i                                                                                          
+           staged     unstaged path
+  1:    unchanged        +1/-1 .git-annex/GM/GV/WORM-s183630166-m1301072171--somefile.log
+  2:    unchanged        +1/-1 .git-annex/Gm/GV/WORM-s183630166-m1301072171--somefile.log
+

+
+
+this has somewhat confused git when it tries to stage/merge files, I didn't notice this at first, but it is definately a problem for someone using case insensitive filesystems  like the default OSX HFS+ formats or vfat/fat32.

From 3aaf1fcc549791c19688a8d91cacc7c037506de5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 13:02:00 +0000
Subject: [PATCH 1224/2835]

---
 doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index 9320b30a99..b11cfca427 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -11,3 +11,5 @@ $ git add -i
 
 
 this has somewhat confused git when it tries to stage/merge files, I didn't notice this at first, but it is definately a problem for someone using case insensitive filesystems  like the default OSX HFS+ formats or vfat/fat32.
+
+Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.

From 3d84ec1bda71771c5d04c4c8ade37977eecf5a65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 12:41:53 -0400
Subject: [PATCH 1225/2835] response

---
 ...annex_directory_hashing_problems_on_osx.mdwn | 17 +++++++++++++++++
 ...th_git_when_staging__47__commiting_logs.mdwn |  6 ++++++
 2 files changed, 23 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index b11cfca427..d1a3c06077 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -12,4 +12,21 @@ $ git add -i
 
 this has somewhat confused git when it tries to stage/merge files, I didn't notice this at first, but it is definately a problem for someone using case insensitive filesystems  like the default OSX HFS+ formats or vfat/fat32.
 
+> I feel a bit stupid to not have considered case-insensative filesystems.
+> They are just so far from where I have lived for 20 years that it's hard
+> to keep them in mind.
+> 
+> I guess that
+> [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]] is
+> somehow a consequence (or cause?) of this, but I don't quite understand
+> how this is causing git to fail to stage files, or stage the same file
+> twice under different capitalizations. git-annex always will run git add
+> on the path with the "correct" capitalization. So unless something else
+> has added the path with the other capitalization (perhaps git add
+> .git-annex manually?) I don't understand how you get to this state.
+> --[[Joey]]
+
 Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.
+
+> There is a guard against `git annex add .git-annex/foo`, but it doesn't
+> notice `cd .git-annex; git annex add foo`. --[[Joey]]
diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index d4e2635f56..5c654ff2ba 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -3,3 +3,9 @@ After a series of pretty convoluted copying files around between annex'd repos a
 I tried running a *`git rm --cached -f -r *`* then *git add -u .git-annex/* or the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.
 
 For now it's just a bit of extra work for me when it does occur but it does not seem to occur often.
+
+> What do you mean when you say that git "got wedged"? It hung somehow?
+>
+> If git-annex runs concurrently with another git command that locks
+> the repository its git add of log files can fail.
+> --[[Joey]] 

From 5daf33d557291f1467a995f6c26f193e9323dd95 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 16:49:10 +0000
Subject: [PATCH 1226/2835]

---
 ...ex_has_issues_with_git_when_staging__47__commiting_logs.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 5c654ff2ba..4b61950dfe 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -9,3 +9,5 @@ For now it's just a bit of extra work for me when it does occur but it does not
 > If git-annex runs concurrently with another git command that locks
 > the repository its git add of log files can fail.
 > --[[Joey]] 
+
+>> It "got wedged" as in git doesn't let me commit anything, even though it tells me that there is stuff to be committed in the staging area.

From bb7688233bd10773ab75eba7135497ae10af9061 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 16:53:51 +0000
Subject: [PATCH 1227/2835]

---
 doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index d1a3c06077..2315bbb83f 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -26,6 +26,8 @@ this has somewhat confused git when it tries to stage/merge files, I didn't noti
 > .git-annex manually?) I don't understand how you get to this state.
 > --[[Joey]]
 
+>> I think I got myself into this situation when I copied some files over from a HFS+ partition to a GPFS network share (which is pretty posix compliant) over samba. It probably is related to the [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]]. I thought they were unique enough to have two bug reports logged as one is a git behavioural thing and the other is git-annex specific.
+
 Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.
 
 > There is a guard against `git annex add .git-annex/foo`, but it doesn't

From 41d660c88f60e982bf3f954d70b074ad1dd0c13a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 13:15:25 -0400
Subject: [PATCH 1228/2835] response

---
 ...x_has_issues_with_git_when_staging__47__commiting_logs.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 4b61950dfe..f1290a8186 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -11,3 +11,6 @@ For now it's just a bit of extra work for me when it does occur but it does not
 > --[[Joey]] 
 
 >> It "got wedged" as in git doesn't let me commit anything, even though it tells me that there is stuff to be committed in the staging area.
+
+>>> I've never seen git refuse to commit staged files. There would have to
+>>> be some error message? --[[Joey]] 

From 2821effce9ae95a2ef12a083ce0806fe058ac987 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 13:30:18 -0400
Subject: [PATCH 1229/2835] response

---
 ...nex_directory_hashing_problems_on_osx.mdwn | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index 2315bbb83f..2e6b138c16 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -28,6 +28,29 @@ this has somewhat confused git when it tries to stage/merge files, I didn't noti
 
 >> I think I got myself into this situation when I copied some files over from a HFS+ partition to a GPFS network share (which is pretty posix compliant) over samba. It probably is related to the [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]]. I thought they were unique enough to have two bug reports logged as one is a git behavioural thing and the other is git-annex specific.
 
+>>> If you copied `.git/` over, perhaps you got a git repo without
+>>> core.ignorecase set right for the filesystem it landed on?
+>>> 
+>>> Something like this might reproduce it:
+
+
+# mkdir test; cd test; git init
+# git config core.ignorecase false
+# mkdir Foo
+# touch Foo/bar
+# git add Foo/bar
+# git add foo/bar
+# git add fOo/bar
+# git status
+# touch foo/other
+# git add fOo/other
+# git status
+

+
+>>>> And then either git commit or git clone would probably get confused
+>>>> if it thought 3 distinct files had been committed.
+>>>> --[[Joey]]
+
 Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.
 
 > There is a guard against `git annex add .git-annex/foo`, but it doesn't

From 3ded849532216d97ac6004146445fbcc85769054 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 17:37:07 +0000
Subject: [PATCH 1230/2835]

---
 ...ex_has_issues_with_git_when_staging__47__commiting_logs.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index f1290a8186..b7944b418c 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -14,3 +14,5 @@ For now it's just a bit of extra work for me when it does occur but it does not
 
 >>> I've never seen git refuse to commit staged files. There would have to
 >>> be some error message? --[[Joey]] 
+
+>>>> there were no error messages at all

From 1fccea5154ac5b8e10fa387e2230e3a4103cef3b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 17:48:51 +0000
Subject: [PATCH 1231/2835]

---
 ...nex_directory_hashing_problems_on_osx.mdwn | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index 2e6b138c16..e5937fe83f 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -30,6 +30,9 @@ this has somewhat confused git when it tries to stage/merge files, I didn't noti
 
 >>> If you copied `.git/` over, perhaps you got a git repo without
 >>> core.ignorecase set right for the filesystem it landed on?
+
+>>>> I usually git clone or do a fresh repository and pull things in, I was also unaware of this ignorecase setting as well.
+
 >>> 
 >>> Something like this might reproduce it:
 
@@ -51,6 +54,47 @@ this has somewhat confused git when it tries to stage/merge files, I didn't noti
 >>>> if it thought 3 distinct files had been committed.
 >>>> --[[Joey]]
 
+>>>>> Doing the above test on a HFS+ partition yields this
+
+
+## with ignorecase=false
+commit bb024c6fd7482b2d10f60ae899cb7a949aca1ad8
+Author: Jimmy Tang 
+Date:   Sun Mar 27 18:40:24 2011 +0100
+
+    commit
+
+diff --git a/Foo/bar b/Foo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/fOo/bar b/fOo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/fOo/other b/fOo/other
+new file mode 100644
+index 0000000..e69de29
+diff --git a/foo/bar b/foo/bar
+new file mode 100644
+index 0000000..e69de29
+

+
+>>>>> and without changing ignorecase
+
+
+commit 909a089158ffb98f8e91f98905e2bfdc7234666f
+Author: Jimmy Tang 
+Date:   Sun Mar 27 18:46:57 2011 +0100
+
+    commit
+
+diff --git a/Foo/bar b/Foo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/Foo/other b/Foo/other
+new file mode 100644
+index 0000000..e69de29
+

+
 Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.
 
 > There is a guard against `git annex add .git-annex/foo`, but it doesn't

From b40f253d6e126d699e9f298bf670fc5e875bfd86 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 15:56:43 -0400
Subject: [PATCH 1232/2835] start of generalizing remotes

Goal is to support multiple different types of remotes, some of which
are not git repositories. To that end, added a Remote class, and moved
git remote specific code into Remote.GitRemote.

Remotes.hs is still present as some code has not been converted to use the
new Remote class yet.
---
 Backend/File.hs     |  42 +++----
 Remote.hs           |  66 +++++++++++
 Remote/GitRemote.hs | 263 ++++++++++++++++++++++++++++++++++++++++++++
 RemoteClass.hs      |  46 ++++++++
 Remotes.hs          |   9 ++
 UUID.hs             |  18 ---
 6 files changed, 405 insertions(+), 39 deletions(-)
 create mode 100644 Remote.hs
 create mode 100644 Remote/GitRemote.hs
 create mode 100644 RemoteClass.hs

diff --git a/Backend/File.hs b/Backend/File.hs
index fb8a052553..743d8d6278 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -14,14 +14,14 @@
 
 module Backend.File (backend, checkKey) where
 
-import Control.Monad.State
-import System.Directory
+import Control.Monad.State (liftIO)
 import Data.List
+import Data.String.Utils
 
 import BackendClass
 import LocationLog
-import Locations
-import qualified Remotes
+import qualified Remote
+import qualified RemoteClass
 import qualified GitRepo as Git
 import Content
 import qualified Annex
@@ -51,10 +51,10 @@ dummyStore :: FilePath -> Key -> Annex Bool
 dummyStore _ _ = return True
 
 {- Try to find a copy of the file in one of the remotes,
- - and copy it over to this one. -}
+ - and copy it to here. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(remotes, _) <- Remotes.keyPossibilities key
+	(remotes, _) <- Remote.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -72,18 +72,18 @@ copyKeyFile key file = do
 				then docopy r (trycopy full rs)
 				else trycopy full rs
 		-- This check is to avoid an ugly message if a remote is a
-		-- drive that is not mounted. Avoid checking inAnnex for ssh
-		-- remotes because that is unnecessarily slow, and the
-		-- locationlog should be trusted. (If the ssh remote is down
-		-- or really lacks the file, it's ok to show an ugly message
-		-- before going on to the next remote.)
+		-- drive that is not mounted.
 		probablyPresent r =
-			if not $ Git.repoIsUrl r
-				then liftIO $ doesFileExist $ gitAnnexLocation r key
+			if RemoteClass.hasKeyCheap r
+				then do
+					res <- RemoteClass.hasKey r key
+					case res of
+						Right b -> return b
+						Left _ -> return False
 				else return True
 		docopy r continue = do
-			showNote $ "copying from " ++ Git.repoDescribe r ++ "..."
-			copied <- Remotes.copyFromRemote r key file
+			showNote $ "copying from " ++ RemoteClass.name r ++ "..."
+			copied <- RemoteClass.retrieveKeyFile r key file
 			if copied
 				then return True
 				else continue
@@ -97,9 +97,9 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- Remotes.keyPossibilities key
+			(remotes, trusteduuids) <- Remote.keyPossibilities key
 			untrusteduuids <- trustGet UnTrusted
-			tocheck <- reposWithoutUUID remotes (trusteduuids++untrusteduuids)
+			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
 			findcopies numcopies trusteduuids tocheck []
 	where
@@ -109,9 +109,9 @@ checkRemoveKey key numcopiesM = do
 		findcopies need have (r:rs) bad
 			| length have >= need = return True
 			| otherwise = do
-				u <- getUUID r
+				let u = RemoteClass.uuid r
 				let dup = u `elem` have
-				haskey <- Remotes.inAnnex r key
+				haskey <- (RemoteClass.hasKey r) key
 				case (dup, haskey) of
 					(False, Right True)	-> findcopies need (u:have) rs bad
 					(False, Left _)		-> findcopies need have rs (r:bad)
@@ -147,11 +147,11 @@ showLocations key exclude = do
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
 		message rs us = message rs [] ++ message [] us
 
-showTriedRemotes :: [Git.Repo] -> Annex ()
+showTriedRemotes :: [RemoteClass.Remote] -> Annex ()
 showTriedRemotes [] = return ()	
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
-		Remotes.list remotes
+		(join ", " $ map RemoteClass.name remotes)
 
 getNumCopies :: Maybe Int -> Annex Int
 getNumCopies (Just n) = return n
diff --git a/Remote.hs b/Remote.hs
new file mode 100644
index 0000000000..9eff5556c9
--- /dev/null
+++ b/Remote.hs
@@ -0,0 +1,66 @@
+{- git-annex remotes
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote (
+	generate,
+	keyPossibilities,
+	remotesWithUUID,
+	remotesWithoutUUID
+) where
+
+import Control.Monad.State (liftIO)
+import Data.List
+
+import RemoteClass
+import qualified Remote.GitRemote
+import Types
+import UUID
+import qualified Annex
+import Trust
+import LocationLog
+
+{- add generators for new Remotes here -}
+generators :: [Annex [Remote]]
+generators = [Remote.GitRemote.generate]
+
+{- generates a list of all available Remotes -}
+generate :: Annex [Remote]
+generate = do
+	lists <- sequence generators
+	return $ concat lists
+
+{- Filters a list of remotes to ones that have the listed uuids. -}
+remotesWithUUID :: [Remote] -> [UUID] -> [Remote]
+remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
+
+{- Filters a list of remotes to ones that do not have the listed uuids. -}
+remotesWithoutUUID :: [Remote] -> [UUID] -> [Remote]
+remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
+
+{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+ -
+ - Also returns a list of UUIDs that are trusted to have the key
+ - (some may not have configured remotes).
+ -}
+keyPossibilities :: Key -> Annex ([Remote], [UUID])
+keyPossibilities key = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+	trusted <- trustGet Trusted
+
+	-- get uuids of all remotes that are recorded to have the key
+	uuids <- liftIO $ keyLocations g key
+	let validuuids = filter (/= u) uuids
+
+	-- note that validuuids is assumed to not have dups
+	let validtrusteduuids = intersect validuuids trusted
+
+	-- remotes that match uuids that have the key
+	allremotes <- generate
+	let validremotes = remotesWithUUID allremotes validuuids
+
+	return (sort validremotes, validtrusteduuids)
diff --git a/Remote/GitRemote.hs b/Remote/GitRemote.hs
new file mode 100644
index 0000000000..ccc5f7b42f
--- /dev/null
+++ b/Remote/GitRemote.hs
@@ -0,0 +1,263 @@
+{- Standard git remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.GitRemote (generate) where
+
+import Control.Exception.Extensible
+import Control.Monad.State (liftIO)
+import qualified Data.Map as Map
+import Data.String.Utils
+import System.Cmd.Utils
+import Control.Monad (unless, filterM)
+
+import RemoteClass
+import Types
+import qualified GitRepo as Git
+import qualified Annex
+import Locations
+import UUID
+import Utility
+import qualified Content
+import Messages
+import CopyFile
+import RsyncFile
+import Ssh
+
+generate :: Annex [Remote]
+generate = do
+	readConfigs
+	g <- Annex.gitRepo
+	rs <- filterM repoNotIgnored (Git.remotes g)
+	mapM genRemote rs
+	
+genRemote :: Git.Repo -> Annex Remote
+genRemote r = do
+	u <- getUUID r
+	c <- repoCost r
+	return Remote {
+		uuid = u,
+		cost = c,
+		name = Git.repoDescribe r,
+		storeKey = copyToRemote r,
+		retrieveKeyFile = copyFromRemote r,
+		removeKey = error "TODO Remote.GitRemote.removeKey",
+		hasKey = inAnnex r,
+		hasKeyCheap = not (Git.repoIsUrl r)
+	}
+
+{- Reads the configs of all remotes.
+ -
+ - As reading the config of remotes can be expensive, this
+ - function will only read configs once per git-annex run. It's
+ - assumed to be cheap to read the config of non-URL remotes,
+ - so this is done each time git-annex is run. Conversely,
+ - the config of an URL remote is only read when there is no
+ - cached UUID value.
+ - -}
+readConfigs :: Annex ()
+readConfigs = do
+	remotesread <- Annex.getState Annex.remotesread
+	unless remotesread $ do
+		g <- Annex.gitRepo
+		allremotes <- filterM repoNotIgnored $ Git.remotes g
+		let cheap = filter (not . Git.repoIsUrl) allremotes
+		let expensive = filter Git.repoIsUrl allremotes
+		doexpensive <- filterM cachedUUID expensive
+		unless (null doexpensive) $
+			showNote $ "getting UUID for " ++
+				list doexpensive ++ "..."
+		let todo = cheap ++ doexpensive
+		unless (null todo) $ do
+			mapM_ tryGitConfigRead todo
+			Annex.changeState $ \s -> s { Annex.remotesread = True }
+	where
+		cachedUUID r = do
+			u <- getUUID r
+			return $ null u
+
+{- The git configs for the git repo's remotes is not read on startup
+ - because reading it may be expensive. This function tries to read the
+ - config for a specified remote, and updates state. If successful, it
+ - returns the updated git repo. -}
+tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo)
+tryGitConfigRead r 
+	| not $ Map.null $ Git.configMap r = return $ Right r -- already read
+	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
+	| Git.repoIsUrl r = return $ Left r
+	| otherwise = store $ safely $ Git.configRead r
+	where
+		-- Reading config can fail due to IO error or
+		-- for other reasons; catch all possible exceptions.
+		safely a = do
+			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
+			case result of
+				Left _ -> return r
+				Right r' -> return r'
+		pipedconfig cmd params = safely $
+			pOpen ReadFromPipe cmd (toCommand params) $
+				Git.hConfigRead r
+		store a = do
+			r' <- a
+			g <- Annex.gitRepo
+			let l = Git.remotes g
+			let g' = Git.remotesAdd g $ exchange l r'
+			Annex.changeState $ \s -> s { Annex.repo = g' }
+			return $ Right r'
+		exchange [] _ = []
+		exchange (old:ls) new =
+			if Git.repoRemoteName old == Git.repoRemoteName new
+				then new : exchange ls new
+				else old : exchange ls new
+
+{- Calculates cost for a repo.
+ -
+ - The default cost is 100 for local repositories, and 200 for remote
+ - repositories; it can also be configured by remote..annex-cost
+ -}
+repoCost :: Git.Repo -> Annex Int
+repoCost r = do
+	c <- Annex.repoConfig r "cost" ""
+	if not $ null c
+		then return $ read c
+		else if Git.repoIsUrl r
+			then return 200
+			else return 100
+
+{- Checks if a repo should be ignored, based either on annex-ignore
+ - setting, or on command-line options. Allows command-line to override
+ - annex-ignore. -}
+repoNotIgnored :: Git.Repo -> Annex Bool
+repoNotIgnored r = do
+	ignored <- Annex.repoConfig r "ignore" "false"
+	to <- match Annex.toremote
+	from <- match Annex.fromremote
+	if to || from
+		then return True
+		else return $ not $ Git.configTrue ignored
+	where
+		match a = do
+			n <- Annex.getState a
+			return $ n == Git.repoRemoteName r
+
+{- Checks if a given remote has the content for a key inAnnex.
+ - If the remote cannot be accessed, returns a Left error.
+ -}
+inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
+inAnnex r key = if Git.repoIsUrl r
+		then checkremote
+		else liftIO (try checklocal ::IO (Either IOException Bool))
+	where
+		checklocal = do
+			-- run a local check inexpensively,
+			-- by making an Annex monad using the remote
+			a <- Annex.new r []
+			Annex.eval a (Content.inAnnex key)
+		checkremote = do
+			showNote ("checking " ++ Git.repoDescribe r ++ "...")
+			inannex <- onRemote r (boolSystem, False) "inannex" 
+				[Param (show key)]
+			return $ Right inannex
+
+{- Tries to copy a key's content from a remote's annex to a file. -}
+copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
+copyFromRemote r key file
+	| not $ Git.repoIsUrl r = liftIO $ copyFile (gitAnnexLocation r key) file
+	| Git.repoIsSsh r = rsynchelper r True key file
+	| otherwise = error "copying from non-ssh repo not supported"
+
+{- Tries to copy a key's content to a remote's annex. -}
+copyToRemote :: Git.Repo -> Key -> Annex Bool
+copyToRemote r key
+	| not $ Git.repoIsUrl r = do
+		g <- Annex.gitRepo
+		let keysrc = gitAnnexLocation g key
+		-- run copy from perspective of remote
+		liftIO $ do
+			a <- Annex.new r []
+			Annex.eval a $ do
+				ok <- Content.getViaTmp key $
+					\f -> liftIO $ copyFile keysrc f
+				Annex.queueRun
+				return ok
+	| Git.repoIsSsh r = do
+		g <- Annex.gitRepo
+		let keysrc = gitAnnexLocation g key
+		rsynchelper r False key keysrc
+	| otherwise = error "copying to non-ssh repo not supported"
+
+rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool)
+rsynchelper r sending key file = do
+	showProgress -- make way for progress bar
+	p <- rsyncParams r sending key file
+	res <- liftIO $ boolSystem "rsync" p
+	if res
+		then return res
+		else do
+			showLongNote "rsync failed -- run git annex again to resume file transfer"
+			return res
+
+{- Generates rsync parameters that ssh to the remote and asks it
+ - to either receive or send the key's content. -}
+rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]
+rsyncParams r sending key file = do
+	Just (shellcmd, shellparams) <- git_annex_shell r
+		(if sending then "sendkey" else "recvkey")
+		[ Param $ show key
+		-- Command is terminated with "--", because
+		-- rsync will tack on its own options afterwards,
+		-- and they need to be ignored.
+		, Param "--"
+		]
+	-- Convert the ssh command into rsync command line.
+	let eparam = rsyncShell (Param shellcmd:shellparams)
+	o <- Annex.repoConfig r "rsync-options" ""
+	let base = options ++ map Param (words o) ++ eparam
+	if sending
+		then return $ base ++ [dummy, File file]
+		else return $ base ++ [File file, dummy]
+	where
+		-- inplace makes rsync resume partial files
+		options = [Params "-p --progress --inplace"]
+		-- the rsync shell parameter controls where rsync
+		-- goes, so the source/dest parameter can be a dummy value,
+		-- that just enables remote rsync mode.
+		dummy = Param ":"
+
+{- Uses a supplied function to run a git-annex-shell command on a remote.
+ -
+ - Or, if the remote does not support running remote commands, returns
+ - a specified error value. -}
+onRemote 
+	:: Git.Repo
+	-> (FilePath -> [CommandParam] -> IO a, a)
+	-> String
+	-> [CommandParam]
+	-> Annex a
+onRemote r (with, errorval) command params = do
+	s <- git_annex_shell r command params
+	case s of
+		Just (c, ps) -> liftIO $ with c ps
+		Nothing -> return errorval
+
+{- Generates parameters to run a git-annex-shell command on a remote. -}
+git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam]))
+git_annex_shell r command params
+	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)
+	| Git.repoIsSsh r = do
+		sshparams <- sshToRepo r [Param sshcmd]
+		return $ Just ("ssh", sshparams)
+	| otherwise = return Nothing
+	where
+		dir = Git.workTree r
+		shellcmd = "git-annex-shell"
+		shellopts = (Param command):(File dir):params
+		sshcmd = shellcmd ++ " " ++ 
+			unwords (map shellEscape $ toCommand shellopts)
+
+{- Human visible list of remotes. -}
+list :: [Git.Repo] -> String
+list remotes = join ", " $ map Git.repoDescribe remotes
diff --git a/RemoteClass.hs b/RemoteClass.hs
new file mode 100644
index 0000000000..df2aefb715
--- /dev/null
+++ b/RemoteClass.hs
@@ -0,0 +1,46 @@
+{- git-annex remotes class
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module RemoteClass where
+
+import Control.Exception
+
+import Annex
+import UUID
+import Key
+
+data Remote = Remote {
+	-- each Remote has a unique uuid
+	uuid :: UUID,
+	-- each Remote has a human visible name
+	name :: String,
+	-- Remotes have a use cost; higher is more expensive
+	cost :: Int,
+	-- Transfers a key to the remote.
+	storeKey :: Key -> Annex Bool,
+	-- retrieves a key's contents to a file
+	retrieveKeyFile :: Key -> FilePath -> Annex Bool,
+	-- removes a key's contents
+	removeKey :: Key -> Annex Bool,
+	-- Checks if a key is present in the remote; if the remote
+	-- cannot be accessed returns a Left error.
+	hasKey :: Key -> Annex (Either IOException Bool),
+	-- Some remotes can check hasKey without an expensive network
+	-- operation.
+	hasKeyCheap :: Bool
+}
+
+instance Show Remote where
+	show remote = "Remote { uuid =\"" ++ uuid remote ++ "\" }"
+
+-- two remotes are the same if they have the same uuid
+instance Eq Remote where
+	a == b = uuid a == uuid b
+
+-- order remotes by cost
+instance Ord Remote where
+	compare a b = compare (cost a) (cost b)
diff --git a/Remotes.hs b/Remotes.hs
index 5a65e4fc7d..5fc594ee29 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -322,3 +322,12 @@ git_annex_shell r command params
 		shellopts = (Param command):(File dir):params
 		sshcmd = shellcmd ++ " " ++ 
 			unwords (map shellEscape $ toCommand shellopts)
+
+{- Filters a list of repos to ones that have listed UUIDs. -}
+reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
+reposByUUID repos uuids = filterM match repos
+	where
+		match r = do
+			u <- getUUID r
+			return $ u `elem` uuids
+
diff --git a/UUID.hs b/UUID.hs
index 239d373f14..42afd7ba86 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -14,8 +14,6 @@ module UUID (
 	getUncachedUUID,
 	prepUUID,
 	genUUID,
-	reposByUUID,
-	reposWithoutUUID,
 	prettyPrintUUIDs,
 	describeUUID,
 	uuidLog,
@@ -87,22 +85,6 @@ prepUUID = do
 		uuid <- liftIO $ genUUID
 		Annex.setConfig configkey uuid
 
-{- Filters a list of repos to ones that have listed UUIDs. -}
-reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
-reposByUUID repos uuids = filterM match repos
-	where
-		match r = do
-			u <- getUUID r
-			return $ u `elem` uuids
-
-{- Filters a list of repos to ones that do not have the listed UUIDs. -}
-reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
-reposWithoutUUID repos uuids = filterM unmatch repos
-	where
-		unmatch r = do
-			u <- getUUID r
-			return $ u `notElem` uuids
-
 {- Pretty-prints a list of UUIDs -}
 prettyPrintUUIDs :: [UUID] -> Annex String
 prettyPrintUUIDs uuids = do

From cd451760c33b702616727f91b0364601609491c5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 27 Mar 2011 20:11:20 +0000
Subject: [PATCH 1233/2835]

---
 doc/forum/batch_check_on_remote_when_using_copy.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/batch_check_on_remote_when_using_copy.mdwn

diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
new file mode 100644
index 0000000000..355d98f04b
--- /dev/null
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -0,0 +1,5 @@
+When I copy my local repository with SHA* to a remote repo with SHA*, every single file is checked by itself which seems rather inefficient. When my remote is accessed via ssh, git-annex opens a new connections for every check. If you are not using a ssh key or key agent, this gets tedious...
+
+For all locked files, either git's built-in mechanisms should be used or, if that's not possible, a few hundred checksums (assuming SHA* backend) should be transfered at once and then checked locally before deciding that to transfer.
+
+-- RichiH

From bec63186ea0b9b25e2387685c209578bc316397d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 27 Mar 2011 20:14:47 +0000
Subject: [PATCH 1234/2835]

---
 doc/forum/batch_check_on_remote_when_using_copy.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
index 355d98f04b..0f20ab6454 100644
--- a/doc/forum/batch_check_on_remote_when_using_copy.mdwn
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -2,4 +2,7 @@ When I copy my local repository with SHA* to a remote repo with SHA*, every sing
 
 For all locked files, either git's built-in mechanisms should be used or, if that's not possible, a few hundred checksums (assuming SHA* backend) should be transfered at once and then checked locally before deciding that to transfer.
 
+Once all checks are done, one single transfer session should be started. Creating new sessions and waiting for TCP's slowstart to get going is a lot less than efficient.
+
+
 -- RichiH

From f30320aa75d6fa590f60030f13df6b3899816196 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 16:17:56 -0400
Subject: [PATCH 1235/2835] add remotes slot to Annex

This required parameterizing the type for Remote, to avoid a cycle.
---
 Annex.hs            |  5 +++--
 Backend/File.hs     |  2 +-
 Remote.hs           | 10 +++++-----
 Remote/GitRemote.hs | 37 ++++++++++++++++---------------------
 RemoteClass.hs      | 24 +++++++++++-------------
 Remotes.hs          |  5 +++--
 UUID.hs             |  4 ++--
 7 files changed, 41 insertions(+), 46 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index f45415a72f..bb26608f4a 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -27,6 +27,7 @@ import Data.Maybe
 import qualified GitRepo as Git
 import qualified GitQueue
 import qualified BackendClass
+import qualified RemoteClass
 import Utility
 
 -- git-annex's monad
@@ -37,6 +38,7 @@ data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [BackendClass.Backend Annex]
 	, supportedBackends :: [BackendClass.Backend Annex]
+	, remotes :: [RemoteClass.Remote Annex]
 	, repoqueue :: GitQueue.Queue
 	, quiet :: Bool
 	, force :: Bool
@@ -46,13 +48,13 @@ data AnnexState = AnnexState
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
 	, exclude :: [String]
-	, remotesread :: Bool
 	} deriving (Show)
 
 newState :: Git.Repo -> [BackendClass.Backend Annex] -> AnnexState
 newState gitrepo allbackends = AnnexState
 	{ repo = gitrepo
 	, backends = []
+	, remotes = []
 	, supportedBackends = allbackends
 	, repoqueue = GitQueue.empty
 	, quiet = False
@@ -63,7 +65,6 @@ newState gitrepo allbackends = AnnexState
 	, toremote = Nothing
 	, fromremote = Nothing
 	, exclude = []
-	, remotesread = False
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
diff --git a/Backend/File.hs b/Backend/File.hs
index 743d8d6278..9c102cf509 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -147,7 +147,7 @@ showLocations key exclude = do
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
 		message rs us = message rs [] ++ message [] us
 
-showTriedRemotes :: [RemoteClass.Remote] -> Annex ()
+showTriedRemotes :: [RemoteClass.Remote Annex] -> Annex ()
 showTriedRemotes [] = return ()	
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
diff --git a/Remote.hs b/Remote.hs
index 9eff5556c9..078a603bbf 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -24,21 +24,21 @@ import Trust
 import LocationLog
 
 {- add generators for new Remotes here -}
-generators :: [Annex [Remote]]
+generators :: [Annex [Remote Annex]]
 generators = [Remote.GitRemote.generate]
 
 {- generates a list of all available Remotes -}
-generate :: Annex [Remote]
+generate :: Annex [Remote Annex]
 generate = do
 	lists <- sequence generators
 	return $ concat lists
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
-remotesWithUUID :: [Remote] -> [UUID] -> [Remote]
+remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 
 {- Filters a list of remotes to ones that do not have the listed uuids. -}
-remotesWithoutUUID :: [Remote] -> [UUID] -> [Remote]
+remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
@@ -46,7 +46,7 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
  -}
-keyPossibilities :: Key -> Annex ([Remote], [UUID])
+keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
 keyPossibilities key = do
 	g <- Annex.gitRepo
 	u <- getUUID g
diff --git a/Remote/GitRemote.hs b/Remote/GitRemote.hs
index ccc5f7b42f..0ec0c70e8d 100644
--- a/Remote/GitRemote.hs
+++ b/Remote/GitRemote.hs
@@ -27,14 +27,14 @@ import CopyFile
 import RsyncFile
 import Ssh
 
-generate :: Annex [Remote]
+generate :: Annex [Remote Annex]
 generate = do
 	readConfigs
 	g <- Annex.gitRepo
 	rs <- filterM repoNotIgnored (Git.remotes g)
 	mapM genRemote rs
 	
-genRemote :: Git.Repo -> Annex Remote
+genRemote :: Git.Repo -> Annex (Remote Annex)
 genRemote r = do
 	u <- getUUID r
 	c <- repoCost r
@@ -49,31 +49,26 @@ genRemote r = do
 		hasKeyCheap = not (Git.repoIsUrl r)
 	}
 
-{- Reads the configs of all remotes.
+{- Reads the configs of git remotes.
  -
- - As reading the config of remotes can be expensive, this
- - function will only read configs once per git-annex run. It's
- - assumed to be cheap to read the config of non-URL remotes,
+ - It's assumed to be cheap to read the config of non-URL remotes,
  - so this is done each time git-annex is run. Conversely,
  - the config of an URL remote is only read when there is no
  - cached UUID value.
- - -}
+ -}
 readConfigs :: Annex ()
 readConfigs = do
-	remotesread <- Annex.getState Annex.remotesread
-	unless remotesread $ do
-		g <- Annex.gitRepo
-		allremotes <- filterM repoNotIgnored $ Git.remotes g
-		let cheap = filter (not . Git.repoIsUrl) allremotes
-		let expensive = filter Git.repoIsUrl allremotes
-		doexpensive <- filterM cachedUUID expensive
-		unless (null doexpensive) $
-			showNote $ "getting UUID for " ++
-				list doexpensive ++ "..."
-		let todo = cheap ++ doexpensive
-		unless (null todo) $ do
-			mapM_ tryGitConfigRead todo
-			Annex.changeState $ \s -> s { Annex.remotesread = True }
+	g <- Annex.gitRepo
+	allremotes <- filterM repoNotIgnored $ Git.remotes g
+	let cheap = filter (not . Git.repoIsUrl) allremotes
+	let expensive = filter Git.repoIsUrl allremotes
+	doexpensive <- filterM cachedUUID expensive
+	unless (null doexpensive) $
+		showNote $ "getting UUID for " ++
+			list doexpensive ++ "..."
+	let todo = cheap ++ doexpensive
+	unless (null todo) $ do
+		mapM_ tryGitConfigRead todo
 	where
 		cachedUUID r = do
 			u <- getUUID r
diff --git a/RemoteClass.hs b/RemoteClass.hs
index df2aefb715..9fef0e44a0 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -9,38 +9,36 @@ module RemoteClass where
 
 import Control.Exception
 
-import Annex
-import UUID
 import Key
 
-data Remote = Remote {
+data Remote a = Remote {
 	-- each Remote has a unique uuid
-	uuid :: UUID,
+	uuid :: String,
 	-- each Remote has a human visible name
 	name :: String,
 	-- Remotes have a use cost; higher is more expensive
 	cost :: Int,
 	-- Transfers a key to the remote.
-	storeKey :: Key -> Annex Bool,
+	storeKey :: Key -> a Bool,
 	-- retrieves a key's contents to a file
-	retrieveKeyFile :: Key -> FilePath -> Annex Bool,
+	retrieveKeyFile :: Key -> FilePath -> a Bool,
 	-- removes a key's contents
-	removeKey :: Key -> Annex Bool,
+	removeKey :: Key -> a Bool,
 	-- Checks if a key is present in the remote; if the remote
 	-- cannot be accessed returns a Left error.
-	hasKey :: Key -> Annex (Either IOException Bool),
+	hasKey :: Key -> a (Either IOException Bool),
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
 	hasKeyCheap :: Bool
 }
 
-instance Show Remote where
+instance Show (Remote a) where
 	show remote = "Remote { uuid =\"" ++ uuid remote ++ "\" }"
 
 -- two remotes are the same if they have the same uuid
-instance Eq Remote where
-	a == b = uuid a == uuid b
+instance Eq (Remote a) where
+	x == y = uuid x == uuid y
 
 -- order remotes by cost
-instance Ord Remote where
-	compare a b = compare (cost a) (cost b)
+instance Ord (Remote a) where
+	compare x y = compare (cost x) (cost y)
diff --git a/Remotes.hs b/Remotes.hs
index 5fc594ee29..7f6a6718b3 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -91,7 +91,8 @@ tryGitConfigRead r
  - -}
 readConfigs :: Annex ()
 readConfigs = do
-	remotesread <- Annex.getState Annex.remotesread
+--	remotesread <- Annex.getState Annex.remotesread
+	let remotesread = False
 	unless remotesread $ do
 		g <- Annex.gitRepo
 		allremotes <- filterM repoNotIgnored $ Git.remotes g
@@ -104,7 +105,7 @@ readConfigs = do
 		let todo = cheap ++ doexpensive
 		unless (null todo) $ do
 			mapM_ tryGitConfigRead todo
-			Annex.changeState $ \s -> s { Annex.remotesread = True }
+--			Annex.changeState $ \s -> s { Annex.remotesread = True }
 	where
 		cachedUUID r = do
 			u <- getUUID r
diff --git a/UUID.hs b/UUID.hs
index 42afd7ba86..3f28434851 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -36,7 +36,7 @@ import qualified SysConfig
 type UUID = String
 
 configkey :: String
-configkey="annex.uuid"
+configkey = "annex.uuid"
 
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
@@ -74,7 +74,7 @@ getUUID r = do
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = Git.configGet r "annex.uuid" ""
+getUncachedUUID r = Git.configGet r configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()

From 3470260a8500b42f805b8263af9c73b99706bb92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 16:24:46 -0400
Subject: [PATCH 1236/2835] clean up remote list generation to only run once

---
 Remote.hs | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 078a603bbf..f4b56846b1 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -6,7 +6,6 @@
  -}
 
 module Remote (
-	generate,
 	keyPossibilities,
 	remotesWithUUID,
 	remotesWithoutUUID
@@ -27,11 +26,19 @@ import LocationLog
 generators :: [Annex [Remote Annex]]
 generators = [Remote.GitRemote.generate]
 
-{- generates a list of all available Remotes -}
-generate :: Annex [Remote Annex]
-generate = do
-	lists <- sequence generators
-	return $ concat lists
+{- Builds a list of all available Remotes.
+ - Since doing so can be expensive, the list is cached in the Annex. -}
+genList :: Annex [Remote Annex]
+genList = do
+	liftIO $ putStrLn "Remote.genList"
+	rs <- Annex.getState Annex.remotes
+	if null rs
+		then do
+			lists <- sequence generators
+			let rs' = concat lists
+			Annex.changeState $ \s -> s { Annex.remotes = rs' }
+			return rs'
+		else return rs
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
@@ -60,7 +67,7 @@ keyPossibilities key = do
 	let validtrusteduuids = intersect validuuids trusted
 
 	-- remotes that match uuids that have the key
-	allremotes <- generate
+	allremotes <- genList
 	let validremotes = remotesWithUUID allremotes validuuids
 
 	return (sort validremotes, validtrusteduuids)

From 834309d1d0c96f5f1ca9530c9622a24234242e28 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 27 Mar 2011 20:39:14 +0000
Subject: [PATCH 1237/2835]

---
 ..._version_upgrade_leaves_repo_unusable.mdwn | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
new file mode 100644
index 0000000000..16a0ca3744
--- /dev/null
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -0,0 +1,28 @@
+foo is a local repo, bar is a bare remote.
+
+I upgraded foo's git-annex to 0.20110325 and upgraded a local repo backend to version 2. I then ran `git annex copy . --to bar` and checked the remote. This created WORM:SHA512--123123 files in annex/objects. Understandable but unwanted. So I upgraded git-annex on bar's machine, as well.
+
+    % git annex copy . --to bar
+    copy quux (checking bar) git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade (to bar)
+    git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.7]
+    
+      rsync failed -- run git annex again to resume file transfer
+    failed
+
+Running `git annex upgrade` on bar's machine I get:
+
+    % git annex upgrade
+    upgrade  (v1 to v2) (moving content...) git-annex: Prelude.read: no parse
+
+Again, bar is a bare repo.
+Running the copy job again, I am still getting the same error as above (as expected). Partial contents of annex/objects on bar:
+
+    [...]
+    SHA512:123
+    WORM:SHA512--234
+    [...]
+
+
+-- RichiH

From 30f427700f9e9d59a83775a049e670cc05f2dee6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 16:55:43 -0400
Subject: [PATCH 1238/2835] converted several commands to use Remote

only move and map still to convert
---
 Command/Describe.hs  | 14 +++++---------
 Command/Semitrust.hs | 13 +++++--------
 Command/Trust.hs     | 13 +++++--------
 Command/Untrust.hs   | 13 +++++--------
 Remote.hs            | 40 +++++++++++++++++++++++++++++++++-------
 UUID.hs              |  9 ++++-----
 test.hs              |  2 +-
 7 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/Command/Describe.hs b/Command/Describe.hs
index 32aef4f245..9e98a81437 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -7,10 +7,8 @@
 
 module Command.Describe where
 
-
 import Command
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import UUID
 import Messages
 import qualified Command.Init
@@ -30,12 +28,10 @@ start params = notBareRepo $ do
 			_ -> error "Specify a repository and a description."
 	
 	showStart "describe" name
-	Remotes.readConfigs
-	r <- Remotes.byName name
-	return $ Just $ perform r description
+	u <- Remote.nameToUUID name
+	return $ Just $ perform u description
 
-perform :: Git.Repo -> String -> CommandPerform
-perform repo description = do
-	u <- getUUID repo
+perform :: UUID -> String -> CommandPerform
+perform u description = do
 	describeUUID u description
 	return $ Just $ Command.Init.cleanup
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 351336b899..e64d418f83 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -8,8 +8,7 @@
 module Command.Semitrust where
 
 import Command
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import UUID
 import Trust
 import Messages
@@ -24,12 +23,10 @@ seek = [withString start]
 start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "semitrust" name
-	Remotes.readConfigs
-	r <- Remotes.byName name
-	return $ Just $ perform r
+	u <- Remote.nameToUUID name
+	return $ Just $ perform u
 
-perform :: Git.Repo -> CommandPerform
-perform repo = do
-	uuid <- getUUID repo
+perform :: UUID -> CommandPerform
+perform uuid = do
 	trustSet uuid SemiTrusted
 	return $ Just $ return True
diff --git a/Command/Trust.hs b/Command/Trust.hs
index f7dba56485..05505cd045 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -8,8 +8,7 @@
 module Command.Trust where
 
 import Command
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import Trust
 import UUID
 import Messages
@@ -24,12 +23,10 @@ seek = [withString start]
 start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "trust" name
-	Remotes.readConfigs
-	r <- Remotes.byName name
-	return $ Just $ perform r
+	u <- Remote.nameToUUID name
+	return $ Just $ perform u
 
-perform :: Git.Repo -> CommandPerform
-perform repo = do
-	uuid <- getUUID repo
+perform :: UUID -> CommandPerform
+perform uuid = do
 	trustSet uuid Trusted
 	return $ Just $ return True
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 9c11efe465..311ec6eeb7 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -8,8 +8,7 @@
 module Command.Untrust where
 
 import Command
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import UUID
 import Trust
 import Messages
@@ -24,12 +23,10 @@ seek = [withString start]
 start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "untrust" name
-	Remotes.readConfigs
-	r <- Remotes.byName name
-	return $ Just $ perform r
+	u <- Remote.nameToUUID name
+	return $ Just $ perform u
 
-perform :: Git.Repo -> CommandPerform
-perform repo = do
-	uuid <- getUUID repo
+perform :: UUID -> CommandPerform
+perform uuid = do
 	trustSet uuid UnTrusted
 	return $ Just $ return True
diff --git a/Remote.hs b/Remote.hs
index f4b56846b1..64ad85d62e 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -6,12 +6,15 @@
  -}
 
 module Remote (
+	byName,
+	nameToUUID,
 	keyPossibilities,
 	remotesWithUUID,
 	remotesWithoutUUID
 ) where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (when, liftM)
 import Data.List
 
 import RemoteClass
@@ -21,6 +24,7 @@ import UUID
 import qualified Annex
 import Trust
 import LocationLog
+import Messages
 
 {- add generators for new Remotes here -}
 generators :: [Annex [Remote Annex]]
@@ -30,7 +34,9 @@ generators = [Remote.GitRemote.generate]
  - Since doing so can be expensive, the list is cached in the Annex. -}
 genList :: Annex [Remote Annex]
 genList = do
-	liftIO $ putStrLn "Remote.genList"
+	g <- Annex.gitRepo
+	u <- getUUID g
+	showNote $ "Remote.genList " ++ u
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
@@ -40,13 +46,24 @@ genList = do
 			return rs'
 		else return rs
 
-{- Filters a list of remotes to ones that have the listed uuids. -}
-remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
-remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
+{- Looks up a remote by name. (Or by UUID.) -}
+byName :: String -> Annex (Remote Annex)
+byName "" = error "no remote specified"
+byName n = do
+	allremotes <- genList
+	let match = filter matching allremotes
+	when (null match) $ error $
+		"there is no git remote named \"" ++ n ++ "\""
+	return $ head match
+	where
+		matching r = n == name r || n == uuid r
 
-{- Filters a list of remotes to ones that do not have the listed uuids. -}
-remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
-remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
+{- Looks up a remote by name (or by UUID), and returns its UUID. -}
+nameToUUID :: String -> Annex UUID
+nameToUUID "." = do -- special case for current repo
+	g <- Annex.gitRepo
+	getUUID g
+nameToUUID n = liftM uuid (byName n)
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
@@ -71,3 +88,12 @@ keyPossibilities key = do
 	let validremotes = remotesWithUUID allremotes validuuids
 
 	return (sort validremotes, validtrusteduuids)
+
+{- Filters a list of remotes to ones that have the listed uuids. -}
+remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
+remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
+
+{- Filters a list of remotes to ones that do not have the listed uuids. -}
+remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
+remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
+
diff --git a/UUID.hs b/UUID.hs
index 3f28434851..5caf110453 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -3,6 +3,9 @@
  - Each git repository used by git-annex has an annex.uuid setting that
  - uniquely identifies that repository.
  -
+ - UUIDs of remotes are cached in git config, using keys named
+ - remote..annex-uuid
+ -
  - Copyright 2010 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
@@ -51,11 +54,7 @@ genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
 			else []
 
 {- Looks up a repo's UUID. May return "" if none is known.
- -
- - UUIDs of remotes are cached in git config, using keys named
- - remote..annex-uuid
- -
- - -}
+ -}
 getUUID :: Git.Repo -> Annex UUID
 getUUID r = do
 	g <- Annex.gitRepo
diff --git a/test.hs b/test.hs
index 49f7f2ab99..57eb5e664e 100644
--- a/test.hs
+++ b/test.hs
@@ -334,6 +334,7 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 	git_annex "semitrust" ["-q", repo] @? "semitrust of semitrusted failed"
 	trustcheck Trust.SemiTrusted "semitrusted 2"
 	where
+		repo = "origin"
 		trustcheck expected msg = do
 			present <- annexeval $ do
 				Remotes.readConfigs
@@ -342,7 +343,6 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 				u <- UUID.getUUID r
 				return $ u `elem` l
 			assertBool msg present
-		repo = "origin"
 
 test_fsck :: Test
 test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withremoteuntrusted]

From 688e94fd30ca1aa908f5387e488451b9ed2f35e5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 16:58:28 -0400
Subject: [PATCH 1239/2835] convert test to Remote

---
 test.hs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/test.hs b/test.hs
index 57eb5e664e..30ebe6e593 100644
--- a/test.hs
+++ b/test.hs
@@ -35,7 +35,7 @@ import qualified GitAnnex
 import qualified LocationLog
 import qualified UUID
 import qualified Trust
-import qualified Remotes
+import qualified Remote
 import qualified Content
 import qualified Command.DropUnused
 import qualified Key
@@ -337,10 +337,8 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 		repo = "origin"
 		trustcheck expected msg = do
 			present <- annexeval $ do
-				Remotes.readConfigs
 				l <- Trust.trustGet expected
-				r <- Remotes.byName repo
-				u <- UUID.getUUID r
+				u <- Remote.nameToUUID repo
 				return $ u `elem` l
 			assertBool msg present
 

From 0d83d17f0455e664f06e3e46b994abadceb7b879 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:00:05 -0400
Subject: [PATCH 1240/2835] convert map to use new code

---
 Command/Map.hs      | 4 ++--
 Remote/GitRemote.hs | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 4ae947b15e..6206aeeb5e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -16,7 +16,7 @@ import Data.List.Utils
 import Command
 import qualified Annex
 import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote.GitRemote
 import Messages
 import Types
 import Utility
@@ -203,7 +203,7 @@ tryScan r
 				Git.hConfigRead r
 
 		configlist =
-			Remotes.onRemote r (pipedconfig, Nothing) "configlist" []
+			Remote.GitRemote.onRemote r (pipedconfig, Nothing) "configlist" []
 		manualconfiglist = do
 			let sshcmd =
 				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
diff --git a/Remote/GitRemote.hs b/Remote/GitRemote.hs
index 0ec0c70e8d..8671ef7fa2 100644
--- a/Remote/GitRemote.hs
+++ b/Remote/GitRemote.hs
@@ -5,7 +5,10 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.GitRemote (generate) where
+module Remote.GitRemote (
+	generate,
+	onRemote
+) where
 
 import Control.Exception.Extensible
 import Control.Monad.State (liftIO)

From 48418cb92bd2548d333350ac0b7bf2a04540d621 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:12:32 -0400
Subject: [PATCH 1241/2835] reexport RemoteClass from Remote for cleanliness

---
 Backend/File.hs | 17 ++++++++---------
 Remote.hs       |  9 +++++++++
 RemoteClass.hs  |  2 ++
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 9c102cf509..b86413e400 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -21,7 +21,6 @@ import Data.String.Utils
 import BackendClass
 import LocationLog
 import qualified Remote
-import qualified RemoteClass
 import qualified GitRepo as Git
 import Content
 import qualified Annex
@@ -74,16 +73,16 @@ copyKeyFile key file = do
 		-- This check is to avoid an ugly message if a remote is a
 		-- drive that is not mounted.
 		probablyPresent r =
-			if RemoteClass.hasKeyCheap r
+			if Remote.hasKeyCheap r
 				then do
-					res <- RemoteClass.hasKey r key
+					res <- Remote.hasKey r key
 					case res of
 						Right b -> return b
 						Left _ -> return False
 				else return True
 		docopy r continue = do
-			showNote $ "copying from " ++ RemoteClass.name r ++ "..."
-			copied <- RemoteClass.retrieveKeyFile r key file
+			showNote $ "copying from " ++ Remote.name r ++ "..."
+			copied <- Remote.retrieveKeyFile r key file
 			if copied
 				then return True
 				else continue
@@ -109,9 +108,9 @@ checkRemoveKey key numcopiesM = do
 		findcopies need have (r:rs) bad
 			| length have >= need = return True
 			| otherwise = do
-				let u = RemoteClass.uuid r
+				let u = Remote.uuid r
 				let dup = u `elem` have
-				haskey <- (RemoteClass.hasKey r) key
+				haskey <- Remote.hasKey r key
 				case (dup, haskey) of
 					(False, Right True)	-> findcopies need (u:have) rs bad
 					(False, Left _)		-> findcopies need have rs (r:bad)
@@ -147,11 +146,11 @@ showLocations key exclude = do
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
 		message rs us = message rs [] ++ message [] us
 
-showTriedRemotes :: [RemoteClass.Remote Annex] -> Annex ()
+showTriedRemotes :: [Remote.Remote Annex] -> Annex ()
 showTriedRemotes [] = return ()	
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
-		(join ", " $ map RemoteClass.name remotes)
+		(join ", " $ map Remote.name remotes)
 
 getNumCopies :: Maybe Int -> Annex Int
 getNumCopies (Just n) = return n
diff --git a/Remote.hs b/Remote.hs
index 64ad85d62e..1d9a828d50 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -6,6 +6,15 @@
  -}
 
 module Remote (
+	Remote,
+	uuid,
+	name,
+	storeKey,
+	retrieveKeyFile,
+	removeKey,
+	hasKey,
+	hasKeyCheap,
+
 	byName,
 	nameToUUID,
 	keyPossibilities,
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 9fef0e44a0..38e8407a54 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -1,4 +1,6 @@
 {- git-annex remotes class
+ -
+ - Most things should not need this, using Remote instead
  -
  - Copyright 2011 Joey Hess 
  -

From a70035e981d44a3d5f1dd224be643f4eebb0f243 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:24:20 -0400
Subject: [PATCH 1242/2835] converted move to use Remote

Drop old Remotes.hs, now unused!
---
 Command/Move.hs     |  60 ++++----
 Remote/GitRemote.hs |   9 +-
 Remotes.hs          | 334 --------------------------------------------
 3 files changed, 36 insertions(+), 367 deletions(-)
 delete mode 100644 Remotes.hs

diff --git a/Command/Move.hs b/Command/Move.hs
index 8056e95dbe..907bbf00ef 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -15,8 +15,7 @@ import qualified Annex
 import LocationLog
 import Types
 import Content
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import UUID
 import Messages
 import Utility
@@ -34,16 +33,15 @@ seek = [withFilesInGit $ start True]
  - moving data in the key-value backend. -}
 start :: Bool -> CommandStartString
 start move file = do
-	Remotes.readConfigs
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
 	case (from, to) of
 		(Nothing, Nothing) -> error "specify either --from or --to"
 		(Nothing, Just name) -> do
-			dest <- Remotes.byName name
+			dest <- Remote.byName name
 			toStart dest move file
 		(Just name, Nothing) -> do
-			src <- Remotes.byName name
+			src <- Remote.byName name
 			fromStart src move file
 		(_ ,  _) -> error "only one of --from or --to can be specified"
 
@@ -56,88 +54,86 @@ showAction False file = showStart "copy" file
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot be relied on. For example, it's not done
  - for bare repos. -}
-remoteHasKey :: Git.Repo -> Key -> Bool -> Annex ()
+remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()
 remoteHasKey remote key present	= do
 	g <- Annex.gitRepo
-	remoteuuid <- getUUID remote
+	let remoteuuid = Remote.uuid remote
 	logfile <- liftIO $ logChange g key remoteuuid status
 	Annex.queue "add" [Param "--"] logfile
 	where
 		status = if present then ValuePresent else ValueMissing
 
-{- Moves (or copies) the content of an annexed file to another repository,
- - and updates locationlog information on both.
+{- Moves (or copies) the content of an annexed file to a remote.
  -
- - When moving, if the destination already has the content, it is
- - still removed from the current repository.
+ - If the remote already has the content, it is still removed from
+ - the current repository.
  -
  - Note that unlike drop, this does not honor annex.numcopies.
  - A file's content can be moved even if there are insufficient copies to
  - allow it to be dropped.
  -}
-toStart :: Git.Repo -> Bool -> CommandStartString
+toStart :: Remote.Remote Annex -> Bool -> CommandStartString
 toStart dest move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
+	u <- getUUID g
 	ishere <- inAnnex key
-	if not ishere || g == dest
+	if not ishere || u == Remote.uuid dest
 		then return Nothing -- not here, so nothing to do
 		else do
 			showAction move file
 			return $ Just $ toPerform dest move key
-toPerform :: Git.Repo -> Bool -> Key -> CommandPerform
+toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
 	-- checking the remote is expensive, so not done in the start step
-	isthere <- Remotes.inAnnex dest key
+	isthere <- Remote.hasKey dest key
 	case isthere of
 		Left err -> do
 			showNote $ show err
 			return Nothing
 		Right False -> do
-			showNote $ "to " ++ Git.repoDescribe dest ++ "..."
-			ok <- Remotes.copyToRemote dest key
+			showNote $ "to " ++ Remote.name dest ++ "..."
+			ok <- Remote.storeKey dest key
 			if ok
 				then return $ Just $ toCleanup dest move key
 				else return Nothing -- failed
 		Right True -> return $ Just $ toCleanup dest move key
-toCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup
+toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 toCleanup dest move key = do
 	remoteHasKey dest key True
 	if move
 		then Command.Drop.cleanup key
 		else return True
 
-{- Moves (or copies) the content of an annexed file from another repository
- - to the current repository and updates locationlog information on both.
+{- Moves (or copies) the content of an annexed file from a remote
+ - to the current repository.
  -
  - If the current repository already has the content, it is still removed
- - from the other repository when moving.
+ - from the remote.
  -}
-fromStart :: Git.Repo -> Bool -> CommandStartString
+fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
-	(remotes, _) <- Remotes.keyPossibilities key
-	if (g == src) || (null $ filter (\r -> Remotes.same r src) remotes)
+	u <- getUUID g
+	(remotes, _) <- Remote.keyPossibilities key
+	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
 		then return Nothing
 		else do
 			showAction move file
 			return $ Just $ fromPerform src move key
-fromPerform :: Git.Repo -> Bool -> Key -> CommandPerform
+fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = do
 	ishere <- inAnnex key
 	if ishere
 		then return $ Just $ fromCleanup src move key
 		else do
-			showNote $ "from " ++ Git.repoDescribe src ++ "..."
-			ok <- getViaTmp key $ Remotes.copyFromRemote src key
+			showNote $ "from " ++ Remote.name src ++ "..."
+			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
 			if ok
 				then return $ Just $ fromCleanup src move key
 				else return Nothing -- fail
-fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup
+fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 fromCleanup src True key = do
-	ok <- Remotes.onRemote src (boolSystem, False) "dropkey" 
-		[ Params "--quiet --force"
-		, Param $ show key
-		]
+	ok <- Remote.removeKey src key
 	-- better safe than sorry: assume the src dropped the key
 	-- even if it seemed to fail; the failure could have occurred
 	-- after it really dropped it
diff --git a/Remote/GitRemote.hs b/Remote/GitRemote.hs
index 8671ef7fa2..43e75b97bd 100644
--- a/Remote/GitRemote.hs
+++ b/Remote/GitRemote.hs
@@ -47,7 +47,7 @@ genRemote r = do
 		name = Git.repoDescribe r,
 		storeKey = copyToRemote r,
 		retrieveKeyFile = copyFromRemote r,
-		removeKey = error "TODO Remote.GitRemote.removeKey",
+		removeKey = dropKey r,
 		hasKey = inAnnex r,
 		hasKeyCheap = not (Git.repoIsUrl r)
 	}
@@ -159,6 +159,13 @@ inAnnex r key = if Git.repoIsUrl r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
 				[Param (show key)]
 			return $ Right inannex
+	
+dropKey :: Git.Repo -> Key -> Annex Bool
+dropKey r key = 
+	onRemote r (boolSystem, False) "dropkey"
+		[ Params "--quiet --force"
+		, Param $ show key
+		]
 
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
diff --git a/Remotes.hs b/Remotes.hs
deleted file mode 100644
index 7f6a6718b3..0000000000
--- a/Remotes.hs
+++ /dev/null
@@ -1,334 +0,0 @@
-{- git-annex remote repositories
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Remotes (
-	list,
-	readConfigs,
-	keyPossibilities,
-	inAnnex,
-	same,
-	byName,
-	copyFromRemote,
-	copyToRemote,
-	onRemote
-) where
-
-import Control.Exception.Extensible
-import Control.Monad.State (liftIO)
-import qualified Data.Map as Map
-import Data.String.Utils
-import System.Cmd.Utils
-import Data.List (intersect, sortBy)
-import Control.Monad (when, unless, filterM)
-
-import Types
-import qualified GitRepo as Git
-import qualified Annex
-import LocationLog
-import Locations
-import UUID
-import Trust
-import Utility
-import qualified Content
-import Messages
-import CopyFile
-import RsyncFile
-import Ssh
-
-{- Human visible list of remotes. -}
-list :: [Git.Repo] -> String
-list remotes = join ", " $ map Git.repoDescribe remotes 
-
-{- The git configs for the git repo's remotes is not read on startup
- - because reading it may be expensive. This function tries to read the
- - config for a specified remote, and updates state. If successful, it
- - returns the updated git repo. -}
-tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo)
-tryGitConfigRead r 
-	| not $ Map.null $ Git.configMap r = return $ Right r -- already read
-	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
-	| Git.repoIsUrl r = return $ Left r
-	| otherwise = store $ safely $ Git.configRead r
-	where
-		-- Reading config can fail due to IO error or
-		-- for other reasons; catch all possible exceptions.
-		safely a = do
-			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
-			case result of
-				Left _ -> return r
-				Right r' -> return r'
-		pipedconfig cmd params = safely $
-			pOpen ReadFromPipe cmd (toCommand params) $
-				Git.hConfigRead r
-		store a = do
-			r' <- a
-			g <- Annex.gitRepo
-			let l = Git.remotes g
-			let g' = Git.remotesAdd g $ exchange l r'
-			Annex.changeState $ \s -> s { Annex.repo = g' }
-			return $ Right r'
-		exchange [] _ = []
-		exchange (old:ls) new =
-			if Git.repoRemoteName old == Git.repoRemoteName new
-				then new : exchange ls new
-				else old : exchange ls new
-
-{- Reads the configs of all remotes.
- -
- - This has to be called before things that rely on eg, the UUID of
- - remotes. Most such things will take care of running this themselves.
- -
- - As reading the config of remotes can be expensive, this
- - function will only read configs once per git-annex run. It's
- - assumed to be cheap to read the config of non-URL remotes,
- - so this is done each time git-annex is run. Conversely,
- - the config of an URL remote is only read when there is no
- - cached UUID value.
- - -}
-readConfigs :: Annex ()
-readConfigs = do
---	remotesread <- Annex.getState Annex.remotesread
-	let remotesread = False
-	unless remotesread $ do
-		g <- Annex.gitRepo
-		allremotes <- filterM repoNotIgnored $ Git.remotes g
-		let cheap = filter (not . Git.repoIsUrl) allremotes
-		let expensive = filter Git.repoIsUrl allremotes
-		doexpensive <- filterM cachedUUID expensive
-		unless (null doexpensive) $
-			showNote $ "getting UUID for " ++
-				list doexpensive ++ "..."
-		let todo = cheap ++ doexpensive
-		unless (null todo) $ do
-			mapM_ tryGitConfigRead todo
---			Annex.changeState $ \s -> s { Annex.remotesread = True }
-	where
-		cachedUUID r = do
-			u <- getUUID r
-			return $ null u
-
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
- -
- - Also returns a list of UUIDs that are trusted to have the key
- - (some may not have configured remotes).
- -}
-keyPossibilities :: Key -> Annex ([Git.Repo], [UUID])
-keyPossibilities key = do
-	readConfigs
-
-	allremotes <- remotesByCost
-	g <- Annex.gitRepo
-	u <- getUUID g
-	trusted <- trustGet Trusted
-
-	-- get uuids of all repositories that are recorded to have the key
-	uuids <- liftIO $ keyLocations g key
-	let validuuids = filter (/= u) uuids
-
-	-- note that validuuids is assumed to not have dups
-	let validtrusteduuids = intersect validuuids trusted
-
-	-- remotes that match uuids that have the key
-	validremotes <- reposByUUID allremotes validuuids
-
-	return (validremotes, validtrusteduuids)
-
-{- Checks if a given remote has the content for a key inAnnex.
- - If the remote cannot be accessed, returns a Left error.
- -}
-inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
-inAnnex r key = if Git.repoIsUrl r
-		then checkremote
-		else liftIO (try checklocal ::IO (Either IOException Bool))
-	where
-		checklocal = do
-			-- run a local check inexpensively,
-			-- by making an Annex monad using the remote
-			a <- Annex.new r []
-			Annex.eval a (Content.inAnnex key)
-		checkremote = do
-			showNote ("checking " ++ Git.repoDescribe r ++ "...")
-			inannex <- onRemote r (boolSystem, False) "inannex" 
-				[Param (show key)]
-			return $ Right inannex
-
-{- Cost Ordered list of remotes. -}
-remotesByCost :: Annex [Git.Repo]
-remotesByCost = do
-	g <- Annex.gitRepo
-	reposByCost $ Git.remotes g
-
-{- Orders a list of git repos by cost. Throws out ignored ones. -}
-reposByCost :: [Git.Repo] -> Annex [Git.Repo]
-reposByCost l = do
-	notignored <- filterM repoNotIgnored l
-	costpairs <- mapM costpair notignored
-	return $ fst $ unzip $ sortBy cmpcost costpairs
-	where
-		costpair r = do
-			cost <- repoCost r
-			return (r, cost)
-		cmpcost (_, c1) (_, c2) = compare c1 c2
-
-{- Calculates cost for a repo.
- -
- - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost
- -}
-repoCost :: Git.Repo -> Annex Int
-repoCost r = do
-	cost <- Annex.repoConfig r "cost" ""
-	if not $ null cost
-		then return $ read cost
-		else if Git.repoIsUrl r
-			then return 200
-			else return 100
-
-{- Checks if a repo should be ignored, based either on annex-ignore
- - setting, or on command-line options. Allows command-line to override
- - annex-ignore. -}
-repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = do
-	ignored <- Annex.repoConfig r "ignore" "false"
-	to <- match Annex.toremote
-	from <- match Annex.fromremote
-	if to || from
-		then return True
-		else return $ not $ Git.configTrue ignored
-	where
-		match a = do
-			name <- Annex.getState a
-			case name of
-				Nothing -> return False
-				n -> return $ n == Git.repoRemoteName r
-
-{- Checks if two repos are the same, by comparing their remote names. -}
-same :: Git.Repo -> Git.Repo -> Bool
-same a b = Git.repoRemoteName a == Git.repoRemoteName b
-
-{- Looks up a remote by name. (Or by UUID.) -}
-byName :: String -> Annex Git.Repo
-byName "." = Annex.gitRepo -- special case to refer to current repository
-byName name = do
-	when (null name) $ error "no remote specified"
-	g <- Annex.gitRepo
-	match <- filterM matching $ Git.remotes g
-	when (null match) $ error $
-		"there is no git remote named \"" ++ name ++ "\""
-	return $ head match
-	where
-		matching r = do
-			if Just name == Git.repoRemoteName r
-				then return True
-				else do
-					u <- getUUID r
-					return $ (name == u)
-
-{- Tries to copy a key's content from a remote's annex to a file. -}
-copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
-copyFromRemote r key file
-	| not $ Git.repoIsUrl r = liftIO $ copyFile (gitAnnexLocation r key) file
-	| Git.repoIsSsh r = rsynchelper r True key file
-	| otherwise = error "copying from non-ssh repo not supported"
-
-{- Tries to copy a key's content to a remote's annex. -}
-copyToRemote :: Git.Repo -> Key -> Annex Bool
-copyToRemote r key
-	| not $ Git.repoIsUrl r = do
-		g <- Annex.gitRepo
-		let keysrc = gitAnnexLocation g key
-		-- run copy from perspective of remote
-		liftIO $ do
-			a <- Annex.new r []
-			Annex.eval a $ do
-				ok <- Content.getViaTmp key $
-					\f -> liftIO $ copyFile keysrc f
-				Annex.queueRun
-				return ok
-	| Git.repoIsSsh r = do
-		g <- Annex.gitRepo
-		let keysrc = gitAnnexLocation g key
-		rsynchelper r False key keysrc
-	| otherwise = error "copying to non-ssh repo not supported"
-
-rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool)
-rsynchelper r sending key file = do
-	showProgress -- make way for progress bar
-	p <- rsyncParams r sending key file
-	res <- liftIO $ boolSystem "rsync" p
-	if res
-		then return res
-		else do
-			showLongNote "rsync failed -- run git annex again to resume file transfer"
-			return res
-
-{- Generates rsync parameters that ssh to the remote and asks it
- - to either receive or send the key's content. -}
-rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]
-rsyncParams r sending key file = do
-	Just (shellcmd, shellparams) <- git_annex_shell r
-		(if sending then "sendkey" else "recvkey")
-		[ Param $ show key
-		-- Command is terminated with "--", because
-		-- rsync will tack on its own options afterwards,
-		-- and they need to be ignored.
-		, Param "--"
-		]
-	-- Convert the ssh command into rsync command line.
-	let eparam = rsyncShell (Param shellcmd:shellparams)
-	o <- Annex.repoConfig r "rsync-options" ""
-	let base = options ++ map Param (words o) ++ eparam
-	if sending
-		then return $ base ++ [dummy, File file]
-		else return $ base ++ [File file, dummy]
-	where
-		-- inplace makes rsync resume partial files
-		options = [Params "-p --progress --inplace"]
-		-- the rsync shell parameter controls where rsync
-		-- goes, so the source/dest parameter can be a dummy value,
-		-- that just enables remote rsync mode.
-		dummy = Param ":"
-
-{- Uses a supplied function to run a git-annex-shell command on a remote.
- -
- - Or, if the remote does not support running remote commands, returns
- - a specified error value. -}
-onRemote 
-	:: Git.Repo
-	-> (FilePath -> [CommandParam] -> IO a, a)
-	-> String
-	-> [CommandParam]
-	-> Annex a
-onRemote r (with, errorval) command params = do
-	s <- git_annex_shell r command params
-	case s of
-		Just (c, ps) -> liftIO $ with c ps
-		Nothing -> return errorval
-
-{- Generates parameters to run a git-annex-shell command on a remote. -}
-git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam]))
-git_annex_shell r command params
-	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)
-	| Git.repoIsSsh r = do
-		sshparams <- sshToRepo r [Param sshcmd]
-		return $ Just ("ssh", sshparams)
-	| otherwise = return Nothing
-	where
-		dir = Git.workTree r
-		shellcmd = "git-annex-shell"
-		shellopts = (Param command):(File dir):params
-		sshcmd = shellcmd ++ " " ++ 
-			unwords (map shellEscape $ toCommand shellopts)
-
-{- Filters a list of repos to ones that have listed UUIDs. -}
-reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
-reposByUUID repos uuids = filterM match repos
-	where
-		match r = do
-			u <- getUUID r
-			return $ u `elem` uuids
-

From 45dbfbd02f20c211f70061a93d382ee333ef27ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:26:46 -0400
Subject: [PATCH 1243/2835] remove debug

---
 Remote.hs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 1d9a828d50..a7136ea651 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -33,7 +33,6 @@ import UUID
 import qualified Annex
 import Trust
 import LocationLog
-import Messages
 
 {- add generators for new Remotes here -}
 generators :: [Annex [Remote Annex]]
@@ -43,9 +42,6 @@ generators = [Remote.GitRemote.generate]
  - Since doing so can be expensive, the list is cached in the Annex. -}
 genList :: Annex [Remote Annex]
 genList = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-	showNote $ "Remote.genList " ++ u
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do

From f8693facabdaa81cbebab7141151d306615fd6a5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:30:44 -0400
Subject: [PATCH 1244/2835] doc update

---
 debian/changelog  | 2 ++
 doc/backends.mdwn | 9 +++------
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e9bc896bf0..e995009db0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (0.20110326) UNRELEASED; urgency=low
 
   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
+  * Generalized remotes handling, laying groundwork for remotes that are
+    not regular git remotes.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 5d02ad3a1c..22164850ab 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -9,16 +9,13 @@ to retrieve the file's content (its value).
 Multiple pluggable backends are supported, and a single repository
 can use different backends for different files.
 
-* `WORM` ("Write Once, Read Many") This backend stores the file's content
-  only in `.git/annex/objects/`, and assumes that any file with the same
-  basename, size, and modification time has the same content. So with
+* `WORM` ("Write Once, Read Many") This backend assumes that any file with
+  the same basename, size, and modification time has the same content. So with
   this backend, files can be moved around, but should never be added to
   or changed. This is the default, and the least expensive backend.
-* `SHA1` -- This backend stores the file's content in
-  `.git/annex/objects/`, with a name based on its sha1 checksum. This backend
+* `SHA1` -- This backend uses a key based on a sha1 checksum. This backend
   allows modifications of files to be tracked. Its need to generate checksums
   can make it slower for large files.
-  for use.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
   researching checksum collisions and wants to annex their colliding data. ;)

From fb47e884040c3d95ceaa9e9bbc442fdf14abdd3a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:45:11 -0400
Subject: [PATCH 1245/2835] revamp s3 design

looking very doable now
---
 doc/todo/S3.mdwn | 103 ++++++++++++-----------------------------------
 1 file changed, 25 insertions(+), 78 deletions(-)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 946fa68170..09a64f1a7a 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -2,90 +2,37 @@ Support Amazon S3 as a file storage backend.
 
 There's a haskell library that looks good. Not yet in Debian.
 
-Multiple ways of using S3 are possible. Current plan is to have a S3BUCKET
-backend, that is derived from Backend.File, so it caches files locally and
-can transfer files between systems too, without involving S3.
+Multiple ways of using S3 are possible. Current plan is to 
+have a special type of git remote (though git won't know how to use it;
+only git-annex will) that uses a S3 bucket.
 
-get will try to get it from S3 or from a remote. A annex.s3.cost can
-configure the cost of S3 vs the cost of other remotes.
+Something like:
 
-add will always upload a copy to S3.
+	[remote "s3"]
+		annex-s3bucket = mybucket
+		annex-s3datacenter = Europe
+		annex-uuid = 1a586cf6-45e9-11e0-ba9c-3b0a3397aec2
+		annex-cost = 500
 
-Each file in the S3 bucket is assumed to be in the annex. So unused
-will show files in the bucket that nothing points to, and dropunused remove
-them.
+The UUID will be stored as a special file in the S3 bucket.
 
-For numcopies counting, S3 will count as 1 copy (or maybe more?), so if
-numcopies=2, then you don't fully trust S3 and request git-annex assure
-one other copy.
+Using a different type of remote like this will allow S3 to be used
+anywhere a regular remote would be used. `git annex get` will transparently
+download a file from S3 if S3 has it and is the cheapest remote.
 
-drop will remove a file locally, but keep it in S3. drop --force *might*
-remove it from S3. TBD.
+	git annex copy --to s3
+	git annex move --from s3
+	git annex drop --from s3 # not currently allowed, will need adding
 
-annex.s3.bucket would configure the bucket the use. (And an env var or
-something configure the password.) Although the bucket
-would also be encoded in the keys. So, the configured bucket would be used
-when adding new files. A system could move from one bucket to another over
-time while still having legacy files in an earlier one; 
-perhaps you move to Europe and want new files to be put in that region.
+Each s3 remote will count as one copy for numcopies handling, just like
+any other remote.
 
-And git annex `migrate --backend=S3BUCKET --force` could move files
-between datacenters!
+## unused checking
 
-Problem: Then the only way for unused to know what buckets are in use
-is to see what keys point to them -- but if the last file from a bucket is
-deleted, it would then not be able to say that the files in that bucket are
-all unused. Need cached list of recently seen S3 buckets?
+One problem is `git annex unused`. Currently it only looks at the local
+repository, not remotes. But if something is dropped from the local repo,
+and you forget to drop it from S3, cruft can build up there.
 
------
-
-One problem with this is what key metadata to include. Should it be like
-WORM? Or like SHA1? Or just a new unique identifier for each file? It might
-be worth having S3 variants of *all* the Backend.File derived backends.
-
-More blue-sky, it might be nice to be able to union or stack together
-multiple backends, so S3BUCKET+SHA1 or S3BUCKET+WORM. That would likely
-be hard to get right.
-
-Less blue-sky, if the S3 capability were added directly to Backend.File,
-and bucket name was configured by annex.s3.bucket, then any existing
-annexed file could be upgraded to also store on S3.
-
-## alternate approach
-
-The above assumes S3 should be a separate backend somehow. What if,
-instead a S3 bucket is treated as a separate **remote**.
-
-* Could "git annex add" while offline, and "git annex push --to S3" when
-  online.
-* No need to choose whether a file goes to S3 at add time; no need to
-  migrate to move files there.
-* numcopies counting Just Works
-* Could have multiple S3 buckets as desired.
-
-The bucket name could 1:1 map with its annex.uuid, so not much
-configuration would be needed when cloning a repo to get it using S3 --
-just configure the S3 access token(s) to use for various UUIDs.
-
-Implementing this might not be as conceptually nice as making S3 a separate
-backend. It would need some changes to the remotes code, perhaps lifting
-some of it into backend-specific hooks. Then the S3 backend could be
-implicitly stacked in front of a backend like WORM.
-
----
-
-Maybe the right way to look at this is that a list of Stores
-should be a property of the Backend. Backend.File is a Backend, that
-uses various Stores, which can be of different types (the local
-git repo, remote git repos, S3, etc). Backend.URL is a backend that uses
-other Stores (the local git repo, and the web).
-
-Operations on Stores are:
-
-* uuid -- each store has a unique uuid value
-* cost -- each store has a use cost value
-* getConfig -- attempts to look up values (uuid, possibly cost)
-* copyToStore -- store a file's contents to a key
-* copyFromStore -- retrieve a key's contents to a file
-* removeFromStore -- removes a key's contents from the store
-* hasKey -- checks if the key's content is available
+This could be fixed by adding a hook to list all keys present in a remote.
+Then unused could scan remotes for keys, and if they were not used locally,
+offer the possibility to drop them from the remote.

From 9a4127f0fed9fa9cd684fff78b3a2af9da3c62ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 18:02:42 -0400
Subject: [PATCH 1246/2835] response

---
 Upgrade/V1.hs                                              | 1 +
 doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c0470a3bc8..a876142220 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -226,6 +226,7 @@ getKeysPresent1' dir = do
 			return $ map fileKey1 files
 	where
 		present d = do
+			liftIO $ putStrLn $ dir ++ "/" ++ d ++ "/"  ++ takeFileName d
 			result <- try $
 				getFileStatus $ dir ++ "/" ++ d ++ "/"  ++ takeFileName d
 			case result of
diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 16a0ca3744..1c18e585c7 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -26,3 +26,8 @@ Running the copy job again, I am still getting the same error as above (as expec
 
 
 -- RichiH
+
+> Upgrading bare repos to v2 generally works fine, so I actually need
+> to see the full content of annex/, not a fragment, in order to debug this.
+> (Filename contents I don't need to see.) Feel free to email me the details at
+> joey@kitenet.net if you don't want to post them here. --[[Joey]]

From 4868b64868747455a9c5d512650f9e7074e6009e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 18:34:30 -0400
Subject: [PATCH 1247/2835] Provide a less expensive version of `git annex copy
 --to`, enabled via --fast. This assumes that location tracking information is
 correct, rather than contacting the remote for every file.

---
 Command/Move.hs                               | 10 ++++++++--
 debian/changelog                              |  3 +++
 ...batch_check_on_remote_when_using_copy.mdwn | 19 +++++++++++++++++++
 doc/git-annex.mdwn                            | 19 ++++++++++++-------
 4 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 907bbf00ef..3ac5a7ab2c 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -84,8 +84,14 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 			return $ Just $ toPerform dest move key
 toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
-	-- checking the remote is expensive, so not done in the start step
-	isthere <- Remote.hasKey dest key
+	-- Checking the remote is expensive, so not done in the start step.
+	-- In fast mode, location tracking is assumed to be correct,
+	-- and an explicit check is not done, when copying. When moving,
+	-- it has to be done, to avoid inaverdent data loss.
+	fast <- Annex.getState Annex.fast
+	isthere <- if fast && not move
+		then return $ Right True
+		else Remote.hasKey dest key
 	case isthere of
 		Left err -> do
 			showNote $ show err
diff --git a/debian/changelog b/debian/changelog
index e995009db0..2f532784d4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (0.20110326) UNRELEASED; urgency=low
   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
   * Generalized remotes handling, laying groundwork for remotes that are
     not regular git remotes.
+  * Provide a less expensive version of `git annex copy --to`, enabled
+    via --fast. This assumes that location tracking information is correct,
+    rather than contacting the remote for every file.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
index 0f20ab6454..b08c33b8ba 100644
--- a/doc/forum/batch_check_on_remote_when_using_copy.mdwn
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -6,3 +6,22 @@ Once all checks are done, one single transfer session should be started. Creatin
 
 
 -- RichiH
+
+> (Use of SHA is irrelevant here, copy does not checksum anything.)
+> 
+> I think what you're seeing is
+> that `git annex copy --to remote` is slow, going to the remote repository
+> every time to see if it has the file, while `git annex copy --from remote`
+> is fast, since it looks at what files are locally present.
+> 
+> That is something I mean to improve. At least `git annex copy --fast --to remote`
+> could easily do a fast copy of all files that are known to be missing from
+> the remote repository. When local and remote git repos are not 100% in sync,
+> relying on that data could miss some files that the remote doesn't have anymore,
+> but local doesn't know it dropped. That's why it's a candidate for `--fast`.
+> 
+> I've just implemented that.
+> 
+> While I do hope to improve ssh usage so that it sshs once, and feeds
+> `git-annex-shell` a series of commands to run, that is a much longer-term
+> thing. --[[Joey]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 32f190e75d..8afe93c109 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -84,20 +84,22 @@ Many git-annex commands will stage changes for later `git commit` by you.
   it is safe to do so, typically because of the setting of annex.numcopies.
 
 * move [path ...]
+  
+  When used with the --from option, moves the content of annexed files
+  from the specified repository to the current one.
 
   When used with the --to option, moves the content of annexed files from
   the current repository to the specified one.
 
-  When used with the --from option, moves the content of annexed files
-  from the specified repository to the current one.
-
 * copy [path ...]
 
+  When used with the --from option, copies the content of annexed files
+  from the specified repository to the current one.
+
   When used with the --to option, copies the content of annexed files from
   the current repository to the specified one.
 
-  When used with the --from option, copies the content of annexed files
-  from the specified repository to the current one.
+  To avoid contacting the remote to check if it has every file, specify --fast
 
 * unlock [path ...]
 
@@ -137,11 +139,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   With parameters, only the specified files are checked.
 
+  To avoid expensive checksum calculations, specify --fast
+
 * unused
 
   Checks the annex for data that is not used by any files currently
   in the annex, and prints a numbered list of the data.
 
+  To only show unused temp files, specify --fast
+
 * dropunused [number ...]
 
   Drops the data corresponding to the numbers, as listed by the last
@@ -286,8 +292,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * --fast
 
   Enables less expensive, but also less thorough versions of some commands.
-  What is avoided depends on the command. A fast fsck avoids calculating
-  checksums; a fast unused only shows temp files and not other unused files.
+  What is avoided depends on the command.
 
 * --quiet
 

From be3f6a9acf6cfea9fcba861bf51e10a59039d575 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 27 Mar 2011 23:19:45 +0000
Subject: [PATCH 1248/2835]

---
 doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 1c18e585c7..1218b0e517 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -31,3 +31,5 @@ Running the copy job again, I am still getting the same error as above (as expec
 > to see the full content of annex/, not a fragment, in order to debug this.
 > (Filename contents I don't need to see.) Feel free to email me the details at
 > joey@kitenet.net if you don't want to post them here. --[[Joey]]
+
+>> Sent. -- RichiH

From 28bf28a73c503c7c2d9add38e964149355bb9e50 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 19:23:00 -0400
Subject: [PATCH 1249/2835] rename

---
 Command/Map.hs                  | 4 ++--
 Remote.hs                       | 5 +++--
 Remote/{GitRemote.hs => Git.hs} | 0
 3 files changed, 5 insertions(+), 4 deletions(-)
 rename Remote/{GitRemote.hs => Git.hs} (100%)

diff --git a/Command/Map.hs b/Command/Map.hs
index 6206aeeb5e..dc3acb56e4 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -16,7 +16,7 @@ import Data.List.Utils
 import Command
 import qualified Annex
 import qualified GitRepo as Git
-import qualified Remote.GitRemote
+import qualified Remote.Git
 import Messages
 import Types
 import Utility
@@ -203,7 +203,7 @@ tryScan r
 				Git.hConfigRead r
 
 		configlist =
-			Remote.GitRemote.onRemote r (pipedconfig, Nothing) "configlist" []
+			Remote.Git.onRemote r (pipedconfig, Nothing) "configlist" []
 		manualconfiglist = do
 			let sshcmd =
 				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
diff --git a/Remote.hs b/Remote.hs
index a7136ea651..b3f1a0c6b8 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -27,7 +27,8 @@ import Control.Monad (when, liftM)
 import Data.List
 
 import RemoteClass
-import qualified Remote.GitRemote
+import qualified Remote.Git
+import qualified Remote.S3
 import Types
 import UUID
 import qualified Annex
@@ -36,7 +37,7 @@ import LocationLog
 
 {- add generators for new Remotes here -}
 generators :: [Annex [Remote Annex]]
-generators = [Remote.GitRemote.generate]
+generators = [Remote.Git.generate]
 
 {- Builds a list of all available Remotes.
  - Since doing so can be expensive, the list is cached in the Annex. -}
diff --git a/Remote/GitRemote.hs b/Remote/Git.hs
similarity index 100%
rename from Remote/GitRemote.hs
rename to Remote/Git.hs

From 221251b4584824637db2c97b97efb919c9dd57fe Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 01:19:22 +0000
Subject: [PATCH 1250/2835] Added a comment

---
 ..._97848f9a3db89c0427cfb671ba13300e._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment b/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment
new file mode 100644
index 0000000000..be34473c0a
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-28T01:16:21Z"
+ content="""
+Another nice thing would be a summary of _what_ is wrong. I.e.
+
+    % git fsck
+    [...]
+    git-annex: 100 total failed
+      50 checksum failed
+      50 not enough copies exit
+
+And the same/similar for all other failure modes.
+
+
+-- RichiH
+"""]]

From 029a50a7ce4ff8032fd1d24d94f4e97c667455ce Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 01:22:25 +0000
Subject: [PATCH 1251/2835] Added a comment

---
 ..._ea3c828c5f51c3fcca2de16bce4a7561._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment b/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment
new file mode 100644
index 0000000000..dced87287a
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2011-03-28T01:19:25Z"
+ content="""
+Another nice thing would be a summary of _what_ is wrong. I.e.
+
+    % git fsck
+    [...]
+    git-annex: 100 total failed
+      50 checksum failed
+      50 not enough copies exit
+
+And the same/similar for all other failure modes.
+
+
+-- RichiH
+"""]]

From 68005f25f9446603ae84cae109ea76a69a2bd156 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 01:27:05 +0000
Subject: [PATCH 1252/2835] removed

---
 ..._ea3c828c5f51c3fcca2de16bce4a7561._comment | 19 -------------------
 1 file changed, 19 deletions(-)
 delete mode 100644 doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment b/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment
deleted file mode 100644
index dced87287a..0000000000
--- a/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment
+++ /dev/null
@@ -1,19 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
- nickname="Richard"
- subject="comment 4"
- date="2011-03-28T01:19:25Z"
- content="""
-Another nice thing would be a summary of _what_ is wrong. I.e.
-
-    % git fsck
-    [...]
-    git-annex: 100 total failed
-      50 checksum failed
-      50 not enough copies exit
-
-And the same/similar for all other failure modes.
-
-
--- RichiH
-"""]]

From 6b5918c295715d0599005c9367f5dab5468169c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 21:43:25 -0400
Subject: [PATCH 1253/2835] some reorg and further remote generalization

---
 Annex.hs       |  23 -----------
 Config.hs      |  68 +++++++++++++++++++++++++++++++
 Content.hs     |   3 +-
 GitRepo.hs     |  30 ++++++++++----
 Remote.hs      |  24 ++++++++---
 Remote/Git.hs  | 106 ++++++++++++++-----------------------------------
 RemoteClass.hs |   5 +++
 Ssh.hs         |   4 +-
 UUID.hs        |   5 ++-
 Version.hs     |   3 +-
 10 files changed, 154 insertions(+), 117 deletions(-)
 create mode 100644 Config.hs

diff --git a/Annex.hs b/Annex.hs
index bb26608f4a..2723c6a008 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -17,12 +17,9 @@ module Annex (
 	queue,
 	queueRun,
 	queueRunAt,
-	setConfig,
-	repoConfig
 ) where
 
 import Control.Monad.State
-import Data.Maybe
 
 import qualified GitRepo as Git
 import qualified GitQueue
@@ -119,23 +116,3 @@ queueRunAt n = do
 	state <- get
 	let q = repoqueue state
 	when (GitQueue.size q >= n) queueRun
-
-{- Changes a git config setting in both internal state and .git/config -}
-setConfig :: String -> String -> Annex ()
-setConfig k value = do
-	g <- Annex.gitRepo
-	liftIO $ Git.run g "config" [Param k, Param value]
-	-- re-read git config and update the repo's state
-	g' <- liftIO $ Git.configRead g
-	Annex.changeState $ \s -> s { Annex.repo = g' }
-
-{- Looks up a per-remote config option in git config.
- - Failing that, tries looking for a global config option. -}
-repoConfig :: Git.Repo -> String -> String -> Annex String
-repoConfig r key def = do
-	g <- Annex.gitRepo
-	let def' = Git.configGet g global def
-	return $ Git.configGet g local def'
-	where
-		local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
-		global = "annex." ++ key
diff --git a/Config.hs b/Config.hs
new file mode 100644
index 0000000000..aae7d82915
--- /dev/null
+++ b/Config.hs
@@ -0,0 +1,68 @@
+{- Git configuration
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Config where
+
+import Data.Maybe
+import Control.Monad.State (liftIO)
+
+import qualified GitRepo as Git
+import qualified Annex
+import Types
+import Utility
+
+type ConfigKey = String
+
+{- Changes a git config setting in both internal state and .git/config -}
+setConfig :: ConfigKey -> String -> Annex ()
+setConfig k value = do
+	g <- Annex.gitRepo
+	liftIO $ Git.run g "config" [Param k, Param value]
+	-- re-read git config and update the repo's state
+	g' <- liftIO $ Git.configRead g
+	Annex.changeState $ \s -> s { Annex.repo = g' }
+
+{- Looks up a per-remote config setting in git config.
+ - Failing that, tries looking for a global config option. -}
+getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
+getConfig r key def = do
+	g <- Annex.gitRepo
+	let def' = Git.configGet g global def
+	return $ Git.configGet g local def'
+	where
+		local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
+		global = "annex." ++ key
+
+{- Calculates cost for a remote.
+ -
+ - The default cost is 100 for local repositories, and 200 for remote
+ - repositories; it can also be configured by remote..annex-cost
+ -}
+remoteCost :: Git.Repo -> Annex Int
+remoteCost r = do
+	c <- getConfig r "cost" ""
+	if not $ null c
+		then return $ read c
+		else if not $ Git.repoIsUrl r
+			then return 100
+			else return 200
+
+{- Checks if a repo should be ignored, based either on annex-ignore
+ - setting, or on command-line options. Allows command-line to override
+ - annex-ignore. -}
+remoteNotIgnored :: Git.Repo -> Annex Bool
+remoteNotIgnored r = do
+	ignored <- getConfig r "ignore" "false"
+	to <- match Annex.toremote
+	from <- match Annex.fromremote
+	if to || from
+		then return True
+		else return $ not $ Git.configTrue ignored
+	where
+		match a = do
+			n <- Annex.getState a
+			return $ n == Git.repoRemoteName r
diff --git a/Content.hs b/Content.hs
index 7aa30f7ffc..88e8dbc007 100644
--- a/Content.hs
+++ b/Content.hs
@@ -40,6 +40,7 @@ import Utility
 import StatFS
 import Key
 import DataUnits
+import Config
 
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
@@ -121,7 +122,7 @@ checkDiskSpace = checkDiskSpace' 0
 checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- Annex.gitRepo
-	r <- Annex.repoConfig g "diskreserve" ""
+	r <- getConfig g "diskreserve" ""
 	let reserve = case readSize dataUnits r of
 		Nothing -> megabyte
 		Just v -> v
diff --git a/GitRepo.hs b/GitRepo.hs
index ad58b28a00..1b14e4a636 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -12,6 +12,7 @@ module GitRepo (
 	Repo,
 	repoFromCwd,
 	repoFromAbsPath,
+	repoFromUnknown,
 	repoFromUrl,
 	localToUrl,
 	repoIsUrl,
@@ -41,6 +42,7 @@ module GitRepo (
 	remotes,
 	remotesAdd,
 	repoRemoteName,
+	repoRemoteNameSet,
 	inRepo,
 	notInRepo,
 	stagedFiles,
@@ -81,7 +83,7 @@ import Utility
 
 {- There are two types of repositories; those on local disk and those
  - accessed via an URL. -}
-data RepoLocation = Dir FilePath | Url URI
+data RepoLocation = Dir FilePath | Url URI | Unknown
 	deriving (Show, Eq)
 
 data Repo = Repo {
@@ -123,6 +125,10 @@ repoFromUrl url
 				Just v -> v
 				Nothing -> error $ "bad url " ++ url
 
+{- Creates a repo that has an unknown location. -}
+repoFromUnknown :: Repo
+repoFromUnknown = newFrom Unknown
+
 {- Converts a Local Repo into a remote repo, using the reference repo
  - which is assumed to be on the same host. -}
 localToUrl :: Repo -> Repo -> Repo
@@ -141,11 +147,13 @@ repoDescribe :: Repo -> String
 repoDescribe Repo { remoteName = Just name } = name
 repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
+repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
 {- Location of the repo, either as a path or url. -}
 repoLocation :: Repo -> String
 repoLocation Repo { location = Url url } = show url
 repoLocation Repo { location = Dir dir } = dir
+repoLocation Repo { location = Unknown } = undefined
 
 {- Constructs and returns an updated version of a repo with
  - different remotes list. -}
@@ -158,6 +166,14 @@ repoRemoteName :: Repo -> Maybe String
 repoRemoteName Repo { remoteName = Just name } = Just name
 repoRemoteName _ = Nothing
 
+{- Sets the name of a remote based on the git config key, such as
+   "remote.foo.url". -}
+repoRemoteNameSet :: Repo -> String -> Repo
+repoRemoteNameSet r k = r { remoteName = Just basename }
+	where
+		basename = join "." $ reverse $ drop 1 $
+				reverse $ drop 1 $ split "." k
+
 {- Some code needs to vary between URL and normal repos,
  - or bare and non-bare, these functions help with that. -}
 repoIsUrl :: Repo -> Bool
@@ -218,6 +234,7 @@ gitDir repo
 workTree :: Repo -> FilePath
 workTree r@(Repo { location = Url _ }) = urlPath r
 workTree (Repo { location = Dir d }) = d
+workTree Repo { location = Unknown } = undefined
 
 {- Given a relative or absolute filename in a repository, calculates the
  - name to use to refer to the file relative to a git repository's top.
@@ -393,10 +410,6 @@ configStore repo s = do
 	where
 		r = repo { config = configParse s }
 
-{- Checks if a string from git config is a true value. -}
-configTrue :: String -> Bool
-configTrue s = map toLower s == "true"
-
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
 configRemotes :: Repo -> IO [Repo]
 configRemotes repo = mapM construct remotepairs
@@ -404,10 +417,9 @@ configRemotes repo = mapM construct remotepairs
 		remotepairs = Map.toList $ filterremotes $ config repo
 		filterremotes = Map.filterWithKey (\k _ -> isremote k)
 		isremote k = startswith "remote." k && endswith ".url" k
-		remotename k = join "." $ reverse $ drop 1 $ reverse $ drop 1 $ split "." k
 		construct (k,v) = do
 			r <- gen v
-			return $ r { remoteName = Just $ remotename k }
+			return $ repoRemoteNameSet r k
 		gen v	| scpstyle v = repoFromUrl $ scptourl v
 			| isURI v = repoFromUrl v
 			| otherwise = repoFromRemotePath v repo
@@ -423,6 +435,10 @@ configRemotes repo = mapM construct remotepairs
 					| d !! 0 == '~' = '/':dir
 					| otherwise = "/~/" ++ dir
 
+{- Checks if a string from git config is a true value. -}
+configTrue :: String -> Bool
+configTrue s = map toLower s == "true"
+
 {- Parses git config --list output into a config map. -}
 configParse :: String -> Map.Map String String
 configParse s = Map.fromList $ map pair $ lines s
diff --git a/Remote.hs b/Remote.hs
index b3f1a0c6b8..5508e0d123 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -25,20 +25,35 @@ module Remote (
 import Control.Monad.State (liftIO)
 import Control.Monad (when, liftM)
 import Data.List
+import Data.String.Utils
 
 import RemoteClass
 import qualified Remote.Git
-import qualified Remote.S3
+--import qualified Remote.S3
 import Types
 import UUID
 import qualified Annex
 import Trust
 import LocationLog
+import Messages
 
-{- add generators for new Remotes here -}
-generators :: [Annex [Remote Annex]]
+{- Add generators for new Remotes here. -}
+generators :: [Annex (RemoteGenerator Annex)]
 generators = [Remote.Git.generate]
 
+{- Runs a list of generators. -}
+runGenerators :: [Annex (RemoteGenerator Annex)] -> Annex [Remote Annex]
+runGenerators gs = do
+	(actions, expensive) <- collect ([], []) gs
+	when (not $ null expensive) $
+		showNote $ "getting UUID for " ++ join ", " expensive
+	sequence actions
+	where
+		collect v [] = return v
+		collect (actions, expensive) (x:xs) = do
+			(a, e) <- x
+			collect (a++actions, e++expensive) xs
+
 {- Builds a list of all available Remotes.
  - Since doing so can be expensive, the list is cached in the Annex. -}
 genList :: Annex [Remote Annex]
@@ -46,8 +61,7 @@ genList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
-			lists <- sequence generators
-			let rs' = concat lists
+			rs' <- runGenerators generators
 			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 43e75b97bd..9021a2230c 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.GitRemote (
+module Remote.Git (
 	generate,
 	onRemote
 ) where
@@ -13,9 +13,8 @@ module Remote.GitRemote (
 import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as Map
-import Data.String.Utils
 import System.Cmd.Utils
-import Control.Monad (unless, filterM)
+import Control.Monad (filterM, liftM)
 
 import RemoteClass
 import Types
@@ -29,18 +28,34 @@ import Messages
 import CopyFile
 import RsyncFile
 import Ssh
+import Config
 
-generate :: Annex [Remote Annex]
+generate :: Annex (RemoteGenerator Annex)
 generate = do
-	readConfigs
 	g <- Annex.gitRepo
-	rs <- filterM repoNotIgnored (Git.remotes g)
-	mapM genRemote rs
+	allremotes <- filterM remoteNotIgnored $ Git.remotes g
+
+ 	{- It's assumed to be cheap to read the config of non-URL remotes,
+	 - so this is done each time git-annex is run. Conversely,
+	 - the config of an URL remote is only read when there is no
+	 - cached UUID value. -}
+	let cheap = filter (not . Git.repoIsUrl) allremotes
+	let expensive = filter Git.repoIsUrl allremotes
+	expensive_todo <- filterM cachedUUID expensive
+	let skip = filter (`notElem` expensive_todo) expensive
+	let todo = cheap++expensive_todo
+
+	let actions = map genRemote skip ++ 
+		map (\r -> genRemote =<< tryGitConfigRead r) todo
+	return (actions, map Git.repoDescribe expensive_todo)
+
+	where
+		cachedUUID r = liftM null $ getUUID r
 	
 genRemote :: Git.Repo -> Annex (Remote Annex)
 genRemote r = do
 	u <- getUUID r
-	c <- repoCost r
+	c <- remoteCost r
 	return Remote {
 		uuid = u,
 		cost = c,
@@ -52,40 +67,13 @@ genRemote r = do
 		hasKeyCheap = not (Git.repoIsUrl r)
 	}
 
-{- Reads the configs of git remotes.
- -
- - It's assumed to be cheap to read the config of non-URL remotes,
- - so this is done each time git-annex is run. Conversely,
- - the config of an URL remote is only read when there is no
- - cached UUID value.
- -}
-readConfigs :: Annex ()
-readConfigs = do
-	g <- Annex.gitRepo
-	allremotes <- filterM repoNotIgnored $ Git.remotes g
-	let cheap = filter (not . Git.repoIsUrl) allremotes
-	let expensive = filter Git.repoIsUrl allremotes
-	doexpensive <- filterM cachedUUID expensive
-	unless (null doexpensive) $
-		showNote $ "getting UUID for " ++
-			list doexpensive ++ "..."
-	let todo = cheap ++ doexpensive
-	unless (null todo) $ do
-		mapM_ tryGitConfigRead todo
-	where
-		cachedUUID r = do
-			u <- getUUID r
-			return $ null u
-
-{- The git configs for the git repo's remotes is not read on startup
- - because reading it may be expensive. This function tries to read the
- - config for a specified remote, and updates state. If successful, it
- - returns the updated git repo. -}
-tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo)
+{- Tries to read the config for a specified remote, updates state, and
+ - returns the updated repo. -}
+tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
-	| not $ Map.null $ Git.configMap r = return $ Right r -- already read
+	| not $ Map.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
-	| Git.repoIsUrl r = return $ Left r
+	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
@@ -104,43 +92,13 @@ tryGitConfigRead r
 			let l = Git.remotes g
 			let g' = Git.remotesAdd g $ exchange l r'
 			Annex.changeState $ \s -> s { Annex.repo = g' }
-			return $ Right r'
+			return r'
 		exchange [] _ = []
 		exchange (old:ls) new =
 			if Git.repoRemoteName old == Git.repoRemoteName new
 				then new : exchange ls new
 				else old : exchange ls new
 
-{- Calculates cost for a repo.
- -
- - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost
- -}
-repoCost :: Git.Repo -> Annex Int
-repoCost r = do
-	c <- Annex.repoConfig r "cost" ""
-	if not $ null c
-		then return $ read c
-		else if Git.repoIsUrl r
-			then return 200
-			else return 100
-
-{- Checks if a repo should be ignored, based either on annex-ignore
- - setting, or on command-line options. Allows command-line to override
- - annex-ignore. -}
-repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = do
-	ignored <- Annex.repoConfig r "ignore" "false"
-	to <- match Annex.toremote
-	from <- match Annex.fromremote
-	if to || from
-		then return True
-		else return $ not $ Git.configTrue ignored
-	where
-		match a = do
-			n <- Annex.getState a
-			return $ n == Git.repoRemoteName r
-
 {- Checks if a given remote has the content for a key inAnnex.
  - If the remote cannot be accessed, returns a Left error.
  -}
@@ -219,7 +177,7 @@ rsyncParams r sending key file = do
 		]
 	-- Convert the ssh command into rsync command line.
 	let eparam = rsyncShell (Param shellcmd:shellparams)
-	o <- Annex.repoConfig r "rsync-options" ""
+	o <- getConfig r "rsync-options" ""
 	let base = options ++ map Param (words o) ++ eparam
 	if sending
 		then return $ base ++ [dummy, File file]
@@ -262,7 +220,3 @@ git_annex_shell r command params
 		shellopts = (Param command):(File dir):params
 		sshcmd = shellcmd ++ " " ++ 
 			unwords (map shellEscape $ toCommand shellopts)
-
-{- Human visible list of remotes. -}
-list :: [Git.Repo] -> String
-list remotes = join ", " $ map Git.repoDescribe remotes
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 38e8407a54..eb4a017486 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -13,6 +13,11 @@ import Control.Exception
 
 import Key
 
+{- A remote generator identifies configured remotes, and returns an action
+ - that can be run to set up each remote, and a list of names of remotes
+ - that are not cheap to set up. -}
+type RemoteGenerator a = ([a (Remote a)], [String])
+
 data Remote a = Remote {
 	-- each Remote has a unique uuid
 	uuid :: String,
diff --git a/Ssh.hs b/Ssh.hs
index 04cd9bec83..6d01a56423 100644
--- a/Ssh.hs
+++ b/Ssh.hs
@@ -7,17 +7,17 @@
 
 module Ssh where
 
-import qualified Annex
 import qualified GitRepo as Git
 import Utility
 import Types
+import Config
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
  - passed command. -}
 sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam]
 sshToRepo repo sshcmd = do
-	s <- Annex.repoConfig repo "ssh-options" ""
+	s <- getConfig repo "ssh-options" ""
 	let sshoptions = map Param (words s)
 	let sshport = case Git.urlPort repo of
 		Nothing -> []
diff --git a/UUID.hs b/UUID.hs
index 5caf110453..eb1fb319c6 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -35,6 +35,7 @@ import Locations
 import qualified Annex
 import Utility
 import qualified SysConfig
+import Config
 
 type UUID = String
 
@@ -69,7 +70,7 @@ getUUID r = do
 		else return c
 	where
 		cached g = Git.configGet g cachekey ""
-		updatecache g u = when (g /= r) $ Annex.setConfig cachekey u
+		updatecache g u = when (g /= r) $ setConfig cachekey u
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
@@ -82,7 +83,7 @@ prepUUID = do
 	u <- getUUID g
 	when ("" == u) $ do
 		uuid <- liftIO $ genUUID
-		Annex.setConfig configkey uuid
+		setConfig configkey uuid
 
 {- Pretty-prints a list of UUIDs -}
 prettyPrintUUIDs :: [UUID] -> Annex String
diff --git a/Version.hs b/Version.hs
index d061a2eab2..947af8cef5 100644
--- a/Version.hs
+++ b/Version.hs
@@ -15,6 +15,7 @@ import Types
 import qualified Annex
 import qualified GitRepo as Git
 import Locations
+import Config
 
 type Version = String
 
@@ -54,7 +55,7 @@ getVersion = do
 					return defaultVersion
 
 setVersion :: Annex ()
-setVersion = Annex.setConfig versionField defaultVersion
+setVersion = setConfig versionField defaultVersion
 
 checkVersion :: Annex ()
 checkVersion = do

From 65b72604d73b1d92dea1d81984964394235834bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 22:00:44 -0400
Subject: [PATCH 1254/2835] skeleton of S3 remote

---
 Remote.hs    |  7 ++++--
 Remote/S3.hs | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 2 deletions(-)
 create mode 100644 Remote/S3.hs

diff --git a/Remote.hs b/Remote.hs
index 5508e0d123..6aab4a741c 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -29,7 +29,7 @@ import Data.String.Utils
 
 import RemoteClass
 import qualified Remote.Git
---import qualified Remote.S3
+import qualified Remote.S3
 import Types
 import UUID
 import qualified Annex
@@ -39,7 +39,10 @@ import Messages
 
 {- Add generators for new Remotes here. -}
 generators :: [Annex (RemoteGenerator Annex)]
-generators = [Remote.Git.generate]
+generators =
+	[ Remote.Git.generate
+	, Remote.S3.generate
+	]
 
 {- Runs a list of generators. -}
 runGenerators :: [Annex (RemoteGenerator Annex)] -> Annex [Remote Annex]
diff --git a/Remote/S3.hs b/Remote/S3.hs
new file mode 100644
index 0000000000..818cde2030
--- /dev/null
+++ b/Remote/S3.hs
@@ -0,0 +1,61 @@
+{- Amazon S3 remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.S3 (generate) where
+
+import qualified Data.Map as Map
+import Data.String.Utils
+import Control.Monad (filterM, liftM)
+
+import RemoteClass
+import Types
+import qualified GitRepo as Git
+import qualified Annex
+import UUID
+import Config
+
+generate :: Annex (RemoteGenerator Annex)
+generate = do
+	g <- Annex.gitRepo
+	remotes <- filterM remoteNotIgnored $ findS3Remotes g
+	todo <- filterM cachedUUID remotes
+	let ok = filter (`notElem` todo) remotes
+	
+	let actions = map genRemote ok ++
+		map (\r -> genRemote =<< tryS3ConfigRead r) todo
+	return (actions, map Git.repoDescribe todo)
+
+	where
+		cachedUUID r = liftM null $ getUUID r
+
+genRemote :: Git.Repo -> Annex (Remote Annex)
+genRemote r = do
+	return Remote {
+		uuid = error "TODO",
+		cost = error "TODO",
+		name = Git.repoDescribe r,
+		storeKey = error "TODO",
+		retrieveKeyFile = error "TODO",
+		removeKey = error "TODO",
+		hasKey = error "TODO",
+		hasKeyCheap = False
+	}
+
+{- S3 remotes have a remote..annex-s3bucket config setting.
+ - Git.Repo does not normally generate remotes for things that
+ - have no configured url, so the Git.Repo objects have to be
+ - constructed as coming from an unknown location. -}
+findS3Remotes :: Git.Repo -> [Git.Repo]
+findS3Remotes r = map construct remotepairs
+	where
+		remotepairs = Map.toList $ filterremotes $ Git.configMap r
+		filterremotes = Map.filterWithKey (\k _ -> s3remote k)
+		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
+		s3remote k = startswith "remote." k && endswith ".annex-s3bucket" k
+
+tryS3ConfigRead :: Git.Repo -> Annex Git.Repo
+tryS3ConfigRead r = error "TODO"

From c0fd38bfa9ade9dc4515140cf5cf5619582c5a89 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 22:52:13 -0400
Subject: [PATCH 1255/2835] document S3 remotes

---
 Remote/S3.hs                                  |  4 +-
 doc/backends.mdwn                             |  7 ++++
 doc/special_remotes.mdwn                      | 38 +++++++++++++++++++
 doc/walkthrough.mdwn                          |  1 +
 ...ing_file_content_between_repositories.mdwn |  4 +-
 doc/walkthrough/using_Amazon_S3.mdwn          | 15 ++++++++
 6 files changed, 65 insertions(+), 4 deletions(-)
 create mode 100644 doc/special_remotes.mdwn
 create mode 100644 doc/walkthrough/using_Amazon_S3.mdwn

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 818cde2030..bc010bf0bb 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -45,7 +45,7 @@ genRemote r = do
 		hasKeyCheap = False
 	}
 
-{- S3 remotes have a remote..annex-s3bucket config setting.
+{- S3 remotes have a remote..annex-s3-bucket config setting.
  - Git.Repo does not normally generate remotes for things that
  - have no configured url, so the Git.Repo objects have to be
  - constructed as coming from an unknown location. -}
@@ -55,7 +55,7 @@ findS3Remotes r = map construct remotepairs
 		remotepairs = Map.toList $ filterremotes $ Git.configMap r
 		filterremotes = Map.filterWithKey (\k _ -> s3remote k)
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
-		s3remote k = startswith "remote." k && endswith ".annex-s3bucket" k
+		s3remote k = startswith "remote." k && endswith ".annex-s3-bucket" k
 
 tryS3ConfigRead :: Git.Repo -> Annex Git.Repo
 tryS3ConfigRead r = error "TODO"
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 22164850ab..b0a2c882aa 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -9,6 +9,10 @@ to retrieve the file's content (its value).
 Multiple pluggable backends are supported, and a single repository
 can use different backends for different files.
 
+These backends can transfer file contents in configured git remotes.
+It's also possible to use [[special_remotes]], such as Amazon S3 with
+these backends.
+
 * `WORM` ("Write Once, Read Many") This backend assumes that any file with
   the same basename, size, and modification time has the same content. So with
   this backend, files can be moved around, but should never be added to
@@ -19,6 +23,9 @@ can use different backends for different files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
   researching checksum collisions and wants to annex their colliding data. ;)
+
+These backends store file contents in other key/value stores.
+
 * `URL` -- This backend downloads the file's content from an external URL.
 
 The `annex.backends` git-config setting can be used to list the backends
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
new file mode 100644
index 0000000000..a62d55f5bb
--- /dev/null
+++ b/doc/special_remotes.mdwn
@@ -0,0 +1,38 @@
+Most [[backends]] can transfer data to and from configured git remotes.
+Normally those remotes are normal git repositories (bare and non-bare),
+that store the file contents in their own git annex directory.
+
+But, git-annex also extends git's concept of remotes, with these special
+types of remotes. These can be used just like any normal remote by git-annex.
+They cannot be used by other git commands though.
+
+## Amazon S3
+
+Stores file contents in a bucket in Amazon S3 or a similar service.
+
+Example of configuring such a remote:
+
+	git config remote.mys3.annex-s3-bucket myannex
+	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+
+That creates a remote named "mys3" using the bucket named "myannex",
+which will be created if it doesn't already exist.
+
+Here is the full set of configurable settings for Amazon S3.
+Each setting can be configured on a per-remote basis in git-config,
+or globally in an environment variable.
+
+* `remote.$name.annex-s3-secret-access-key` `ANNEX_S3_SECRET_ACCESS_KEY` 
+  Your S3 password. Usually stored in the environment variable
+  to avoid it being exposed.
+* `remote.$name.annex-s3-access-key-id` `ANNEX_S3_ACCESS_KEY_ID` 
+  Your S3 access key. For example, "". Does not need to be kept
+  private.
+* `remote.$name.annex-s3-host` `ANNEX_S3_HOST` 
+  Host to connect to. Default is s3.amazonaws.com.
+* `remote.$name.annex-s3-port` `ANNEX_S3_PORT` 
+  Port to connect to. Default is 80.
+* `remote.$name.annex-s3-datacenter` `ANNEX_S3_DATACENTER` 
+  Name of the datacenter to use. Default is "US";
+  other valid values include "EU", "us-west-1", and "ap-southeast-1".
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index 3b4f7d56a4..53f0be6bb4 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -14,6 +14,7 @@ A walkthrough of the basic features of git-annex.
 	modifying_annexed_files
 	using_ssh_remotes
 	moving_file_content_between_repositories
+	using_Amazon_S3
 	using_the_URL_backend
 	using_the_SHA1_backend
 	migrating_data_to_a_new_backend
diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn
index 6b3e3f4e80..27dffe9138 100644
--- a/doc/walkthrough/moving_file_content_between_repositories.mdwn
+++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn
@@ -6,8 +6,8 @@ server to your laptop. Doing that by hand (by using `git annex get` and
 makes it very easy.
 
 	# git annex move my_cool_big_file --to usbdrive
-	move my_cool_big_file (moving to usbdrive...) ok
+	move my_cool_big_file (to usbdrive...) ok
 	# git annex move video/hackity_hack_and_kaxxt.mov --from fileserver
-	move video/hackity_hack_and_kaxxt.mov (moving from fileserver...)
+	move video/hackity_hack_and_kaxxt.mov (from fileserver...)
 	WORM-s86050597-m1274316523--hackity_hack_and_kax 100%   82MB 199.1KB/s   07:02
 	ok
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
new file mode 100644
index 0000000000..d7222731b3
--- /dev/null
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -0,0 +1,15 @@
+git-annex extends git's usual remotes with some [[special_remotes]], that
+are not git repositories. This way you can set up a remote using say,
+Amazon S3, and use git-annex to transfer files into the cloud.
+
+	# git config remote.mys3.annex-s3-bucket myannex
+	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+	# git annex copy my_cool_big_file --to mys3
+	copy my_cool_big_file (to mys3...) ok
+	# git annex move video/hackity_hack_and_kaxxt.mov --to mys3
+	move video/hackity_hack_and_kaxxt.mov (to mys3...) ok
+
+An Amazon S3 remote works just like a ssh remote, except it does not have
+a git repository at the other end, and it costs you money. :) For full
+details about setting them up, see [[special_remotes]].

From 026c76914e21c768a38e86461849213e33b70046 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 23:11:09 -0400
Subject: [PATCH 1256/2835] update

---
 doc/special_remotes.mdwn | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index a62d55f5bb..3849b0e841 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -23,16 +23,20 @@ Here is the full set of configurable settings for Amazon S3.
 Each setting can be configured on a per-remote basis in git-config,
 or globally in an environment variable.
 
-* `remote.$name.annex-s3-secret-access-key` `ANNEX_S3_SECRET_ACCESS_KEY` 
-  Your S3 password. Usually stored in the environment variable
-  to avoid it being exposed.
 * `remote.$name.annex-s3-access-key-id` `ANNEX_S3_ACCESS_KEY_ID` 
-  Your S3 access key. For example, "". Does not need to be kept
-  private.
+  Your S3 Access Key ID. Does not need to be kept private.
+* `remote.$name.annex-s3-secret-access-key` `ANNEX_S3_SECRET_ACCESS_KEY` 
+  Your S3 Secret Access Key. This is a password. Usually stored in the
+  environment variable to avoid it being exposed.
 * `remote.$name.annex-s3-host` `ANNEX_S3_HOST` 
   Host to connect to. Default is s3.amazonaws.com.
 * `remote.$name.annex-s3-port` `ANNEX_S3_PORT` 
   Port to connect to. Default is 80.
 * `remote.$name.annex-s3-datacenter` `ANNEX_S3_DATACENTER` 
-  Name of the datacenter to use. Default is "US";
+  Name of the datacenter to use when creating a new bucket. Default is "US";
   other valid values include "EU", "us-west-1", and "ap-southeast-1".
+* `remote.$name.annex-s3-storageclass` `ANNEX_S3_STORAGECLASS` 
+  Name of storage class to use when adding new content to the bucket.
+  Default is "STANDARD". If you have configured git-annex to preserve
+  multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY" to
+  save money.

From a7bd63eb0100fd282da9058acc28935bdfdf25df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 01:32:47 -0400
Subject: [PATCH 1257/2835] basic s3 remote start

But bucket name is not handled right; it needs to be globally unique.
---
 Config.hs        |  10 ++---
 Remote/S3.hs     | 100 ++++++++++++++++++++++++++++++++++++++---------
 debian/changelog |   3 +-
 3 files changed, 89 insertions(+), 24 deletions(-)

diff --git a/Config.hs b/Config.hs
index aae7d82915..c821364ced 100644
--- a/Config.hs
+++ b/Config.hs
@@ -31,11 +31,11 @@ setConfig k value = do
 getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
 getConfig r key def = do
 	g <- Annex.gitRepo
-	let def' = Git.configGet g global def
-	return $ Git.configGet g local def'
-	where
-		local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
-		global = "annex." ++ key
+	let def' = Git.configGet g ("annex." ++ key) def
+	return $ Git.configGet g (remoteConfig r key) def'
+
+remoteConfig :: Git.Repo -> ConfigKey -> String
+remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 
 {- Calculates cost for a remote.
  -
diff --git a/Remote/S3.hs b/Remote/S3.hs
index bc010bf0bb..23ec33bb59 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -7,9 +7,18 @@
 
 module Remote.S3 (generate) where
 
+import Network.AWS.AWSConnection
+import Network.AWS.S3Object
+import Network.AWS.S3Bucket
+import Network.AWS.AWSResult
+import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as Map
 import Data.String.Utils
-import Control.Monad (filterM, liftM)
+import Control.Monad (filterM, liftM, when)
+import Control.Monad.State (liftIO)
+import System.Environment
+import Data.Char
+import Messages
 
 import RemoteClass
 import Types
@@ -25,26 +34,13 @@ generate = do
 	todo <- filterM cachedUUID remotes
 	let ok = filter (`notElem` todo) remotes
 	
-	let actions = map genRemote ok ++
-		map (\r -> genRemote =<< tryS3ConfigRead r) todo
+	let actions = map (\r -> genRemote r =<< getUUID r) ok ++
+		map (\r -> genRemote r =<< getS3UUID r) todo
 	return (actions, map Git.repoDescribe todo)
 
 	where
 		cachedUUID r = liftM null $ getUUID r
 
-genRemote :: Git.Repo -> Annex (Remote Annex)
-genRemote r = do
-	return Remote {
-		uuid = error "TODO",
-		cost = error "TODO",
-		name = Git.repoDescribe r,
-		storeKey = error "TODO",
-		retrieveKeyFile = error "TODO",
-		removeKey = error "TODO",
-		hasKey = error "TODO",
-		hasKeyCheap = False
-	}
-
 {- S3 remotes have a remote..annex-s3-bucket config setting.
  - Git.Repo does not normally generate remotes for things that
  - have no configured url, so the Git.Repo objects have to be
@@ -57,5 +53,73 @@ findS3Remotes r = map construct remotepairs
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
 		s3remote k = startswith "remote." k && endswith ".annex-s3-bucket" k
 
-tryS3ConfigRead :: Git.Repo -> Annex Git.Repo
-tryS3ConfigRead r = error "TODO"
+genRemote :: Git.Repo -> UUID -> Annex (Remote Annex)
+genRemote r u = do
+	c <- remoteCost r
+	return Remote {
+		uuid = u,
+		cost = c,
+		name = Git.repoDescribe r,
+		storeKey = error "TODO",
+		retrieveKeyFile = error "TODO",
+		removeKey = error "TODO",
+		hasKey = error "TODO",
+		hasKeyCheap = False
+	}
+
+s3Connection :: Git.Repo -> Annex (Maybe AWSConnection)
+s3Connection r = do
+	host <- getS3Config r "s3-host" (Just defaultAmazonS3Host)
+	port <- getS3Config r "s3-port" (Just $ show defaultAmazonS3Port)
+	accesskey <- getS3Config r "s3-access-key-id" Nothing
+	secretkey <- getS3Config r "s3-secret-access-key" Nothing
+	case reads port of
+		[(p, _)] -> return $ Just $ AWSConnection host p accesskey secretkey
+		_ -> error $ "bad S3 port value: " ++ port
+
+withS3Connection :: Git.Repo -> Annex a -> ((AWSConnection, String) -> Annex a) -> Annex a
+withS3Connection r def a = do
+	c <- s3Connection r
+	case c of
+		Nothing -> def
+		Just c' -> do
+			b <- getConfig r "s3-bucket" ""
+			a (c', b)
+
+getS3Config :: Git.Repo -> String -> Maybe String-> Annex String
+getS3Config r s def = do
+	e <- liftIO $ catch (liftM Just $ getEnv envvar) (const $ return def)
+	v <- case e of
+		Nothing -> getConfig r s ""
+		Just d -> getConfig r s d
+	when (null v) $ error $ "set " ++ envvar ++ " or " ++ remoteConfig r s
+	return v
+	where
+		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
+
+{- The UUID of a S3 bucket is stored in a file "git-annex-uuid" in the
+ - bucket. Gets the UUID, or if there is none, sets a new UUID, possibly
+ - also creating the bucket. -}
+getS3UUID :: Git.Repo -> Annex UUID
+getS3UUID r = withS3Connection r disable $ \(c, b) -> do
+	res <- liftIO $
+		getObject c $ S3Object b uuidfile "" [] L.empty
+	case res of
+		Right o -> return $ L.unpack $ obj_data o
+		Left _ -> do
+			location <- getS3Config r "s3-datacenter" (Just "EU")
+			-- bucket may already exist, or not
+			_ <- liftIO $ createBucketIn c b location
+			u <- getUUID r
+			res' <- liftIO $ sendObject c $
+				S3Object b uuidfile "" [] $
+					L.pack u
+			case res' of
+				Right _ -> return u
+				Left e -> do
+					warning $ prettyReqError e
+					disable
+					
+	where
+		uuidfile = "git-annex-uuid"
+		disable = return "" -- empty uuid will disable this remote
diff --git a/debian/changelog b/debian/changelog
index 2f532784d4..0469f2242b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,12 @@
 git-annex (0.20110326) UNRELEASED; urgency=low
 
-  * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
+  * Amazon is S3 now supported as a special type of remote.
   * Generalized remotes handling, laying groundwork for remotes that are
     not regular git remotes.
   * Provide a less expensive version of `git annex copy --to`, enabled
     via --fast. This assumes that location tracking information is correct,
     rather than contacting the remote for every file.
+  * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 

From 1878745a4693913417f65c7c5182e939512e9b22 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 02:12:05 -0400
Subject: [PATCH 1258/2835] more s3 docs

---
 debian/changelog                     |  2 +-
 doc/git-annex.mdwn                   | 31 +++++++++++++++++++++++++++
 doc/internals.mdwn                   | 11 ++++++++++
 doc/special_remotes.mdwn             | 32 +---------------------------
 doc/todo/S3.mdwn                     |  2 ++
 doc/walkthrough/using_Amazon_S3.mdwn | 22 ++++++++++++++-----
 6 files changed, 63 insertions(+), 37 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 0469f2242b..945a4ed6b0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,6 @@
 git-annex (0.20110326) UNRELEASED; urgency=low
 
-  * Amazon is S3 now supported as a special type of remote.
+  * Amazon S3 is now supported as a special type of remote.
   * Generalized remotes handling, laying groundwork for remotes that are
     not regular git remotes.
   * Provide a less expensive version of `git annex copy --to`, enabled
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 8afe93c109..6960c19662 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,6 +132,16 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
+* s3bucket name description [datacenter host port]
+
+  Creates a bucket in Amazon S3. The bucket's name can be used
+  to configure git remote using the bucket.
+
+  The datacenter defaults to "US". Other values include "EU",
+  "us-west-1", and "ap-southeast-1".
+
+  To use a different, S3-compatable service, specify a host and port.
+
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
@@ -387,6 +397,25 @@ Here are all the supported configuration settings.
   Default ssh and rsync options to use if a remote does not have
   specific options.
 
+* `remote..annex-s3-access-key-id`
+
+  Your S3 Access Key ID. Does not need to be kept private.
+  If not set, the environment variable `AWS_ACCESS_KEY_ID`
+  will be used.
+
+* `remote..annex-s3-secret-access-key`
+
+  Your S3 Secret Access Key. This is a password.
+  If not set, the environment variable `AWS_SECRET_ACCESS_KEY`
+  will be used.
+
+* `remote..annex-s3-storageclass`
+
+  Storage class to use when adding new content to S3. The default
+  is "STANDARD". If you have configured git-annex to preserve
+  multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY"
+  to save money.
+
 * `annex.diskreserve`
 
   Amount of disk space to reserve. Disk space is checked when transferring
@@ -401,6 +430,8 @@ Here are all the supported configuration settings.
 
   Automatically maintained, and used to automate upgrades between versions.
 
+  
+
 # CONFIGURATION VIA .gitattributes
 
 The backend used when adding a new file to the annex can be configured
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index a133320b4b..55b1045a11 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -30,6 +30,17 @@ space and then the description through to the end of the line. Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 laptop
 	26339d22-446b-11e0-9101-002170d25c55 usb disk
 
+## `git-annex/s3.log`
+
+Associates the UUIDs of Amazon S3 buckets with a bucket nickname and connection
+information. Example:
+
+	be72acb8-5901-11e0-b600-002170d25c55 mybucket s3.amazonaws.com 80
+
+Note that the actual bucket name used on S3 in the above example 
+is "mybucket-be72acb8-5901-11e0-b600-002170d25c55". The UUID is included
+in the bucket name to ensure it is globally unique.
+
 ## `.git-annex/trust.log`
 
 Records the [[trust]] information for repositories. Does not exist unless
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 3849b0e841..7dc54fd9bf 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -9,34 +9,4 @@ They cannot be used by other git commands though.
 ## Amazon S3
 
 Stores file contents in a bucket in Amazon S3 or a similar service.
-
-Example of configuring such a remote:
-
-	git config remote.mys3.annex-s3-bucket myannex
-	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
-
-That creates a remote named "mys3" using the bucket named "myannex",
-which will be created if it doesn't already exist.
-
-Here is the full set of configurable settings for Amazon S3.
-Each setting can be configured on a per-remote basis in git-config,
-or globally in an environment variable.
-
-* `remote.$name.annex-s3-access-key-id` `ANNEX_S3_ACCESS_KEY_ID` 
-  Your S3 Access Key ID. Does not need to be kept private.
-* `remote.$name.annex-s3-secret-access-key` `ANNEX_S3_SECRET_ACCESS_KEY` 
-  Your S3 Secret Access Key. This is a password. Usually stored in the
-  environment variable to avoid it being exposed.
-* `remote.$name.annex-s3-host` `ANNEX_S3_HOST` 
-  Host to connect to. Default is s3.amazonaws.com.
-* `remote.$name.annex-s3-port` `ANNEX_S3_PORT` 
-  Port to connect to. Default is 80.
-* `remote.$name.annex-s3-datacenter` `ANNEX_S3_DATACENTER` 
-  Name of the datacenter to use when creating a new bucket. Default is "US";
-  other valid values include "EU", "us-west-1", and "ap-southeast-1".
-* `remote.$name.annex-s3-storageclass` `ANNEX_S3_STORAGECLASS` 
-  Name of storage class to use when adding new content to the bucket.
-  Default is "STANDARD". If you have configured git-annex to preserve
-  multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY" to
-  save money.
+See [[walkthrough/using_Amazon_S3]] for examples.
diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 09a64f1a7a..356b2af2eb 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -1,3 +1,5 @@
+[[done]]
+
 Support Amazon S3 as a file storage backend.
 
 There's a haskell library that looks good. Not yet in Debian.
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index d7222731b3..cadd78582c 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -2,14 +2,26 @@ git-annex extends git's usual remotes with some [[special_remotes]], that
 are not git repositories. This way you can set up a remote using say,
 Amazon S3, and use git-annex to transfer files into the cloud.
 
-	# git config remote.mys3.annex-s3-bucket myannex
-	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+First, export your S3 credentials:
+
+	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+
+Next, create a bucket, giving it a name and a description:
+
+	git annex s3bucket mybucket "my Amazon S3 bucket"
+	s3bucket (creating mybucket...) ok
+
+Finally, configure a git remote to use the bucket you created:
+
+	git config remote.mys3.annex-s3-bucket mybucket
+
+Now the remote can be used like any other remote.
+
 	# git annex copy my_cool_big_file --to mys3
 	copy my_cool_big_file (to mys3...) ok
 	# git annex move video/hackity_hack_and_kaxxt.mov --to mys3
 	move video/hackity_hack_and_kaxxt.mov (to mys3...) ok
 
 An Amazon S3 remote works just like a ssh remote, except it does not have
-a git repository at the other end, and it costs you money. :) For full
-details about setting them up, see [[special_remotes]].
+a git repository at the other end, and it costs you money. :)

From 61063dee6cc50652b393979358142f246894de58 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 07:26:41 +0000
Subject: [PATCH 1259/2835] Added a comment

---
 .../comment_1_9a7b09de132097100c1a68ea7b846727._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment
new file mode 100644
index 0000000000..aa5e46ca2b
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-03-28T07:23:41Z"
+ content="""
+One possible work around is to just create a loopback file system with a case sensitive filesystem. I think I might do that for anything that I really care about for now.
+"""]]

From ee9973019c933a58568040f8fd12fb8736dbc965 Mon Sep 17 00:00:00 2001
From: "http://ertai.myopenid.com/" 
Date: Mon, 28 Mar 2011 12:32:30 +0000
Subject: [PATCH 1260/2835]

---
 ...d_check__47__fix_the_permissions_of_.git__47__annex.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn

diff --git a/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
new file mode 100644
index 0000000000..ec8b6d2330
--- /dev/null
+++ b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
@@ -0,0 +1,6 @@
+git annex carefully setup restrictive permissions of .git/annex directories and files.
+
+The fsck command should check that they are still correct.
+The fix command should fix them.
+
+PS: Thanks for this nice tool!

From 3bdc5eb29077add9f2de18ba587ca88bb98cb63e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 08:40:05 -0400
Subject: [PATCH 1261/2835] will need gpg encryption for s3

---
 doc/git-annex.mdwn                   | 6 +++++-
 doc/special_remotes.mdwn             | 1 +
 doc/walkthrough/using_Amazon_S3.mdwn | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 6960c19662..c01f4fbc59 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,7 +132,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
-* s3bucket name description [datacenter host port]
+* s3bucket name description [datacenter host port] [--key=gpgkey]
 
   Creates a bucket in Amazon S3. The bucket's name can be used
   to configure git remote using the bucket.
@@ -142,6 +142,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   To use a different, S3-compatable service, specify a host and port.
 
+  By default, data (including filenames) is encrypted using gpg.
+  To use a key other than the default gpg key, specify it with
+  the --key option. To disable encryption, specify "none".
+
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 7dc54fd9bf..717ec48404 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -9,4 +9,5 @@ They cannot be used by other git commands though.
 ## Amazon S3
 
 Stores file contents in a bucket in Amazon S3 or a similar service.
+Content is stored encrypted by gpg.
 See [[walkthrough/using_Amazon_S3]] for examples.
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index cadd78582c..8cb77ab6cd 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -24,4 +24,6 @@ Now the remote can be used like any other remote.
 	move video/hackity_hack_and_kaxxt.mov (to mys3...) ok
 
 An Amazon S3 remote works just like a ssh remote, except it does not have
-a git repository at the other end, and it costs you money. :)
+a git repository at the other end, and it costs you money. :) In particular,
+all data is stored encrypted with gpg, so neither Amazon nor anyone in
+between can see it.

From 02601c6b9f2129e80323abea5ad76bf64c27b574 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 08:41:23 -0400
Subject: [PATCH 1262/2835] analysis; workaround

---
 ..._version_upgrade_leaves_repo_unusable.mdwn | 26 ++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 1218b0e517..964a176bd1 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -1,6 +1,10 @@
 foo is a local repo, bar is a bare remote.
 
-I upgraded foo's git-annex to 0.20110325 and upgraded a local repo backend to version 2. I then ran `git annex copy . --to bar` and checked the remote. This created WORM:SHA512--123123 files in annex/objects. Understandable but unwanted. So I upgraded git-annex on bar's machine, as well.
+I upgraded foo's git-annex to 0.20110325 and upgraded a local repo backend
+to version 2. I then ran `git annex copy . --to bar` and checked the
+remote. This created WORM:SHA512--123123 files in annex/objects.
+Understandable but unwanted. So I upgraded git-annex on bar's machine, as
+well.
 
     % git annex copy . --to bar
     copy quux (checking bar) git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade (to bar)
@@ -33,3 +37,23 @@ Running the copy job again, I am still getting the same error as above (as expec
 > joey@kitenet.net if you don't want to post them here. --[[Joey]]
 
 >> Sent. -- RichiH
+
+>>> Ok, I'm going to go work on my reading comprehension. I see now
+>>> that you
+>>> explained the problem pretty well. The problem is caused by these
+>>> few weird v1 mixed with v2 keys in the annex.
+>>> Ones like "annex/objects/WORM:SHA512--$sha512".
+>>>
+>>> That's a v1 key, but a corrupt form of the key; it's missing the 
+>>> size and mtime fields that all WORM keys have in v1. And 
+>>> the filename is itself a key, a v2 SHA512 key. My guess at what
+>>> happened is that these were created when you did the `git annex copy`
+>>> to the v1 bare repo.
+>>>
+>>> So, assuming none of these are the only copy of your data
+>>> (which you should check), the best thing to do is delete those
+>>> keys, and re-run the copy now that it's upgraded. Not sure 
+>>> it even makes sense to fix the crash. I should probably focus
+>>> on fixing what let these mixed keys be created by the mixed-version
+>>> copy.
+>>> --[[Joey]]

From d2ed1b3a99da45ae25e84cbc832693442bab7de6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 08:52:42 -0400
Subject: [PATCH 1263/2835] second thought

---
 doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 964a176bd1..7bbd02364f 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -56,4 +56,7 @@ Running the copy job again, I am still getting the same error as above (as expec
 >>> it even makes sense to fix the crash. I should probably focus
 >>> on fixing what let these mixed keys be created by the mixed-version
 >>> copy.
+>>>
+>>> On second thought, you shouldn't delete anything. I'll simply
+>>> make the v2 upgrade detect and work around this bug.
 >>> --[[Joey]]

From 9d86d02b3db23f0b8848f4a9a044befa58e1ecbb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 08:55:00 -0400
Subject: [PATCH 1264/2835] update

---
 ..._version_upgrade_leaves_repo_unusable.mdwn | 20 ++++++++-----------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 7bbd02364f..64c18f202c 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -46,17 +46,13 @@ Running the copy job again, I am still getting the same error as above (as expec
 >>>
 >>> That's a v1 key, but a corrupt form of the key; it's missing the 
 >>> size and mtime fields that all WORM keys have in v1. And 
->>> the filename is itself a key, a v2 SHA512 key. My guess at what
->>> happened is that these were created when you did the `git annex copy`
->>> to the v1 bare repo.
+>>> the filename is itself a key, a v2 SHA512 key. These were
+>>> created when you did the `git annex copy to the v1 bare repo.
+>>> In v2, git-annex-shell takes a full key object, while in v1,
+>>> it takes a key name and a backend name. This incompatability
+>>> leads to the weird behavior seen.
 >>>
->>> So, assuming none of these are the only copy of your data
->>> (which you should check), the best thing to do is delete those
->>> keys, and re-run the copy now that it's upgraded. Not sure 
->>> it even makes sense to fix the crash. I should probably focus
->>> on fixing what let these mixed keys be created by the mixed-version
->>> copy.
->>>
->>> On second thought, you shouldn't delete anything. I'll simply
->>> make the v2 upgrade detect and work around this bug.
+>>> I had suggested you delete data.. don't. On second thought,
+>>> you shouldn't delete anything. I'll simply make the v2 upgrade
+>>> detect and work around this bug.
 >>> --[[Joey]]

From 016eea028086f2e8c1733ac77612f4397297d1a3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 09:27:28 -0400
Subject: [PATCH 1265/2835] Bugfix: Keys could be received into v1 annexes from
 v2 annexes, via v1 git-annex-shell. This results in some oddly named keys in
 the v1 annex. Recognise and fix those keys when upgrading, instead of
 crashing.

---
 Upgrade/V1.hs                                 | 44 +++++++++++--------
 debian/changelog                              |  3 ++
 ..._version_upgrade_leaves_repo_unusable.mdwn |  3 ++
 3 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index a876142220..4ce2612d60 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -16,6 +16,7 @@ import System.FilePath
 import Data.String.Utils
 import System.Posix.Types
 import Data.Maybe
+import Data.Char
 
 import Key
 import Content
@@ -79,12 +80,11 @@ upgrade = do
 moveContent :: Annex ()
 moveContent = do
 	showNote "moving content..."
-	keys <- getKeysPresent1
-	forM_ keys move
+	files <- getKeyFilesPresent1
+	forM_ files move
 	where
-		move k = do
-			g <- Annex.gitRepo
-			let f = gitAnnexObjectDir g  keyFile1 k  keyFile1 k
+		move f = do
+			let k = fileKey1 (takeFileName f)
 			let d = parentDir f
 			liftIO $ allowWrite d
 			liftIO $ allowWrite f
@@ -154,8 +154,15 @@ oldlog2key l =
 
 -- WORM backend keys: "WORM:mtime:size:filename"
 -- all the rest: "backend:key"
+--
+-- If the file looks like "WORM:XXX-...", then it was created by mixing
+-- v2 and v1; that infelicity is worked around by treating the value
+-- as the v2 key that it is.
 readKey1 :: String -> Key
-readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
+readKey1 v = 
+	if mixup
+		then fromJust $ readKey $ join ":" $ tail bits
+		else Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
 	where
 		bits = split ":" v
 		b = head bits
@@ -166,7 +173,8 @@ readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
 		s = if wormy
 			then Just (read (bits !! 2) :: Integer)
 			else Nothing
-		wormy = b == "WORM"
+		wormy = head bits == "WORM"
+		mixup = wormy && (isUpper $ head $ bits !! 1)
 
 showKey1 :: Key -> String
 showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } =
@@ -211,24 +219,22 @@ lookupFile1 file = do
 				skip = "skipping " ++ file ++ 
 					" (unknown backend " ++ bname ++ ")"
 
-getKeysPresent1 :: Annex [Key]
-getKeysPresent1  = do
+getKeyFilesPresent1 :: Annex [FilePath]
+getKeyFilesPresent1  = do
 	g <- Annex.gitRepo
-	getKeysPresent1' $ gitAnnexObjectDir g
-getKeysPresent1' :: FilePath -> Annex [Key]
-getKeysPresent1' dir = do
+	getKeyFilesPresent1' $ gitAnnexObjectDir g
+getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
+getKeyFilesPresent1' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
 	if (not exists)
 		then return []
 		else do
-			contents <- liftIO $ getDirectoryContents dir
-			files <- liftIO $ filterM present contents
-			return $ map fileKey1 files
+			dirs <- liftIO $ getDirectoryContents dir
+			let files = map (\d -> dir ++ "/" ++ d ++ "/" ++ takeFileName d) dirs
+			liftIO $ filterM present files
 	where
-		present d = do
-			liftIO $ putStrLn $ dir ++ "/" ++ d ++ "/"  ++ takeFileName d
-			result <- try $
-				getFileStatus $ dir ++ "/" ++ d ++ "/"  ++ takeFileName d
+		present f = do
+			result <- try $ getFileStatus f
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
diff --git a/debian/changelog b/debian/changelog
index 2f532784d4..517cf59695 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,9 @@ git-annex (0.20110326) UNRELEASED; urgency=low
   * Provide a less expensive version of `git annex copy --to`, enabled
     via --fast. This assumes that location tracking information is correct,
     rather than contacting the remote for every file.
+  * Bugfix: Keys could be received into v1 annexes from v2 annexes, via
+    v1 git-annex-shell. This results in some oddly named keys in the v1
+    annex. Recognise and fix those keys when upgrading, instead of crashing.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 64c18f202c..1eebd9ecd4 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -56,3 +56,6 @@ Running the copy job again, I am still getting the same error as above (as expec
 >>> you shouldn't delete anything. I'll simply make the v2 upgrade
 >>> detect and work around this bug.
 >>> --[[Joey]]
+
+>>>> This should be fixed in current git. The scambled keys will be 
+>>>> fixed up on upgrade. Thanks for your patience! [[done]] --[[Joey]]

From 2933860847c8092cd2f447e29f3da8b00a15cb18 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Mon, 28 Mar 2011 14:01:00 +0000
Subject: [PATCH 1266/2835] not actively pursuing this issue any more

---
 doc/forum/relying_on_git_for_numcopies.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/relying_on_git_for_numcopies.mdwn b/doc/forum/relying_on_git_for_numcopies.mdwn
index b7ebba805a..37b46cf4e7 100644
--- a/doc/forum/relying_on_git_for_numcopies.mdwn
+++ b/doc/forum/relying_on_git_for_numcopies.mdwn
@@ -1,3 +1,5 @@
+**<out-of-date-warning>**The main problems this is supposed to solve are addressed in a different way with [[todo/hidden files]] and the `--fast` option introduced in [[batch check on remote when using copy]], so while this is not technically obsolete, the main reasons for it are gone. --[[chrysn]]**</out-of-date-warning>**
+
 This is a rough sketch of a modification of git-annex to rely more on git commit semantics. It might be flawed due to my lack of understanding of git-annex internals. --[[chrysn]]
 
 Summary

From fc363b9ca39f264566ec9299358f201c348b792a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 14:44:26 +0000
Subject: [PATCH 1267/2835]

---
 .../backend_version_upgrade_leaves_repo_unusable.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 1eebd9ecd4..122224a8f3 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -59,3 +59,14 @@ Running the copy job again, I am still getting the same error as above (as expec
 
 >>>> This should be fixed in current git. The scambled keys will be 
 >>>> fixed up on upgrade. Thanks for your patience! [[done]] --[[Joey]]
+
+>>>>> I should stop reading your answers via git; by the time I got to
+>>>>> "second thoughts", I had already deleted the files & directories
+>>>>> in question, upgraded the bare repo and was busy uploading from my
+>>>>> local repo. I agree that taking care of this in the upgrade code
+>>>>> is the cleanest approach, by the way.
+>>>>> No need to thank me for my patience; thank you for your quickness!
+>>>>> RichiH
+>>>>> 
+>>>>> PS: If I get a handle on the mtime issue in the SHA backend, git
+>>>>> annex will be pretty much perfect :)

From 1b6927995de5e25ec6c5c464c6444d2ba61ec748 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 11:12:32 -0400
Subject: [PATCH 1268/2835] releasing version 0.20110328

---
 debian/changelog | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 517cf59695..7251ab6653 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,8 @@
-git-annex (0.20110326) UNRELEASED; urgency=low
+git-annex (0.20110328) experimental; urgency=low
 
   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
   * Generalized remotes handling, laying groundwork for remotes that are
-    not regular git remotes.
+    not regular git remotes. (Think Amazon S3.)
   * Provide a less expensive version of `git annex copy --to`, enabled
     via --fast. This assumes that location tracking information is correct,
     rather than contacting the remote for every file.
@@ -10,7 +10,7 @@ git-annex (0.20110326) UNRELEASED; urgency=low
     v1 git-annex-shell. This results in some oddly named keys in the v1
     annex. Recognise and fix those keys when upgrading, instead of crashing.
 
- -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
+ -- Joey Hess   Mon, 28 Mar 2011 10:47:29 -0400
 
 git-annex (0.20110325) experimental; urgency=low
 

From 659f0fe980b6bff6900037d9ae52024b989613e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 11:13:15 -0400
Subject: [PATCH 1269/2835] add news item for git-annex 0.20110328

---
 doc/news/version_0.20110328.mdwn | 11 +++++++++++
 doc/news/version_0.24.mdwn       | 12 ------------
 2 files changed, 11 insertions(+), 12 deletions(-)
 create mode 100644 doc/news/version_0.20110328.mdwn
 delete mode 100644 doc/news/version_0.24.mdwn

diff --git a/doc/news/version_0.20110328.mdwn b/doc/news/version_0.20110328.mdwn
new file mode 100644
index 0000000000..512ce4647a
--- /dev/null
+++ b/doc/news/version_0.20110328.mdwn
@@ -0,0 +1,11 @@
+git-annex 0.20110328 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
+   * Generalized remotes handling, laying groundwork for remotes that are
+     not regular git remotes. (Think Amazon S3.)
+   * Provide a less expensive version of `git annex copy --to`, enabled
+     via --fast. This assumes that location tracking information is correct,
+     rather than contacting the remote for every file.
+   * Bugfix: Keys could be received into v1 annexes from v2 annexes, via
+     v1 git-annex-shell. This results in some oddly named keys in the v1
+     annex. Recognise and fix those keys when upgrading, instead of crashing."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.24.mdwn b/doc/news/version_0.24.mdwn
deleted file mode 100644
index 81b013a26f..0000000000
--- a/doc/news/version_0.24.mdwn
+++ /dev/null
@@ -1,12 +0,0 @@
-Branched the 0.24 series, which will be maintained for a while (in the
-stable branch in git) to support v1 git-annex repos, while main development
-moves to the 0.2011 series, with v2 git-annex repos.
-
-git-annex 0.24 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-* Add Suggests on graphviz. Closes: #[618039](http://bugs.debian.org/618039)
-* When adding files to the annex, the symlinks pointing at the annexed
-  content are made to have the same mtime as the original file.
-  While git does not preserve that information, this allows a tool
-  like metastore to be used with annexed files.
-  (Currently this is only done on systems supporting POSIX 200809.)"""]]

From b6d40c119a9982f05db4134e182e762cb1953697 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 15:13:54 +0000
Subject: [PATCH 1270/2835] Added a comment

---
 ..._174952fc3e3be12912e5fcfe78f2dd13._comment | 185 ++++++++++++++++++
 1 file changed, 185 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment
new file mode 100644
index 0000000000..6e6e5dc6be
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment
@@ -0,0 +1,185 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-28T15:09:45Z"
+ content="""
+I think I know how I got myself into this mess... I was on my mac workstation and I had just pulled in a change set from another repo on a linux workstation after I had a made a bunch of moves. here's a bit of a log of what happened...
+
+
+
+jtang@x00:~/sources $ git pull cports-devel master
+Warning: untrusted X11 forwarding setup failed: xauth key data not generated
+Warning: No xauth data; using fake authentication data for X11 forwarding.
+remote: Counting objects: 4195, done.
+remote: Compressing objects: 100% (1135/1135), done.
+remote: Total 2582 (delta 866), reused 2576 (delta 860)
+Receiving objects: 100% (2582/2582), 229.42 KiB | 111 KiB/s, done.
+Resolving deltas: 100% (866/866), completed with 9 local objects.
+From cports-devel:/home/people/jtang/sources
+ * branch            master     -> FETCH_HEAD
+Updating 319df99..ab0a98c
+error: Your local changes to the following files would be overwritten by merge:
+	.git-annex/09/5X/WORM-s361516678-m1301310614--l_fcompxe_intel64_2011.2.137.tgz.log
+	.git-annex/43/2g/WORM-s19509673-m1301310496--l_fcompxe_2011.2.137_redist.tgz.log
+	.git-annex/4J/qF/WORM-s18891115-m1301310934--w_flm_p_1.0.011_ia64.zip.log
+	.git-annex/87/w1/WORM-s12212473-m1301310909--w_flm_p_1.0.011_ia32.zip.log
+	.git-annex/99/Jq/WORM-s194345957-m1301310926--l_mkl_10.3.2.137_ia32.log
+	.git-annex/99/kf/WORM-s9784531-m1301311680--l_ccompxe_2011.2.137_redist.log
+	.git-annex/FF/f3/WORM-s93033394-m1301311706--l_gen_ipp_7.0.2.137.log
+	.git-annex/MF/xZ/WORM-s515140733-m1301310936--l_cprof_p_11.1.075.log
+	.git-annex/XW/X8/WORM-s355559731-m1301310797--l_mkl_10.3.2.137.log
+	.git-annex/fJ/mZ/WORM-s1372886477-m1301313368--l_cproc_p_11.1.075.log
+	.git-annex/j7/Q9/WORM-s44423202-m1301310622--l_cprof_p_11.1.075_redist.log
+	.git-annex/k4/K7/WORM-s239539070-m1301310760--l_mkl_10.3.2.137_intel64.log
+	.git-annex/kz/01/WORM-s279573314-m1301310783--l_cprof_p_11.1.075_ia32.log
+	.git-annex/p6/Kq/WORM-s31199343-m1301311829--l_cproc_p_11.1.075_redist.log
+	.git-annex/pz/J5/WORM-s626995277-m1301312301--l_ccompxe_ia32_2011.2.137.log
+	.git-annex/v3/kX/WORM-s339693045-m1301310851--l_cprof_p_11.1.075_intel64.log
+Please, commit your changes or stash them before you can merge.
+error: Your local changes to the following files would be overwritten by merge:
+	.git-annex/12/3W/WORM-s3058814-m1276699694--Botan-1.8.9.tgz.log
+	.git-annex/1G/qV/WORM-s9122-m1251558854--Array-Compare-2.01.tar.gz.log
+	.git-annex/3W/W5/WORM-s231523-m1270740744--DBD-Pg-2.17.1.tar.gz.log
+	.git-annex/3x/PX/WORM-s380310-m1293025187--HTSeq-0.4.7.tar.gz.log
+	.git-annex/45/gk/WORM-s67337-m1248732018--ExtUtils-Install-1.54.tar.gz.log
+	.git-annex/4J/7Q/WORM-s8608-m1224694862--Algorithm-Munkres-0.08.tar.gz.log
+	.git-annex/4g/XQ/WORM-s89208-m1278682033--HTML-Parser-3.66.tar.gz.log
+	.git-annex/54/jw/WORM-s300163-m1226422051--AcePerl-1.92.tar.gz.log
+	.git-annex/63/kj/WORM-s1213460-m1262942058--DBD-SQLite-1.29.tar.gz.log
+	.git-annex/6Z/42/WORM-s4074-m943766010--File-Sync-0.09.tar.gz.log
+	.git-annex/8F/M5/WORM-s6989-m1263161127--Digest-HMAC-1.02.tar.gz.log
+	.git-annex/G2/FK/WORM-s3309-m1163872981--Bundle-BioPerl-2.1.8.tar.gz.log
+	.git-annex/Gk/XF/WORM-s23572243-m1279546902--EMBOSS-6.3.1.tar.gz.log
+	.git-annex/Jk/X6/WORM-s566429-m1279309002--DBI-1.612.tar.gz.log
+	.git-annex/K6/fV/WORM-s1561451-m1240055295--Convert-Binary-C-0.74.tar.gz.log
+	.git-annex/KM/4q/WORM-s146959-m1268515086--Graph-0.94.tar.gz.log
+	.git-annex/MF/m2/WORM-s425766-m1212514609--Data-Stag-0.11.tar.gz.log
+	.git-annex/QJ/P6/WORM-s1045868-m1282215033--9base-6.tar.gz.log
+	.git-annex/Qm/WG/WORM-s39078-m1278163547--Digest-SHA1-2.13.tar.gz.log
+	.git-annex/Wq/Fj/WORM-s45680640-m1297862101--BclConverter-1.7.1.tar.log
+	.git-annex/Wq/Wm/WORM-s263536640-m1295025537--CASAVA_v1.7.0.tar.log
+	.git-annex/XW/qm/WORM-s36609-m1276050470--Bio-ASN1-EntrezGene-1.10-withoutworldwriteables.tar.gz.log
+	.git-annex/f7/g0/WORM-s40872-m1278273227--ExtUtils-ParseXS-2.2206.tar.gz.log
+	.git-annex/j3/JF/WORM-s11753-m1232427595--Clone-0.31.tar.gz.log
+	.git-annex/kX/9g/WORM-s84690-m1229117599--GraphViz-2.04.tar.gz.log
+	.git-annex/km/z5/WORM-s44634-m1275505134--Authen-SASL-2.15.tar.gz.log
+	.git-annex/kw/J3/WORM-s132396-m1278780649--DBD-mysql-4.016.tar.gz.log
+	.git-annex/p5/1P/WORM-s53736-m1278673485--Archive-Tar-1.64.tar.gz.log
+	.git-annex/wv/zG/WORM-s30584-m1268774021--ExtUtils-CBuilder-0.2703.tar.gz.log
+	.git-annex/x5/7v/WORM-s10462526-m1254242591--BioPerl-1.6.1.tar.gz.log
+Please, commit your changes or stash them before you can merge.
+error: The following untracked working tree files would be overwritten by merge:
+	.git-annex/1g/X3/WORM-s309910751-m1301311322--l_fcompxe_ia32_2011.2.137.tgz.log
+	.git-annex/3w/Xf/WORM-s805764902-m1301312756--l_cproc_p_11.1.075_intel64.log
+	.git-annex/9Q/Wz/WORM-s1234430253-m1301311891--l_ccompxe_2011.2.137.log
+	.git-annex/FQ/4z/WORM-s318168323-m1301310848--l_cprof_p_11.1.075_ia64.log
+	.git-annex/FV/0P/WORM-s710135470-m1301311835--l_ccompxe_intel64_2011.2.137.log
+	.git-annex/Jx/qM/WORM-s599386592-m1301310731--l_fcompxe_2011.2.137.tgz.log
+	.git-annex/KX/w1/WORM-s35976002-m1301312193--l_tbb_3.0.6.174.log
+	.git-annex/Vw/jK/WORM-s15795178-m1301310913--w_flm_p_1.0.011_intel64.zip.log
+	.git-annex/jK/zK/WORM-s374617670-m1301312705--l_ipp_7.0.2.137_intel64.log
+	.git-annex/vK/kv/WORM-s584342291-m1301312669--l_cproc_p_11.1.075_ia64.log
+	.git-annex/vw/v1/WORM-s736986678-m1301312794--l_cproc_p_11.1.075_ia32.log
+	.git-annex/zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+Please move or remove them before you can merge.
+Aborting
+1|jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 2 commits.
+#
+# Changes to be committed:
+#   (use \"git reset HEAD ...\" to unstage)
+#
+#	modified:   .git-annex/09/5X/WORM-s361516678-m1301310614--l_fcompxe_intel64_2011.2.137.tgz.log
+#	modified:   .git-annex/43/2g/WORM-s19509673-m1301310496--l_fcompxe_2011.2.137_redist.tgz.log
+#	modified:   .git-annex/4J/qF/WORM-s18891115-m1301310934--w_flm_p_1.0.011_ia64.zip.log
+#	modified:   .git-annex/87/w1/WORM-s12212473-m1301310909--w_flm_p_1.0.011_ia32.zip.log
+#	modified:   .git-annex/99/Jq/WORM-s194345957-m1301310926--l_mkl_10.3.2.137_ia32.log
+#	modified:   .git-annex/99/kf/WORM-s9784531-m1301311680--l_ccompxe_2011.2.137_redist.log
+#	modified:   .git-annex/FF/f3/WORM-s93033394-m1301311706--l_gen_ipp_7.0.2.137.log
+#	modified:   .git-annex/MF/xZ/WORM-s515140733-m1301310936--l_cprof_p_11.1.075.log
+#	modified:   .git-annex/XW/X8/WORM-s355559731-m1301310797--l_mkl_10.3.2.137.log
+#	modified:   .git-annex/fJ/mZ/WORM-s1372886477-m1301313368--l_cproc_p_11.1.075.log
+#	modified:   .git-annex/j7/Q9/WORM-s44423202-m1301310622--l_cprof_p_11.1.075_redist.log
+#	modified:   .git-annex/k4/K7/WORM-s239539070-m1301310760--l_mkl_10.3.2.137_intel64.log
+#	modified:   .git-annex/kz/01/WORM-s279573314-m1301310783--l_cprof_p_11.1.075_ia32.log
+#	modified:   .git-annex/p6/Kq/WORM-s31199343-m1301311829--l_cproc_p_11.1.075_redist.log
+#	modified:   .git-annex/pz/J5/WORM-s626995277-m1301312301--l_ccompxe_ia32_2011.2.137.log
+#	modified:   .git-annex/v3/kX/WORM-s339693045-m1301310851--l_cprof_p_11.1.075_intel64.log
+#
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   .git-annex/12/3W/WORM-s3058814-m1276699694--Botan-1.8.9.tgz.log
+#	modified:   .git-annex/1G/qV/WORM-s9122-m1251558854--Array-Compare-2.01.tar.gz.log
+#	modified:   .git-annex/3W/W5/WORM-s231523-m1270740744--DBD-Pg-2.17.1.tar.gz.log
+#	modified:   .git-annex/3x/PX/WORM-s380310-m1293025187--HTSeq-0.4.7.tar.gz.log
+#	modified:   .git-annex/45/gk/WORM-s67337-m1248732018--ExtUtils-Install-1.54.tar.gz.log
+#	modified:   .git-annex/4J/7Q/WORM-s8608-m1224694862--Algorithm-Munkres-0.08.tar.gz.log
+#	modified:   .git-annex/4g/XQ/WORM-s89208-m1278682033--HTML-Parser-3.66.tar.gz.log
+#	modified:   .git-annex/54/jw/WORM-s300163-m1226422051--AcePerl-1.92.tar.gz.log
+#	modified:   .git-annex/63/kj/WORM-s1213460-m1262942058--DBD-SQLite-1.29.tar.gz.log
+#	modified:   .git-annex/6Z/42/WORM-s4074-m943766010--File-Sync-0.09.tar.gz.log
+#	modified:   .git-annex/8F/M5/WORM-s6989-m1263161127--Digest-HMAC-1.02.tar.gz.log
+#	modified:   .git-annex/G2/FK/WORM-s3309-m1163872981--Bundle-BioPerl-2.1.8.tar.gz.log
+#	modified:   .git-annex/Gk/XF/WORM-s23572243-m1279546902--EMBOSS-6.3.1.tar.gz.log
+#	modified:   .git-annex/Jk/X6/WORM-s566429-m1279309002--DBI-1.612.tar.gz.log
+#	modified:   .git-annex/K6/fV/WORM-s1561451-m1240055295--Convert-Binary-C-0.74.tar.gz.log
+#	modified:   .git-annex/KM/4q/WORM-s146959-m1268515086--Graph-0.94.tar.gz.log
+#	modified:   .git-annex/MF/m2/WORM-s425766-m1212514609--Data-Stag-0.11.tar.gz.log
+#	modified:   .git-annex/QJ/P6/WORM-s1045868-m1282215033--9base-6.tar.gz.log
+#	modified:   .git-annex/Qm/WG/WORM-s39078-m1278163547--Digest-SHA1-2.13.tar.gz.log
+#	modified:   .git-annex/Wq/Fj/WORM-s45680640-m1297862101--BclConverter-1.7.1.tar.log
+#	modified:   .git-annex/Wq/Wm/WORM-s263536640-m1295025537--CASAVA_v1.7.0.tar.log
+#	modified:   .git-annex/XW/qm/WORM-s36609-m1276050470--Bio-ASN1-EntrezGene-1.10-withoutworldwriteables.tar.gz.log
+#	modified:   .git-annex/Zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+#	modified:   .git-annex/f7/g0/WORM-s40872-m1278273227--ExtUtils-ParseXS-2.2206.tar.gz.log
+#	modified:   .git-annex/j3/JF/WORM-s11753-m1232427595--Clone-0.31.tar.gz.log
+#	modified:   .git-annex/kX/9g/WORM-s84690-m1229117599--GraphViz-2.04.tar.gz.log
+#	modified:   .git-annex/km/z5/WORM-s44634-m1275505134--Authen-SASL-2.15.tar.gz.log
+#	modified:   .git-annex/kw/J3/WORM-s132396-m1278780649--DBD-mysql-4.016.tar.gz.log
+#	modified:   .git-annex/p5/1P/WORM-s53736-m1278673485--Archive-Tar-1.64.tar.gz.log
+#	modified:   .git-annex/wv/zG/WORM-s30584-m1268774021--ExtUtils-CBuilder-0.2703.tar.gz.log
+#	modified:   .git-annex/x5/7v/WORM-s10462526-m1254242591--BioPerl-1.6.1.tar.gz.log
+#
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	.git-annex/1G/X3/
+#	.git-annex/3W/Xf/
+#	.git-annex/9q/Wz/
+#	.git-annex/Fq/4z/
+#	.git-annex/Jk/zK/
+#	.git-annex/Kx/w1/
+#	.git-annex/VK/kv/
+#	.git-annex/fv/0P/
+#	.git-annex/jX/qM/
+#	.git-annex/vW/jK/
+#	.git-annex/vW/v1/
+jtang@x00:~/sources $ git commit -a -m \"snap\"
+[master 45f254a] snap
+ 47 files changed, 64 insertions(+), 30 deletions(-)
+jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 3 commits.
+#
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	.git-annex/1G/X3/
+#	.git-annex/3W/Xf/
+#	.git-annex/9q/Wz/
+#	.git-annex/Fq/4z/
+#	.git-annex/Jk/zK/
+#	.git-annex/Kx/w1/
+#	.git-annex/VK/kv/
+#	.git-annex/fv/0P/
+#	.git-annex/jX/qM/
+#	.git-annex/vW/jK/
+#	.git-annex/vW/v1/
+nothing added to commit but untracked files present (use \"git add\" to track)
+jtang@x00:~/sources $ git pull
+

+"""]]

From 8db6b4f655096d141af5b8a25cf76826c02d9fdd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 11:16:36 -0400
Subject: [PATCH 1271/2835] response

---
 ...has_issues_with_git_when_staging__47__commiting_logs.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index b7944b418c..554cfa41e2 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -16,3 +16,8 @@ For now it's just a bit of extra work for me when it does occur but it does not
 >>> be some error message? --[[Joey]] 
 
 >>>> there were no error messages at all
+
+>>>>> Can I see a transcript? I'm having difficulty getting my head around
+>>>>> what git is doing. Sounds like the files could just not be `git
+>>>>> added` yet, but I get the impression from other things that you say
+>>>>> that it's not so simple. --[[Joey]] 

From 73b4fe5d61e492fe537f04d3971f25970b7ebf3f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 15:20:59 +0000
Subject: [PATCH 1272/2835]

---
 doc/forum/wishlist:_git_backend_for_git-annex.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex.mdwn

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex.mdwn b/doc/forum/wishlist:_git_backend_for_git-annex.mdwn
new file mode 100644
index 0000000000..63ae83097e
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex.mdwn
@@ -0,0 +1,7 @@
+Preamble: Obviously, the core feature of git-annex is the ability to keep a subset of files in a local repo. The main trade-off is that you don't get version tracking.
+
+Use case: On my laptop, I might not have enough disk space to store everything. Not so for my main box nor my backup server. And I would _really_ like to have proper version tracking for many of my files. Thus...
+
+Wish: ...why not use git as a version backend? That way, I could just push all my stuff to the central instance(s) and have the best of both worlds. Depending on what backend is used in the local repos, it might make sense to define a list of supported client backends with pre-computed keys.
+
+-- RichiH

From 70d4df79ee1ada9a773d03579a86b798f3e1ff1d Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 15:25:18 +0000
Subject: [PATCH 1273/2835] Added a comment

---
 ...3_a18ada7ac74c63be5753fdb2fe68dae5._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
new file mode 100644
index 0000000000..4c2f609d9c
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-28T15:25:18Z"
+ content="""
+So, there is evidence here of a circumstance caused by the [[other_bug|git-annex_has_issues_with_git_when_staging__47__commiting_logs]], as I suspected.
+
+I don't think that manual `git commit -a` caused the problem. I suspect it was a subsequent `git add` that caused git to follow the wrong case paths and add the files in the wrong place. Ie, when you run \"git add .git-annex\", it recurses into `.git-annex/Gm/`, and adds files using that case, that were previously added from `.git-annex/GM/`.
+
+For completeness, can you verify this repo's core.ignorecase setting?
+
+---
+
+I hate that you are stuck using loop filesystems to work around this bug. If my guess is correct, you don't need to, as long as you avoid manually running \"git add .git-annex\". I take this bug seriously. While I'm currently very involved in adding Amazon S3 support to git-annex (which will take days more of solid work), I do plan to make a look filesystem of my own, probably vfat, so I can try and reproduce this on a case-insensative filesystem. If you could confirm my above hypothesis, that would speed things up for me.
+
+It's possible I will have to tweak the hash directories. Hopefully if so, I will only tweak them for *new* keys; if I had to do a v3 backend just to fix this stupid thing, I'd murder myself -- upgrading all my offline disks from v1 to v2 took me many days.
+"""]]

From dd4004c43b95389440d9eadbcefc72dd726269b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 11:28:34 -0400
Subject: [PATCH 1274/2835] wording

---
 .../comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
index 4c2f609d9c..00988ab58c 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
@@ -12,7 +12,7 @@ For completeness, can you verify this repo's core.ignorecase setting?
 
 ---
 
-I hate that you are stuck using loop filesystems to work around this bug. If my guess is correct, you don't need to, as long as you avoid manually running \"git add .git-annex\". I take this bug seriously. While I'm currently very involved in adding Amazon S3 support to git-annex (which will take days more of solid work), I do plan to make a look filesystem of my own, probably vfat, so I can try and reproduce this on a case-insensative filesystem. If you could confirm my above hypothesis, that would speed things up for me.
+I hate that you are stuck using loop filesystems to work around this bug. If my guess is correct, you don't need to, as long as you avoid manually running \"git add .git-annex\". I take this bug seriously. While I'm currently very involved in adding Amazon S3 support to git-annex (which will take days more of solid work), I do plan to make a loop filesystem of my own, probably vfat, so I can try and reproduce this on a case-insensative filesystem. If you could confirm my above hypothesis, that would speed things up for me.
 
-It's possible I will have to tweak the hash directories. Hopefully if so, I will only tweak them for *new* keys; if I had to do a v3 backend just to fix this stupid thing, I'd murder myself -- upgrading all my offline disks from v1 to v2 took me many days.
+It's possible I will have to tweak the hash directories. Hopefully if so, I will only tweak them for *new* keys; if I had to do a v3 backend just to fix this stupid thing, I'd be sad -- upgrading all my offline disks from v1 to v2 took me many days.
 """]]

From 2b8abd920101be74da8559694125a19f062e2c71 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 15:31:52 +0000
Subject: [PATCH 1275/2835]

---
 doc/forum/git-annex_communication_channels.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels.mdwn

diff --git a/doc/forum/git-annex_communication_channels.mdwn b/doc/forum/git-annex_communication_channels.mdwn
new file mode 100644
index 0000000000..8c56ac36a2
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels.mdwn
@@ -0,0 +1,10 @@
+Thought I'd ask how y'all are finding the current communication by this forum/website/git repo only.
+
+Would there be a benefit to having an irc channel for git-annex?
+
+Maybe a mailing list? (Any persuasive reason why it would be better than this forum?)
+
+Are the existing RSS feeds on this site, for eg, new [[comments]] and posts to this forum, sufficient to keep up with
+things?
+
+--[[Joey]]

From 06342e016930562f5e233b6521fad3a2997fd700 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 15:44:56 +0000
Subject: [PATCH 1276/2835] Added a comment

---
 .../comment_4_039e945617a6c1852c96974a402db29c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment
new file mode 100644
index 0000000000..d045f71205
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2011-03-28T15:41:56Z"
+ content="""
+In my \"sources\" repo on x00, the current setting is this \"ignorecase = true\" it was the first repo that I created before I clone it elsewhere and pull my changes back, it is on a HFS+ partition which is case insensitive and it is replicated on a portable hdd with a bare repo on a exfat partition. I wonder if my portable disk has a partially borked repo :P
+"""]]

From 62cf351e5afc96a94a4a1ec010649a40b339bf62 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 15:51:08 +0000
Subject: [PATCH 1277/2835] Added a comment

---
 ..._1_198325d2e9337c90f026396de89eec0e._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment

diff --git a/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment b/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment
new file mode 100644
index 0000000000..163aae02cc
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-28T15:48:08Z"
+ content="""
+No matter what you end up doing, I would appreciate a git-annex-announce@ list.
+
+I really like the persistence of ikiwiki, but it's not ideal for quick communication. I would be fine with IRC and/or ML. The advantage of a ML over ikiwiki is that it doesn't seem to be as \"wasteful\" to mix normal chat with actual problem-solving.  But maybe that's merely my own perception.
+
+Speaking of RSS: I thought I had added a wishlist item to ikiwiki about providing per-subsite RSS feeds. For example there is no (obvious) way to subscribe to changes in http://git-annex.branchable.com/forum/git-annex_communication_channels/ .
+
+FWIW, I resorted to tagging my local clone of git-annex to keep track of what I've read, already.
+
+
+-- RichiH
+"""]]

From 8a20d0d7440c7668bbc977d74bb99e14c79cd606 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 15:51:32 +0000
Subject: [PATCH 1278/2835] Added a comment

---
 ..._eacd0b18475c05ab9feed8cf7290b79a._comment | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment
new file mode 100644
index 0000000000..7127a6eef8
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2011-03-28T15:51:11Z"
+ content="""
+I also failed to mention, that in the case when i have stray log files after what has happened in comment 2, I get this left over after a commit when git is confused...
+
+
+
+jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 1 commit.
+#
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   .git-annex/1G/X3/WORM-s309910751-m1301311322--l_fcompxe_ia32_2011.2.137.tgz.log
+#	modified:   .git-annex/3W/Xf/WORM-s805764902-m1301312756--l_cproc_p_11.1.075_intel64.log
+#	modified:   .git-annex/9Q/Wz/WORM-s1234430253-m1301311891--l_ccompxe_2011.2.137.log
+#	modified:   .git-annex/FQ/4z/WORM-s318168323-m1301310848--l_cprof_p_11.1.075_ia64.log
+#	modified:   .git-annex/FV/0P/WORM-s710135470-m1301311835--l_ccompxe_intel64_2011.2.137.log
+#	modified:   .git-annex/Jk/zK/WORM-s374617670-m1301312705--l_ipp_7.0.2.137_intel64.log
+#	modified:   .git-annex/Jx/qM/WORM-s599386592-m1301310731--l_fcompxe_2011.2.137.tgz.log
+#	modified:   .git-annex/KX/w1/WORM-s35976002-m1301312193--l_tbb_3.0.6.174.log
+#	modified:   .git-annex/VK/kv/WORM-s584342291-m1301312669--l_cproc_p_11.1.075_ia64.log
+#	modified:   .git-annex/Vw/jK/WORM-s15795178-m1301310913--w_flm_p_1.0.011_intel64.zip.log
+#	modified:   .git-annex/Zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+#	modified:   .git-annex/vW/v1/WORM-s736986678-m1301312794--l_cproc_p_11.1.075_ia32.log
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+

+
+
+Up until now I have just been updating the status of the staged files by hand and commiting it on my mac x00, this probably isn't helping. I'd rather not lose the tracking information.
+"""]]

From d8ac58a15ea48bfed8338000ac44833c37a11e82 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 16:01:30 +0000
Subject: [PATCH 1279/2835] Added a comment

---
 ...comment_1_04319051fedc583e6c326bb21fcce5a5._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment
new file mode 100644
index 0000000000..a691393b1a
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-28T16:01:30Z"
+ content="""
+Indeed, see [[todo/add_a_git_backend]], where you and I have already discussed this idea. :)
+
+With the new support for special remotes, which will be used by S3, it would be possible to make such a git repo, using bup, be a special remote. I think it would be pretty easy to implement now. Not a priority for me though.
+"""]]

From cf99575d74d2a68004efa152ea9e7ac1c2866935 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 12:09:47 -0400
Subject: [PATCH 1280/2835] update

---
 Makefile                                                  | 1 +
 ..._issues_with_git_when_staging__47__commiting_logs.mdwn | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e1aaf8ec37..8e16645034 100644
--- a/Makefile
+++ b/Makefile
@@ -58,6 +58,7 @@ docs: $(mans)
 		--no-usedirs --disable-plugin=openid --plugin=sidebar \
 		--underlaydir=/dev/null --disable-plugin=shortcut \
 		--disable-plugin=smiley \
+		--plugin=comments --set comments_pagespec="*" \
 		--exclude='news/.*'
 
 clean:
diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 554cfa41e2..774ca6a16c 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -7,7 +7,13 @@ For now it's just a bit of extra work for me when it does occur but it does not
 > What do you mean when you say that git "got wedged"? It hung somehow?
 >
 > If git-annex runs concurrently with another git command that locks
-> the repository its git add of log files can fail.
+> the repository, its git add of log files can fail.
+> 
+> Update: Also, of course, if you are running a "got annex get" or
+> similar, and ctrl-c it after it has gotten some files, it can
+> end up with unstaged or in some cases un-added log files that git-annex
+> wrote -- since git-annex only stages log files in git on shutdown, and
+> ctrl-c bypasses that.
 > --[[Joey]] 
 
 >> It "got wedged" as in git doesn't let me commit anything, even though it tells me that there is stuff to be committed in the staging area.

From 0cb63c0737a0a4896c78a45ab8bfd56125c95c50 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 16:57:50 +0000
Subject: [PATCH 1281/2835] Comment moderation

---
 .../comment_1_7cb5561f11dfc7726a537ddde2477489._comment   | 8 ++++++++
 .../comment_4_e4911dc6793f98fb81151daacbe49968._comment   | 8 ++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment

diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment
new file mode 100644
index 0000000000..35a2cdb3fe
--- /dev/null
+++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-21T13:15:03Z"
+ content="""
+Maybe I will run into issues myself somewhere down the road, but generally speaking, I really really like the fact that files are immutable by default.
+"""]]
diff --git a/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment b/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment
new file mode 100644
index 0000000000..e8c9837462
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-03-25T11:23:04Z"
+ content="""
+FWIW, I wanted to suggest exactly the same thing.
+"""]]

From 0591ecb2a108ca8d3d1cb6b649c4a2913c96b593 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 13:20:12 -0400
Subject: [PATCH 1282/2835] docs

---
 doc/transferring_data.mdwn | 3 ++-
 doc/use_case/Alice.mdwn    | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
index 9526a3e48c..f6ae9bfcde 100644
--- a/doc/transferring_data.mdwn
+++ b/doc/transferring_data.mdwn
@@ -1,7 +1,8 @@
 git-annex can transfer data to or from any of a repository's git remotes.
 Depending on where the remote is, the data transfer is done using rsync
 (over ssh, with automatic resume), or plain cp (with copy-on-write
-optimisations on supported filesystems).
+optimisations on supported filesystems). Some [[special_remotes]]
+are also supported that are not traditional git remotes.
 
 It's equally easy to transfer a single file to or from a repository,
 or to launch a retrievel of a massive pile of files from whatever
diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn
index ccc0727bce..ee97efa43a 100644
--- a/doc/use_case/Alice.mdwn
+++ b/doc/use_case/Alice.mdwn
@@ -2,9 +2,9 @@
 
 Alice is always on the move, often with her trusty netbook and a small
 handheld terabyte USB drive, or a smaller USB keydrive. She has a server
-out there on the net. All these things can have different files on them,
-but Alice no longer has to deal with the tedious process of keeping them
-manually in sync.
+out there on the net. She stores data in the Cloud. All these things can
+have different files on them, but Alice no longer has to deal with the
+tedious process of keeping them manually in sync.
 
 When she has 1 bar on her cell, Alice queues up interesting files on her
 server for later. At a coffee shop, she has git-annex download them to her

From 3162a724f1bfdc15efadd939a49ba8740d553d69 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 13:47:29 -0400
Subject: [PATCH 1283/2835] S3 updates; gpg keys

---
 debian/changelog                     |  2 ++
 doc/git-annex.mdwn                   | 16 +++++-----
 doc/special_remotes.mdwn             |  6 +---
 doc/special_remotes/Amazon_S3.mdwn   | 45 ++++++++++++++++++++++++++++
 doc/walkthrough/using_Amazon_S3.mdwn | 12 ++++----
 5 files changed, 63 insertions(+), 18 deletions(-)
 create mode 100644 doc/special_remotes/Amazon_S3.mdwn

diff --git a/debian/changelog b/debian/changelog
index 03a0091cbf..825382256d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (0.20110329) UNRELEASED; urgency=low
 
   * Amazon S3 is now supported as a special type of remote.
+    Warning: Encrypting data before sending it to S3 is not currently
+    supported.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index c01f4fbc59..3985addc6c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,20 +132,22 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
-* s3bucket name description [datacenter host port] [--key=gpgkey]
+* s3bucket name gpgkey [datacenter host port]
 
-  Creates a bucket in Amazon S3. The bucket's name can be used
-  to configure git remote using the bucket.
+  Create or updates the key of a bucket in Amazon S3. The bucket's
+  name can be used to configure git remote using the bucket.
+
+  The gpgkey is a value that can be looked up (using gpg -k) to
+  find a gpg encryption key that will be given access to the bucket.
+  To disable encryption, specify "unencrypted". Note that additional gpg
+  keys can be given access to a bucket by running s3bucket on an existing
+  bucket, with a new key.
 
   The datacenter defaults to "US". Other values include "EU",
   "us-west-1", and "ap-southeast-1".
 
   To use a different, S3-compatable service, specify a host and port.
 
-  By default, data (including filenames) is encrypted using gpg.
-  To use a key other than the default gpg key, specify it with
-  the --key option. To disable encryption, specify "none".
-
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 717ec48404..651b24afa4 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -6,8 +6,4 @@ But, git-annex also extends git's concept of remotes, with these special
 types of remotes. These can be used just like any normal remote by git-annex.
 They cannot be used by other git commands though.
 
-## Amazon S3
-
-Stores file contents in a bucket in Amazon S3 or a similar service.
-Content is stored encrypted by gpg.
-See [[walkthrough/using_Amazon_S3]] for examples.
+* [[Amazon_S3]]
diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
new file mode 100644
index 0000000000..dce0a9241f
--- /dev/null
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -0,0 +1,45 @@
+This special remote type stores file contents in a bucket in Amazon S3
+or a similar service.
+
+See [[walkthrough/using_Amazon_S3]] for usage examples.
+
+## bucket names
+
+When `git annex s3bucket` is used to create a new bucket, it generates a
+UUID, and the name of the bucket includes that UUID, as well as the name
+specified by the user. This makes for some unweidly bucket names, but
+since S3 requires that bucket names be globally unique, it avoids needing
+to hunt for a unused bucket name.
+
+## data security
+
+When `git annex s3bucket` is used to create an unencrypted bucket,
+there is **no** protection against your data being read as it is sent
+to/from S3, or by Amazon when it is stored in S3. This should only be used
+for public data.
+
+** Encryption is not yet supported. **
+
+When an encrypted bucket is created, all files stored in the bucket are
+encrypted with gpg. Additionally, the filenames themselves are hashed
+to obfuscate them. The size of the encrypted files, and access patterns of
+the data, should be the only clues to what type of data you are storing in
+S3.
+
+[[!template id=note text="""
+This scheme was originally developed by Lars Wirzenius at al [for Obnam](http://braawi.org/obnam/encryption/).
+"""]]
+The data stored in S3 is encrypted by gpg with a symmetric cipher. The
+passphrase of the cipher is itself checked into your git repository,
+encrypted using one or more gpg public keys. This scheme allows new public
+keys to be given access to a bucket's content, after the bucket is created
+and is in use. It also allows revoking compromised public keys without
+having to throw out the contents of the bucket. The symmetric cipher
+is also hashed together with filenames used in the bucket, obfuscate
+the filenames.
+
+To add a new gpg key to an existing bucket, just re-run `git annex
+s3bucket`, specifying the new key id. For example:
+
+	# git annex s3bucket mybucket 16D0B8EF
+	s3bucket (adding gpg key 16D0B8EF) ok
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 8cb77ab6cd..2833a9c5a4 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -9,8 +9,11 @@ First, export your S3 credentials:
 
 Next, create a bucket, giving it a name and a description:
 
-	git annex s3bucket mybucket "my Amazon S3 bucket"
-	s3bucket (creating mybucket...) ok
+	git annex s3bucket mybucket unencrypted
+	s3bucket (creating mybucket...) (no encryption!) ok
+
+**Note that encrypted buckets are not (yet) supported. Data sent to S3
+is susceptible to snooping.**
 
 Finally, configure a git remote to use the bucket you created:
 
@@ -23,7 +26,4 @@ Now the remote can be used like any other remote.
 	# git annex move video/hackity_hack_and_kaxxt.mov --to mys3
 	move video/hackity_hack_and_kaxxt.mov (to mys3...) ok
 
-An Amazon S3 remote works just like a ssh remote, except it does not have
-a git repository at the other end, and it costs you money. :) In particular,
-all data is stored encrypted with gpg, so neither Amazon nor anyone in
-between can see it.
+See [[special_remotes/Amazon_S3]] for details.

From bd40e0c777ec164a0edbb716f12587acdfcd09e8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 13:49:48 -0400
Subject: [PATCH 1284/2835] thinko

---
 doc/special_remotes/Amazon_S3.mdwn | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index dce0a9241f..67bea3b1c0 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -27,13 +27,14 @@ the data, should be the only clues to what type of data you are storing in
 S3.
 
 [[!template id=note text="""
-This scheme was originally developed by Lars Wirzenius at al [for Obnam](http://braawi.org/obnam/encryption/).
+This scheme was originally developed by Lars Wirzenius at al
+[for Obnam](http://braawi.org/obnam/encryption/).
 """]]
 The data stored in S3 is encrypted by gpg with a symmetric cipher. The
 passphrase of the cipher is itself checked into your git repository,
-encrypted using one or more gpg public keys. This scheme allows new public
+encrypted using one or more gpg public keys. This scheme allows new private
 keys to be given access to a bucket's content, after the bucket is created
-and is in use. It also allows revoking compromised public keys without
+and is in use. It also allows revoking compromised private keys without
 having to throw out the contents of the bucket. The symmetric cipher
 is also hashed together with filenames used in the bucket, obfuscate
 the filenames.

From 6ba8a53de19263c7dcfd2e5e4683a716ac358f24 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 17:50:38 +0000
Subject: [PATCH 1285/2835] Added a comment

---
 ...ent_2_7f529f19a47e10b571f65ab382e97fd5._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment
new file mode 100644
index 0000000000..14798e7a71
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-03-28T17:47:38Z"
+ content="""
+On the plus side, the past me wanted exactly what I had in mind.
+
+On the meh side, I really forgot about this conversation :/
+
+When you say this todo is not a priority, does that mean there's no ETA at all and that it will most likely sleep for a long time? Or the almost usual \"what the heck, I will just wizard it up in two lines of haskell\"?
+
+-- RichiH
+"""]]

From 0212126f67660ce2753aa2fb32c643107243f2af Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 18:35:51 +0000
Subject: [PATCH 1286/2835] Added a comment

---
 .../comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment

diff --git a/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment b/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment
new file mode 100644
index 0000000000..09a2b8c1a5
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-28T18:35:50Z"
+ content="""
+I think the forums/website currently is sufficient, I do at times wish there was a mailing list or anonymous git push to the wiki as I find editing posts through the web browser is some times tedious (the lack of !fmt or alt-q bugs me at times ;) ). The main advantage of keeping stuff on the site/forum is that everything gets saved and passed on to anyone who checks out the git repo of the code base.
+"""]]

From 974b1e18d14fa0a311a7379d0ce6eb34ca3c2cf9 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 20:05:13 +0000
Subject: [PATCH 1287/2835] Added a comment

---
 ...comment_3_a077bbad3e4b07cce019eb55a45330e7._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment
new file mode 100644
index 0000000000..8c3286d27b
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-28T20:05:13Z"
+ content="""
+Probably more like 150 lines of haskell. Maybe just 50 lines if the bup repository is required to be on the same computer as the git-annex repository.
+
+Since I do have some repositories where I'd appreciate this level of assurance that data not be lost, it's mostly a matter of me finding a free day.
+"""]]

From 0956f0dd15f3114d7227787578499210f0c17db8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 16:19:20 -0400
Subject: [PATCH 1288/2835] fsck: Ensure that files and directories in
 .git/annex/objects have proper permissions.

---
 Command/Fsck.hs                                       | 11 ++++++++++-
 debian/changelog                                      |  7 +++++++
 ...k__47__fix_the_permissions_of_.git__47__annex.mdwn |  2 ++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 216c87493b..bedb9fb992 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,6 +7,7 @@
 
 module Command.Fsck where
 
+import Control.Monad (when)
 import Control.Monad.State (liftIO)
 
 import Command
@@ -18,6 +19,7 @@ import Messages
 import Utility
 import Content
 import LocationLog
+import Locations
 
 command :: [Command]
 command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
@@ -47,9 +49,16 @@ perform key file backend numcopies = do
    in this repository only. -}
 verifyLocationLog :: Key -> FilePath -> Annex Bool
 verifyLocationLog key file = do
+	g <- Annex.gitRepo
 	present <- inAnnex key
 	
-	g <- Annex.gitRepo
+	-- Since we're checking that a key's file is present, throw
+	-- in a permission fixup here too.
+	when present $ liftIO $ do
+		let f = gitAnnexLocation g key
+		preventWrite f
+		preventWrite (parentDir f)
+
 	u <- getUUID g
         uuids <- liftIO $ keyLocations g key
 
diff --git a/debian/changelog b/debian/changelog
index 7251ab6653..1b09165538 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (0.20110329) UNRELEASED; urgency=low
+
+  * fsck: Ensure that files and directories in .git/annex/objects
+    have proper permissions.
+
+ -- Joey Hess   Mon, 28 Mar 2011 16:17:59 -0400
+
 git-annex (0.20110328) experimental; urgency=low
 
   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
diff --git a/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
index ec8b6d2330..c649ff9f7c 100644
--- a/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
+++ b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
@@ -4,3 +4,5 @@ The fsck command should check that they are still correct.
 The fix command should fix them.
 
 PS: Thanks for this nice tool!
+
+> Good idea, [[done]] (actually, fsck just fixes them too)! --[[Joey]]

From caef7c82213e00695679bd9c934a4edef0a04eaa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 16:35:59 -0400
Subject: [PATCH 1289/2835] nix on revocation

---
 doc/special_remotes/Amazon_S3.mdwn | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index 67bea3b1c0..ae3990a76e 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -34,10 +34,8 @@ The data stored in S3 is encrypted by gpg with a symmetric cipher. The
 passphrase of the cipher is itself checked into your git repository,
 encrypted using one or more gpg public keys. This scheme allows new private
 keys to be given access to a bucket's content, after the bucket is created
-and is in use. It also allows revoking compromised private keys without
-having to throw out the contents of the bucket. The symmetric cipher
-is also hashed together with filenames used in the bucket, obfuscate
-the filenames.
+and is in use. The symmetric cipher is also hashed together with filenames
+used in the bucket, in order to obfuscate the filenames.
 
 To add a new gpg key to an existing bucket, just re-run `git annex
 s3bucket`, specifying the new key id. For example:

From 216817e0d42cb69768b42c79cbdb7f6072f5a47e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 20:45:36 +0000
Subject: [PATCH 1290/2835] Added a comment

---
 .../comment_4_ecca429e12d734b509c671166a676c9d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment
new file mode 100644
index 0000000000..cf649a8a25
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2011-03-28T20:45:35Z"
+ content="""
+Personally, I would not mind a requirement to keep a local bup repo. I wouldn't want my data to to unncessarily complex setups, anyway. -- RichiH
+"""]]

From 42483ba19856aa1e51054d100364e45fdaf8696c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 20:47:23 +0000
Subject: [PATCH 1291/2835] Added a comment

---
 .../comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment

diff --git a/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment b/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment
new file mode 100644
index 0000000000..72a48445ed
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-28T20:47:23Z"
+ content="""
+Push access to the non-code bits of git-annex' ikiwiki would be very welcome indeed. Given the choice, I would rather edit everything in Vim than in a browser. -- RichiH
+"""]]

From 58af57493418d80eb9fb3c4719f20442725aa7f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 19:08:12 -0400
Subject: [PATCH 1292/2835] generalize special remote configuration storage

---
 doc/git-annex.mdwn                   | 39 ++++++---------------------
 doc/internals.mdwn                   | 14 +++++-----
 doc/special_remotes/Amazon_S3.mdwn   | 40 ++++++++++++++++------------
 doc/walkthrough/using_Amazon_S3.mdwn | 10 +++----
 4 files changed, 40 insertions(+), 63 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3985addc6c..ce5b380d0e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,21 +132,17 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
-* s3bucket name gpgkey [datacenter host port]
+* initremote name [type=value param=value ...]
 
-  Create or updates the key of a bucket in Amazon S3. The bucket's
-  name can be used to configure git remote using the bucket.
+  Sets up a [[special_remote|special_remotes]] of some type. The remote's
+  type and configuration is specified by the parameters. If a remote
+  with the specified name has already been configured, its configuration
+  is modified by any values specified. In either case, the remote will be
+  added added to `.git/config`.
 
-  The gpgkey is a value that can be looked up (using gpg -k) to
-  find a gpg encryption key that will be given access to the bucket.
-  To disable encryption, specify "unencrypted". Note that additional gpg
-  keys can be given access to a bucket by running s3bucket on an existing
-  bucket, with a new key.
+  Example Amazon S3 remote:
 
-  The datacenter defaults to "US". Other values include "EU",
-  "us-west-1", and "ap-southeast-1".
-
-  To use a different, S3-compatable service, specify a host and port.
+	initremote mys3 type=S3 encryption=none datacenter=EU
 
 * fsck [path ...]
 
@@ -403,25 +399,6 @@ Here are all the supported configuration settings.
   Default ssh and rsync options to use if a remote does not have
   specific options.
 
-* `remote..annex-s3-access-key-id`
-
-  Your S3 Access Key ID. Does not need to be kept private.
-  If not set, the environment variable `AWS_ACCESS_KEY_ID`
-  will be used.
-
-* `remote..annex-s3-secret-access-key`
-
-  Your S3 Secret Access Key. This is a password.
-  If not set, the environment variable `AWS_SECRET_ACCESS_KEY`
-  will be used.
-
-* `remote..annex-s3-storageclass`
-
-  Storage class to use when adding new content to S3. The default
-  is "STANDARD". If you have configured git-annex to preserve
-  multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY"
-  to save money.
-
 * `annex.diskreserve`
 
   Amount of disk space to reserve. Disk space is checked when transferring
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 55b1045a11..6296095035 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -30,16 +30,14 @@ space and then the description through to the end of the line. Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 laptop
 	26339d22-446b-11e0-9101-002170d25c55 usb disk
 
-## `git-annex/s3.log`
+## `git-annex/remotes.log`
 
-Associates the UUIDs of Amazon S3 buckets with a bucket nickname and connection
-information. Example:
+Holds persistent configuration settings for [[special_remotes]] such as
+Amazon S3.
 
-	be72acb8-5901-11e0-b600-002170d25c55 mybucket s3.amazonaws.com 80
-
-Note that the actual bucket name used on S3 in the above example 
-is "mybucket-be72acb8-5901-11e0-b600-002170d25c55". The UUID is included
-in the bucket name to ensure it is globally unique.
+The file format is one line per remote, starting with the uuid of the
+remote, followed by a space, the name of the remote, a space, and then
+a series of key=value pairs, each separated by whitespace.
 
 ## `.git-annex/trust.log`
 
diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index ae3990a76e..42c4a54534 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -3,24 +3,36 @@ or a similar service.
 
 See [[walkthrough/using_Amazon_S3]] for usage examples.
 
-## bucket names
+## initremote parameters
 
-When `git annex s3bucket` is used to create a new bucket, it generates a
-UUID, and the name of the bucket includes that UUID, as well as the name
-specified by the user. This makes for some unweidly bucket names, but
-since S3 requires that bucket names be globally unique, it avoids needing
-to hunt for a unused bucket name.
+A number of parameters can be passed to `git annex initremote` to configure
+the S3 remote.
+
+* `encryption` - Either "none" to disable encryption,
+  or a value that can be looked up (using gpg -k) to find a gpg encryption
+  key that will be given access to the remote. Note that additional gpg
+  keys can be given access to a remote by rerunning initremote with
+  the new key id.
+
+* `datacenter` - Defaults to "US". Other values include "EU",
+  "us-west-1", and "ap-southeast-1".
+
+* `storageclass` - Default is "STANDARD". If you have configured git-annex
+  to preserve multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY"
+  to save money.
+
+* `host` and `port` - Specify in order to use a different, S3 compatable
+  service.
 
 ## data security
 
-When `git annex s3bucket` is used to create an unencrypted bucket,
-there is **no** protection against your data being read as it is sent
-to/from S3, or by Amazon when it is stored in S3. This should only be used
-for public data.
+When encryption=none, there is **no** protection against your data being read
+as it is sent to/from S3, or by Amazon when it is stored in S3. This should
+only be used for public data.
 
 ** Encryption is not yet supported. **
 
-When an encrypted bucket is created, all files stored in the bucket are
+When encryption is enabled, all files stored in the bucket are
 encrypted with gpg. Additionally, the filenames themselves are hashed
 to obfuscate them. The size of the encrypted files, and access patterns of
 the data, should be the only clues to what type of data you are storing in
@@ -36,9 +48,3 @@ encrypted using one or more gpg public keys. This scheme allows new private
 keys to be given access to a bucket's content, after the bucket is created
 and is in use. The symmetric cipher is also hashed together with filenames
 used in the bucket, in order to obfuscate the filenames.
-
-To add a new gpg key to an existing bucket, just re-run `git annex
-s3bucket`, specifying the new key id. For example:
-
-	# git annex s3bucket mybucket 16D0B8EF
-	s3bucket (adding gpg key 16D0B8EF) ok
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 2833a9c5a4..b87238a327 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -7,18 +7,14 @@ First, export your S3 credentials:
 	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
 	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
 
-Next, create a bucket, giving it a name and a description:
+Next, create the remote.
 
-	git annex s3bucket mybucket unencrypted
-	s3bucket (creating mybucket...) (no encryption!) ok
+	git annex initremote mys3 encryption=none
+	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
 
 **Note that encrypted buckets are not (yet) supported. Data sent to S3
 is susceptible to snooping.**
 
-Finally, configure a git remote to use the bucket you created:
-
-	git config remote.mys3.annex-s3-bucket mybucket
-
 Now the remote can be used like any other remote.
 
 	# git annex copy my_cool_big_file --to mys3

From 235720d27e5c1044ddd8904d7140c9e8841e5715 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 22:05:11 -0400
Subject: [PATCH 1293/2835] tweak

---
 doc/git-annex.mdwn                   | 20 ++++++++++----------
 doc/walkthrough/using_Amazon_S3.mdwn |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index ce5b380d0e..4d14623942 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -126,23 +126,23 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * describe repository description
 
-  Changes the description of a git repository.
+  Changes the description of a repository.
 
   The repository to describe can be specified by git remote name or
   by uuid. To change the description of the current repository, use
   "."
 
-* initremote name [type=value param=value ...]
+* initremote type name [param=value ...]
 
   Sets up a [[special_remote|special_remotes]] of some type. The remote's
-  type and configuration is specified by the parameters. If a remote
+  configuration is configured by the parameters. If a remote
   with the specified name has already been configured, its configuration
   is modified by any values specified. In either case, the remote will be
   added added to `.git/config`.
 
   Example Amazon S3 remote:
 
-	initremote mys3 type=S3 encryption=none datacenter=EU
+	initremote s3 mys3 type=S3 encryption=none datacenter=EU
 
 * fsck [path ...]
 
@@ -318,12 +318,12 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * --from=repository
 
   Specifies a repository that content will be retrieved from.
-  It should be specified using the name of a configured git remote.
+  It should be specified using the name of a configured remote.
 
 * --to=repository
 
-  Specifies a git repository that content will be sent to. 
-  It should be specified using the name of a configured git remote.
+  Specifies a repository that content will be sent to. 
+  It should be specified using the name of a configured remote.
 
 * --exclude=glob
 
@@ -382,16 +382,16 @@ Here are all the supported configuration settings.
 
 * `remote..annex-uuid`
 
-  git-annex caches UUIDs of repositories here.
+  git-annex caches UUIDs of remote repositories here.
 
 * `remote..annex-ssh-options`
 
-  Options to use when using ssh to talk to this repository.
+  Options to use when using ssh to talk to this remote.
 
 * `remote..annex-rsync-options`
 
   Options to use when using rsync
-  to or from this repository. For example, to force ipv6, and limit
+  to or from this remote. For example, to force ipv6, and limit
   the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100"
 
 * `annex.ssh-options`, `annex.rsync-options`
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index b87238a327..38a6a6de59 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -9,11 +9,11 @@ First, export your S3 credentials:
 
 Next, create the remote.
 
-	git annex initremote mys3 encryption=none
+	git annex initremote s3 mys3 encryption=none
 	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
 
 **Note that encrypted buckets are not (yet) supported. Data sent to S3
-is susceptible to snooping.**
+is without encryption susceptible to snooping.**
 
 Now the remote can be used like any other remote.
 

From b1db436816b6b70ff0b9891bbc4a5468d9b895b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 23:22:31 -0400
Subject: [PATCH 1294/2835] started on initremote

---
 Command.hs                           |  4 ++
 Command/InitRemote.hs                | 48 ++++++++++++++++++
 GitAnnex.hs                          |  2 +
 Remote.hs                            | 74 +++++++++++++++++++++++++++-
 doc/git-annex.mdwn                   |  6 +--
 doc/walkthrough/using_Amazon_S3.mdwn |  2 +-
 6 files changed, 131 insertions(+), 5 deletions(-)
 create mode 100644 Command/InitRemote.hs

diff --git a/Command.hs b/Command.hs
index 446b1b55fe..9c908b8004 100644
--- a/Command.hs
+++ b/Command.hs
@@ -238,6 +238,10 @@ paramGlob :: String
 paramGlob = "GLOB"
 paramName :: String
 paramName = "NAME"
+paramType :: String
+paramType = "TYPE"
+paramKeyValue :: String
+paramKeyValue = "K=V"
 paramNothing :: String
 paramNothing = ""
 
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
new file mode 100644
index 0000000000..cf6a341c53
--- /dev/null
+++ b/Command/InitRemote.hs
@@ -0,0 +1,48 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.InitRemote where
+
+import qualified Data.Map as M
+import Control.Monad (when)
+import Control.Monad.State (liftIO)
+
+import Command
+import qualified Remote
+import UUID
+import Messages
+
+command :: [Command]
+command = [repoCommand "initremote"
+	(paramPair paramName $
+		paramOptional $ paramRepeating $ paramKeyValue) seek
+	"sets up a special (non-git) remote"]
+
+seek :: [CommandSeek]
+seek = [withString start]
+
+start :: CommandStartString
+start params = notBareRepo $ do
+	when (null ws) $ error "Specify a name for the remote"
+	showStart "initremote" name
+	r <- Remote.configGet name
+	(u, c) <- case r of
+		Just t -> return t
+		Nothing -> do
+			uuid <- liftIO $ genUUID
+			return $ (uuid, M.empty)
+	return $ Just $ perform name u $ M.union config c
+
+	where
+		ws = words params
+		name = head ws
+		config = Remote.keyValToMap $ tail ws
+
+perform :: String -> UUID -> M.Map String String -> CommandPerform
+perform name uuid config = do
+	liftIO $ putStrLn $ show $ (uuid, config)
+	return Nothing
diff --git a/GitAnnex.hs b/GitAnnex.hs
index adf07e5b3e..736b430e60 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -27,6 +27,7 @@ import qualified Command.SetKey
 import qualified Command.Fix
 import qualified Command.Init
 import qualified Command.Describe
+import qualified Command.InitRemote
 import qualified Command.Fsck
 import qualified Command.Unused
 import qualified Command.DropUnused
@@ -55,6 +56,7 @@ cmds = concat
 	, Command.Lock.command
 	, Command.Init.command
 	, Command.Describe.command
+	, Command.InitRemote.command
 	, Command.Unannex.command
 	, Command.Uninit.command
 	, Command.PreCommit.command
diff --git a/Remote.hs b/Remote.hs
index 6aab4a741c..f281d565a4 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -19,13 +19,19 @@ module Remote (
 	nameToUUID,
 	keyPossibilities,
 	remotesWithUUID,
-	remotesWithoutUUID
+	remotesWithoutUUID,
+
+	configGet,
+	configSet,
+	keyValToMap
 ) where
 
 import Control.Monad.State (liftIO)
 import Control.Monad (when, liftM)
 import Data.List
 import Data.String.Utils
+import qualified Data.Map as M
+import Data.Maybe
 
 import RemoteClass
 import qualified Remote.Git
@@ -35,6 +41,7 @@ import UUID
 import qualified Annex
 import Trust
 import LocationLog
+import Locations
 import Messages
 
 {- Add generators for new Remotes here. -}
@@ -120,3 +127,68 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
+{- Filename of remote.log. -}
+remoteLog :: Annex FilePath
+remoteLog = do
+	g <- Annex.gitRepo
+	return $ gitStateDir g ++ "remote.log"
+
+{- Reads the uuid and config of the specified remote from the remoteLog. -}
+configGet :: String -> Annex (Maybe (UUID, M.Map String String))
+configGet n = do
+	rs <- readRemoteLog
+	let matches = filter (matchName n) rs
+	case matches of
+		[] -> return Nothing
+		((u, _, c):_) -> return $ Just (u, c)
+
+{- Changes or adds a remote's config in the remoteLog. -}
+configSet :: String -> UUID -> M.Map String String -> Annex ()
+configSet n u c = do
+	rs <- readRemoteLog
+	let others = filter (not . matchName n) rs
+	writeRemoteLog $ (u, n, c):others
+
+matchName :: String -> (UUID, String, M.Map String String) -> Bool
+matchName n (_, n', _) = n == n'
+
+readRemoteLog :: Annex [(UUID, String, M.Map String String)]
+readRemoteLog = do
+	l <- remoteLog
+	s <- liftIO $ catch (readFile l) ignoreerror
+	return $ remoteLogParse s
+	where
+		ignoreerror _ = return []
+
+writeRemoteLog :: [(UUID, String, M.Map String String)] -> Annex ()
+writeRemoteLog rs = do
+	l <- remoteLog
+	liftIO $ writeFile l $ unlines $ map toline rs
+	where
+		toline (u, n, c) = u ++ " " ++ n ++ (unwords $ mapToKeyVal c)
+
+remoteLogParse :: String -> [(UUID, String, M.Map String String)]
+remoteLogParse s = catMaybes $ map parseline $ filter (not . null) $ lines s
+	where
+		parseline l
+			| length w > 2 = Just (u, n, c)
+			| otherwise = Nothing
+			where
+				w = words l
+				u = w !! 0
+				n = w !! 1
+				c = keyValToMap $ drop 2 w
+
+{- Given Strings like "key=value", generates a Map. -}
+keyValToMap :: [String] -> M.Map String String
+keyValToMap ws = M.fromList $ map (/=/) ws
+	where
+		(/=/) s = (k, v)
+			where
+				k = takeWhile (/= '=') s
+				v = drop (1 + length k) s
+
+mapToKeyVal :: M.Map String String -> [String]
+mapToKeyVal m = map toword $ M.toList m
+	where
+		toword (k, v) = k ++ "=" ++ v
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 4d14623942..0f548fa8a5 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,17 +132,17 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
-* initremote type name [param=value ...]
+* initremote name [param=value ...]
 
   Sets up a [[special_remote|special_remotes]] of some type. The remote's
-  configuration is configured by the parameters. If a remote
+  type and configuration is specified by the parameters. If a remote
   with the specified name has already been configured, its configuration
   is modified by any values specified. In either case, the remote will be
   added added to `.git/config`.
 
   Example Amazon S3 remote:
 
-	initremote s3 mys3 type=S3 encryption=none datacenter=EU
+	initremote mys3 type=S3 encryption=none datacenter=EU
 
 * fsck [path ...]
 
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 38a6a6de59..a99746c955 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -9,7 +9,7 @@ First, export your S3 credentials:
 
 Next, create the remote.
 
-	git annex initremote s3 mys3 encryption=none
+	git annex initremote mys3 type=S3 encryption=none
 	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
 
 **Note that encrypted buckets are not (yet) supported. Data sent to S3

From a3b6586902d6689b07c050b1fc50e19f4115c42e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 23:51:07 -0400
Subject: [PATCH 1295/2835] update

---
 Remote.hs      | 19 +++++++++----------
 Remote/Git.hs  | 14 ++++++++++----
 Remote/S3.hs   | 14 ++++++++++----
 RemoteClass.hs | 17 ++++++++++++++++-
 4 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index f281d565a4..71bc08c8ae 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -44,17 +44,16 @@ import LocationLog
 import Locations
 import Messages
 
-{- Add generators for new Remotes here. -}
-generators :: [Annex (RemoteGenerator Annex)]
-generators =
-	[ Remote.Git.generate
-	, Remote.S3.generate
+remoteTypes :: [RemoteType Annex]
+remoteTypes =
+	[ Remote.Git.remote
+	, Remote.S3.remote
 	]
 
-{- Runs a list of generators. -}
-runGenerators :: [Annex (RemoteGenerator Annex)] -> Annex [Remote Annex]
-runGenerators gs = do
-	(actions, expensive) <- collect ([], []) gs
+{- Runs the generators of each type of Remote -}
+runGenerators :: Annex [Remote Annex]
+runGenerators = do
+	(actions, expensive) <- collect ([], []) $ map generator remoteTypes
 	when (not $ null expensive) $
 		showNote $ "getting UUID for " ++ join ", " expensive
 	sequence actions
@@ -71,7 +70,7 @@ genList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
-			rs' <- runGenerators generators
+			rs' <- runGenerators
 			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 9021a2230c..68bd172e91 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -6,7 +6,7 @@
  -}
 
 module Remote.Git (
-	generate,
+	remote,
 	onRemote
 ) where
 
@@ -30,8 +30,11 @@ import RsyncFile
 import Ssh
 import Config
 
-generate :: Annex (RemoteGenerator Annex)
-generate = do
+remote :: RemoteType Annex
+remote = RemoteType { typename = "git", generator = gen }
+
+gen :: Annex (RemoteGenerator Annex)
+gen = do
 	g <- Annex.gitRepo
 	allremotes <- filterM remoteNotIgnored $ Git.remotes g
 
@@ -64,7 +67,10 @@ genRemote r = do
 		retrieveKeyFile = copyFromRemote r,
 		removeKey = dropKey r,
 		hasKey = inAnnex r,
-		hasKeyCheap = not (Git.repoIsUrl r)
+		hasKeyCheap = not (Git.repoIsUrl r),
+		hasConfig = False,
+		config = Nothing,
+		setup = \_ -> return ()
 	}
 
 {- Tries to read the config for a specified remote, updates state, and
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 23ec33bb59..4aa1bc639b 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.S3 (generate) where
+module Remote.S3 (remote) where
 
 import Network.AWS.AWSConnection
 import Network.AWS.S3Object
@@ -27,8 +27,11 @@ import qualified Annex
 import UUID
 import Config
 
-generate :: Annex (RemoteGenerator Annex)
-generate = do
+remote :: RemoteType Annex
+remote = RemoteType { typename = "S3", generator = gen }
+
+gen :: Annex (RemoteGenerator Annex)
+gen = do
 	g <- Annex.gitRepo
 	remotes <- filterM remoteNotIgnored $ findS3Remotes g
 	todo <- filterM cachedUUID remotes
@@ -64,7 +67,10 @@ genRemote r u = do
 		retrieveKeyFile = error "TODO",
 		removeKey = error "TODO",
 		hasKey = error "TODO",
-		hasKeyCheap = False
+		hasKeyCheap = False,
+		hasConfig = True,
+		config = Nothing,
+		setup = \_ -> return ()
 	}
 
 s3Connection :: Git.Repo -> Annex (Maybe AWSConnection)
diff --git a/RemoteClass.hs b/RemoteClass.hs
index eb4a017486..f3cc9379b0 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -10,6 +10,7 @@
 module RemoteClass where
 
 import Control.Exception
+import Data.Map as M
 
 import Key
 
@@ -18,6 +19,15 @@ import Key
  - that are not cheap to set up. -}
 type RemoteGenerator a = ([a (Remote a)], [String])
 
+{- There are different types of remotes. -}
+data RemoteType a = RemoteType {
+	-- human visible type name
+	typename :: String,
+	-- generates remotes of this type
+	generator :: a (RemoteGenerator a)
+}
+
+{- An individual remote. -}
 data Remote a = Remote {
 	-- each Remote has a unique uuid
 	uuid :: String,
@@ -36,7 +46,12 @@ data Remote a = Remote {
 	hasKey :: Key -> a (Either IOException Bool),
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
-	hasKeyCheap :: Bool
+	hasKeyCheap :: Bool,
+	-- a Remote may have a persistent configuration store
+	hasConfig :: Bool,
+	config :: Maybe (M.Map String String),
+	-- initializes or changes the config of a remote
+	setup :: M.Map String String -> a ()
 }
 
 instance Show (Remote a) where

From 97e3486f15eee4ac038b3e21d295b789091b0aef Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 29 Mar 2011 12:11:38 +0000
Subject: [PATCH 1296/2835]

---
 doc/forum/batch_check_on_remote_when_using_copy.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
index b08c33b8ba..633b61d6f9 100644
--- a/doc/forum/batch_check_on_remote_when_using_copy.mdwn
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -25,3 +25,10 @@ Once all checks are done, one single transfer session should be started. Creatin
 > While I do hope to improve ssh usage so that it sshs once, and feeds
 > `git-annex-shell` a series of commands to run, that is a much longer-term
 > thing. --[[Joey]]
+
+>> FYI, in a repo with 1228 files, all small, repos _completely in sync_.
+
+    % git annex copy . --to foo # 1200 seconds
+    % git annex copy . --to foo --fast # 20 seconds
+
+>> RichiH

From 05751d55cd8002e6a2a2afc520622fb6697472e3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 13:49:54 -0400
Subject: [PATCH 1297/2835] clean up remote.log handling

---
 Command/InitRemote.hs | 20 +++++++++++---
 Remote.hs             | 63 +++++++++++++++++++++----------------------
 Remote/Git.hs         |  5 ++--
 Remote/S3.hs          |  7 +++--
 RemoteClass.hs        |  3 +--
 doc/internals.mdwn    |  4 +--
 6 files changed, 55 insertions(+), 47 deletions(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index cf6a341c53..0d9a40cd33 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -29,12 +29,12 @@ start :: CommandStartString
 start params = notBareRepo $ do
 	when (null ws) $ error "Specify a name for the remote"
 	showStart "initremote" name
-	r <- Remote.configGet name
-	(u, c) <- case r of
+	m <- Remote.readRemoteLog
+	(u, c) <- case findByName name m of
 		Just t -> return t
 		Nothing -> do
 			uuid <- liftIO $ genUUID
-			return $ (uuid, M.empty)
+			return $ (uuid, M.insert nameKey name M.empty)
 	return $ Just $ perform name u $ M.union config c
 
 	where
@@ -46,3 +46,17 @@ perform :: String -> UUID -> M.Map String String -> CommandPerform
 perform name uuid config = do
 	liftIO $ putStrLn $ show $ (uuid, config)
 	return Nothing
+
+findByName :: String ->  M.Map UUID (M.Map String String) -> Maybe (UUID, M.Map String String)
+findByName n m = if null matches then Nothing else Just $ head matches
+	where
+		matches = filter (matching . snd) $ M.toList m
+		matching c = case M.lookup nameKey c of
+			Nothing -> False
+			Just n'
+				| n' == n -> True
+				| otherwise -> False
+
+{- The name of a configured remote is stored in its config using this key. -}
+nameKey :: String
+nameKey = "name"
diff --git a/Remote.hs b/Remote.hs
index 71bc08c8ae..f79b512625 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -21,7 +21,7 @@ module Remote (
 	remotesWithUUID,
 	remotesWithoutUUID,
 
-	configGet,
+	readRemoteLog,
 	configSet,
 	keyValToMap
 ) where
@@ -71,7 +71,8 @@ genList = do
 	if null rs
 		then do
 			rs' <- runGenerators
-			Annex.changeState $ \s -> s { Annex.remotes = rs' }
+			rs'' <- getConfigs rs'
+			Annex.changeState $ \s -> s { Annex.remotes = rs'' }
 			return rs'
 		else return rs
 
@@ -132,51 +133,47 @@ remoteLog = do
 	g <- Annex.gitRepo
 	return $ gitStateDir g ++ "remote.log"
 
-{- Reads the uuid and config of the specified remote from the remoteLog. -}
-configGet :: String -> Annex (Maybe (UUID, M.Map String String))
-configGet n = do
-	rs <- readRemoteLog
-	let matches = filter (matchName n) rs
-	case matches of
-		[] -> return Nothing
-		((u, _, c):_) -> return $ Just (u, c)
+{- Load stored config into remotes.
+ -
+ - This way, the log is read once, lazily, so if no remotes access
+ - their config, no work is done.
+ -}
+getConfigs :: [Remote Annex] -> Annex [Remote Annex]
+getConfigs rs = do
+	m <- readRemoteLog
+	return $ map (get m) rs	
+	where
+		get m r = r { config = M.lookup (uuid r) m }
 
-{- Changes or adds a remote's config in the remoteLog. -}
-configSet :: String -> UUID -> M.Map String String -> Annex ()
-configSet n u c = do
-	rs <- readRemoteLog
-	let others = filter (not . matchName n) rs
-	writeRemoteLog $ (u, n, c):others
+{- Adds or updates a remote's config in the log. -}
+configSet :: UUID -> M.Map String String -> Annex ()
+configSet u c = do
+	m <- readRemoteLog
+	l <- remoteLog
+	liftIO $ writeFile l $ unlines $ map toline $ M.toList $ M.insert u c m
+	where
+		toline (u', c') = u' ++ " " ++ (unwords $ mapToKeyVal c')
 
-matchName :: String -> (UUID, String, M.Map String String) -> Bool
-matchName n (_, n', _) = n == n'
-
-readRemoteLog :: Annex [(UUID, String, M.Map String String)]
+{- Map of remotes by uuid containing key/value config maps. -}
+readRemoteLog :: Annex (M.Map UUID (M.Map String String))
 readRemoteLog = do
 	l <- remoteLog
 	s <- liftIO $ catch (readFile l) ignoreerror
 	return $ remoteLogParse s
 	where
-		ignoreerror _ = return []
+		ignoreerror _ = return ""
 
-writeRemoteLog :: [(UUID, String, M.Map String String)] -> Annex ()
-writeRemoteLog rs = do
-	l <- remoteLog
-	liftIO $ writeFile l $ unlines $ map toline rs
-	where
-		toline (u, n, c) = u ++ " " ++ n ++ (unwords $ mapToKeyVal c)
-
-remoteLogParse :: String -> [(UUID, String, M.Map String String)]
-remoteLogParse s = catMaybes $ map parseline $ filter (not . null) $ lines s
+remoteLogParse :: String -> M.Map UUID (M.Map String String)
+remoteLogParse s =
+	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s
 	where
 		parseline l
-			| length w > 2 = Just (u, n, c)
+			| length w > 2 = Just (u, c)
 			| otherwise = Nothing
 			where
 				w = words l
 				u = w !! 0
-				n = w !! 1
-				c = keyValToMap $ drop 2 w
+				c = keyValToMap $ tail w
 
 {- Given Strings like "key=value", generates a Map. -}
 keyValToMap :: [String] -> M.Map String String
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 68bd172e91..b686e47af5 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -12,7 +12,7 @@ module Remote.Git (
 
 import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
-import qualified Data.Map as Map
+import qualified Data.Map as M
 import System.Cmd.Utils
 import Control.Monad (filterM, liftM)
 
@@ -68,7 +68,6 @@ genRemote r = do
 		removeKey = dropKey r,
 		hasKey = inAnnex r,
 		hasKeyCheap = not (Git.repoIsUrl r),
-		hasConfig = False,
 		config = Nothing,
 		setup = \_ -> return ()
 	}
@@ -77,7 +76,7 @@ genRemote r = do
  - returns the updated repo. -}
 tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
-	| not $ Map.null $ Git.configMap r = return r -- already read
+	| not $ M.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ Git.configRead r
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 4aa1bc639b..7971faa8fb 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -12,7 +12,7 @@ import Network.AWS.S3Object
 import Network.AWS.S3Bucket
 import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
-import qualified Data.Map as Map
+import qualified Data.Map as M
 import Data.String.Utils
 import Control.Monad (filterM, liftM, when)
 import Control.Monad.State (liftIO)
@@ -51,8 +51,8 @@ gen = do
 findS3Remotes :: Git.Repo -> [Git.Repo]
 findS3Remotes r = map construct remotepairs
 	where
-		remotepairs = Map.toList $ filterremotes $ Git.configMap r
-		filterremotes = Map.filterWithKey (\k _ -> s3remote k)
+		remotepairs = M.toList $ filterremotes $ Git.configMap r
+		filterremotes = M.filterWithKey (\k _ -> s3remote k)
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
 		s3remote k = startswith "remote." k && endswith ".annex-s3-bucket" k
 
@@ -68,7 +68,6 @@ genRemote r u = do
 		removeKey = error "TODO",
 		hasKey = error "TODO",
 		hasKeyCheap = False,
-		hasConfig = True,
 		config = Nothing,
 		setup = \_ -> return ()
 	}
diff --git a/RemoteClass.hs b/RemoteClass.hs
index f3cc9379b0..0482faac70 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -47,8 +47,7 @@ data Remote a = Remote {
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
 	hasKeyCheap :: Bool,
-	-- a Remote may have a persistent configuration store
-	hasConfig :: Bool,
+	-- a Remote can have a persistent configuration store
 	config :: Maybe (M.Map String String),
 	-- initializes or changes the config of a remote
 	setup :: M.Map String String -> a ()
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 6296095035..2e9f253831 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -36,8 +36,8 @@ Holds persistent configuration settings for [[special_remotes]] such as
 Amazon S3.
 
 The file format is one line per remote, starting with the uuid of the
-remote, followed by a space, the name of the remote, a space, and then
-a series of key=value pairs, each separated by whitespace.
+remote, followed by a space, and then a series of key=value pairs,
+each separated by whitespace.
 
 ## `.git-annex/trust.log`
 

From 0a4c610b4fd78f7d1589117cb723d7d8c15c120c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 14:55:59 -0400
Subject: [PATCH 1298/2835] initremote works

---
 Command/InitRemote.hs | 66 ++++++++++++++++++++++++++++++++++---------
 Remote.hs             | 13 ++++++---
 Remote/Git.hs         |  9 ++++--
 Remote/S3.hs          | 13 +++++++--
 RemoteClass.hs        |  8 +++---
 doc/git-annex.mdwn    |  6 ++--
 6 files changed, 85 insertions(+), 30 deletions(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 0d9a40cd33..39ec366539 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -12,7 +12,12 @@ import Control.Monad (when)
 import Control.Monad.State (liftIO)
 
 import Command
+import qualified Annex
 import qualified Remote
+import qualified RemoteClass
+import qualified GitRepo as Git
+import Utility
+import Types
 import UUID
 import Messages
 
@@ -28,27 +33,49 @@ seek = [withString start]
 start :: CommandStartString
 start params = notBareRepo $ do
 	when (null ws) $ error "Specify a name for the remote"
+
+	(u, c) <- findByName name
+	let fullconfig = M.union config c	
+	t <- findType fullconfig
+	
 	showStart "initremote" name
-	m <- Remote.readRemoteLog
-	(u, c) <- case findByName name m of
-		Just t -> return t
-		Nothing -> do
-			uuid <- liftIO $ genUUID
-			return $ (uuid, M.insert nameKey name M.empty)
-	return $ Just $ perform name u $ M.union config c
+	return $ Just $ perform t u $ M.union config c
 
 	where
 		ws = words params
 		name = head ws
 		config = Remote.keyValToMap $ tail ws
 
-perform :: String -> UUID -> M.Map String String -> CommandPerform
-perform name uuid config = do
-	liftIO $ putStrLn $ show $ (uuid, config)
-	return Nothing
+perform :: RemoteClass.RemoteType Annex -> UUID -> M.Map String String -> CommandPerform
+perform t u c = do
+	c' <- RemoteClass.setup t u c
+	return $ Just $ cleanup u c'
 
-findByName :: String ->  M.Map UUID (M.Map String String) -> Maybe (UUID, M.Map String String)
-findByName n m = if null matches then Nothing else Just $ head matches
+cleanup :: UUID -> M.Map String String -> CommandCleanup
+cleanup u c = do
+	Remote.configSet u c
+	g <- Annex.gitRepo
+	logfile <- Remote.remoteLog
+	liftIO $ Git.run g "add" [File logfile]
+        liftIO $ Git.run g "commit"
+                [ Params "-q --allow-empty -m"
+                , Param "git annex initremote"
+                , File logfile
+                ]
+        return True
+
+{- Look up existing remote's UUID and config by name, or generate a new one -}
+findByName :: String -> Annex (UUID, M.Map String String)
+findByName name = do
+	m <- Remote.readRemoteLog
+	case findByName' name m of
+		Just i -> return i
+		Nothing -> do
+			uuid <- liftIO $ genUUID
+			return $ (uuid, M.insert nameKey name M.empty)
+
+findByName' :: String ->  M.Map UUID (M.Map String String) -> Maybe (UUID, M.Map String String)
+findByName' n m = if null matches then Nothing else Just $ head matches
 	where
 		matches = filter (matching . snd) $ M.toList m
 		matching c = case M.lookup nameKey c of
@@ -57,6 +84,19 @@ findByName n m = if null matches then Nothing else Just $ head matches
 				| n' == n -> True
 				| otherwise -> False
 
+{- find the specified remote type -}
+findType :: M.Map String String -> Annex (RemoteClass.RemoteType Annex)
+findType config = 
+	case M.lookup typeKey config of
+		Nothing -> error "Specify the type of remote with type="
+		Just s -> case filter (\i -> RemoteClass.typename i == s) Remote.remoteTypes of
+			[] -> error $ "Unknown remote type " ++ s
+			(t:_) -> return t
+
 {- The name of a configured remote is stored in its config using this key. -}
 nameKey :: String
 nameKey = "name"
+
+{- The type of a remote is stored in its config using this key. -}
+typeKey :: String
+typeKey = "type"
diff --git a/Remote.hs b/Remote.hs
index f79b512625..1ca05d77ba 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,12 +15,14 @@ module Remote (
 	hasKey,
 	hasKeyCheap,
 
+	remoteTypes,
 	byName,
 	nameToUUID,
 	keyPossibilities,
 	remotesWithUUID,
 	remotesWithoutUUID,
 
+	remoteLog,
 	readRemoteLog,
 	configSet,
 	keyValToMap
@@ -34,8 +36,6 @@ import qualified Data.Map as M
 import Data.Maybe
 
 import RemoteClass
-import qualified Remote.Git
-import qualified Remote.S3
 import Types
 import UUID
 import qualified Annex
@@ -43,6 +43,10 @@ import Trust
 import LocationLog
 import Locations
 import Messages
+import Utility
+
+import qualified Remote.Git
+import qualified Remote.S3
 
 remoteTypes :: [RemoteType Annex]
 remoteTypes =
@@ -150,7 +154,8 @@ configSet :: UUID -> M.Map String String -> Annex ()
 configSet u c = do
 	m <- readRemoteLog
 	l <- remoteLog
-	liftIO $ writeFile l $ unlines $ map toline $ M.toList $ M.insert u c m
+	liftIO $ safeWriteFile l $ unlines $ sort $
+		map toline $ M.toList $ M.insert u c m
 	where
 		toline (u', c') = u' ++ " " ++ (unwords $ mapToKeyVal c')
 
@@ -185,6 +190,6 @@ keyValToMap ws = M.fromList $ map (/=/) ws
 				v = drop (1 + length k) s
 
 mapToKeyVal :: M.Map String String -> [String]
-mapToKeyVal m = map toword $ M.toList m
+mapToKeyVal m = map toword $ sort $ M.toList m
 	where
 		toword (k, v) = k ++ "=" ++ v
diff --git a/Remote/Git.hs b/Remote/Git.hs
index b686e47af5..2d7a0c8ff7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -31,7 +31,11 @@ import Ssh
 import Config
 
 remote :: RemoteType Annex
-remote = RemoteType { typename = "git", generator = gen }
+remote = RemoteType {
+	typename = "git",
+	generator = gen,
+	setup = error "not supported"
+}
 
 gen :: Annex (RemoteGenerator Annex)
 gen = do
@@ -68,8 +72,7 @@ genRemote r = do
 		removeKey = dropKey r,
 		hasKey = inAnnex r,
 		hasKeyCheap = not (Git.repoIsUrl r),
-		config = Nothing,
-		setup = \_ -> return ()
+		config = Nothing
 	}
 
 {- Tries to read the config for a specified remote, updates state, and
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 7971faa8fb..489114b126 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -28,7 +28,11 @@ import UUID
 import Config
 
 remote :: RemoteType Annex
-remote = RemoteType { typename = "S3", generator = gen }
+remote = RemoteType {
+	typename = "S3",
+	generator = gen,
+	setup = s3Setup
+}
 
 gen :: Annex (RemoteGenerator Annex)
 gen = do
@@ -68,8 +72,7 @@ genRemote r u = do
 		removeKey = error "TODO",
 		hasKey = error "TODO",
 		hasKeyCheap = False,
-		config = Nothing,
-		setup = \_ -> return ()
+		config = Nothing
 	}
 
 s3Connection :: Git.Repo -> Annex (Maybe AWSConnection)
@@ -102,6 +105,10 @@ getS3Config r s def = do
 	where
 		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
 
+s3Setup :: UUID -> M.Map String String -> Annex (M.Map String String)
+s3Setup u c = do
+	return c
+
 {- The UUID of a S3 bucket is stored in a file "git-annex-uuid" in the
  - bucket. Gets the UUID, or if there is none, sets a new UUID, possibly
  - also creating the bucket. -}
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 0482faac70..825197a4bc 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -24,7 +24,9 @@ data RemoteType a = RemoteType {
 	-- human visible type name
 	typename :: String,
 	-- generates remotes of this type
-	generator :: a (RemoteGenerator a)
+	generator :: a (RemoteGenerator a),
+	-- initializes or changes a remote
+	setup :: String -> M.Map String String -> a (M.Map String String)
 }
 
 {- An individual remote. -}
@@ -48,9 +50,7 @@ data Remote a = Remote {
 	-- operation.
 	hasKeyCheap :: Bool,
 	-- a Remote can have a persistent configuration store
-	config :: Maybe (M.Map String String),
-	-- initializes or changes the config of a remote
-	setup :: M.Map String String -> a ()
+	config :: Maybe (M.Map String String)
 }
 
 instance Show (Remote a) where
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0f548fa8a5..d890b518b8 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -134,11 +134,11 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * initremote name [param=value ...]
 
-  Sets up a [[special_remote|special_remotes]] of some type. The remote's
-  type and configuration is specified by the parameters. If a remote
+  Sets up a [[special_remote|special_remotes]]. The remote's
+  configuration is specified by the parameters. If a remote
   with the specified name has already been configured, its configuration
   is modified by any values specified. In either case, the remote will be
-  added added to `.git/config`.
+  added to `.git/config`.
 
   Example Amazon S3 remote:
 

From e62f9816ab29dceb5489d520b9cf569ede2ffb52 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 15:12:07 -0400
Subject: [PATCH 1299/2835] doc update for S3

---
 doc/special_remotes/Amazon_S3.mdwn   |  6 +++++-
 doc/walkthrough/using_Amazon_S3.mdwn | 28 ++++++++++++++++++----------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index 42c4a54534..c8e44b609c 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -8,7 +8,7 @@ See [[walkthrough/using_Amazon_S3]] for usage examples.
 A number of parameters can be passed to `git annex initremote` to configure
 the S3 remote.
 
-* `encryption` - Either "none" to disable encryption,
+* `encryption` - Required. Either "none" to disable encryption,
   or a value that can be looked up (using gpg -k) to find a gpg encryption
   key that will be given access to the remote. Note that additional gpg
   keys can be given access to a remote by rerunning initremote with
@@ -24,6 +24,10 @@ the S3 remote.
 * `host` and `port` - Specify in order to use a different, S3 compatable
   service.
 
+* `bucket` - S3 requires that buckets have a globally unique name, 
+  so by default, a bucket name is chosen based on the remote name
+  and UUID. This can be specified to pick a bucket name.
+
 ## data security
 
 When encryption=none, there is **no** protection against your data being read
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index a99746c955..34c843b18f 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -2,19 +2,27 @@ git-annex extends git's usual remotes with some [[special_remotes]], that
 are not git repositories. This way you can set up a remote using say,
 Amazon S3, and use git-annex to transfer files into the cloud.
 
-First, export your S3 credentials:
-
-	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
-
-Next, create the remote.
-
-	git annex initremote mys3 type=S3 encryption=none
-	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
-
 **Note that encrypted buckets are not (yet) supported. Data sent to S3
 is without encryption susceptible to snooping.**
 
+First, export your S3 credentials:
+
+	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+
+Next, create the S3 remote.
+
+	# git annex initremote mys3 type=S3 encryption=none
+	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
+
+The configuration for the S3 remote is stored in git. So to make a different
+repository use the same S3 remote is easy:
+
+	# cd /media/usb/annex
+        # git pull laptop master
+	# git annex initremote mys3
+	initremote ok
+
 Now the remote can be used like any other remote.
 
 	# git annex copy my_cool_big_file --to mys3

From 475f7073613b7164302e3f826f60929cf4cd38f0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 16:21:21 -0400
Subject: [PATCH 1300/2835] initremote now creates buckets

---
 Remote.hs                            |  21 +---
 Remote/Git.hs                        |  16 +--
 Remote/S3.hs                         | 147 ++++++++++++++++-----------
 RemoteClass.hs                       |   7 +-
 doc/walkthrough/using_Amazon_S3.mdwn |   4 +-
 5 files changed, 106 insertions(+), 89 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 1ca05d77ba..03615ac6ef 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -31,7 +31,6 @@ module Remote (
 import Control.Monad.State (liftIO)
 import Control.Monad (when, liftM)
 import Data.List
-import Data.String.Utils
 import qualified Data.Map as M
 import Data.Maybe
 
@@ -42,7 +41,6 @@ import qualified Annex
 import Trust
 import LocationLog
 import Locations
-import Messages
 import Utility
 
 import qualified Remote.Git
@@ -54,19 +52,6 @@ remoteTypes =
 	, Remote.S3.remote
 	]
 
-{- Runs the generators of each type of Remote -}
-runGenerators :: Annex [Remote Annex]
-runGenerators = do
-	(actions, expensive) <- collect ([], []) $ map generator remoteTypes
-	when (not $ null expensive) $
-		showNote $ "getting UUID for " ++ join ", " expensive
-	sequence actions
-	where
-		collect v [] = return v
-		collect (actions, expensive) (x:xs) = do
-			(a, e) <- x
-			collect (a++actions, e++expensive) xs
-
 {- Builds a list of all available Remotes.
  - Since doing so can be expensive, the list is cached in the Annex. -}
 genList :: Annex [Remote Annex]
@@ -74,9 +59,9 @@ genList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
-			rs' <- runGenerators
-			rs'' <- getConfigs rs'
-			Annex.changeState $ \s -> s { Annex.remotes = rs'' }
+			l <- mapM generator remoteTypes
+			rs' <- getConfigs (concat l)
+			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 2d7a0c8ff7..85bd04a23a 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -15,6 +15,8 @@ import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
 import Control.Monad (filterM, liftM)
+import Data.String.Utils
+import Maybe
 
 import RemoteClass
 import Types
@@ -37,7 +39,7 @@ remote = RemoteType {
 	setup = error "not supported"
 }
 
-gen :: Annex (RemoteGenerator Annex)
+gen :: Annex [Remote Annex]
 gen = do
 	g <- Annex.gitRepo
 	allremotes <- filterM remoteNotIgnored $ Git.remotes g
@@ -52,18 +54,20 @@ gen = do
 	let skip = filter (`notElem` expensive_todo) expensive
 	let todo = cheap++expensive_todo
 
-	let actions = map genRemote skip ++ 
-		map (\r -> genRemote =<< tryGitConfigRead r) todo
-	return (actions, map Git.repoDescribe expensive_todo)
+	showNote $ "getting UUID for " ++ (join ", " $
+		map Git.repoDescribe expensive_todo)
+	done <- mapM tryGitConfigRead todo
 
+	generated <- mapM genRemote $ skip ++ done
+	return $ catMaybes generated
 	where
 		cachedUUID r = liftM null $ getUUID r
 	
-genRemote :: Git.Repo -> Annex (Remote Annex)
+genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
 genRemote r = do
 	u <- getUUID r
 	c <- remoteCost r
-	return Remote {
+	return $ Just $ Remote {
 		uuid = u,
 		cost = c,
 		name = Git.repoDescribe r,
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 489114b126..16b3992da8 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -13,12 +13,12 @@ import Network.AWS.S3Bucket
 import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
+import Data.Maybe
 import Data.String.Utils
 import Control.Monad (filterM, liftM, when)
 import Control.Monad.State (liftIO)
 import System.Environment
 import Data.Char
-import Messages
 
 import RemoteClass
 import Types
@@ -26,6 +26,8 @@ import qualified GitRepo as Git
 import qualified Annex
 import UUID
 import Config
+import Utility
+import Messages
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -34,21 +36,14 @@ remote = RemoteType {
 	setup = s3Setup
 }
 
-gen :: Annex (RemoteGenerator Annex)
+gen :: Annex [Remote Annex]
 gen = do
 	g <- Annex.gitRepo
-	remotes <- filterM remoteNotIgnored $ findS3Remotes g
-	todo <- filterM cachedUUID remotes
-	let ok = filter (`notElem` todo) remotes
-	
-	let actions = map (\r -> genRemote r =<< getUUID r) ok ++
-		map (\r -> genRemote r =<< getS3UUID r) todo
-	return (actions, map Git.repoDescribe todo)
+	l <- filterM remoteNotIgnored $ findS3Remotes g
+	generated <- mapM genRemote l
+	return $ catMaybes generated
 
-	where
-		cachedUUID r = liftM null $ getUUID r
-
-{- S3 remotes have a remote..annex-s3-bucket config setting.
+{- S3 remotes have a remote..annex-s3 config setting.
  - Git.Repo does not normally generate remotes for things that
  - have no configured url, so the Git.Repo objects have to be
  - constructed as coming from an unknown location. -}
@@ -58,56 +53,81 @@ findS3Remotes r = map construct remotepairs
 		remotepairs = M.toList $ filterremotes $ Git.configMap r
 		filterremotes = M.filterWithKey (\k _ -> s3remote k)
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
-		s3remote k = startswith "remote." k && endswith ".annex-s3-bucket" k
+		s3remote k = startswith "remote." k && endswith ".annex-s3" k
 
-genRemote :: Git.Repo -> UUID -> Annex (Remote Annex)
-genRemote r u = do
-	c <- remoteCost r
-	return Remote {
-		uuid = u,
-		cost = c,
-		name = Git.repoDescribe r,
-		storeKey = error "TODO",
-		retrieveKeyFile = error "TODO",
-		removeKey = error "TODO",
-		hasKey = error "TODO",
-		hasKeyCheap = False,
-		config = Nothing
-	}
+genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
+genRemote r = do
+	u <- getUUID r
+	if (u == "")
+		then return Nothing
+		else do
+			c <- remoteCost r
+			return $ Just $ Remote {
+				uuid = u,
+				cost = c,
+				name = Git.repoDescribe r,
+				storeKey = error "TODO",
+				retrieveKeyFile = error "TODO",
+				removeKey = error "TODO",
+				hasKey = error "TODO",
+				hasKeyCheap = False,
+				config = Nothing
+			}
 
-s3Connection :: Git.Repo -> Annex (Maybe AWSConnection)
-s3Connection r = do
-	host <- getS3Config r "s3-host" (Just defaultAmazonS3Host)
-	port <- getS3Config r "s3-port" (Just $ show defaultAmazonS3Port)
-	accesskey <- getS3Config r "s3-access-key-id" Nothing
-	secretkey <- getS3Config r "s3-secret-access-key" Nothing
-	case reads port of
-		[(p, _)] -> return $ Just $ AWSConnection host p accesskey secretkey
-		_ -> error $ "bad S3 port value: " ++ port
-
-withS3Connection :: Git.Repo -> Annex a -> ((AWSConnection, String) -> Annex a) -> Annex a
-withS3Connection r def a = do
-	c <- s3Connection r
-	case c of
-		Nothing -> def
-		Just c' -> do
-			b <- getConfig r "s3-bucket" ""
-			a (c', b)
-
-getS3Config :: Git.Repo -> String -> Maybe String-> Annex String
-getS3Config r s def = do
-	e <- liftIO $ catch (liftM Just $ getEnv envvar) (const $ return def)
-	v <- case e of
-		Nothing -> getConfig r s ""
-		Just d -> getConfig r s d
-	when (null v) $ error $ "set " ++ envvar ++ " or " ++ remoteConfig r s
-	return v
+s3Connection :: M.Map String String -> IO AWSConnection
+s3Connection c = do
+	ak <- getEnvKey "AWS_ACCESS_KEY_ID"
+	sk <- getEnvKey "AWS_SECRET_ACCESS_KEY"
+	return $ AWSConnection host port ak sk
 	where
-		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
+		host = fromJust $ (M.lookup "host" c)
+		port = let s = fromJust $ (M.lookup "port" c) in
+			case reads s of
+			[(p, _)] -> p
+			_ -> error $ "bad S3 port value: " ++ s
+		getEnvKey s = catch (getEnv s) (error $ "Set " ++ s)
 
 s3Setup :: UUID -> M.Map String String -> Annex (M.Map String String)
 s3Setup u c = do
-	return c
+	-- verify configuration is sane
+	case M.lookup "encryption" c of
+		Nothing -> error "Specify encryption=key or encryption=none"
+		Just "none" -> return ()
+		Just k -> error "encryption keys not yet supported"
+	let fullconfig = M.union c defaults
+
+	-- check bucket location to see if the bucket exists
+	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
+	conn <- liftIO $ s3Connection fullconfig
+	showNote "checking bucket"
+	loc <- liftIO $ getBucketLocation conn bucket 
+	case loc of
+		Right _ -> return ()
+		Left err@(NetworkError _) -> error $ prettyReqError err
+		Left (AWSError _ _) -> do
+			showNote "creating bucket"
+			res <- liftIO $ createBucketIn conn bucket datacenter
+			case res of
+				Right _ -> return ()
+				Left err -> error $ prettyReqError err
+
+	g <- Annex.gitRepo
+	liftIO $ do
+		Git.run g "config" [Param ("remote." ++ name ++ ".annex-s3"), Param "true"]
+		Git.run g "config" [Param ("remote." ++ name ++ ".annex-uuid"), Param u]
+	return fullconfig
+	where
+		name = fromJust (M.lookup "name" c)
+		bucket = name ++ "-" ++ u
+		defaults = M.fromList
+			[ ("datacenter", "US")
+			, ("storageclass", "STANDARD")
+			, ("host", defaultAmazonS3Host)
+			, ("port", show defaultAmazonS3Port)
+			, ("bucket", bucket)
+			]
+
+{-
 
 {- The UUID of a S3 bucket is stored in a file "git-annex-uuid" in the
  - bucket. Gets the UUID, or if there is none, sets a new UUID, possibly
@@ -135,3 +155,16 @@ getS3UUID r = withS3Connection r disable $ \(c, b) -> do
 	where
 		uuidfile = "git-annex-uuid"
 		disable = return "" -- empty uuid will disable this remote
+
+getS3Config :: Git.Repo -> String -> Maybe String-> Annex String
+getS3Config r s def = do
+	e <- liftIO $ catch (liftM Just $ getEnv envvar) (const $ return def)
+	v <- case e of
+		Nothing -> getConfig r s ""
+		Just d -> getConfig r s d
+	when (null v) $ error $ "set " ++ envvar ++ " or " ++ remoteConfig r s
+	return v
+	where
+		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
+
+-}
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 825197a4bc..e16cbdbb01 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -14,17 +14,12 @@ import Data.Map as M
 
 import Key
 
-{- A remote generator identifies configured remotes, and returns an action
- - that can be run to set up each remote, and a list of names of remotes
- - that are not cheap to set up. -}
-type RemoteGenerator a = ([a (Remote a)], [String])
-
 {- There are different types of remotes. -}
 data RemoteType a = RemoteType {
 	-- human visible type name
 	typename :: String,
 	-- generates remotes of this type
-	generator :: a (RemoteGenerator a),
+	generator :: a [Remote a],
 	-- initializes or changes a remote
 	setup :: String -> M.Map String String -> a (M.Map String String)
 }
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 34c843b18f..5f2766868b 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -13,7 +13,7 @@ First, export your S3 credentials:
 Next, create the S3 remote.
 
 	# git annex initremote mys3 type=S3 encryption=none
-	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
+	initremote mys3 (checking bucket) (creating bucket) ok
 
 The configuration for the S3 remote is stored in git. So to make a different
 repository use the same S3 remote is easy:
@@ -21,7 +21,7 @@ repository use the same S3 remote is easy:
 	# cd /media/usb/annex
         # git pull laptop master
 	# git annex initremote mys3
-	initremote ok
+	initremote mys3 (checking bucket) ok
 
 Now the remote can be used like any other remote.
 

From 72f94cc42eca1a6aaa7cc95daf423915761805ff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 17:20:22 -0400
Subject: [PATCH 1301/2835] progress

---
 Remote.hs                            |   4 +-
 Remote/Git.hs                        |  11 +--
 Remote/S3.hs                         | 114 +++++++++++++--------------
 doc/walkthrough/using_Amazon_S3.mdwn |   2 +-
 4 files changed, 66 insertions(+), 65 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 03615ac6ef..1474811853 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -53,7 +53,7 @@ remoteTypes =
 	]
 
 {- Builds a list of all available Remotes.
- - Since doing so can be expensive, the list is cached in the Annex. -}
+ - Since doing so can be expensive, the list is cached. -}
 genList :: Annex [Remote Annex]
 genList = do
 	rs <- Annex.getState Annex.remotes
@@ -130,7 +130,7 @@ remoteLog = do
 getConfigs :: [Remote Annex] -> Annex [Remote Annex]
 getConfigs rs = do
 	m <- readRemoteLog
-	return $ map (get m) rs	
+	return $ map (get m) rs
 	where
 		get m r = r { config = M.lookup (uuid r) m }
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 85bd04a23a..e5f2aa62d0 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -14,7 +14,7 @@ import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
-import Control.Monad (filterM, liftM)
+import Control.Monad (filterM, liftM, when)
 import Data.String.Utils
 import Maybe
 
@@ -50,18 +50,19 @@ gen = do
 	 - cached UUID value. -}
 	let cheap = filter (not . Git.repoIsUrl) allremotes
 	let expensive = filter Git.repoIsUrl allremotes
-	expensive_todo <- filterM cachedUUID expensive
+	expensive_todo <- filterM noCachedUUID expensive
 	let skip = filter (`notElem` expensive_todo) expensive
 	let todo = cheap++expensive_todo
 
-	showNote $ "getting UUID for " ++ (join ", " $
-		map Git.repoDescribe expensive_todo)
+	when (not $ null expensive_todo) $
+		showNote $ "getting UUID for " ++ (join ", " $
+			map Git.repoDescribe expensive_todo)
 	done <- mapM tryGitConfigRead todo
 
 	generated <- mapM genRemote $ skip ++ done
 	return $ catMaybes generated
 	where
-		cachedUUID r = liftM null $ getUUID r
+		noCachedUUID r = liftM null $ getUUID r
 	
 genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
 genRemote r = do
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 16b3992da8..887b19e731 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -7,6 +7,7 @@
 
 module Remote.S3 (remote) where
 
+import Control.Exception.Extensible (IOException)
 import Network.AWS.AWSConnection
 import Network.AWS.S3Object
 import Network.AWS.S3Bucket
@@ -15,10 +16,9 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.Maybe
 import Data.String.Utils
-import Control.Monad (filterM, liftM, when)
+import Control.Monad (filterM, when)
 import Control.Monad.State (liftIO)
 import System.Environment
-import Data.Char
 
 import RemoteClass
 import Types
@@ -62,17 +62,21 @@ genRemote r = do
 		then return Nothing
 		else do
 			c <- remoteCost r
-			return $ Just $ Remote {
-				uuid = u,
-				cost = c,
-				name = Git.repoDescribe r,
-				storeKey = error "TODO",
-				retrieveKeyFile = error "TODO",
-				removeKey = error "TODO",
-				hasKey = error "TODO",
-				hasKeyCheap = False,
-				config = Nothing
-			}
+			return $ Just $ newremote u c
+	where
+		newremote u c = this
+			where
+				this = Remote {
+					uuid = u,
+					cost = c,
+					name = Git.repoDescribe r,
+			 		storeKey = s3Store this,
+					retrieveKeyFile = error "TODO retrievekey",
+					removeKey = error "TODO removekey",
+					hasKey = s3CheckPresent this,
+					hasKeyCheap = False,
+					config = Nothing
+				}
 
 s3Connection :: M.Map String String -> IO AWSConnection
 s3Connection c = do
@@ -93,10 +97,10 @@ s3Setup u c = do
 	case M.lookup "encryption" c of
 		Nothing -> error "Specify encryption=key or encryption=none"
 		Just "none" -> return ()
-		Just k -> error "encryption keys not yet supported"
+		Just _ -> error "encryption keys not yet supported"
 	let fullconfig = M.union c defaults
 
-	-- check bucket location to see if the bucket exists
+	-- check bucket location to see if the bucket exists, and create it
 	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
 	conn <- liftIO $ s3Connection fullconfig
 	showNote "checking bucket"
@@ -105,7 +109,7 @@ s3Setup u c = do
 		Right _ -> return ()
 		Left err@(NetworkError _) -> error $ prettyReqError err
 		Left (AWSError _ _) -> do
-			showNote "creating bucket"
+			showNote $ "creating bucket in " ++ datacenter
 			res <- liftIO $ createBucketIn conn bucket datacenter
 			case res of
 				Right _ -> return ()
@@ -113,12 +117,13 @@ s3Setup u c = do
 
 	g <- Annex.gitRepo
 	liftIO $ do
-		Git.run g "config" [Param ("remote." ++ name ++ ".annex-s3"), Param "true"]
-		Git.run g "config" [Param ("remote." ++ name ++ ".annex-uuid"), Param u]
+		Git.run g "config" [Param (configsetting "annex-s3"), Param "true"]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
 	return fullconfig
 	where
-		name = fromJust (M.lookup "name" c)
-		bucket = name ++ "-" ++ u
+		remotename = fromJust (M.lookup "name" c)
+		bucket = remotename ++ "-" ++ u
+		configsetting s = "remote." ++ remotename ++ "." ++ s
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
@@ -127,44 +132,39 @@ s3Setup u c = do
 			, ("bucket", bucket)
 			]
 
-{-
+s3Action :: Remote Annex -> ((AWSConnection, String) -> Annex a) -> Annex a
+s3Action r a = do
+	when (config r == Nothing) $
+		error $ "Missing configuration for special remote " ++ name r
+	conn <- liftIO $ s3Connection (fromJust $ config r)
+	let bucket = fromJust $ M.lookup "bucket" $ fromJust $ config r
+	a (conn, bucket)
 
-{- The UUID of a S3 bucket is stored in a file "git-annex-uuid" in the
- - bucket. Gets the UUID, or if there is none, sets a new UUID, possibly
- - also creating the bucket. -}
-getS3UUID :: Git.Repo -> Annex UUID
-getS3UUID r = withS3Connection r disable $ \(c, b) -> do
-	res <- liftIO $
-		getObject c $ S3Object b uuidfile "" [] L.empty
+s3File :: Key -> FilePath
+s3File k = show k
+
+s3CheckPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
+s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
+	let object = S3Object bucket (s3File k) "" [] L.empty
+	showNote ("checking " ++ name r ++ "...")
+	res <- liftIO $ getObjectInfo conn object
 	case res of
-		Right o -> return $ L.unpack $ obj_data o
-		Left _ -> do
-			location <- getS3Config r "s3-datacenter" (Just "EU")
-			-- bucket may already exist, or not
-			_ <- liftIO $ createBucketIn c b location
-			u <- getUUID r
-			res' <- liftIO $ sendObject c $
-				S3Object b uuidfile "" [] $
-					L.pack u
-			case res' of
-				Right _ -> return u
-				Left e -> do
-					warning $ prettyReqError e
-					disable
-					
-	where
-		uuidfile = "git-annex-uuid"
-		disable = return "" -- empty uuid will disable this remote
+		Right _ -> return $ Right True
+		Left (AWSError _ _) -> return $ Right False
+		Left e -> return $ Left (error $ prettyReqError e)
 
-getS3Config :: Git.Repo -> String -> Maybe String-> Annex String
-getS3Config r s def = do
-	e <- liftIO $ catch (liftM Just $ getEnv envvar) (const $ return def)
-	v <- case e of
-		Nothing -> getConfig r s ""
-		Just d -> getConfig r s d
-	when (null v) $ error $ "set " ++ envvar ++ " or " ++ remoteConfig r s
-	return v
+s3Store :: Remote Annex -> Key -> Annex Bool
+s3Store r k = s3Action r $ \(conn, bucket) -> do
+	let object = setStorageClass storageclass $
+		S3Object bucket (s3File k) "" [] (error "read content here")
+	res <- liftIO $ sendObject conn object
+	case res of
+		Right _ -> return True
+		Left e -> do
+			warning $ prettyReqError e
+			return False
 	where
-		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
-
--}
+		storageclass =
+			case fromJust $ M.lookup "storageclass" $ fromJust $ config r of
+				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
+				_ -> STANDARD
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 5f2766868b..e0d2296620 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -13,7 +13,7 @@ First, export your S3 credentials:
 Next, create the S3 remote.
 
 	# git annex initremote mys3 type=S3 encryption=none
-	initremote mys3 (checking bucket) (creating bucket) ok
+	initremote mys3 (checking bucket) (creating bucket in US) ok
 
 The configuration for the S3 remote is stored in git. So to make a different
 repository use the same S3 remote is easy:

From 0782d7006365e82c0040b25364fa452b0e00e527 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 17:57:20 -0400
Subject: [PATCH 1302/2835] copy --to S3 works

---
 Remote.hs      | 24 +++++++++-------------
 Remote/Git.hs  | 41 ++++++++++++++++---------------------
 Remote/S3.hs   | 55 +++++++++++++++++++++++++-------------------------
 RemoteClass.hs |  7 +++++--
 4 files changed, 60 insertions(+), 67 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 1474811853..9fd53a2f27 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -59,11 +59,19 @@ genList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
-			l <- mapM generator remoteTypes
-			rs' <- getConfigs (concat l)
+			m <- readRemoteLog
+			l <- mapM (process m) remoteTypes
+			let rs' = concat l
 			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
+	where
+		process m t = do
+			l <- enumerate t
+			mapM (gen m t) l
+		gen m t r = do
+			u <- getUUID r
+			generate t r (M.lookup u m)
 
 {- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex (Remote Annex)
@@ -122,18 +130,6 @@ remoteLog = do
 	g <- Annex.gitRepo
 	return $ gitStateDir g ++ "remote.log"
 
-{- Load stored config into remotes.
- -
- - This way, the log is read once, lazily, so if no remotes access
- - their config, no work is done.
- -}
-getConfigs :: [Remote Annex] -> Annex [Remote Annex]
-getConfigs rs = do
-	m <- readRemoteLog
-	return $ map (get m) rs
-	where
-		get m r = r { config = M.lookup (uuid r) m }
-
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> M.Map String String -> Annex ()
 configSet u c = do
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e5f2aa62d0..984f9c88fe 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -14,9 +14,7 @@ import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
-import Control.Monad (filterM, liftM, when)
-import Data.String.Utils
-import Maybe
+import Control.Monad (filterM)
 
 import RemoteClass
 import Types
@@ -35,40 +33,35 @@ import Config
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "git",
-	generator = gen,
+	enumerate = list,
+	generate = gen,
 	setup = error "not supported"
 }
 
-gen :: Annex [Remote Annex]
-gen = do
+list :: Annex [Git.Repo]
+list = do
 	g <- Annex.gitRepo
-	allremotes <- filterM remoteNotIgnored $ Git.remotes g
+	filterM remoteNotIgnored $ Git.remotes g
 
+gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen repo _ = do
  	{- It's assumed to be cheap to read the config of non-URL remotes,
 	 - so this is done each time git-annex is run. Conversely,
 	 - the config of an URL remote is only read when there is no
 	 - cached UUID value. -}
-	let cheap = filter (not . Git.repoIsUrl) allremotes
-	let expensive = filter Git.repoIsUrl allremotes
-	expensive_todo <- filterM noCachedUUID expensive
-	let skip = filter (`notElem` expensive_todo) expensive
-	let todo = cheap++expensive_todo
-
-	when (not $ null expensive_todo) $
-		showNote $ "getting UUID for " ++ (join ", " $
-			map Git.repoDescribe expensive_todo)
-	done <- mapM tryGitConfigRead todo
-
-	generated <- mapM genRemote $ skip ++ done
-	return $ catMaybes generated
-	where
-		noCachedUUID r = liftM null $ getUUID r
+	let cheap = not $ Git.repoIsUrl repo
+	u <- getUUID repo
+	repo' <- case (cheap, u) of
+		(True, _) -> tryGitConfigRead repo
+		(False, "") -> tryGitConfigRead repo
+		_ -> return repo
+	genRemote repo'
 	
-genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
+genRemote :: Git.Repo -> Annex (Remote Annex)
 genRemote r = do
 	u <- getUUID r
 	c <- remoteCost r
-	return $ Just $ Remote {
+	return $ Remote {
 		uuid = u,
 		cost = c,
 		name = Git.repoDescribe r,
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 887b19e731..4e151e22f3 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -28,20 +28,20 @@ import UUID
 import Config
 import Utility
 import Messages
+import Locations
 
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "S3",
-	generator = gen,
+	enumerate = s3List,
+	generate = s3Gen,
 	setup = s3Setup
 }
 
-gen :: Annex [Remote Annex]
-gen = do
+s3List :: Annex [Git.Repo]
+s3List = do
 	g <- Annex.gitRepo
-	l <- filterM remoteNotIgnored $ findS3Remotes g
-	generated <- mapM genRemote l
-	return $ catMaybes generated
+	filterM remoteNotIgnored $ findS3Remotes g
 
 {- S3 remotes have a remote..annex-s3 config setting.
  - Git.Repo does not normally generate remotes for things that
@@ -55,28 +55,27 @@ findS3Remotes r = map construct remotepairs
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
 		s3remote k = startswith "remote." k && endswith ".annex-s3" k
 
-genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
-genRemote r = do
+s3Gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
+s3Gen r c = do
 	u <- getUUID r
-	if (u == "")
-		then return Nothing
-		else do
-			c <- remoteCost r
-			return $ Just $ newremote u c
+	cst <- remoteCost r
+	return $ genRemote r u c cst
 	where
-		newremote u c = this
-			where
-				this = Remote {
-					uuid = u,
-					cost = c,
-					name = Git.repoDescribe r,
-			 		storeKey = s3Store this,
-					retrieveKeyFile = error "TODO retrievekey",
-					removeKey = error "TODO removekey",
-					hasKey = s3CheckPresent this,
-					hasKeyCheap = False,
-					config = Nothing
-				}
+
+genRemote :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Int -> Remote Annex
+genRemote r u c cst = this
+	where
+		this = Remote {
+			uuid = u,
+			cost = cst,
+			name = Git.repoDescribe r,
+	 		storeKey = s3Store this,
+			retrieveKeyFile = error "TODO retrievekey",
+			removeKey = error "TODO removekey",
+			hasKey = s3CheckPresent this,
+			hasKeyCheap = False,
+			config = c
+		}
 
 s3Connection :: M.Map String String -> IO AWSConnection
 s3Connection c = do
@@ -155,8 +154,10 @@ s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
 
 s3Store :: Remote Annex -> Key -> Annex Bool
 s3Store r k = s3Action r $ \(conn, bucket) -> do
+	g <- Annex.gitRepo
+	content <- liftIO $ L.readFile $ gitAnnexLocation g k
 	let object = setStorageClass storageclass $
-		S3Object bucket (s3File k) "" [] (error "read content here")
+		S3Object bucket (s3File k) "" [] content
 	res <- liftIO $ sendObject conn object
 	case res of
 		Right _ -> return True
diff --git a/RemoteClass.hs b/RemoteClass.hs
index e16cbdbb01..de4c281f44 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -12,14 +12,17 @@ module RemoteClass where
 import Control.Exception
 import Data.Map as M
 
+import qualified GitRepo as Git
 import Key
 
 {- There are different types of remotes. -}
 data RemoteType a = RemoteType {
 	-- human visible type name
 	typename :: String,
-	-- generates remotes of this type
-	generator :: a [Remote a],
+	-- enumerates remotes of this type
+	enumerate :: a [Git.Repo],
+	-- generates a remote of this type
+	generate :: Git.Repo -> Maybe (M.Map String String) -> a (Remote a),
 	-- initializes or changes a remote
 	setup :: String -> M.Map String String -> a (M.Map String String)
 }

From d8154eaad3f39e045d7abba187a7d2c1399b89dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:09:22 -0400
Subject: [PATCH 1303/2835] transfering content back from s3 works!

---
 Remote/S3.hs | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 4e151e22f3..3265ced78c 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -70,7 +70,7 @@ genRemote r u c cst = this
 			cost = cst,
 			name = Git.repoDescribe r,
 	 		storeKey = s3Store this,
-			retrieveKeyFile = error "TODO retrievekey",
+			retrieveKeyFile = s3Retrieve this,
 			removeKey = error "TODO removekey",
 			hasKey = s3CheckPresent this,
 			hasKeyCheap = False,
@@ -139,14 +139,13 @@ s3Action r a = do
 	let bucket = fromJust $ M.lookup "bucket" $ fromJust $ config r
 	a (conn, bucket)
 
-s3File :: Key -> FilePath
-s3File k = show k
+bucketKey :: String -> Key -> L.ByteString -> S3Object
+bucketKey bucket k content = S3Object bucket (show k) "" [] content
 
 s3CheckPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
 s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
-	let object = S3Object bucket (s3File k) "" [] L.empty
 	showNote ("checking " ++ name r ++ "...")
-	res <- liftIO $ getObjectInfo conn object
+	res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty
 	case res of
 		Right _ -> return $ Right True
 		Left (AWSError _ _) -> return $ Right False
@@ -156,8 +155,7 @@ s3Store :: Remote Annex -> Key -> Annex Bool
 s3Store r k = s3Action r $ \(conn, bucket) -> do
 	g <- Annex.gitRepo
 	content <- liftIO $ L.readFile $ gitAnnexLocation g k
-	let object = setStorageClass storageclass $
-		S3Object bucket (s3File k) "" [] content
+	let object = setStorageClass storageclass $ bucketKey bucket k content
 	res <- liftIO $ sendObject conn object
 	case res of
 		Right _ -> return True
@@ -169,3 +167,14 @@ s3Store r k = s3Action r $ \(conn, bucket) -> do
 			case fromJust $ M.lookup "storageclass" $ fromJust $ config r of
 				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
 				_ -> STANDARD
+
+s3Retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
+s3Retrieve r k f = s3Action r $ \(conn, bucket) -> do
+	res <- liftIO $ getObject conn $ bucketKey bucket k L.empty
+	case res of
+		Right o -> do
+			liftIO $ L.writeFile f (obj_data o)
+			return True
+		Left e -> do
+			warning $ prettyReqError e
+			return False

From 3adb48f46a553e8de926e2ce93ea5162dd589111 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:21:05 -0400
Subject: [PATCH 1304/2835] more S3 docs

---
 Remote/S3.hs                         | 11 ++++++++++-
 doc/install.mdwn                     |  1 +
 doc/walkthrough/using_Amazon_S3.mdwn |  6 ++++--
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 3265ced78c..b3a9106396 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -71,7 +71,7 @@ genRemote r u c cst = this
 			name = Git.repoDescribe r,
 	 		storeKey = s3Store this,
 			retrieveKeyFile = s3Retrieve this,
-			removeKey = error "TODO removekey",
+			removeKey = s3Remove this,
 			hasKey = s3CheckPresent this,
 			hasKeyCheap = False,
 			config = c
@@ -178,3 +178,12 @@ s3Retrieve r k f = s3Action r $ \(conn, bucket) -> do
 		Left e -> do
 			warning $ prettyReqError e
 			return False
+
+s3Remove :: Remote Annex -> Key -> Annex Bool
+s3Remove r k = s3Action r $ \(conn, bucket) -> do
+	res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty
+	case res of
+		Right _ -> return True
+		Left e -> do
+			warning $ prettyReqError e
+			return False
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 0501663231..7b2c536c9d 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -13,6 +13,7 @@ To build and use git-annex, you will need:
 * MissingH: 
 * pcre-light: 
 * utf8-string: 
+* hS3: 
 * `uuid`: 
   (or uuidgen from util-linux)
 * `xargs`: 
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index e0d2296620..b8eb7da530 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -10,12 +10,14 @@ First, export your S3 credentials:
 	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
 	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
 
-Next, create the S3 remote.
+Next, create the S3 remote, and describe it.
 
 	# git annex initremote mys3 type=S3 encryption=none
 	initremote mys3 (checking bucket) (creating bucket in US) ok
+	# git annex describe mys3 "at Amazon's US datacenter"
+	describe mys3 ok
 
-The configuration for the S3 remote is stored in git. So to make a different
+The configuration for the S3 remote is stored in git. So to make another
 repository use the same S3 remote is easy:
 
 	# cd /media/usb/annex

From 43bdebbc2d2c22623a0114dca51b8339bf7231c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:24:26 -0400
Subject: [PATCH 1305/2835] update

---
 debian/changelog | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4c0c9f7411..faf2833a84 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,7 @@
 git-annex (0.20110329) UNRELEASED; urgency=low
 
   * Amazon S3 is now supported as a special type of remote.
-    Warning: Encrypting data before sending it to S3 is not currently
-    supported.
+    Warning: Encrypting data before sending it to S3 is not yet supported.
   * fsck: Ensure that files and directories in .git/annex/objects
     have proper permissions.
 

From 8f9951369d5e85e3a1bf323760f0c873a3f21b97 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:28:37 -0400
Subject: [PATCH 1306/2835] refactor

---
 Remote.hs     | 6 ++++--
 Remote/Git.hs | 3 +--
 Remote/S3.hs  | 4 ++--
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 9fd53a2f27..914c69abe5 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -29,7 +29,7 @@ module Remote (
 ) where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when, liftM)
+import Control.Monad (when, liftM, filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
@@ -42,6 +42,7 @@ import Trust
 import LocationLog
 import Locations
 import Utility
+import Config
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -68,7 +69,8 @@ genList = do
 	where
 		process m t = do
 			l <- enumerate t
-			mapM (gen m t) l
+			l' <- filterM remoteNotIgnored l
+			mapM (gen m t) l'
 		gen m t r = do
 			u <- getUUID r
 			generate t r (M.lookup u m)
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 984f9c88fe..d0dedd4fde 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -14,7 +14,6 @@ import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
-import Control.Monad (filterM)
 
 import RemoteClass
 import Types
@@ -41,7 +40,7 @@ remote = RemoteType {
 list :: Annex [Git.Repo]
 list = do
 	g <- Annex.gitRepo
-	filterM remoteNotIgnored $ Git.remotes g
+	return $ Git.remotes g
 
 gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
 gen repo _ = do
diff --git a/Remote/S3.hs b/Remote/S3.hs
index b3a9106396..260c1eee8e 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -16,7 +16,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.Maybe
 import Data.String.Utils
-import Control.Monad (filterM, when)
+import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Environment
 
@@ -41,7 +41,7 @@ remote = RemoteType {
 s3List :: Annex [Git.Repo]
 s3List = do
 	g <- Annex.gitRepo
-	filterM remoteNotIgnored $ findS3Remotes g
+	return $ findS3Remotes g
 
 {- S3 remotes have a remote..annex-s3 config setting.
  - Git.Repo does not normally generate remotes for things that

From e7c1332fa2f447f46781641821d1ff2c71716657 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:30:41 -0400
Subject: [PATCH 1307/2835] updat

---
 doc/todo/S3.mdwn | 28 +++++-----------------------
 1 file changed, 5 insertions(+), 23 deletions(-)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 356b2af2eb..dd0174cc55 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -1,33 +1,15 @@
-[[done]]
-
 Support Amazon S3 as a file storage backend.
 
 There's a haskell library that looks good. Not yet in Debian.
 
-Multiple ways of using S3 are possible. Current plan is to 
-have a special type of git remote (though git won't know how to use it;
-only git-annex will) that uses a S3 bucket.
+Multiple ways of using S3 are possible. Currently implemented as 
+a special type of git remote.
 
-Something like:
+Before this can be close, I need to fix:
 
-	[remote "s3"]
-		annex-s3bucket = mybucket
-		annex-s3datacenter = Europe
-		annex-uuid = 1a586cf6-45e9-11e0-ba9c-3b0a3397aec2
-		annex-cost = 500
+## encryption
 
-The UUID will be stored as a special file in the S3 bucket.
-
-Using a different type of remote like this will allow S3 to be used
-anywhere a regular remote would be used. `git annex get` will transparently
-download a file from S3 if S3 has it and is the cheapest remote.
-
-	git annex copy --to s3
-	git annex move --from s3
-	git annex drop --from s3 # not currently allowed, will need adding
-
-Each s3 remote will count as one copy for numcopies handling, just like
-any other remote.
+TODO
 
 ## unused checking
 

From def137b0cc0c86d9cd976c11b59f7ba0669c0735 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:41:57 -0400
Subject: [PATCH 1308/2835] use HMAC

---
 doc/special_remotes/Amazon_S3.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index c8e44b609c..ad73e0a10d 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -37,10 +37,10 @@ only be used for public data.
 ** Encryption is not yet supported. **
 
 When encryption is enabled, all files stored in the bucket are
-encrypted with gpg. Additionally, the filenames themselves are hashed
-to obfuscate them. The size of the encrypted files, and access patterns of
-the data, should be the only clues to what type of data you are storing in
-S3.
+encrypted with gpg. Additionally, the filenames themselves are encrypted
+(using HMAC). The size of the encrypted files, and
+access patterns of the data, should be the only clues to what type of
+data you are storing in S3.
 
 [[!template id=note text="""
 This scheme was originally developed by Lars Wirzenius at al

From 9c96d86502c521cf78228f816e33ac456fb2ee59 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 01:32:05 -0400
Subject: [PATCH 1309/2835] nasty hack to build when hS3 is not available

So, it would be nicer to just use Cabal and take advantage
of its conditional compilation support. But, Cabal seems to
lack good support for a package with an internal library that is used by
multiple executables. It wants to build everything twice or more.
That's too slow for me.

Anyway, fairly soon, I expect to upgrade hS3 to a requirment, and I
can just revert this.
---
 Makefile                    | 13 ++++++++++---
 Remote/{S3.hs => S3real.hs} |  0
 Remote/S3stub.hs            | 13 +++++++++++++
 debian/changelog            |  2 ++
 doc/install.mdwn            |  8 ++++----
 5 files changed, 29 insertions(+), 7 deletions(-)
 rename Remote/{S3.hs => S3real.hs} (100%)
 create mode 100644 Remote/S3stub.hs

diff --git a/Makefile b/Makefile
index 8e16645034..6a1531a3c8 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,14 @@ SysConfig.hs: configure.hs TestConfig.hs
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 
-$(bins): SysConfig.hs Touch.hs StatFS.hs
+Remote/S3.o:
+	@ln -sf S3real.hs Remote/S3.hs
+	@if ! $(GHCMAKE) Remote/S3.hs; then \
+		ln -sf S3stub.hs Remote/S3.hs; \
+		echo "** building without S3 support"; \
+	fi
+
+$(bins): SysConfig.hs Touch.hs StatFS.hs Remote/S3.o
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
@@ -62,8 +69,8 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure \
-		StatFS.hs Touch.hs SysConfig.hs *.tix .hpc
+	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \
+		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs
 	rm -rf doc/.ikiwiki html
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
diff --git a/Remote/S3.hs b/Remote/S3real.hs
similarity index 100%
rename from Remote/S3.hs
rename to Remote/S3real.hs
diff --git a/Remote/S3stub.hs b/Remote/S3stub.hs
new file mode 100644
index 0000000000..0d6ec47de3
--- /dev/null
+++ b/Remote/S3stub.hs
@@ -0,0 +1,13 @@
+-- stub for when hS3 is not available
+module Remote.S3 (remote) where
+
+import RemoteClass
+import Types
+
+remote :: RemoteType Annex
+remote = RemoteType {
+	typename = "S3",
+	enumerate = return [],
+	generate = error "S3 not enabled",
+	setup = error "S3 not enabled"
+}
diff --git a/debian/changelog b/debian/changelog
index faf2833a84..b03bc1d1b5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (0.20110329) UNRELEASED; urgency=low
 
   * Amazon S3 is now supported as a special type of remote.
     Warning: Encrypting data before sending it to S3 is not yet supported.
+  * Note that Amazon S3 support is not built in by default on Debian yet,
+    as hS3 is not packaged.
   * fsck: Ensure that files and directories in .git/annex/objects
     have proper permissions.
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 7b2c536c9d..70ab8e30b6 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -13,17 +13,17 @@ To build and use git-annex, you will need:
 * MissingH: 
 * pcre-light: 
 * utf8-string: 
-* hS3: 
+* hS3  (optional, but recommended)
 * `uuid`: 
   (or uuidgen from util-linux)
 * `xargs`: 
 * `rsync`: 
 * `curl` :  (optional, but recommended)
 * `sha1sum`:  (optional, but recommended)
-* Then just [[download]] git-annex and run: `make; make install`
+* [Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
+  but that will be skipped if it is not installed.
 
-([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
-but that will be skipped if it is not installed.)
+Then just [[download]] git-annex and run: `make; make install`
 
 Additionally, to run the test suite (via `make test`), you will need:
 

From 5eb5b8721521ee7cde756716563fe89374d2c023 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 01:37:02 -0400
Subject: [PATCH 1310/2835] typo

---
 doc/special_remotes/Amazon_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index ad73e0a10d..b625d9b3aa 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -43,7 +43,7 @@ access patterns of the data, should be the only clues to what type of
 data you are storing in S3.
 
 [[!template id=note text="""
-This scheme was originally developed by Lars Wirzenius at al
+This scheme was originally developed by Lars Wirzenius et al
 [for Obnam](http://braawi.org/obnam/encryption/).
 """]]
 The data stored in S3 is encrypted by gpg with a symmetric cipher. The

From 320a4102d6dfff193fc501e53859b2b3edc397d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 01:45:39 -0400
Subject: [PATCH 1311/2835] update

---
 doc/special_remotes/Amazon_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index b625d9b3aa..e4e57b3689 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -3,7 +3,7 @@ or a similar service.
 
 See [[walkthrough/using_Amazon_S3]] for usage examples.
 
-## initremote parameters
+## configuration
 
 A number of parameters can be passed to `git annex initremote` to configure
 the S3 remote.

From ee84c75de066826c9fbd8351b456bd7c9980bab6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 10:37:27 +0000
Subject: [PATCH 1312/2835]

---
 ..._bucket_uses_the_same_key_for_encryption_and_hashing.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn
new file mode 100644
index 0000000000..0ec66652e2
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn
@@ -0,0 +1,5 @@
+While using HMAC instead of "plain" hash functions is inherently more secure, it's still a bad idea to re-use keys for different purposes.
+
+Also, ttbomk, HMAC needs two keys, not one. Are you re-using the same key twice?
+
+Compability for old buckets and support for different ones can be maintained by introducing a new option and simply copying over the encryption key's identifier into this new option should it be missing.

From 8e32c3a596a8860ad71e51aafddfe8e401e8d364 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 30 Mar 2011 14:32:34 +0000
Subject: [PATCH 1313/2835] Added a comment

---
 ...1_dc5ae7af499203cfd903e866595b8fea._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment
new file mode 100644
index 0000000000..320fb5ef08
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-30T14:32:34Z"
+ content="""
+S3 doesn't support encryption at all, yet.
+
+It certainly makes sense to use a different portion of the encrypted secret key for HMAC than is uses as the gpg symmetric encryption key.
+
+The two keys used in HMAC would be the secret key and the key/value key for the content being stored.
+
+There is a difficult problem with encrypting filenames in S3 buckets, and that is determining when some data in the bucket is unused for dropunused. I've considered two choices:
+
+1. gpg encrypt the filenames. This would allow dropunused to recover the original filenames, and is probably more robust encryption. But it would double the number of times gpg is run when moving content in/out, and to check for unused content, gpg would have to be run once for every item in the bucket, which just feels way excessive, even though it would not be prompting for a passphrase. Still, haven't ruled this out.
+
+2. HMAC or other hash. To determine what data was unused the same hash and secret key would have to be used to hash all filenames currently used, and then that set of hashes could be interested with the set in the bucket. But then git-annex could only say \"here are some opaque hashes of content that appears unused by anything in your current git repository, but there's no way, short of downloading it and examining it to tell what it is\". (This could be improved by keeping a local mapping between filenames and S3 keys, but maintaining and committing that would bring pain of its own.)
+"""]]

From 564d013628c7ef5eab23880be6e6dabff5bc85d4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 17:01:41 +0000
Subject: [PATCH 1314/2835] Added a comment

---
 ...mment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment
new file mode 100644
index 0000000000..dec06c89ff
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-03-30T17:01:40Z"
+ content="""
+After mulling this over, I think actually encrypting the filenames is preferable.
+
+Did you consider encrypting the symmetric key with an asymmetric one? That's what TrueCrypt etc are using to allow different people access to a shared volume. This has the added benefit that you could, potentially, add new keys for data that new people should have access to while making access to old data impossible. Or keys per subdirectory, or, or, or.
+
+As an aside, could the same mechanism be extended to transparently encrypt data for a remote annex repo? A friend of mine is interested to host his data with me, but he wants to encrypt his data for obvious reasons.
+"""]]

From a47ed922e1302480d79f54f553532e85eebae872 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 13:18:46 -0400
Subject: [PATCH 1315/2835] add Remote.Directory

---
 .gitignore                         |   1 +
 Remote.hs                          |   2 +
 Remote/Directory.hs                | 110 +++++++++++++++++++++++++++++
 debian/changelog                   |   2 +
 doc/special_remotes.mdwn           |   1 +
 doc/special_remotes/directory.mdwn |  10 +++
 6 files changed, 126 insertions(+)
 create mode 100644 Remote/Directory.hs
 create mode 100644 doc/special_remotes/directory.mdwn

diff --git a/.gitignore b/.gitignore
index aa677c1335..b73167c925 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ html
 .hpc
 Touch.hs
 StatFS.hs
+Remote/S3.hs
diff --git a/Remote.hs b/Remote.hs
index 914c69abe5..0cfec3c282 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -46,11 +46,13 @@ import Config
 
 import qualified Remote.Git
 import qualified Remote.S3
+import qualified Remote.Directory
 
 remoteTypes :: [RemoteType Annex]
 remoteTypes =
 	[ Remote.Git.remote
 	, Remote.S3.remote
+	, Remote.Directory.remote
 	]
 
 {- Builds a list of all available Remotes.
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
new file mode 100644
index 0000000000..697de5ea7c
--- /dev/null
+++ b/Remote/Directory.hs
@@ -0,0 +1,110 @@
+{- A "remote" that is just a local directory.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Directory (remote) where
+
+import IO
+import Control.Exception.Extensible (IOException)
+import qualified Data.Map as M
+import Data.Maybe
+import Data.String.Utils
+import Control.Monad (when)
+import Control.Monad.State (liftIO)
+import System.Directory (doesDirectoryExist, doesFileExist, removeFile)
+import System.FilePath
+
+import RemoteClass
+import Types
+import qualified GitRepo as Git
+import qualified Annex
+import UUID
+import Config
+import Utility
+import Locations
+import CopyFile
+
+remote :: RemoteType Annex
+remote = RemoteType {
+	typename = "directory",
+	enumerate = list,
+	generate = gen,
+	setup = dosetup
+}
+
+list :: Annex [Git.Repo]
+list = do
+	g <- Annex.gitRepo
+	return $ findDirectoryRemotes g
+
+findDirectoryRemotes :: Git.Repo -> [Git.Repo]
+findDirectoryRemotes r = map construct remotepairs
+	where
+		remotepairs = M.toList $ filterremotes $ Git.configMap r
+		filterremotes = M.filterWithKey (\k _ -> directoryremote k)
+		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
+		directoryremote k = startswith "remote." k && endswith ".annex-directory" k
+
+gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r c = do
+	u <- getUUID r
+	cst <- remoteCost r
+	return $ genRemote r u c cst
+	where
+
+genRemote :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Int -> Remote Annex
+genRemote r u c cst = this
+	where
+		this = Remote {
+			uuid = u,
+			cost = cst,
+			name = Git.repoDescribe r,
+	 		storeKey = store this,
+			retrieveKeyFile = retrieve this,
+			removeKey = remove this,
+			hasKey = checkPresent this,
+			hasKeyCheap = True,
+			config = c
+		}
+
+dosetup :: UUID -> M.Map String String -> Annex (M.Map String String)
+dosetup u c = do
+	-- verify configuration is sane
+	let dir = case M.lookup "directory" c of
+		Nothing -> error "Specify directory="
+		Just d -> d
+	e <- liftIO $ doesDirectoryExist dir
+	when (not e) $ error $ "Directory does not exist: " ++ dir
+
+	g <- Annex.gitRepo
+	liftIO $ do
+		Git.run g "config" [Param (configsetting "annex-directory"), Param "true"]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
+	return c
+	where
+		remotename = fromJust (M.lookup "name" c)
+		configsetting s = "remote." ++ remotename ++ "." ++ s
+
+dirKey :: Remote Annex -> Key -> FilePath
+dirKey r k = dir  show k
+	where
+		dir = fromJust $ M.lookup "directory" $ fromJust $ config r
+
+store :: Remote Annex -> Key -> Annex Bool
+store r k = do
+	g <- Annex.gitRepo
+	liftIO $ copyFile (gitAnnexLocation g k) (dirKey r k)
+
+retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
+retrieve r k f = liftIO $ copyFile (dirKey r k) f
+
+remove :: Remote Annex -> Key -> Annex Bool
+remove r k = liftIO $ catch
+	(removeFile (dirKey r k) >> return True)
+	(const $ return False)
+
+checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
+checkPresent r k = liftIO $ try $ doesFileExist (dirKey r k)
diff --git a/debian/changelog b/debian/changelog
index b03bc1d1b5..0a232220f0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (0.20110329) UNRELEASED; urgency=low
     as hS3 is not packaged.
   * fsck: Ensure that files and directories in .git/annex/objects
     have proper permissions.
+  * Added a special type of remote called a directory remote, which
+    simply stores files in an arbitrary local directory.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 651b24afa4..09b751d0f4 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -7,3 +7,4 @@ types of remotes. These can be used just like any normal remote by git-annex.
 They cannot be used by other git commands though.
 
 * [[Amazon_S3]]
+* [[directory]]
diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn
new file mode 100644
index 0000000000..42dbc5749e
--- /dev/null
+++ b/doc/special_remotes/directory.mdwn
@@ -0,0 +1,10 @@
+This special remote type stores file contents in directory on the system.
+
+One use case for this would be if you have a removable drive, that you
+cannot put a git repository on for some reason, and you want to use it
+to sneakernet files between systems. Just set up both systems to use
+the drive's mountpoint as a directory remote.
+
+Setup example:
+
+	# git annex initremote usbdrive directory=/media/usbdrive/

From 619f07ee6a0ad875f365886096b112b6c18b0606 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 14:00:54 -0400
Subject: [PATCH 1316/2835] boilerplate reduction

---
 Remote.hs           |  3 ++-
 Remote/Directory.hs | 36 ++++++-------------------
 Remote/Git.hs       | 36 +++++++++++--------------
 Remote/S3real.hs    | 66 +++++++++++++--------------------------------
 Remote/Special.hs   | 43 +++++++++++++++++++++++++++++
 RemoteClass.hs      |  6 +++--
 6 files changed, 90 insertions(+), 100 deletions(-)
 create mode 100644 Remote/Special.hs

diff --git a/Remote.hs b/Remote.hs
index 0cfec3c282..4e401ddcc0 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -75,7 +75,8 @@ genList = do
 			mapM (gen m t) l'
 		gen m t r = do
 			u <- getUUID r
-			generate t r (M.lookup u m)
+			cst <- remoteCost r
+			generate t r u cst (M.lookup u m)
 
 {- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex (Remote Annex)
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 697de5ea7c..12736e0507 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -11,7 +11,6 @@ import IO
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Data.Maybe
-import Data.String.Utils
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory (doesDirectoryExist, doesFileExist, removeFile)
@@ -22,41 +21,21 @@ import Types
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
-import Config
 import Utility
 import Locations
 import CopyFile
+import Remote.Special
 
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "directory",
-	enumerate = list,
+	enumerate = findSpecialRemotes "directory",
 	generate = gen,
-	setup = dosetup
+	setup = directorySetup
 }
 
-list :: Annex [Git.Repo]
-list = do
-	g <- Annex.gitRepo
-	return $ findDirectoryRemotes g
-
-findDirectoryRemotes :: Git.Repo -> [Git.Repo]
-findDirectoryRemotes r = map construct remotepairs
-	where
-		remotepairs = M.toList $ filterremotes $ Git.configMap r
-		filterremotes = M.filterWithKey (\k _ -> directoryremote k)
-		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
-		directoryremote k = startswith "remote." k && endswith ".annex-directory" k
-
-gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r c = do
-	u <- getUUID r
-	cst <- remoteCost r
-	return $ genRemote r u c cst
-	where
-
-genRemote :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Int -> Remote Annex
-genRemote r u c cst = this
+gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u cst c = return this
 	where
 		this = Remote {
 			uuid = u,
@@ -70,8 +49,8 @@ genRemote r u c cst = this
 			config = c
 		}
 
-dosetup :: UUID -> M.Map String String -> Annex (M.Map String String)
-dosetup u c = do
+directorySetup :: UUID -> M.Map String String -> Annex (M.Map String String)
+directorySetup u c = do
 	-- verify configuration is sane
 	let dir = case M.lookup "directory" c of
 		Nothing -> error "Specify directory="
@@ -79,6 +58,7 @@ dosetup u c = do
 	e <- liftIO $ doesDirectoryExist dir
 	when (not e) $ error $ "Directory does not exist: " ++ dir
 
+	gitConfigSpecialRemote "directory" u c
 	g <- Annex.gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting "annex-directory"), Param "true"]
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d0dedd4fde..286a8c645c 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -42,33 +42,27 @@ list = do
 	g <- Annex.gitRepo
 	return $ Git.remotes g
 
-gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen repo _ = do
+gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u cst _ = do
  	{- It's assumed to be cheap to read the config of non-URL remotes,
 	 - so this is done each time git-annex is run. Conversely,
 	 - the config of an URL remote is only read when there is no
 	 - cached UUID value. -}
-	let cheap = not $ Git.repoIsUrl repo
-	u <- getUUID repo
-	repo' <- case (cheap, u) of
-		(True, _) -> tryGitConfigRead repo
-		(False, "") -> tryGitConfigRead repo
-		_ -> return repo
-	genRemote repo'
-	
-genRemote :: Git.Repo -> Annex (Remote Annex)
-genRemote r = do
-	u <- getUUID r
-	c <- remoteCost r
+	let cheap = not $ Git.repoIsUrl r
+	r' <- case (cheap, u) of
+		(True, _) -> tryGitConfigRead r
+		(False, "") -> tryGitConfigRead r
+		_ -> return r
+
 	return $ Remote {
 		uuid = u,
-		cost = c,
-		name = Git.repoDescribe r,
-		storeKey = copyToRemote r,
-		retrieveKeyFile = copyFromRemote r,
-		removeKey = dropKey r,
-		hasKey = inAnnex r,
-		hasKeyCheap = not (Git.repoIsUrl r),
+		cost = cst,
+		name = Git.repoDescribe r',
+		storeKey = copyToRemote r',
+		retrieveKeyFile = copyFromRemote r',
+		removeKey = dropKey r',
+		hasKey = inAnnex r',
+		hasKeyCheap = not (Git.repoIsUrl r'),
 		config = Nothing
 	}
 
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 260c1eee8e..4380231fdc 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -15,7 +15,6 @@ import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.Maybe
-import Data.String.Utils
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Environment
@@ -25,54 +24,29 @@ import Types
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
-import Config
-import Utility
 import Messages
 import Locations
+import Remote.Special
 
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "S3",
-	enumerate = s3List,
-	generate = s3Gen,
+	enumerate = findSpecialRemotes "s3",
+	generate = gen,
 	setup = s3Setup
 }
 
-s3List :: Annex [Git.Repo]
-s3List = do
-	g <- Annex.gitRepo
-	return $ findS3Remotes g
-
-{- S3 remotes have a remote..annex-s3 config setting.
- - Git.Repo does not normally generate remotes for things that
- - have no configured url, so the Git.Repo objects have to be
- - constructed as coming from an unknown location. -}
-findS3Remotes :: Git.Repo -> [Git.Repo]
-findS3Remotes r = map construct remotepairs
-	where
-		remotepairs = M.toList $ filterremotes $ Git.configMap r
-		filterremotes = M.filterWithKey (\k _ -> s3remote k)
-		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
-		s3remote k = startswith "remote." k && endswith ".annex-s3" k
-
-s3Gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
-s3Gen r c = do
-	u <- getUUID r
-	cst <- remoteCost r
-	return $ genRemote r u c cst
-	where
-
-genRemote :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Int -> Remote Annex
-genRemote r u c cst = this
+gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u cst c = return this
 	where
 		this = Remote {
 			uuid = u,
 			cost = cst,
 			name = Git.repoDescribe r,
-	 		storeKey = s3Store this,
-			retrieveKeyFile = s3Retrieve this,
-			removeKey = s3Remove this,
-			hasKey = s3CheckPresent this,
+	 		storeKey = store this,
+			retrieveKeyFile = retrieve this,
+			removeKey = remove this,
+			hasKey = checkPresent this,
 			hasKeyCheap = False,
 			config = c
 		}
@@ -114,15 +88,11 @@ s3Setup u c = do
 				Right _ -> return ()
 				Left err -> error $ prettyReqError err
 
-	g <- Annex.gitRepo
-	liftIO $ do
-		Git.run g "config" [Param (configsetting "annex-s3"), Param "true"]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
+	gitConfigSpecialRemote "s3" u fullconfig
 	return fullconfig
 	where
 		remotename = fromJust (M.lookup "name" c)
 		bucket = remotename ++ "-" ++ u
-		configsetting s = "remote." ++ remotename ++ "." ++ s
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
@@ -142,8 +112,8 @@ s3Action r a = do
 bucketKey :: String -> Key -> L.ByteString -> S3Object
 bucketKey bucket k content = S3Object bucket (show k) "" [] content
 
-s3CheckPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
-s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
+checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
+checkPresent r k = s3Action r $ \(conn, bucket) -> do
 	showNote ("checking " ++ name r ++ "...")
 	res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty
 	case res of
@@ -151,8 +121,8 @@ s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
 		Left (AWSError _ _) -> return $ Right False
 		Left e -> return $ Left (error $ prettyReqError e)
 
-s3Store :: Remote Annex -> Key -> Annex Bool
-s3Store r k = s3Action r $ \(conn, bucket) -> do
+store :: Remote Annex -> Key -> Annex Bool
+store r k = s3Action r $ \(conn, bucket) -> do
 	g <- Annex.gitRepo
 	content <- liftIO $ L.readFile $ gitAnnexLocation g k
 	let object = setStorageClass storageclass $ bucketKey bucket k content
@@ -168,8 +138,8 @@ s3Store r k = s3Action r $ \(conn, bucket) -> do
 				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
 				_ -> STANDARD
 
-s3Retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
-s3Retrieve r k f = s3Action r $ \(conn, bucket) -> do
+retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
+retrieve r k f = s3Action r $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey bucket k L.empty
 	case res of
 		Right o -> do
@@ -179,8 +149,8 @@ s3Retrieve r k f = s3Action r $ \(conn, bucket) -> do
 			warning $ prettyReqError e
 			return False
 
-s3Remove :: Remote Annex -> Key -> Annex Bool
-s3Remove r k = s3Action r $ \(conn, bucket) -> do
+remove :: Remote Annex -> Key -> Annex Bool
+remove r k = s3Action r $ \(conn, bucket) -> do
 	res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty
 	case res of
 		Right _ -> return True
diff --git a/Remote/Special.hs b/Remote/Special.hs
new file mode 100644
index 0000000000..d985eef6fe
--- /dev/null
+++ b/Remote/Special.hs
@@ -0,0 +1,43 @@
+{- common functions for special remotes
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Special where
+
+import qualified Data.Map as M
+import Data.Maybe
+import Data.String.Utils
+import Control.Monad.State (liftIO)
+
+import Types
+import qualified GitRepo as Git
+import qualified Annex
+import UUID
+import Utility
+
+{- Special remotes don't have a configured url, so Git.Repo does not
+ - automatically generate remotes for them. This looks for a different
+ - configuration key instead.
+ -}
+findSpecialRemotes :: String -> Annex [Git.Repo]
+findSpecialRemotes s = do
+	g <- Annex.gitRepo
+	return $ map construct $ remotepairs g
+	where
+		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
+		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
+		match k _ = startswith "remote." k && endswith (".annex-"++s) k
+
+{- Sets up configuration for a special remote in .git/config. -}
+gitConfigSpecialRemote :: String -> UUID -> M.Map String String -> Annex ()
+gitConfigSpecialRemote s u c = do
+	g <- Annex.gitRepo
+	liftIO $ do
+		Git.run g "config" [Param (configsetting $ "annex-"++s), Param "true"]
+		Git.run g "config" [Param (configsetting $ "annex-uuid"), Param u]
+	where
+		remotename = fromJust (M.lookup "name" c)
+		configsetting v = "remote." ++ remotename ++ "." ++ v
diff --git a/RemoteClass.hs b/RemoteClass.hs
index de4c281f44..43bf403de4 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -15,6 +15,8 @@ import Data.Map as M
 import qualified GitRepo as Git
 import Key
 
+type Cost = Int
+
 {- There are different types of remotes. -}
 data RemoteType a = RemoteType {
 	-- human visible type name
@@ -22,7 +24,7 @@ data RemoteType a = RemoteType {
 	-- enumerates remotes of this type
 	enumerate :: a [Git.Repo],
 	-- generates a remote of this type
-	generate :: Git.Repo -> Maybe (M.Map String String) -> a (Remote a),
+	generate :: Git.Repo -> String -> Cost -> Maybe (M.Map String String) -> a (Remote a),
 	-- initializes or changes a remote
 	setup :: String -> M.Map String String -> a (M.Map String String)
 }
@@ -34,7 +36,7 @@ data Remote a = Remote {
 	-- each Remote has a human visible name
 	name :: String,
 	-- Remotes have a use cost; higher is more expensive
-	cost :: Int,
+	cost :: Cost,
 	-- Transfers a key to the remote.
 	storeKey :: Key -> a Bool,
 	-- retrieves a key's contents to a file

From 30801372e2c0fbce198154370990a7c463bc5c4a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 30 Mar 2011 18:15:18 +0000
Subject: [PATCH 1317/2835] Added a comment

---
 ...comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment
new file mode 100644
index 0000000000..c5bb26f595
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-30T18:15:18Z"
+ content="""
+Yes, encrypting the symmetric key with users' regular gpg keys is the plan.
+
+I don't think that encryption of content in a git annex remote makes much sense; the filenames obviously cannot be encrypted there. It's more likely that the same encryption would get used for a bup remote, or with the [[special_remotes/directory]] remote I threw in today.
+"""]]

From b7a48de30451e18d13686e3f864a712f0d83b720 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 18:20:57 +0000
Subject: [PATCH 1318/2835] Added a comment

---
 .../comment_4_bb6b814ab961818d514f6553455d2bf3._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment
new file mode 100644
index 0000000000..09b7a8b1ab
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2011-03-30T18:20:56Z"
+ content="""
+Picking up the automagic encryption idea for annex remotes, this would allow you to host a branchable-esque git-annex hosting service. (Nexenta with ZFS is a cheap and reliable option until btrfs becomes stable in a year or five).
+"""]]

From 8b6ef15835087c2b266df624bb24f5e30154dddb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 14:32:08 -0400
Subject: [PATCH 1319/2835] allow directory remotes to be in different
 locations

Two machines might have access to the same directory remote on different
paths, so don't include the path in its persistent config, instead use
the git config to record it.
---
 Remote/Directory.hs | 72 ++++++++++++++++++++-------------------------
 Remote/S3real.hs    |  2 +-
 Remote/Special.hs   |  8 ++---
 3 files changed, 37 insertions(+), 45 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 12736e0507..919dcc295f 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -10,7 +10,6 @@ module Remote.Directory (remote) where
 import IO
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Data.Maybe
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory (doesDirectoryExist, doesFileExist, removeFile)
@@ -21,9 +20,9 @@ import Types
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
-import Utility
 import Locations
 import CopyFile
+import Config
 import Remote.Special
 
 remote :: RemoteType Annex
@@ -35,19 +34,19 @@ remote = RemoteType {
 }
 
 gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r u cst c = return this
-	where
-		this = Remote {
-			uuid = u,
-			cost = cst,
-			name = Git.repoDescribe r,
-	 		storeKey = store this,
-			retrieveKeyFile = retrieve this,
-			removeKey = remove this,
-			hasKey = checkPresent this,
-			hasKeyCheap = True,
-			config = c
-		}
+gen r u cst _ = do
+	dir <- getConfig r "directory" (error "missing directory")
+	return $ Remote {
+		uuid = u,
+		cost = cst,
+		name = Git.repoDescribe r,
+ 		storeKey = store dir,
+		retrieveKeyFile = retrieve dir,
+		removeKey = remove dir,
+		hasKey = checkPresent dir,
+		hasKeyCheap = True,
+		config = Nothing
+	}
 
 directorySetup :: UUID -> M.Map String String -> Annex (M.Map String String)
 directorySetup u c = do
@@ -58,33 +57,26 @@ directorySetup u c = do
 	e <- liftIO $ doesDirectoryExist dir
 	when (not e) $ error $ "Directory does not exist: " ++ dir
 
-	gitConfigSpecialRemote "directory" u c
+	-- The directory is stored in git config, not in this remote's
+	-- persistant state, so it can vary between hosts.
+	gitConfigSpecialRemote u c "directory" dir
+	return $ M.delete "directory" c
+
+dirKey :: FilePath -> Key -> FilePath
+dirKey d k = d  show k
+
+store :: FilePath -> Key -> Annex Bool
+store d k = do
 	g <- Annex.gitRepo
-	liftIO $ do
-		Git.run g "config" [Param (configsetting "annex-directory"), Param "true"]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
-	return c
-	where
-		remotename = fromJust (M.lookup "name" c)
-		configsetting s = "remote." ++ remotename ++ "." ++ s
+	liftIO $ copyFile (gitAnnexLocation g k) (dirKey d k)
 
-dirKey :: Remote Annex -> Key -> FilePath
-dirKey r k = dir  show k
-	where
-		dir = fromJust $ M.lookup "directory" $ fromJust $ config r
+retrieve :: FilePath -> Key -> FilePath -> Annex Bool
+retrieve d k f = liftIO $ copyFile (dirKey d k) f
 
-store :: Remote Annex -> Key -> Annex Bool
-store r k = do
-	g <- Annex.gitRepo
-	liftIO $ copyFile (gitAnnexLocation g k) (dirKey r k)
-
-retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
-retrieve r k f = liftIO $ copyFile (dirKey r k) f
-
-remove :: Remote Annex -> Key -> Annex Bool
-remove r k = liftIO $ catch
-	(removeFile (dirKey r k) >> return True)
+remove :: FilePath -> Key -> Annex Bool
+remove d k = liftIO $ catch
+	(removeFile (dirKey d k) >> return True)
 	(const $ return False)
 
-checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
-checkPresent r k = liftIO $ try $ doesFileExist (dirKey r k)
+checkPresent :: FilePath -> Key -> Annex (Either IOException Bool)
+checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k)
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 4380231fdc..0827c4fbff 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -88,7 +88,7 @@ s3Setup u c = do
 				Right _ -> return ()
 				Left err -> error $ prettyReqError err
 
-	gitConfigSpecialRemote "s3" u fullconfig
+	gitConfigSpecialRemote u fullconfig "s3" "true"
 	return fullconfig
 	where
 		remotename = fromJust (M.lookup "name" c)
diff --git a/Remote/Special.hs b/Remote/Special.hs
index d985eef6fe..b5d5a137fe 100644
--- a/Remote/Special.hs
+++ b/Remote/Special.hs
@@ -32,12 +32,12 @@ findSpecialRemotes s = do
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
-gitConfigSpecialRemote :: String -> UUID -> M.Map String String -> Annex ()
-gitConfigSpecialRemote s u c = do
+gitConfigSpecialRemote :: UUID -> M.Map String String -> String -> String -> Annex ()
+gitConfigSpecialRemote u c k v = do
 	g <- Annex.gitRepo
 	liftIO $ do
-		Git.run g "config" [Param (configsetting $ "annex-"++s), Param "true"]
+		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
 		Git.run g "config" [Param (configsetting $ "annex-uuid"), Param u]
 	where
 		remotename = fromJust (M.lookup "name" c)
-		configsetting v = "remote." ++ remotename ++ "." ++ v
+		configsetting s = "remote." ++ remotename ++ "." ++ s

From fdd455e913964200177530df085f2a7ad7c7f8b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 14:56:31 -0400
Subject: [PATCH 1320/2835] use same directory structure as .git/annex/objects
 for directory remotes

And same file perms.
---
 Locations.hs        |  1 +
 Remote/Directory.hs | 36 +++++++++++++++++++++++++++++-------
 2 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 3cce4c2611..8e10c36b41 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,6 +20,7 @@ module Locations (
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
 	logFile,
+	hashDir,
 
 	prop_idempotent_fileKey
 ) where
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 919dcc295f..cc37e496ea 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -1,4 +1,4 @@
-{- A "remote" that is just a local directory.
+{- A "remote" that is just a filesystem directory.
  -
  - Copyright 2011 Joey Hess 
  -
@@ -12,7 +12,7 @@ import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
-import System.Directory (doesDirectoryExist, doesFileExist, removeFile)
+import System.Directory hiding (copyFile)
 import System.FilePath
 
 import RemoteClass
@@ -23,6 +23,8 @@ import UUID
 import Locations
 import CopyFile
 import Config
+import Content
+import Utility
 import Remote.Special
 
 remote :: RemoteType Annex
@@ -63,20 +65,40 @@ directorySetup u c = do
 	return $ M.delete "directory" c
 
 dirKey :: FilePath -> Key -> FilePath
-dirKey d k = d  show k
+dirKey d k = d  hashDir k  f  f
+	where
+		f = keyFile k
 
 store :: FilePath -> Key -> Annex Bool
 store d k = do
 	g <- Annex.gitRepo
-	liftIO $ copyFile (gitAnnexLocation g k) (dirKey d k)
+	let src = gitAnnexLocation g k
+	liftIO $ catch (copy src) (const $ return False)
+	where
+		dest = dirKey d k
+		dir = parentDir dest
+		copy src = do
+			createDirectoryIfMissing True dir
+			allowWrite dir
+			ok <- copyFile src dest
+			when ok $ do
+				preventWrite dest
+				preventWrite dir
+			return ok
 
 retrieve :: FilePath -> Key -> FilePath -> Annex Bool
 retrieve d k f = liftIO $ copyFile (dirKey d k) f
 
 remove :: FilePath -> Key -> Annex Bool
-remove d k = liftIO $ catch
-	(removeFile (dirKey d k) >> return True)
-	(const $ return False)
+remove d k = liftIO $ catch del (const $ return False)
+	where
+		file = dirKey d k
+		dir = parentDir file
+		del = do
+			allowWrite dir
+			removeFile file
+			removeDirectory dir
+			return True
 
 checkPresent :: FilePath -> Key -> Annex (Either IOException Bool)
 checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k)

From 8f47b38dbc97d2062c21c8b6f883187ffc9f2ef0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 18:59:19 +0000
Subject: [PATCH 1321/2835] Added a comment

---
 .../comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment
new file mode 100644
index 0000000000..49d43ffc63
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 5"
+ date="2011-03-30T18:59:19Z"
+ content="""
+This is brain-storming only so the idea might be crap, but a branch could keep encrypted filenames while master keeps the real deal. This might fit into the whole scheme just nicely or break future stuff in a dozen places, I am not really sure yet. But at least I can't forget the idea, now.
+"""]]

From 07070e6dd938a6d389fa7334e415472b797c61b2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 19:02:20 +0000
Subject: [PATCH 1322/2835] Added a comment

---
 .../comment_6_63fb74da342751fc35e1850409c506f6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment
new file mode 100644
index 0000000000..d994ca77f3
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 6"
+ date="2011-03-30T19:02:20Z"
+ content="""
+OTOH, if encryption makes a bup backend more likely disregard the idea above ;)
+"""]]

From 0c73c08c1c0929f0ba53dcfb6d5d32a73a5f28d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 15:15:46 -0400
Subject: [PATCH 1323/2835] cost bugfixes

---
 Config.hs           | 13 ++++++++-----
 Remote.hs           |  3 +--
 Remote/Directory.hs |  5 +++--
 Remote/Git.hs       |  9 +++++++--
 Remote/S3real.hs    | 26 +++++++++++++++-----------
 RemoteClass.hs      |  8 +++-----
 6 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/Config.hs b/Config.hs
index c821364ced..17a1fa9856 100644
--- a/Config.hs
+++ b/Config.hs
@@ -42,14 +42,17 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex
  - The default cost is 100 for local repositories, and 200 for remote
  - repositories; it can also be configured by remote..annex-cost
  -}
-remoteCost :: Git.Repo -> Annex Int
-remoteCost r = do
+remoteCost :: Git.Repo -> Int -> Annex Int
+remoteCost r def = do
 	c <- getConfig r "cost" ""
 	if not $ null c
 		then return $ read c
-		else if not $ Git.repoIsUrl r
-			then return 100
-			else return 200
+		else return def
+
+cheapRemoteCost :: Int
+cheapRemoteCost = 100
+expensiveRemoteCost :: Int
+expensiveRemoteCost = 200
 
 {- Checks if a repo should be ignored, based either on annex-ignore
  - setting, or on command-line options. Allows command-line to override
diff --git a/Remote.hs b/Remote.hs
index 4e401ddcc0..26097da747 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -75,8 +75,7 @@ genList = do
 			mapM (gen m t) l'
 		gen m t r = do
 			u <- getUUID r
-			cst <- remoteCost r
-			generate t r u cst (M.lookup u m)
+			generate t r u (M.lookup u m)
 
 {- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex (Remote Annex)
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index cc37e496ea..f97449eaa7 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -35,9 +35,10 @@ remote = RemoteType {
 	setup = directorySetup
 }
 
-gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r u cst _ = do
+gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u _ = do
 	dir <- getConfig r "directory" (error "missing directory")
+	cst <- remoteCost r cheapRemoteCost
 	return $ Remote {
 		uuid = u,
 		cost = cst,
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 286a8c645c..c1423bef7a 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -42,8 +42,8 @@ list = do
 	g <- Annex.gitRepo
 	return $ Git.remotes g
 
-gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r u cst _ = do
+gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u _ = do
  	{- It's assumed to be cheap to read the config of non-URL remotes,
 	 - so this is done each time git-annex is run. Conversely,
 	 - the config of an URL remote is only read when there is no
@@ -54,6 +54,11 @@ gen r u cst _ = do
 		(False, "") -> tryGitConfigRead r
 		_ -> return r
 
+	let defcst = if not $ Git.repoIsUrl r
+		then cheapRemoteCost
+		else expensiveRemoteCost
+	cst <- remoteCost r' defcst
+
 	return $ Remote {
 		uuid = u,
 		cost = cst,
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 0827c4fbff..d7a6d507b1 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -26,6 +26,7 @@ import qualified Annex
 import UUID
 import Messages
 import Locations
+import Config
 import Remote.Special
 
 remote :: RemoteType Annex
@@ -36,25 +37,28 @@ remote = RemoteType {
 	setup = s3Setup
 }
 
-gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r u cst c = return this
+gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u c = do
+	cst <- remoteCost r expensiveRemoteCost
+	return $ this cst
 	where
-		this = Remote {
+		this cst = Remote {
 			uuid = u,
 			cost = cst,
 			name = Git.repoDescribe r,
-	 		storeKey = store this,
-			retrieveKeyFile = retrieve this,
-			removeKey = remove this,
-			hasKey = checkPresent this,
+	 		storeKey = store (this cst),
+			retrieveKeyFile = retrieve (this cst),
+			removeKey = remove (this cst),
+			hasKey = checkPresent (this cst),
 			hasKeyCheap = False,
 			config = c
 		}
 
-s3Connection :: M.Map String String -> IO AWSConnection
+s3Connection :: M.Map String String -> Annex AWSConnection
 s3Connection c = do
 	ak <- getEnvKey "AWS_ACCESS_KEY_ID"
 	sk <- getEnvKey "AWS_SECRET_ACCESS_KEY"
+	when (null ak || null sk) $ warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3"
 	return $ AWSConnection host port ak sk
 	where
 		host = fromJust $ (M.lookup "host" c)
@@ -62,7 +66,7 @@ s3Connection c = do
 			case reads s of
 			[(p, _)] -> p
 			_ -> error $ "bad S3 port value: " ++ s
-		getEnvKey s = catch (getEnv s) (error $ "Set " ++ s)
+		getEnvKey s = liftIO $ catch (getEnv s) (const $ return "")
 
 s3Setup :: UUID -> M.Map String String -> Annex (M.Map String String)
 s3Setup u c = do
@@ -75,7 +79,7 @@ s3Setup u c = do
 
 	-- check bucket location to see if the bucket exists, and create it
 	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
-	conn <- liftIO $ s3Connection fullconfig
+	conn <- s3Connection fullconfig
 	showNote "checking bucket"
 	loc <- liftIO $ getBucketLocation conn bucket 
 	case loc of
@@ -105,7 +109,7 @@ s3Action :: Remote Annex -> ((AWSConnection, String) -> Annex a) -> Annex a
 s3Action r a = do
 	when (config r == Nothing) $
 		error $ "Missing configuration for special remote " ++ name r
-	conn <- liftIO $ s3Connection (fromJust $ config r)
+	conn <- s3Connection (fromJust $ config r)
 	let bucket = fromJust $ M.lookup "bucket" $ fromJust $ config r
 	a (conn, bucket)
 
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 43bf403de4..8055c16b06 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -15,8 +15,6 @@ import Data.Map as M
 import qualified GitRepo as Git
 import Key
 
-type Cost = Int
-
 {- There are different types of remotes. -}
 data RemoteType a = RemoteType {
 	-- human visible type name
@@ -24,7 +22,7 @@ data RemoteType a = RemoteType {
 	-- enumerates remotes of this type
 	enumerate :: a [Git.Repo],
 	-- generates a remote of this type
-	generate :: Git.Repo -> String -> Cost -> Maybe (M.Map String String) -> a (Remote a),
+	generate :: Git.Repo -> String -> Maybe (M.Map String String) -> a (Remote a),
 	-- initializes or changes a remote
 	setup :: String -> M.Map String String -> a (M.Map String String)
 }
@@ -36,7 +34,7 @@ data Remote a = Remote {
 	-- each Remote has a human visible name
 	name :: String,
 	-- Remotes have a use cost; higher is more expensive
-	cost :: Cost,
+	cost :: Int,
 	-- Transfers a key to the remote.
 	storeKey :: Key -> a Bool,
 	-- retrieves a key's contents to a file
@@ -54,7 +52,7 @@ data Remote a = Remote {
 }
 
 instance Show (Remote a) where
-	show remote = "Remote { uuid =\"" ++ uuid remote ++ "\" }"
+	show remote = "Remote { name =\"" ++ name remote ++ "\" }"
 
 -- two remotes are the same if they have the same uuid
 instance Eq (Remote a) where

From 2c7ceceba64a75deb69033199acff8ccbcb49bdf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 15:25:59 -0400
Subject: [PATCH 1324/2835] improve robustness when S3 access tokens are is not
 configured

---
 Remote/S3real.hs | 39 +++++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index d7a6d507b1..bb82d54e0e 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -54,12 +54,22 @@ gen r u c = do
 			config = c
 		}
 
-s3Connection :: M.Map String String -> Annex AWSConnection
+s3ConnectionRequired :: M.Map String String -> Annex AWSConnection
+s3ConnectionRequired c = do
+	conn <- s3Connection c
+	case conn of
+		Nothing -> error "Cannot connect to S3"
+		Just conn' -> return conn'
+
+s3Connection :: M.Map String String -> Annex (Maybe AWSConnection)
 s3Connection c = do
 	ak <- getEnvKey "AWS_ACCESS_KEY_ID"
 	sk <- getEnvKey "AWS_SECRET_ACCESS_KEY"
-	when (null ak || null sk) $ warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3"
-	return $ AWSConnection host port ak sk
+	if (null ak || null sk)
+		then do
+			warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3"
+			return Nothing
+		else return $ Just $ AWSConnection host port ak sk
 	where
 		host = fromJust $ (M.lookup "host" c)
 		port = let s = fromJust $ (M.lookup "port" c) in
@@ -79,7 +89,8 @@ s3Setup u c = do
 
 	-- check bucket location to see if the bucket exists, and create it
 	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
-	conn <- s3Connection fullconfig
+	conn <- s3ConnectionRequired fullconfig
+	
 	showNote "checking bucket"
 	loc <- liftIO $ getBucketLocation conn bucket 
 	case loc of
@@ -105,28 +116,32 @@ s3Setup u c = do
 			, ("bucket", bucket)
 			]
 
-s3Action :: Remote Annex -> ((AWSConnection, String) -> Annex a) -> Annex a
-s3Action r a = do
+s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a
+s3Action r noconn action = do
 	when (config r == Nothing) $
 		error $ "Missing configuration for special remote " ++ name r
+	let bucket = M.lookup "bucket" $ fromJust $ config r
 	conn <- s3Connection (fromJust $ config r)
-	let bucket = fromJust $ M.lookup "bucket" $ fromJust $ config r
-	a (conn, bucket)
+	case (bucket, conn) of
+		(Just b, Just c) -> action (c, b)
+		_ -> return noconn
 
 bucketKey :: String -> Key -> L.ByteString -> S3Object
 bucketKey bucket k content = S3Object bucket (show k) "" [] content
 
 checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
-checkPresent r k = s3Action r $ \(conn, bucket) -> do
+checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
 	showNote ("checking " ++ name r ++ "...")
 	res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty
 	case res of
 		Right _ -> return $ Right True
 		Left (AWSError _ _) -> return $ Right False
 		Left e -> return $ Left (error $ prettyReqError e)
+	where
+		noconn = Left $ error "S3 not configured"
 
 store :: Remote Annex -> Key -> Annex Bool
-store r k = s3Action r $ \(conn, bucket) -> do
+store r k = s3Action r False $ \(conn, bucket) -> do
 	g <- Annex.gitRepo
 	content <- liftIO $ L.readFile $ gitAnnexLocation g k
 	let object = setStorageClass storageclass $ bucketKey bucket k content
@@ -143,7 +158,7 @@ store r k = s3Action r $ \(conn, bucket) -> do
 				_ -> STANDARD
 
 retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
-retrieve r k f = s3Action r $ \(conn, bucket) -> do
+retrieve r k f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey bucket k L.empty
 	case res of
 		Right o -> do
@@ -154,7 +169,7 @@ retrieve r k f = s3Action r $ \(conn, bucket) -> do
 			return False
 
 remove :: Remote Annex -> Key -> Annex Bool
-remove r k = s3Action r $ \(conn, bucket) -> do
+remove r k = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty
 	case res of
 		Right _ -> return True

From 56491a7e134e6c67b08fdc0200835ecc5b95b074 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 15:32:34 -0400
Subject: [PATCH 1325/2835] tweak

---
 doc/special_remotes/directory.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn
index 42dbc5749e..daa0b74128 100644
--- a/doc/special_remotes/directory.mdwn
+++ b/doc/special_remotes/directory.mdwn
@@ -1,4 +1,4 @@
-This special remote type stores file contents in directory on the system.
+This special remote type stores file contents in directory.
 
 One use case for this would be if you have a removable drive, that you
 cannot put a git repository on for some reason, and you want to use it

From 95f5247ee6d412eaa9b20fb2fa383142fb7f7f69 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 31 Mar 2011 18:02:42 +0000
Subject: [PATCH 1326/2835] Added a comment

---
 ...ent_6_e55117cb628dc532e468519252571474._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment
new file mode 100644
index 0000000000..aae020972c
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-03-31T18:02:42Z"
+ content="""
+Alright, I have created a case-insensative HFS+ filesystem here on my linux laptop. 
+
+I have not been able to trick git into staging the same file with 2 different capitalizations yet.
+
+It might be helpful if you can send me a copy of a git repository where 'git add -i' shows the same file staged with two capitalizations. Leaving out .git/annex of course. (joey@kitenet.net; a tarball would probably work)
+
+It seems that `git add` only started properly working on case insensative filesystems quite recently. The commit in question is 5e738ae820ec53c45895b029baa3a1f63e654b1b, \"Support case folding for git add when core.ignorecase=true\", which was first released in git 1.7.4, January 30, 2011. If you don't yet have that version, that could explain the problem entirely. In about half an hour (dialup!) I will have downloaded an older git and will see if I can reproduce the problem with it.
+"""]]

From 6e835afc4e624203ad2d8e4e43c6dd8fee528bd8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:03:28 -0400
Subject: [PATCH 1327/2835] update

---
 .../sharebox_a_FUSE_filesystem_for_git-annex.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
new file mode 100644
index 0000000000..cc4488371c
--- /dev/null
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -0,0 +1,15 @@
+Christophe-Marie Duquesne has just announced 
+[Sharebox|(https://github.com/chmduquesne/sharebox), a FUSE filesystem
+relying on git-annex:
+
+

+What are your goals?
+Seamless synchronization "à la dropbox".
+Ability to use with big binary files such as mp3/movies.
+Entirely decentralized.
+Don't use unnecessary space
+Keep it simple: avoid special VCS commands and keep a filesystem
+interface as much as possible.
+

+
+While still alpha, this is promising. --[[Joey]]

From ea91f32651e3a6ac7e1a4bc94fa6cb408516eb93 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:07:28 -0400
Subject: [PATCH 1328/2835] format

---
 .../sharebox_a_FUSE_filesystem_for_git-annex.mdwn  | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
index cc4488371c..63d71a3632 100644
--- a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -1,14 +1,16 @@
+[[!meta title="sharebox: a FUSE filesystem for git-annex"]]
+
 Christophe-Marie Duquesne has just announced 
 [Sharebox|(https://github.com/chmduquesne/sharebox), a FUSE filesystem
 relying on git-annex:
 
 

-What are your goals?
-Seamless synchronization "à la dropbox".
-Ability to use with big binary files such as mp3/movies.
-Entirely decentralized.
-Don't use unnecessary space
-Keep it simple: avoid special VCS commands and keep a filesystem
+What are your goals?  
+Seamless synchronization "à la dropbox".  
+Ability to use with big binary files such as mp3/movies.  
+Entirely decentralized.  
+Don't use unnecessary space  
+Keep it simple: avoid special VCS commands and keep a filesystem  
 interface as much as possible.
 

 

From 29738265f238e0a8782db0d6690aa98d9be1368a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:08:20 -0400
Subject: [PATCH 1329/2835] link

---
 doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
index 63d71a3632..f8d1a0ad7c 100644
--- a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -1,7 +1,7 @@
 [[!meta title="sharebox: a FUSE filesystem for git-annex"]]
 
 Christophe-Marie Duquesne has just announced 
-[Sharebox|(https://github.com/chmduquesne/sharebox), a FUSE filesystem
+[Sharebox](https://github.com/chmduquesne/sharebox), a FUSE filesystem
 relying on git-annex:
 
 


From 48c102f5225ea35fb12b47830bb4bfa404ca1110 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:10:08 -0400
Subject: [PATCH 1330/2835] format

---
 doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
index f8d1a0ad7c..7386841b2b 100644
--- a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -5,6 +5,7 @@ Christophe-Marie Duquesne has just announced
 relying on git-annex:
 
 

+
 What are your goals?  
 Seamless synchronization "à la dropbox".  
 Ability to use with big binary files such as mp3/movies.  
@@ -12,6 +13,7 @@ Entirely decentralized.
 Don't use unnecessary space  
 Keep it simple: avoid special VCS commands and keep a filesystem  
 interface as much as possible.
+

 

 
 While still alpha, this is promising. --[[Joey]]

From 4e8fc2c5b91065e70605b408aba96fec779b18ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:12:07 -0400
Subject: [PATCH 1331/2835] show more news

---
 doc/index.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 4f967b71f5..e84c56b449 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -15,7 +15,7 @@ To get a feel for it, see the [[walkthrough]].
 
 [[News]]:
 
-[[!inline pages="news/* and !*/discussion" archive=yes show=3 feeds=no]]
+[[!inline pages="news/* and !*/discussion" archive=yes show=6 feeds=no]]
 
 
 
Date: Thu, 31 Mar 2011 14:12:48 -0400
Subject: [PATCH 1332/2835] more news yet

---
 doc/index.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index e84c56b449..5fc642ecd4 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -15,7 +15,7 @@ To get a feel for it, see the [[walkthrough]].
 
 [[News]]:
 
-[[!inline pages="news/* and !*/discussion" archive=yes show=6 feeds=no]]
+[[!inline pages="news/* and !*/discussion" archive=yes show=7 feeds=no]]
 
 
 
Date: Thu, 31 Mar 2011 19:08:01 +0000
Subject: [PATCH 1333/2835] Added a comment

---
 ..._0f4f471102e394ebb01da40e4d0fd9f6._comment | 68 +++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment
new file mode 100644
index 0000000000..c3aee6c579
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment
@@ -0,0 +1,68 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-03-31T19:08:01Z"
+ content="""
+git 1.7.4 does not make things better. With it, if I add first \"X/foo\" and then \"x/bar\", it commits \"X/bar\".
+
+That will *certianly* cause problems when interoperating with a repo clone on a case-sensative filesystem, since
+git-annex there will not see the location log that git committed to the wrong case directory.
+
+It's possible there is some interoperability problem when pulling from linux like you did, onto HFS+, too. I am not quite sure. Ah, I did find one.. if I clone the repo with \"X/foo\" in it to a case-sensative filesystem, and add a \"x/foo\" there,
+and pull that commit back to HFS+, git says:
+
+
+ * branch            master     -> FETCH_HEAD
+Updating 8754149..e3d4640
+Fast-forward
+ x/foo |    1 +
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 100644 x/foo
+joey@gnu:/mnt/r4>ls
+X/
+joey@gnu:/mnt/r4>git st
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory
+
+#	modified:   X/foo
+

+
+Aha -- that lets me reproduce your problem with the same file being staged twice with different capitalizations, too:
+
+
+joey@gnu:/mnt/r4>echo haaai >| x/foo
+joey@gnu:/mnt/r4>git st
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   X/bar
+#	modified:   X/foo
+#	modified:   x/foo
+#
+joey@gnu:/mnt/r4>git commit -a
+fatal: Will not add file alias 'X/Bar' ('x/Bar' already exists in index)
+

+
+And modified files that git refuses to commit, which entirely explains [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]].
+
+
+joey@gnu:/mnt/r4>git add X/foo
+joey@gnu:/mnt/r4>git commit X/foo
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   X/bar
+#	modified:   X/foo
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+

+
+I think git is frankly, buggy. It seems I will need to work around this by stopping using mixed case hashing for location logs.
+"""]]

From d512a86da9170d8c95e3aa0c6c736a77c8a32ea5 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 31 Mar 2011 19:28:02 +0000
Subject: [PATCH 1334/2835] Added a comment

---
 ...mment_8_68e2d6ccdb9622b879e4bc7005804623._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment
new file mode 100644
index 0000000000..05fe4658df
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-03-31T19:28:02Z"
+ content="""
+I've posted about this on the git mailing list. It's possible that these bugs, which can be shown to affect things other than just git-annex, will be fixed in git.
+
+I will wait a while to see. But am considering making git-annex use all-lowercase hash dirs for the log files. Maybe it could first look for .git-annex/aaaa/bbbb/foo.log, but also look for, read, and merge in any info from 
+.git-annex/Aa/Bb/foo.log. And always write to the new style filenames. This would avoid confusing git with changes to
+mixed-case files, and avoid another massive transition.
+"""]]

From ad2b8759dfa0fa599685b7cb26812872afe9d3a0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 31 Mar 2011 19:30:30 +0000
Subject: [PATCH 1335/2835]

---
 ...has_issues_with_git_when_staging__47__commiting_logs.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 774ca6a16c..ed629c4240 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -27,3 +27,8 @@ For now it's just a bit of extra work for me when it does occur but it does not
 >>>>> what git is doing. Sounds like the files could just not be `git
 >>>>> added` yet, but I get the impression from other things that you say
 >>>>> that it's not so simple. --[[Joey]] 
+
+This turns out to be a bug in git, and I have posted a bug report on the mailing list.
+The git-annex behavior that causes this situation is being handled as
+another bug, [[git-annex directory hashing problems on osx]].
+So, closing this bug report. [[done]] --[[Joey]]

From a7680717f6600469c6920e22cc3dffac06993b1b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 31 Mar 2011 19:32:25 +0000
Subject: [PATCH 1336/2835] Added a comment

---
 .../comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment

diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
new file mode 100644
index 0000000000..e46f2388f6
--- /dev/null
+++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-31T19:32:25Z"
+ content="""
+One option would be to use the new [[sharebox_a_FUSE_filesystem_for_git-annex]], which would hide the immutable file details from Calibre, and proxy any changes it made through to git-annex as a series of `git annex unlock; modify; git-annex lock`
+"""]]

From a76d1cda912ebdc31211ef92e1ddfef4b4295e41 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 15:40:08 -0400
Subject: [PATCH 1337/2835] update

---
 .../comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment   | 2 +-
 doc/bugs/check_for_curl_in_configure.hs.mdwn              | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
index e46f2388f6..719451976b 100644
--- a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
+++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
@@ -4,5 +4,5 @@
  subject="comment 2"
  date="2011-03-31T19:32:25Z"
  content="""
-One option would be to use the new [[sharebox_a_FUSE_filesystem_for_git-annex]], which would hide the immutable file details from Calibre, and proxy any changes it made through to git-annex as a series of `git annex unlock; modify; git-annex lock`
+One option would be to use the new [[news/sharebox_a_FUSE_filesystem_for_git-annex]], which would hide the immutable file details from Calibre, and proxy any changes it made through to git-annex as a series of `git annex unlock; modify; git-annex lock`
 """]]
diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn
index 2a5227491a..a880392bf7 100644
--- a/doc/bugs/check_for_curl_in_configure.hs.mdwn
+++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn
@@ -53,6 +53,14 @@ index 772ba54..1a563e0 100644
 >>> A S3 backend that could upload files to S3 in addition to downloading
 >>> them, for example, would be handy. --[[Joey]]
 
+>>>> So, rather than use backends to do this, it instead made more sense
+>>>> to make them [[special_remotes]]. The URL backend remains a bit
+>>>> of a special case, and a bittorrent backend that downloaded a file
+>>>> from a bittorrent url would still be a good use of backend, but for
+>>>> storing files in external data stores like S3, making it a remote
+>>>> makes better sense. I think I can close this bug now, [[done]]
+>>>> --[[Joey]] 
+
 also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not)
 
 > It's correct, typewise, but I don't see any real reason to bother

From 70572b24813a38761c480dfa3de8f96466747d79 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 16:00:33 -0400
Subject: [PATCH 1338/2835] add feed aggregator

---
 doc/feeds.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/feeds.mdwn

diff --git a/doc/feeds.mdwn b/doc/feeds.mdwn
new file mode 100644
index 0000000000..25a3190497
--- /dev/null
+++ b/doc/feeds.mdwn
@@ -0,0 +1,3 @@
+Aggregating git-annex mentions from elsewhere on the net..
+
+* [[!aggregate expirecount=25 name="identica" feedurl="http://identi.ca/api/statusnet/tags/timeline/gitannex.rss" url="http://identi.ca/tag/gitannex"]]

From 779de890c2c09c9bee0589db37d5fd79c23f7aea Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 16:02:36 -0400
Subject: [PATCH 1339/2835] add feeds to sidebar

---
 doc/index.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 5fc642ecd4..60477d0db1 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -22,6 +22,10 @@ To get a feel for it, see the [[walkthrough]].
 alt="Flattr this" title="Flattr this" />
 """]]
 
+[[Feeds]]:
+
+[[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]]
+
[[!inline feeds=no template=bare pages=use_case/bob]]
From 6d4506ab775247a15344d5f6bedec38fa9a0a8c1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:05:51 -0400 Subject: [PATCH 1340/2835] layout --- doc/index.mdwn | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 60477d0db1..a7813f99fb 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -12,19 +12,16 @@ To get a feel for it, see the [[walkthrough]]. * [[forum]] * [[comments]] * [[contact]] +* Flattr this [[News]]: [[!inline pages="news/* and !*/discussion" archive=yes show=7 feeds=no]] - - -"""]] - [[Feeds]]: [[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]] +"""]]
[[!inline feeds=no template=bare pages=use_case/bob]]
From 1e0fbab24e96e3e92845e6825f836d883cca2ae0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:08:25 -0400 Subject: [PATCH 1341/2835] smaller --- doc/index.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index a7813f99fb..4f2a6d0123 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -16,11 +16,15 @@ To get a feel for it, see the [[walkthrough]]. [[News]]: + [[!inline pages="news/* and !*/discussion" archive=yes show=7 feeds=no]] + [[Feeds]]: + [[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]] + """]]
From d71eb6bec6e4c910acc99de0f029f7e8856d9988 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:09:12 -0400 Subject: [PATCH 1342/2835] fix --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 4f2a6d0123..63c8e41661 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -22,7 +22,7 @@ To get a feel for it, see the [[walkthrough]]. [[Feeds]]: - + [[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]] """]] From 80b52afc728c05fb97ae29e1d05819e51f9fb612 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:10:17 -0400 Subject: [PATCH 1343/2835] layout --- doc/index.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 63c8e41661..96b7f0cf0f 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -17,7 +17,7 @@ To get a feel for it, see the [[walkthrough]]. [[News]]: -[[!inline pages="news/* and !*/discussion" archive=yes show=7 feeds=no]] +[[!inline pages="news/* and !*/discussion" archive=yes show=3 feeds=no]] [[Feeds]]: @@ -48,7 +48,7 @@ files with git. * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. ----- +
git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and hosted by [Branchable](http://branchable.com/). From cc17e6ced2e9ae19da363bef49e3db580af13b5e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:11:00 -0400 Subject: [PATCH 1344/2835] layout --- doc/index.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index 96b7f0cf0f..25456b2b55 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -50,5 +50,7 @@ files with git.
+---- + git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and hosted by [Branchable](http://branchable.com/). From 2fb7eb1e6272b17b1fb2c1e41e66ead612dbee9a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 31 Mar 2011 20:12:09 +0000 Subject: [PATCH 1345/2835] rename bugs/softlink_atime.mdwn to bugs/softlink_mtime.mdwn --- doc/bugs/{softlink_atime.mdwn => softlink_mtime.mdwn} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/bugs/{softlink_atime.mdwn => softlink_mtime.mdwn} (100%) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_mtime.mdwn similarity index 100% rename from doc/bugs/softlink_atime.mdwn rename to doc/bugs/softlink_mtime.mdwn From 8e2f5ae50418a54a7e1c79c7a8bdcd7022a839e5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:12:26 -0400 Subject: [PATCH 1346/2835] remove author (always the same..) --- doc/templates/microblog.tmpl | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/templates/microblog.tmpl diff --git a/doc/templates/microblog.tmpl b/doc/templates/microblog.tmpl new file mode 100644 index 0000000000..d42e1fae2a --- /dev/null +++ b/doc/templates/microblog.tmpl @@ -0,0 +1,12 @@ +
+ +
+ +
+ +
+ +— + +
+
From cbabee36e1e794096ad0693d6f65467fc7936894 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:13:24 -0400 Subject: [PATCH 1347/2835] layout --- doc/index.mdwn | 2 +- doc/templates/microblog.tmpl | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 doc/templates/microblog.tmpl diff --git a/doc/index.mdwn b/doc/index.mdwn index 25456b2b55..cb2f485c92 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -23,7 +23,7 @@ To get a feel for it, see the [[walkthrough]]. [[Feeds]]: -[[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]] +[[!inline pages="internal(feeds/*)" archive=yes show=5 feeds=no]] """]] diff --git a/doc/templates/microblog.tmpl b/doc/templates/microblog.tmpl deleted file mode 100644 index d42e1fae2a..0000000000 --- a/doc/templates/microblog.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -
- -
- -
- -
- -— - -
-
From ff97607879817ce441637751f21462ac769fe099 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 31 Mar 2011 20:25:07 +0000 Subject: [PATCH 1348/2835] --- doc/bugs/softlink_mtime.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/bugs/softlink_mtime.mdwn b/doc/bugs/softlink_mtime.mdwn index fe09ade38b..1427fc7147 100644 --- a/doc/bugs/softlink_mtime.mdwn +++ b/doc/bugs/softlink_mtime.mdwn @@ -49,3 +49,6 @@ Optionally, editing the meta-data should change the times in all annexes. >>>>>>> -- RichiH >>>>>>>> After getting to actually play with this from different machines with a bare git as central instance for several distributed repos, the metastore trick does not work. The .metadata is causing merge conflicts for every pull. I removed the "done" tag from this issue. -- RichiH + +>>>>>>>>> softbox sounds _really_ nice. File systems need to preserve mtimes. Oviously, it would be nice if git-annex exposed this to the upper layer instead of relying on this FUSE implementation, or the next, or the other totally cool thing around the corner to implement it again and again. +>>>>>>>>> I talked to the author of metastore; he is aware that the format is merge-unfriendly but never needed merges for himself. He is aware that this is not ideal for something like git. He does not have the time to implement a text storage instead of binary and I lack the skills to do it. If metastore is used, all it would need to do is introduce a new version of the store (it's versioned, apparently) and save metadata in text, one file per line. xattr would need to be ASCII-armoured, the rest could be plain text. I still think storing this directly in git-annex would make the most sense. Introducing a metadata storage file per storage object in .git/annex and using the object file's name as index is impossible because several softlinks might point to one object so it would need to be done per-softlink :/ -- RichiH From ee3be251a6480ea997c1e7c2c0aa86a046332aff Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 31 Mar 2011 21:32:11 +0000 Subject: [PATCH 1349/2835] Added a comment --- .../comment_9_45b11ddd200261115b653c7a14d28aa9._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment new file mode 100644 index 0000000000..8dfe746420 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 9" + date="2011-03-31T21:32:10Z" + content=""" +I'm was running git 1.7.4.1 at the time when I came across it, I have just upgraded to 1.7.4.2. I've also just moved to using a loopback fs for the stuff i care about. Do you still want a repo that exhibits the problem (excluding the .git/annex data) ??? I'm also not sure if 1.7.4.2 has corrected the problem yet as I haven't done much with my repos since. I suspect just making all the .git-annex hashed directories seems to be lower case might be better in the long run. +"""]] From f003603e8cc191efbba31b1f19b0086cd374635f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 1 Apr 2011 10:46:23 +0000 Subject: [PATCH 1350/2835] --- ...it_annex_copy_--fast_does_not_copy_files.mdwn | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn diff --git a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn new file mode 100644 index 0000000000..1a59538360 --- /dev/null +++ b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn @@ -0,0 +1,16 @@ +Workflow: + + % git annex add + # list new files + % git commit -a -m "foo" + # commit summary + % git annex copy . --to remote --fast + # all files listed with "ok + % git annex copy . --to remote + # again, lists all files, _but the new ones are actually copied, this time_. + +This happens no matter if I + + % git push + +before copy or not. From 2f814eb0be5bb8c9d4e99e00e7ca2998649a4d9f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 1 Apr 2011 10:47:49 +0000 Subject: [PATCH 1351/2835] --- doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn index 1a59538360..ada844a14c 100644 --- a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn +++ b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn @@ -5,7 +5,7 @@ Workflow: % git commit -a -m "foo" # commit summary % git annex copy . --to remote --fast - # all files listed with "ok + # all files listed with "ok" % git annex copy . --to remote # again, lists all files, _but the new ones are actually copied, this time_. @@ -14,3 +14,5 @@ This happens no matter if I % git push before copy or not. + +PS: Arguably, a copy should push automagically. From 5a6c69c8388dc80154b50000e608468ea36377d1 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 1 Apr 2011 16:11:52 +0000 Subject: [PATCH 1352/2835] Added a comment --- .../comment_10_f3594de3ba2ab17771a4b116031511bb._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment new file mode 100644 index 0000000000..c3e6b5e598 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 10" + date="2011-04-01T16:11:52Z" + content=""" +No, I don't need a copy of your repo now. +"""]] From bf1e2205a4af4b490d5dd284a411bd180d8d6c9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 12:19:26 -0400 Subject: [PATCH 1353/2835] bugfix for uuid lookup --- GitQueue.hs | 3 +-- Remote/Git.hs | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GitQueue.hs b/GitQueue.hs index 097516c195..dfe2976da1 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -60,8 +60,7 @@ run repo (Queue _ m) = do - - Complicated by commandline length limits. -} runAction :: Git.Repo -> Action -> [FilePath] -> IO () -runAction repo action files = do - unless (null files) runxargs +runAction repo action files = unless (null files) runxargs where runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs params = toCommand $ Git.gitCommandLine repo diff --git a/Remote/Git.hs b/Remote/Git.hs index c1423bef7a..a458455109 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -54,13 +54,15 @@ gen r u _ = do (False, "") -> tryGitConfigRead r _ -> return r + u' <- getUUID r' + let defcst = if not $ Git.repoIsUrl r then cheapRemoteCost else expensiveRemoteCost cst <- remoteCost r' defcst return $ Remote { - uuid = u, + uuid = u', cost = cst, name = Git.repoDescribe r', storeKey = copyToRemote r', From ed7fc4fce919af6b10a4ab098f72862060ed750f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 12:34:06 -0400 Subject: [PATCH 1354/2835] Bugfix: copy --to --fast never really copied, fixed. --- Command/Move.hs | 6 ++++-- debian/changelog | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index 3ac5a7ab2c..951695d66e 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -89,8 +89,10 @@ toPerform dest move key = do -- and an explicit check is not done, when copying. When moving, -- it has to be done, to avoid inaverdent data loss. fast <- Annex.getState Annex.fast - isthere <- if fast && not move - then return $ Right True + isthere <- if fast && not move && not (Remote.hasKeyCheap dest) + then do + (remotes, _) <- Remote.keyPossibilities key + return $ Right $ dest `elem` remotes else Remote.hasKey dest key case isthere of Left err -> do diff --git a/debian/changelog b/debian/changelog index 0a232220f0..1f951064b1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.20110329) UNRELEASED; urgency=low have proper permissions. * Added a special type of remote called a directory remote, which simply stores files in an arbitrary local directory. + * Bugfix: copy --to --fast never really copied, fixed. -- Joey Hess Sat, 26 Mar 2011 14:36:16 -0400 From dd5591781db3d47123c51ad5935c651f6a8ecf86 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 12:35:02 -0400 Subject: [PATCH 1355/2835] fixed --- doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn index ada844a14c..9b84c21fdb 100644 --- a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn +++ b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn @@ -16,3 +16,7 @@ This happens no matter if I before copy or not. PS: Arguably, a copy should push automagically. + +> Whups, not supposed to be that fast! [[Fixed|done]], and +> you should run `git annex fsck --fast` on the repo you ran the +> copy in. --[[Joey]] From 3e33e489e8256787e847616730802657ea8cab38 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 1 Apr 2011 23:00:53 +0000 Subject: [PATCH 1356/2835] --- ...ce_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn new file mode 100644 index 0000000000..a04c8b040b --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn @@ -0,0 +1,3 @@ +This works with bind-mount, I might try with softlinks as well. + +Going through git's data on push/pull can take ages on a spindle disk even if the repo is rather small in size. This is especially true if you are used to ssd speeds, but ssd storage is expensive. Storing the annex objects on a cheap spindle disk and everything else on a ssd makes things a _lot_ faster. From fd9b90fea1bd9b0052d1cd6205ee3406b5032b24 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 1 Apr 2011 23:06:06 +0000 Subject: [PATCH 1357/2835] --- doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn diff --git a/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn b/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn new file mode 100644 index 0000000000..3ec4f68aae --- /dev/null +++ b/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn @@ -0,0 +1 @@ +As git annex keeps logs about file transfers anyway, it should be relatively easy to add traffic accounting to a repo. That would allow me to monitor how much traffic a given repo creates. As I might end up hosting git-annex repos for a few personal friends, I need/want a way to track the heavy hitters. -- RichiH From 6ab63c438d3628dbcad201f775a18f4b6c8e758f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 21:24:06 -0400 Subject: [PATCH 1358/2835] not really a bug This can occur if a local remote repo has not been initted, so has no uuid yet. --- LocationLog.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LocationLog.hs b/LocationLog.hs index f1e54432ca..d989cad611 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -90,7 +90,8 @@ instance Read LogLine where logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath logChange repo key u s = do when (null u) $ - error $ "bug detected: unknown UUID for " ++ Git.repoDescribe repo + error $ "unknown UUID for " ++ Git.repoDescribe repo ++ + " (have you run git annex init there?)" line <- logNow s u ls <- readLog logfile writeLog logfile (compactLog $ line:ls) From 1283ef73f8f5de109b350a52c07d2e8f52736679 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 21:31:37 -0400 Subject: [PATCH 1359/2835] releasing version 0.20110401 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 1f951064b1..6d56eeb460 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110329) UNRELEASED; urgency=low +git-annex (0.20110401) experimental; urgency=low * Amazon S3 is now supported as a special type of remote. Warning: Encrypting data before sending it to S3 is not yet supported. @@ -10,7 +10,7 @@ git-annex (0.20110329) UNRELEASED; urgency=low simply stores files in an arbitrary local directory. * Bugfix: copy --to --fast never really copied, fixed. - -- Joey Hess Sat, 26 Mar 2011 14:36:16 -0400 + -- Joey Hess Fri, 01 Apr 2011 21:27:22 -0400 git-annex (0.20110328) experimental; urgency=low From 5ec33dc6f90e81f6cc139a25104e96753203ae0d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 21:32:05 -0400 Subject: [PATCH 1360/2835] add news item for git-annex 0.20110401 --- doc/news/version_0.20110401.mdwn | 11 +++++++++++ doc/news/version_0.25.mdwn | 6 ------ 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 doc/news/version_0.20110401.mdwn delete mode 100644 doc/news/version_0.25.mdwn diff --git a/doc/news/version_0.20110401.mdwn b/doc/news/version_0.20110401.mdwn new file mode 100644 index 0000000000..7c9ca6f5ca --- /dev/null +++ b/doc/news/version_0.20110401.mdwn @@ -0,0 +1,11 @@ +git-annex 0.20110401 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Amazon S3 is now supported as a special type of remote. + Warning: Encrypting data before sending it to S3 is not yet supported. + * Note that Amazon S3 support is not built in by default on Debian yet, + as hS3 is not packaged. + * fsck: Ensure that files and directories in .git/annex/objects + have proper permissions. + * Added a special type of remote called a directory remote, which + simply stores files in an arbitrary local directory. + * Bugfix: copy --to --fast never really copied, fixed."""]] \ No newline at end of file diff --git a/doc/news/version_0.25.mdwn b/doc/news/version_0.25.mdwn deleted file mode 100644 index 13abd66cd5..0000000000 --- a/doc/news/version_0.25.mdwn +++ /dev/null @@ -1,6 +0,0 @@ -git-annex 0.25 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix dropping of files using the URL backend. - * Fix support for remotes with '.' in their names. - * Add version command to show git-annex version as well as repository - version information."""]] \ No newline at end of file From 66ab18325e93b244b27b1c80269c943388622716 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 13:35:57 -0400 Subject: [PATCH 1361/2835] mention archive.org's S3 server git-annex + archive.org could be an interesting combo for public archivists --- doc/special_remotes/Amazon_S3.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn index e4e57b3689..384110d1df 100644 --- a/doc/special_remotes/Amazon_S3.mdwn +++ b/doc/special_remotes/Amazon_S3.mdwn @@ -1,5 +1,6 @@ This special remote type stores file contents in a bucket in Amazon S3 -or a similar service. +or a similar service, such as +[Archive.org's S3 API](http://www.archive.org/help/abouts3.txt). See [[walkthrough/using_Amazon_S3]] for usage examples. From b159c10d9c839dc6dffc10ce4a7c51b456b65b28 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 2 Apr 2011 17:48:29 +0000 Subject: [PATCH 1362/2835] Added a comment --- .../comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment new file mode 100644 index 0000000000..124993bcf1 --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-02T17:48:29Z" + content=""" +Either option should work fine, but git gc --aggressive will probably avoid most of git's seeking. +"""]] From 616e6f8a840ef4d99632d12a2e7ea15c3cfb1805 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 13:49:03 -0400 Subject: [PATCH 1363/2835] Use lowercase hash directories for locationlog files to avoid some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still read; new information is written to the new directories. --- LocationLog.hs | 23 ++++++++--------------- Locations.hs | 33 ++++++++++++++++++++++++--------- Remote/Directory.hs | 2 +- debian/changelog | 9 +++++++++ doc/internals.mdwn | 2 +- 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index d989cad611..8a47db2da3 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -23,7 +23,6 @@ module LocationLog ( LogStatus(..), logChange, - logFile, readLog, writeLog, keyLocations @@ -33,7 +32,6 @@ import Data.Time.Clock.POSIX import Data.Time import System.Locale import qualified Data.Map as Map -import System.Directory import Control.Monad (when) import qualified GitRepo as Git @@ -93,22 +91,16 @@ logChange repo key u s = do error $ "unknown UUID for " ++ Git.repoDescribe repo ++ " (have you run git annex init there?)" line <- logNow s u - ls <- readLog logfile - writeLog logfile (compactLog $ line:ls) - return logfile - where - logfile = logFile repo key + let f = logFile repo key + ls' <- readLog $ logFileOld repo key + ls <- readLog f + writeLog f (compactLog $ line:ls'++ls) + return f {- Reads a log file. - Note that the LogLines returned may be in any order. -} readLog :: FilePath -> IO [LogLine] -readLog file = do - exists <- doesFileExist file - if exists - then do - s <- readFile file - return $ parseLog s - else return [] +readLog file = catch (return . parseLog =<< readFile file) (const $ return []) parseLog :: String -> [LogLine] parseLog s = filter parsable $ map read $ lines s @@ -131,7 +123,8 @@ logNow s u = do keyLocations :: Git.Repo -> Key -> IO [UUID] keyLocations thisrepo key = do ls <- readLog $ logFile thisrepo key - return $ map uuid $ filterPresent ls + ls' <- readLog $ logFileOld thisrepo key + return $ map uuid $ filterPresent $ ls'++ls {- Filters the list of LogLines to find ones where the value - is (or should still be) present. -} diff --git a/Locations.hs b/Locations.hs index 8e10c36b41..6c413a2183 100644 --- a/Locations.hs +++ b/Locations.hs @@ -20,7 +20,8 @@ module Locations ( gitAnnexUnusedLog, isLinkToAnnex, logFile, - hashDir, + logFileOld, + hashDirMixed, prop_idempotent_fileKey ) where @@ -68,7 +69,7 @@ objectDir = addTrailingPathSeparator $ annexDir "objects" {- Annexed file's location relative to the .git directory. -} annexLocation :: Key -> FilePath -annexLocation key = objectDir hashDir key f f +annexLocation key = objectDir hashDirMixed key f f where f = keyFile key @@ -113,8 +114,18 @@ isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s {- The filename of the log file for a given key. -} logFile :: Git.Repo -> Key -> String -logFile repo key = - gitStateDir repo ++ hashDir key ++ keyFile key ++ ".log" +logFile = logFile' hashDirLower + +{- The old filename of the log file for a key. These can have mixed + - case, which turned out to be a bad idea for directories whose contents + - are checked into git. There was no conversion, so these have to be checked + - for and merged in at runtime. -} +logFileOld :: Git.Repo -> Key -> String +logFileOld = logFile' hashDirMixed + +logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String +logFile' hasher repo key = + gitStateDir repo ++ hasher key ++ keyFile key ++ ".log" {- Converts a key into a filename fragment. - @@ -147,13 +158,17 @@ prop_idempotent_fileKey s = Just k == fileKey (keyFile k) {- Given a key, generates a short directory name to put it in, - to do hashing to protect against filesystems that dislike having - many items in a single directory. -} -hashDir :: Key -> FilePath -hashDir k = addTrailingPathSeparator $ take 2 dir drop 2 dir +hashDirMixed :: Key -> FilePath +hashDirMixed k = addTrailingPathSeparator $ take 2 dir drop 2 dir where - dir = take 4 $ abcd_to_dir $ md5 $ Str $ show k + dir = take 4 $ concat $ map display_32bits_as_dir [a,b,c,d] + ABCD (a,b,c,d) = md5 $ Str $ show k -abcd_to_dir :: ABCD -> String -abcd_to_dir (ABCD (a,b,c,d)) = concat $ map display_32bits_as_dir [a,b,c,d] +{- Generates a hash directory that is all lower case. -} +hashDirLower :: Key -> FilePath +hashDirLower k = addTrailingPathSeparator $ take 3 dir drop 3 dir + where + dir = take 6 $ md5s $ Str $ show k {- modified version of display_32bits_as_hex from Data.Hash.MD5 - Copyright (C) 2001 Ian Lynagh diff --git a/Remote/Directory.hs b/Remote/Directory.hs index f97449eaa7..0d3478b79d 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -66,7 +66,7 @@ directorySetup u c = do return $ M.delete "directory" c dirKey :: FilePath -> Key -> FilePath -dirKey d k = d hashDir k f f +dirKey d k = d hashDirMixed k f f where f = keyFile k diff --git a/debian/changelog b/debian/changelog index 6d56eeb460..29f60063e8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +git-annex (0.20110402) UNRELEASED; urgency=low + + * Use lowercase hash directories for locationlog files, to avoid + some issues with git on OSX with the mixed-case directories. + No migration is needed; the old mixed case hash directories are still + read; new information is written to the new directories. + + -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 + git-annex (0.20110401) experimental; urgency=low * Amazon S3 is now supported as a special type of remote. diff --git a/doc/internals.mdwn b/doc/internals.mdwn index 2e9f253831..b362e68e1e 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -53,7 +53,7 @@ Example: e605dca6-446a-11e0-8b2a-002170d25c55 1 26339d22-446b-11e0-9101-002170d25c55 ? -## `.git-annex/aa/bb/*.log` +## `.git-annex/aaa/bbb/*.log` The remainder of the log files record [[location_tracking]] information for file contents. Again these are placed in two levels of subdirectories From 623a071fdba9061caed2fa39d0353508df593d06 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 2 Apr 2011 17:53:58 +0000 Subject: [PATCH 1364/2835] Added a comment --- ...ent_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment new file mode 100644 index 0000000000..db605f9650 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 11" + date="2011-04-02T17:53:58Z" + content=""" +I have pushed out a preliminary fix. The old mixed-case directories will be left where they are, and still read from by git-annex. New data will be written to new, lower-case directories. I think that once git stops seeing changes being made +to mixed-case, colliding directories, the bugs you ran into won't manifest any more. + +You will need to find a way to get your git repository out of the state where it complains about uncommitted files (and won't let you commit them). I have not found a reliable way to do that; git reset --hard worked in one case but not in another. May need to clone a fresh git repository. + +Let me know how it works out. +"""]] From ba423c5c0b07161a819ee6b21a37c0f7d983b83a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 2 Apr 2011 17:58:24 +0000 Subject: [PATCH 1365/2835] Added a comment --- .../comment_12_f1c53c3058a587185e7a78d84987539d._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment new file mode 100644 index 0000000000..5f9a0ae275 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 12" + date="2011-04-02T17:58:24Z" + content=""" +Also, you can delete `.git-annex/??` if you want to, then running `git annex fsck --fast` in each of your clones would regenerate the data using only the lower-case hash directories. +"""]] From 7048a126e2c9a5aa1c5a7781c00e627190183bd1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 14:26:47 -0400 Subject: [PATCH 1366/2835] tweak --- doc/use_case/Alice.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index ee97efa43a..7199258a6f 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -4,7 +4,8 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server out there on the net. She stores data in the Cloud. All these things can have different files on them, but Alice no longer has to deal with the -tedious process of keeping them manually in sync. +tedious process of keeping them manually in sync, or rembering where +she put a file. When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her From c4b3081f7c972fe8fd45c4dfa34d37a659b207e3 Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Sat, 2 Apr 2011 19:11:11 +0000 Subject: [PATCH 1367/2835] --- ...tic__41__:_strange_closure_type_30799.mdwn | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn new file mode 100644 index 0000000000..60f7d9ea9b --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn @@ -0,0 +1,45 @@ +I ran git-annex (git version) on three machines with ghc-7.0.2 for about a month, but recently (no more than a week ago) I've started getting this error for every file on "git annex get": + + git-annex-shell: internal error: evacuate(static): strange closure type 30799 + (GHC version 7.0.2 for i386_unknown_linux) + Please report this as a GHC bug: http://www.haskell.org/ghc/reportabug + +There were no changes to ghc or it's modules, so I assume something has changed in git-annex itself. + +strace shows "git annnex get" (on "host1") performing following exec's: + + [pid 9481] execve("/usr/bin/rsync", ["rsync", "-p", "--progress", "--inplace", "-e", "'ssh' 'user@host2' 'git-annex-shell ''sendkey'' ''/remote/path'' ''SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b'' ''--'''", ":", "/local/path/.git/annex/tmp/SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b"], [/* 41 vars */]) = 0 + [pid 9482] execve("/usr/bin/ssh", ["ssh", "user@host2", "git-annex-shell 'sendkey' '/remote/path' 'SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b' '--'", "", "rsync", "--server", "--sender", "-vpe.Lsf", "--inplace", ".", ""], [/* 41 vars */] + +I've tried running the second command directly from the shell and got the same error message from a remote GHC. +Adding strace before git-annex-shell to remote command yielded something like this in the end: + + stat64("/local/path.git", 0xb727d610) = -1 ENOENT (No such file or directory) + stat64("/local/path.git", 0xb727d6b0) = -1 ENOENT (No such file or directory) + waitpid(7525, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0) = 7525 + chdir("/home/user") = 0 + rt_sigprocmask(SIG_BLOCK, [INT], [], 8) = 0 + write(2, "git-annex-shell: internal error: ", 33git-annex-shell: internal error: ) = 33 + ... + +Note that "/local/path" here is not what's specified in rsync arguments at all, and git repo with files-to-be-fetched on "host2" is in "/remote/path", but "/local/path" is present in git remotes there since I mount it via nfs from "host1" (yes, to the same path as it's there): + + [remote "nfs"] + url = /local/path + fetch = +refs/heads/*:refs/remotes/nfs/* + push = refs/heads/*:refs/remotes/host2/* + annex-uuid = 0a4e14ba-5236-11e0-9004-7f24452c0f05 + +If I comment that remote out from "/remote/path/.git/config", "git annex get" works fine. +The only git-command git-annex-shell seem to exec there (on "host2") is "git config --list", so it's shouldn't be git trying to do something with it's remotes - it's git-annex itself, right? + +Anyways, looks like a simple path-joining error, if "/local/path.git" should be "/local/path/.git" there. + +I'm actually quite confused about what it's trying to do with that path. +Connect from "host1" to "host2" just to connect back to "host1"? +What for, when it should just fetch files from "host2"? + +Not sure if it's a bug or I'm doing something wrong, but if git-annex really need to check something in git remotes' paths, error message (the one at the top of this post) can be a more descriptive, I guess. +Something like "error: failed to do something with git remote X on a remote host" would've been a lot less confusing than that GHC thing. + +Thanks! From f005a84e5675cb3e551b2922ad42642df28264d6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 15:50:51 -0400 Subject: [PATCH 1368/2835] add loggedKeys --- Content.hs | 10 +++------- LocationLog.hs | 32 ++++++++++++++++++++++++++------ Locations.hs | 9 +++++++++ Utility.hs | 12 ++++++++++++ 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/Content.hs b/Content.hs index 88e8dbc007..ba265c9307 100644 --- a/Content.hs +++ b/Content.hs @@ -219,9 +219,9 @@ getKeysPresent' dir = do then return [] else do -- 2 levels of hashing - levela <- liftIO $ subdirContent dir - levelb <- liftIO $ mapM subdirContent levela - contents <- liftIO $ mapM subdirContent (concat levelb) + levela <- liftIO $ dirContents dir + levelb <- liftIO $ mapM dirContents levela + contents <- liftIO $ mapM dirContents (concat levelb) files <- liftIO $ filterM present (concat contents) return $ catMaybes $ map (fileKey . takeFileName) files where @@ -231,7 +231,3 @@ getKeysPresent' dir = do case result of Right s -> return $ isRegularFile s Left _ -> return False - subdirContent d = do - c <- getDirectoryContents d - return $ map (d ) $ filter notcruft c - notcruft f = f /= "." && f /= ".." diff --git a/LocationLog.hs b/LocationLog.hs index 8a47db2da3..c2d956a291 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -1,9 +1,7 @@ {- git-annex location log - - - git-annex keeps track of on which repository it last saw a value. - - This can be useful when using it for archiving with offline storage. - - When you indicate you --want a file, git-annex will tell you which - - repositories have the value. + - git-annex keeps track of which repositories have the contents of annexed + - files. - - Location tracking information is stored in `.git-annex/key.log`. - Repositories record their UUID and the date when they --get or --drop @@ -15,7 +13,7 @@ - Git is configured to use a union merge for this file, - so the lines may be in arbitrary order, but it will never conflict. - - - Copyright 2010 Joey Hess + - Copyright 2010-2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -25,14 +23,19 @@ module LocationLog ( logChange, readLog, writeLog, - keyLocations + keyLocations, + loggedKeys, + logFile ) where import Data.Time.Clock.POSIX import Data.Time import System.Locale +import System.Directory +import System.FilePath import qualified Data.Map as Map import Control.Monad (when) +import Data.Maybe import qualified GitRepo as Git import Utility @@ -153,3 +156,20 @@ mapLog m l = Just l' -> (date l' <= date l) Nothing -> True u = uuid l + +{- Finds all keys that have location log information. -} +loggedKeys :: Git.Repo -> IO [Key] +loggedKeys repo = do + let dir = gitStateDir repo + exists <- doesDirectoryExist dir + if exists + then do + -- 2 levels of hashing + levela <- dirContents dir + levelb <- mapM tryDirContents levela + files <- mapM tryDirContents (concat levelb) + return $ catMaybes $ + map (logFileKey . takeFileName) (concat files) + else return [] + where + tryDirContents d = catch (dirContents d) (return . const []) diff --git a/Locations.hs b/Locations.hs index 6c413a2183..f263ea526b 100644 --- a/Locations.hs +++ b/Locations.hs @@ -21,6 +21,7 @@ module Locations ( isLinkToAnnex, logFile, logFileOld, + logFileKey, hashDirMixed, prop_idempotent_fileKey @@ -127,6 +128,14 @@ logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String logFile' hasher repo key = gitStateDir repo ++ hasher key ++ keyFile key ++ ".log" +{- Converts a log filename into a key. -} +logFileKey :: FilePath -> Maybe Key +logFileKey file + | end == ".log" = readKey beginning + | otherwise = Nothing + where + (beginning, end) = splitAt (length file - 4) file + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid diff --git a/Utility.hs b/Utility.hs index 8312335f86..72f5c50638 100644 --- a/Utility.hs +++ b/Utility.hs @@ -22,6 +22,7 @@ module Utility ( readMaybe, safeWriteFile, dirContains, + dirContents, prop_idempotent_shellEscape, prop_idempotent_shellEscape_multiword, @@ -235,3 +236,14 @@ safeWriteFile file content = do createDirectoryIfMissing True (parentDir file) writeFile tmpfile content renameFile tmpfile file + +{- Lists the contents of a directory. + - Unlike getDirectoryContents, paths are not relative to the directory. -} +dirContents :: FilePath -> IO [FilePath] +dirContents d = do + c <- getDirectoryContents d + return $ map (d ) $ filter notcruft c + where + notcruft "." = False + notcruft ".." = False + notcruft _ = True From c5f8284af1518799bbb76d1ea189a286f03b9bd0 Mon Sep 17 00:00:00 2001 From: "http://dieter-be.myopenid.com/" Date: Sat, 2 Apr 2011 20:24:34 +0000 Subject: [PATCH 1369/2835] Added a comment --- .../comment_3_60691af4400521b5a8c8d75efe3b44cb._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment diff --git a/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment b/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment new file mode 100644 index 0000000000..9280f2dccf --- /dev/null +++ b/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://dieter-be.myopenid.com/" + nickname="dieter" + subject="comment 3" + date="2011-04-02T20:24:33Z" + content=""" + * why the `git remote add laptop ~/annex` ? this remote already exists under the name origin. + * doesn't the last command need to be `git remote add usbdrive /media/usb/annex`? because the actual repo would be in /media/usb/annex, not /media/usb? +"""]] From d1b18047501c8c815ad8cc910e96fae2cb64d4ca Mon Sep 17 00:00:00 2001 From: gernot Date: Sat, 2 Apr 2011 20:54:43 +0000 Subject: [PATCH 1370/2835] --- ..._untracked_.git-annex__47____42___directories.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn new file mode 100644 index 0000000000..ab8b255a0f --- /dev/null +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn @@ -0,0 +1,11 @@ +I upgraded another one of my git-annex clones. The upgrade worked fine (i.e. +according to the manual) on two other clones before, but this time something is +different. + +After 'git pull' and 'git annex upgrade', which took a long time and seemed to +have succeeded, there are no staged changes in git. Instead there are lots of +untracked directories in .git-annex. Aside from that, nothing seems to be +wrong. + +At the time I had git-annex version 0.20110329 and I've been using the SHA1 +backend since version 1. From 61a22d3fee29f4c8d46ab7996391f1b539974944 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 2 Apr 2011 21:29:56 +0000 Subject: [PATCH 1371/2835] typo --- doc/use_case/Alice.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 7199258a6f..80280580e3 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -4,7 +4,7 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server out there on the net. She stores data in the Cloud. All these things can have different files on them, but Alice no longer has to deal with the -tedious process of keeping them manually in sync, or rembering where +tedious process of keeping them manually in sync, or remembering where she put a file. When she has 1 bar on her cell, Alice queues up interesting files on her From f69a4c56e970ccc991c6970d9ead225da2d4200b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 2 Apr 2011 21:34:24 +0000 Subject: [PATCH 1372/2835] Added a comment --- ..._f94abce32ef818176b42a3cc860691ae._comment | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment new file mode 100644 index 0000000000..eddc8c6315 --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-02T21:34:24Z" + content=""" +I'll give it a try as soon as I get rid of this: + + % git annex fsck +fatal: index file smaller than expected +fatal: index file smaller than expected + % git status +fatal: index file smaller than expected + % + +And no, I am not sure where that is coming from all of a sudden... (it might have to do with a hard lockup of the whole system due to a faulty hdd I tested, but I didn't do anything to it for ages before that lock-up. So meh. Also, this is prolly off topic in here) + + +Richard +"""]] From 2a08107ce46a0ebd444af274b7923f0b3043de9d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 2 Apr 2011 21:53:18 +0000 Subject: [PATCH 1373/2835] --- doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn b/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn index 3ec4f68aae..4b661101d7 100644 --- a/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn +++ b/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn @@ -1 +1,3 @@ -As git annex keeps logs about file transfers anyway, it should be relatively easy to add traffic accounting to a repo. That would allow me to monitor how much traffic a given repo creates. As I might end up hosting git-annex repos for a few personal friends, I need/want a way to track the heavy hitters. -- RichiH +As git annex keeps logs about file transfers anyway, it should be relatively easy to add traffic accounting to a repo. That would allow me to monitor how much traffic a given repo generates. As I might end up hosting git-annex repos for a few personal friends, I need/want a way to track the heavy hitters. -- RichiH + +PS: If you ever plan to host git-annex similar branchable, this would probably be of interest to you, as well :) From 6c320804395b3bcb043e2d73322465c2ab7212fc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 00:27:30 +0000 Subject: [PATCH 1374/2835] --- .../No_easy_way_to_re-inject_a_file_into_an_annex.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn new file mode 100644 index 0000000000..ced4fc5a0f --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn @@ -0,0 +1,10 @@ +My local git index got corrupted and I needed to clone and annex get all data from my main repo. + +Some files were never copied anywhere so I am stuck with symlinks to nowhere. + +I tried to copy over the symlink with a copy of the actual file, which did not work. Trying to unlock, copying over the symlink, and relock did not work, either. + +Then, I copied the annex object to the correct place in .git/annex/objects/..., set all modes, re-ran fsck and the file re-appeared. + + +Long story short, I think there should be a `git annex reinject $file` or similar which will take a file, either one replacing the symlink or with an arbitrary path, and put it into the correct place in the object store. Called normally, it should reject all reinjects where the checksum does not match. With --force, this should be overridden. For reasons of safety, WORM should always require --force. From 09a16176dea5ef2a51e3a3d00d77180966c597d9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 20:36:01 -0400 Subject: [PATCH 1375/2835] read log files strictly This avoids leaking fds when an operation needs to read a lot of log files, as unused will. --- LocationLog.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index c2d956a291..e0ccb642b3 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -103,7 +103,7 @@ logChange repo key u s = do {- Reads a log file. - Note that the LogLines returned may be in any order. -} readLog :: FilePath -> IO [LogLine] -readLog file = catch (return . parseLog =<< readFile file) (const $ return []) +readLog file = catch (return . parseLog =<< readFileStrict file) (const $ return []) parseLog :: String -> [LogLine] parseLog s = filter parsable $ map read $ lines s @@ -157,7 +157,8 @@ mapLog m l = Nothing -> True u = uuid l -{- Finds all keys that have location log information. -} +{- Finds all keys that have location log information. + - (There may be duplicate keys in the list.) -} loggedKeys :: Git.Repo -> IO [Key] loggedKeys repo = do let dir = gitStateDir repo From 5eaa90ab3514e4417b0e002ee47da0a0c3c1d716 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 00:36:04 +0000 Subject: [PATCH 1376/2835] --- doc/bugs/Displayed_copy_speed_is_wrong.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/bugs/Displayed_copy_speed_is_wrong.mdwn diff --git a/doc/bugs/Displayed_copy_speed_is_wrong.mdwn b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn new file mode 100644 index 0000000000..c1209c73c0 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn @@ -0,0 +1,5 @@ +When copying data to my remote, I regularly see speeds in excess of 100 MB/s on my home DSL line. + + 2073939 100% 176.96MB/s 0:00:00 (xfer#1, to-check=0/1) + +This is definitely not correct. From a31fb09eafb73897d63d7527f2e66bdd11f6802f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 00:50:48 +0000 Subject: [PATCH 1377/2835] --- ...ninit_do_not_work_when_git_index_is_broken.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn diff --git a/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn new file mode 100644 index 0000000000..509e12aebb --- /dev/null +++ b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn @@ -0,0 +1,14 @@ +git's index broke and I was unable to restore it. While this is not git-annex' problem, it should still be possible to get my data in an un-annexed state. + + % git status + fatal: index file smaller than expected + % git annex unannex foo + fatal: index file smaller than expected + % git annex uninit + fatal: index file smaller than expected + uninit + pre-commit hook (/path/to/git-annex/.git/hooks/pre-commit) contents modified; not deleting. Edit it to remove call to git annex. + ok + % + +Ttbomk, the softlinks and objects are enough to un-annex the files; side-stepping git's index if necessary. From 868300d4c1dafd2c4b91ad3f369cfb48f14bb82a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 20:59:41 -0400 Subject: [PATCH 1378/2835] unused/dropunused: support --from --- Command/DropUnused.hs | 31 ++++++--- Command/Unused.hs | 116 ++++++++++++++++++++++--------- debian/changelog | 3 + doc/git-annex.mdwn | 16 +++-- doc/special_remotes.mdwn | 23 ++++++ doc/walkthrough/unused_data.mdwn | 2 +- 6 files changed, 147 insertions(+), 44 deletions(-) diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 932a8b8635..1eec688202 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -19,6 +19,8 @@ import Messages import Locations import qualified Annex import qualified Command.Drop +import qualified Command.Move +import qualified Remote import Backend import Key @@ -40,15 +42,28 @@ start m s = notBareRepo $ do case M.lookup s m of Nothing -> return Nothing Just key -> do - g <- Annex.gitRepo showStart "dropunused" s - backend <- keyBackend key - -- drop both content in the backend and any tmp - -- file for the key - let tmp = gitAnnexTmpLocation g key - tmp_exists <- liftIO $ doesFileExist tmp - when tmp_exists $ liftIO $ removeFile tmp - return $ Just $ Command.Drop.perform key backend (Just 0) + from <- Annex.getState Annex.fromremote + case from of + Just name -> do + r <- Remote.byName name + return $ Just $ performRemote r key + _ -> return $ Just $ perform key + +{- drop both content in the backend and any tmp file for the key -} +perform :: Key -> CommandPerform +perform key = do + g <- Annex.gitRepo + let tmp = gitAnnexTmpLocation g key + tmp_exists <- liftIO $ doesFileExist tmp + when tmp_exists $ liftIO $ removeFile tmp + backend <- keyBackend key + Command.Drop.perform key backend (Just 0) -- force drop + +performRemote :: Remote.Remote Annex -> Key -> CommandPerform +performRemote r key = do + showNote $ "from " ++ Remote.name r ++ "..." + return $ Just $ Command.Move.fromCleanup r True key readUnusedLog :: Annex (M.Map String Key) readUnusedLog = do diff --git a/Command/Unused.hs b/Command/Unused.hs index 83d8757cff..a3fb6fe232 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -20,9 +20,11 @@ import Content import Messages import Locations import Utility +import LocationLog import qualified Annex import qualified GitRepo as Git import qualified Backend +import qualified Remote command :: [Command] command = [repoCommand "unused" paramNothing seek @@ -39,35 +41,54 @@ start = notBareRepo $ do perform :: CommandPerform perform = do - _ <- checkUnused + from <- Annex.getState Annex.fromremote + case from of + Just name -> do + r <- Remote.byName name + checkRemoteUnused r + _ -> checkUnused return $ Just $ return True -checkUnused :: Annex Bool +checkUnused :: Annex () checkUnused = do (unused, staletmp) <- unusedKeys let unusedlist = number 0 unused let staletmplist = number (length unused) staletmp let list = unusedlist ++ staletmplist - g <- Annex.gitRepo - liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ unlines $ - map (\(n, k) -> show n ++ " " ++ show k) list - unless (null unused) $ showLongNote $ unusedmsg unusedlist - unless (null staletmp) $ showLongNote $ staletmpmsg staletmplist + writeUnusedFile list + unless (null unused) $ showLongNote $ unusedMsg unusedlist + unless (null staletmp) $ showLongNote $ staleTmpMsg staletmplist unless (null list) $ showLongNote $ "\n" - return $ null list +checkRemoteUnused :: Remote.Remote Annex -> Annex () +checkRemoteUnused r = do + g <- Annex.gitRepo + showNote $ "checking for unused data on " ++ Remote.name r ++ "..." + referenced <- getKeysReferenced + logged <- liftIO $ loggedKeys g + remotehas <- filterM isthere logged + let remoteunused = remotehas `exclude` referenced + let list = number 0 remoteunused + writeUnusedFile list + unless (null remoteunused) $ do + showLongNote $ remoteUnusedMsg r list + showLongNote $ "\n" + where + isthere k = do + g <- Annex.gitRepo + us <- liftIO $ keyLocations g k + return $ uuid `elem` us + uuid = Remote.uuid r + +writeUnusedFile :: [(Int, Key)] -> Annex () +writeUnusedFile l = do + g <- Annex.gitRepo + liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ + unlines $ map (\(n, k) -> show n ++ " " ++ show k) l + +table :: [(Int, Key)] -> [String] +table l = [" NUMBER KEY"] ++ map cols l where - unusedmsg u = unlines $ - ["Some annexed data is no longer pointed to by any files in the repository:"] - ++ table u ++ - ["(To see where data was previously used, try: git log --stat -S'KEY')"] ++ - dropmsg - staletmpmsg t = unlines $ - ["Some partially transferred data exists in temporary files:"] - ++ table t ++ dropmsg - dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"] - - table l = [" NUMBER KEY"] ++ map cols l cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k pad n s = s ++ replicate (n - length s) ' ' @@ -75,6 +96,39 @@ number :: Int -> [a] -> [(Int, a)] number _ [] = [] number n (x:xs) = (n+1, x):(number (n+1) xs) +staleTmpMsg :: [(Int, Key)] -> String +staleTmpMsg t = unlines $ + ["Some partially transferred data exists in temporary files:"] + ++ table t ++ [dropMsg Nothing] + +unusedMsg :: [(Int, Key)] -> String +unusedMsg u = unusedMsg' u + ["Some annexed data is no longer used by any files in the repository:"] + [dropMsg Nothing] + +remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String +remoteUnusedMsg r u = unusedMsg' u + ["Some annexed data on " ++ name ++ + " is not used by any files in this repository."] + [dropMsg $ Just r, + "Please be cautious -- are you sure that the remote repository", + "does not use this data?"] + where + name = Remote.name r + +unusedMsg' :: [(Int, Key)] -> [String] -> [String] -> String +unusedMsg' u header trailer = unlines $ + header ++ + table u ++ + ["(To see where data was previously used, try: git log --stat -S'KEY')"] ++ + trailer + +dropMsg :: Maybe (Remote.Remote Annex) -> String +dropMsg Nothing = dropMsg' "" +dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r +dropMsg' :: String -> String +dropMsg' s = "(To remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER)" + {- Finds keys whose content is present, but that do not seem to be used - by any files in the git repo, or that are only present as tmp files. -} unusedKeys :: Annex ([Key], [Key]) @@ -93,7 +147,9 @@ unusedKeys = do referenced <- getKeysReferenced tmps <- tmpKeys - let (unused, staletmp, duptmp) = calcUnusedKeys present referenced tmps + let unused = present `exclude` referenced + let staletmp = tmps `exclude` present + let duptmp = tmps `exclude` staletmp -- Tmp files that are dups of content already present -- can simply be removed. @@ -102,18 +158,16 @@ unusedKeys = do return (unused, staletmp) -calcUnusedKeys :: [Key] -> [Key] -> [Key] -> ([Key], [Key], [Key]) -calcUnusedKeys present referenced tmps = (unused, staletmp, duptmp) +{- Finds items in the first, smaller list, that are not + - present in the second, larger list. + - + - Constructing a single set, of the list that tends to be + - smaller, appears more efficient in both memory and CPU + - than constructing and taking the S.difference of two sets. -} +exclude :: Ord a => [a] -> [a] -> [a] +exclude [] _ = [] -- optimisation +exclude smaller larger = S.toList $ remove larger $ S.fromList smaller where - unused = present `exclude` referenced - staletmp = tmps `exclude` present - duptmp = tmps `exclude` staletmp - - -- Constructing a single set, of the list that tends to be - -- smaller, appears more efficient in both memory and CPU - -- than constructing and taking the S.difference of two sets. - exclude [] _ = [] -- optimisation - exclude smaller larger = S.toList $ remove larger $ S.fromList smaller remove a b = foldl (flip S.delete) b a {- List of keys referenced by symlinks in the git repo. -} diff --git a/debian/changelog b/debian/changelog index 29f60063e8..e504bd8f60 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ git-annex (0.20110402) UNRELEASED; urgency=low some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still read; new information is written to the new directories. + * Unused files on remotes, particulary special remotes, can now be + identified and dropped, by using "--from remote" with git annex unused + and git annex dropunused. -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d890b518b8..7d0fb3e792 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -155,16 +155,21 @@ Many git-annex commands will stage changes for later `git commit` by you. * unused - Checks the annex for data that is not used by any files currently - in the annex, and prints a numbered list of the data. + Checks the annex for data that does not correspond to any files currently + in the respository, and prints a numbered list of the data. To only show unused temp files, specify --fast + To check data on a remote that does not correspond to any files currently + in the local repository, specify --from. + * dropunused [number ...] Drops the data corresponding to the numbers, as listed by the last `git annex unused` + To drop the data from a remote, specify --from. + * find [path ...] Outputs a list of annexed files whose content is currently present. @@ -317,12 +322,15 @@ Many git-annex commands will stage changes for later `git commit` by you. * --from=repository - Specifies a repository that content will be retrieved from. + Specifies a repository that content will be retrieved from, or that + should otherwise be acted on. + It should be specified using the name of a configured remote. * --to=repository - Specifies a repository that content will be sent to. + Specifies a repository that content will be sent to. + It should be specified using the name of a configured remote. * --exclude=glob diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index 09b751d0f4..f4d479aa9c 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -8,3 +8,26 @@ They cannot be used by other git commands though. * [[Amazon_S3]] * [[directory]] + +## Unused content on special remotes + +Over time, special remotes can accumulate file content that is no longer +referred to by files in git. Normally, unused content in the current +repository is found by running `git annex unused`. To detect unused content +on special remotes, instead use `git annex unused --from`. Example: + + $ git annex unused --from mys3 + unused (checking for unused data on mys3...) + Some annexed data on mys3 is not used by any files in this repository. + NUMBER KEY + 1 WORM-s3-m1301674316--foo + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused --from mys3 NUMBER) + Please be cautious -- are you sure that the remote repository + does not use this data? + $ git annex dropunused --from mys3 1 + dropunused 12948 (from mys3...) ok + +Do be cautious when using this; it cannot detect if content in a remote +is used by that remote, or is the last copy of data that is used by +some *other* remote. diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn index 9be32577c3..2f8edcd388 100644 --- a/doc/walkthrough/unused_data.mdwn +++ b/doc/walkthrough/unused_data.mdwn @@ -10,7 +10,7 @@ eliminate it to save space. # git annex unused unused (checking for unused data...) - Some annexed data is no longer pointed to by any files in the repository. + Some annexed data is no longer used by any files in the repository. NUMBER KEY 1 WORM-s3-m1289672605--file 2 WORM-s14-m1289672605--file From dd64c46804be06ae3a1dda3ab353a6241566d5a8 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 01:37:29 +0000 Subject: [PATCH 1379/2835] Added a comment --- .../comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment diff --git a/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment b/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment new file mode 100644 index 0000000000..62a595be77 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T01:37:29Z" + content=""" +That is displayed by rsync. It's not unheard of for rsync to resume a transfer and display extremely high speeds. +"""]] From 9026289e9a1cc20a014945e9d529e7d602a35432 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 01:40:50 +0000 Subject: [PATCH 1380/2835] Added a comment --- .../comment_1_1931e733f0698af5603a8b92267203d4._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment diff --git a/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment new file mode 100644 index 0000000000..84b68bb7ba --- /dev/null +++ b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T01:40:50Z" + content=""" +They rely on git-ls-files to get a list of files that are checked into git, in order to tell what to unannex. +"""]] From 192e988a4422afef08142ab40008bdce6aada8d2 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 01:46:16 +0000 Subject: [PATCH 1381/2835] Added a comment --- .../comment_1_c871605e187f539f3bfe7478433e7fb5._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment new file mode 100644 index 0000000000..9688012a47 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T01:46:16Z" + content=""" +Have you seen [[walkthrough/recover_data_from_lost+found]]? The method described there will also work in this scenario. +"""]] From efefe1397d1cb88b42242bc05c85be1ea7450826 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 01:48:57 +0000 Subject: [PATCH 1382/2835] Added a comment --- ...comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment new file mode 100644 index 0000000000..fc29236c6d --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-03T01:48:57Z" + content=""" +For future reference, git can recover from a corrupted index file with `rm .git/index; git reset --mixed`. + +Of course, you lose any staged changes that were in the old index file, and may need to re-stage some files. +"""]] From 59923c6cf735260d96f99b0a914ce84df122aece Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 21:52:57 -0400 Subject: [PATCH 1383/2835] tweak --- doc/walkthrough/recover_data_from_lost+found.mdwn | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/walkthrough/recover_data_from_lost+found.mdwn b/doc/walkthrough/recover_data_from_lost+found.mdwn index 6e2c241485..f101b3caf6 100644 --- a/doc/walkthrough/recover_data_from_lost+found.mdwn +++ b/doc/walkthrough/recover_data_from_lost+found.mdwn @@ -3,12 +3,14 @@ It's actually very easy to recover from this disaster. First, check out the git repository again. Then, in the new checkout: - mkdir recovered-content - sudo mv ../lost+found/* recovered-content - git annex add recovered-content - git rm recovered-content - git commit -m "recovered some content" - git annex fsck + $ mkdir recovered-content + $ sudo mv ../lost+found/* recovered-content + $ sudo chown you:you recovered-content + $ chmod -R u+w recovered-content + $ git annex add recovered-content + $ git rm recovered-content + $ git commit -m "recovered some content" + $ git annex fsck The way that works is that when git-annex adds the same content that was in the repository before, all the old links to that content start working From 31912a9b3bd0ee31aea0cc3be6acacf230278968 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 02:26:20 +0000 Subject: [PATCH 1384/2835] Added a comment --- ...comment_1_9ca2da52f3c8add0276b72d6099516a6._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment new file mode 100644 index 0000000000..78309df872 --- /dev/null +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T02:26:20Z" + content=""" +I'm not sure how this happened, as far as I can see, and based on my testing, `git annex upgrade` does stage the location log files. OTOH, I vaguely rememeber needing to stage some of them when I was doing my own upgrades, but that was a while ago, and I don't remember the details. + +Your upgrade seems to have gone ok from the file lists you sent, so you can just: `git add .git-annex; git commit` +"""]] From e08fa92dfa1c2bc791b940790c9075d400c518b9 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 02:32:17 +0000 Subject: [PATCH 1385/2835] Added a comment --- ...comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment diff --git a/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment b/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment new file mode 100644 index 0000000000..b4dcb6422a --- /dev/null +++ b/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-04-03T02:32:17Z" + content=""" +Good spotting on the last line, fixed. + +The laptop remote is indeed redundant, but it leads to clearer views of what is going on later in the walkthrough (\"git pull laptop master\", \"(copying from laptop...)\"). And if the original clone is made from a central bare repo, this reinforces that you'll want to set up remotes for other repos on the computer. +"""]] From 248bbf76ca408fe7c89f47917e1cfae5ffc079c7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 22:34:05 -0400 Subject: [PATCH 1386/2835] typo --- doc/walkthrough/adding_a_remote.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn index be8e8e7fe5..97690dfcdf 100644 --- a/doc/walkthrough/adding_a_remote.mdwn +++ b/doc/walkthrough/adding_a_remote.mdwn @@ -8,7 +8,7 @@ Let's start by adding a USB drive as a remote. # git annex init "portable USB drive" # git remote add laptop ~/annex # cd ~/annex - # git remote add usbdrive /media/usb + # git remote add usbdrive /media/usb/annex This is all standard ad-hoc distributed git repository setup. The only git-annex specific part is telling it the name From 757a465b53fb5bf613dc7808e8c29819059d1ff4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 22:55:57 -0400 Subject: [PATCH 1387/2835] analysis --- ...tic__41__:_strange_closure_type_30799.mdwn | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn index 60f7d9ea9b..87a0435208 100644 --- a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn @@ -39,7 +39,33 @@ I'm actually quite confused about what it's trying to do with that path. Connect from "host1" to "host2" just to connect back to "host1"? What for, when it should just fetch files from "host2"? +> git-annex (and git-annex shell) always start up by learning what git +> remotes are locally configured, and this includes checking them to +> try to look up their annex.uuid setting. +> +> Since git will, given a remote like "url = /foo", first look in +> "/foo.git" for a bare git repository, so too does git-annex. +> I do not think this is a path joining error. That seems likely to +> be a red herring. --[[Joey]] + Not sure if it's a bug or I'm doing something wrong, but if git-annex really need to check something in git remotes' paths, error message (the one at the top of this post) can be a more descriptive, I guess. Something like "error: failed to do something with git remote X on a remote host" would've been a lot less confusing than that GHC thing. Thanks! + +> I've never seen anything like this error message. I don't know if the +> problem is caused by building with GHC 7, or what. You didn't say what +> OS you're using. Searching for the error message, it seems to involve +> Mac OS X. + +> For example: +>> The error "strange closure type" indicates some kind of memory corruption, which can have many different causes, from bugs in the GC to hardware failures. +> +> You said that you'd been using git-annex built with that version of GHC +> successfully before. Perhaps you could use `git bisect` to see if you can +> identify a point in git-annex's history where this started happening? +> Since you can reproduce the problem by just running git-annex-shell at +> the command line with the right parameters, it should be easy to bisect it. +> +> Probably your best bet will be changing to a different version or build of +> GHC.. --[[Joey]] From 38598e6f234ba44d997e59ac57e5436063eb1fd0 Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Sun, 3 Apr 2011 04:45:50 +0000 Subject: [PATCH 1388/2835] Added a comment: Bisect it is, then --- ...comment_1_1c19e716069911f17bbebd196d9e4b61._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment new file mode 100644 index 0000000000..98f0adc3db --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Bisect it is, then" + date="2011-04-03T04:45:49Z" + content=""" +Hm, if path's ok, guess there's no way around git-bisect indeed. Wonder if there's some kind of ccache for haskell... + +OS is linux, amd64 on \"host1\" and i386 on \"host2\" where git-annex-shell is crashing. +I'll try to come up with a commit, thanks for clarifications. +"""]] From 702ab4b008cae422bf737106a4f1b8eed105717b Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Sun, 3 Apr 2011 06:22:16 +0000 Subject: [PATCH 1389/2835] Added a comment: Bisect results --- ..._a4d66f29d257044e548313e014ca3dc3._comment | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment new file mode 100644 index 0000000000..fb36581912 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment @@ -0,0 +1,66 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Bisect results" + date="2011-04-03T06:22:15Z" + content=""" +Completed git-bisect twice, getting roughly the same results: + + 828a84ba3341d4b7a84292d8b9002a8095dd2382 is the first bad commit + commit 828a84ba3341d4b7a84292d8b9002a8095dd2382 + Author: Joey Hess + Date: Sat Mar 19 14:33:24 2011 -0400 + + Add version command to show git-annex version as well as repository version information. + + :040000 040000 ed849b7b6e9b177d6887ecebd6a0f146357824f3 1c98699dfd3fc3a3e2ce6b55150c4ef917de96e9 M Command + :100644 100644 b9c22bdfb403b0bdb1999411ccfd34e934f45f5c adf07e5b3e6260b296c982a01a73116b8a9a023c M GitAnnex.hs + :100644 100644 76dd156f83f3d757e1c20c80d689d24d0c533e16 d201cc73edb31f833b6d00edcbe4cf3f48eaecb0 M Upgrade.hs + :100644 100644 5f414e93b84589473af5b093381694090c278e50 d4a58d77a29a6a02daf13cec0df08b5aab74f65e M Version.hs + :100644 100644 f5c2956488a7afafd20374873d79579fb09b1677 f8cd577e992d38c7ec1438ce5c141eb0eb410243 M configure.hs + :040000 040000 f9b7295e997c0a5b1dda352f151417564458bd6e a30008475c1889f4fd8d60d4d9c982563380a692 M debian + :040000 040000 9d87a5d8b9b9fe7b722df303252ffd5760d66f75 08834f61a10d36651b3cdcc38389f45991acdf5e M doc + +contents of final refs/bisect: + + bad (828a84ba3341d4b7a84292d8b9002a8095dd2382) + good-33cb114be5135ce02671d8ce80440d40e97ca824 + good-942480c47f69e13cf053b8f50c98c2ce4eaa256e + good-ca48255495e1b8ef4bda5f7f019c482d2a59b431 + +\"roughly\" because second bisect gave two commits as a result, failing to build one of them (missing .o file on link, guess it's because of -j4 and bad deps in that version's build system): + + There are only 'skip'ped commits left to test. + The first bad commit could be any of: + 828a84ba3341d4b7a84292d8b9002a8095dd2382 + 5022a69e45a073046a2b14b6a4e798910c920ee9 + We cannot bisect more! + +Also noticed that \"git-annex-shell ...\" command succeeds if ran as root user, while failing from unprivileged one. +There are no permission/access errors in \"strace -f git-annex-shell ...\", so I guess it could be some bug in the GHC indeed. + +JIC, logged a whole second bisect operation. +Resulting log: [http://fraggod.net/static/share/git-annex-bisect.log](http://fraggod.net/static/share/git-annex-bisect.log) + +Bisect script I've used (git-annex-shell dies with error code 134 - SIGABRT on GHC error): + + res= + while true; do + if [[ -n \"$res\" ]]; then + cd /var/tmp/paludis/build/dev-scm-git-annex-scm.bak/work/git-annex-scm + echo \"---=== BISECT ($res) ===---\"; git bisect \"$res\" 2>&1; echo '---=== /BISECT ===---' + cd + rm -Rf /var/tmp/paludis/build/dev-scm-git-annex-scm + cp -a --reflink=auto /var/tmp/paludis/build/dev-scm-git-annex-scm{.bak,} + chown -R paludisbuild: /var/tmp/paludis/build/dev-scm-git-annex-scm + fi + res= + cave resolve -zx1 git-annex --skip-until-phase configure || res=skip + if [[ -z \"$res\" ]]; then + cd /remote/path + sudo -u user git-annex-shell 'sendkey' '/remote/path' 'SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b' '--' rsync --server --sender -vpe.Lsf --inplace . '' + if [[ $? -eq 134 ]]; then res=bad; else res=good; fi + cd + fi + done 2>&1 | tee ~/git-annex-bisect.log + +"""]] From 263524e2c45e985eb1f8c75caa4f74218c164d7d Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Sun, 3 Apr 2011 06:57:02 +0000 Subject: [PATCH 1390/2835] Added a comment --- ..._f5f1081eb18143383b2fb1f57d8640f5._comment | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment new file mode 100644 index 0000000000..491b537862 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment @@ -0,0 +1,38 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="comment 3" + date="2011-04-03T06:57:02Z" + content=""" +Repeated bisect with -j1, just to be sure it's not a random error, and it gave me 828a84ba3341d4b7a84292d8b9002a8095dd2382 again. +Guess I'll look through the changes there a bit later and try to revert these until it works. + +Not sure if it's repeatable by anyone but me (and hence worth fixing), but here's a bit more of info about the system: + + Exherbo linux + Linux sacrilege 2.6.38.2-fg.roam #4 SMP PREEMPT Mon Mar 28 21:08:47 YEKST 2011 i686 GNU/Linux + + dev-lang/ghc-7.0.2:7.0.2::installed + dev-haskell/HUnit-1.2.2.3:1.2.2.3::installed + dev-haskell/MissingH-1.1.0.3:1.1.0.3::installed + dev-haskell/QuickCheck-2.4.0.1:2.4.0.1::installed + dev-haskell/array-0.3.0.2:0.3.0.2::installed + dev-haskell/bytestring-0.9.1.7:0.9.1.7::installed + dev-haskell/containers-0.4.0.0:0.4.0.0::installed + dev-haskell/extensible-exceptions-0.1.1.2:0.1.1.2::installed + dev-haskell/filepath-1.2.0.0:1.2.0.0::installed + dev-haskell/hslogger-1.1.3:0::installed + dev-haskell/mtl-2.0.1.0:2.0.1.0::installed + dev-haskell/network-2.3.0.1:2.3.0.1::installed + dev-haskell/old-locale-1.0.0.2:1.0.0.2::installed + dev-haskell/parsec-3.1.0:3.1.0::installed + dev-haskell/pcre-light-0.4:0::installed + dev-haskell/regex-base-0.93.2:0.93.2::installed + dev-haskell/regex-compat-0.93.1:0.93.1::installed + dev-haskell/regex-posix-0.94.4:0.94.4::installed + dev-haskell/syb-0.3:0.3::installed + dev-haskell/transformers-0.2.2.0:0.2.2.0::installed + dev-haskell/utf8-string-0.3.6:0.3.6::installed + +(some stuff listed here as ::installed, but contains no files, since these packages detect whether ghc-7.0.2 already comes with the same/newer package version) + +"""]] From ed40974ed9e388639be0352946f3cab38c621552 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 07:43:37 +0000 Subject: [PATCH 1391/2835] Added a comment --- .../comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment new file mode 100644 index 0000000000..b4a5a72d01 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 13" + date="2011-04-03T07:43:37Z" + content=""" +Ok, thanks for the fix. It seems the fix isn't too reliable with my repos, I get different numbers of \"** No known copies of...\" in the various cloned repos that I have. After all the \"messing\" that I have done to my repos I think git-annex has gotten very confused. I will just leave things as they are and let git-annex slowly migrate over to the new format or re-clone from a linux source and see how things go. I will report back on this issue in abit after I use it more to see. +"""]] From d204b882afc310886b8d490163796a4c392f30ad Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 08:23:43 +0000 Subject: [PATCH 1392/2835] --- ..._-f_REMOTE_._doesn__39__t_work_as_expected.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn diff --git a/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn new file mode 100644 index 0000000000..b57471e5f8 --- /dev/null +++ b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn @@ -0,0 +1,14 @@ +I was testing out the fix/workaround for [[git-annex directory hashing problems on osx]] and I tried using the short forms of some of the commands i.e. + + git annex copy -f externalusb . + +which gives me + + git-annex: user error (option `-f' is ambiguous; could be one of: + -f --force allow actions that may lose annexed data + -f REMOTE --from=REMOTE specify from where to transfer content + + +I would have expected that since *--to* is the same as *-t* and *--from* is the same as *-f* as the in program documentation suggests. But *-f* clashes with the force command, I would suggest that the short form of *--force* be changed to *-F* and possibly rename the *Fast* commands to *Quick* and use *-Q* as the short form of the *Quick* operations. I didn't try the *-f* option with the move command, but it probably suffers from the same issue. It's probably better to avoid clashing short forms of command options. + +I guess this issue is just a documentation issue and a minor interface change if needed and not a bug of git-annex, but a quirk. From 529393a9e2f2954cc549204f501f4425c27e820f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 08:24:18 +0000 Subject: [PATCH 1393/2835] Added a comment --- ...omment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment new file mode 100644 index 0000000000..b92c3ab4ab --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 14" + date="2011-04-03T08:24:17Z" + content=""" +I meant to say in it wasn't reliable when I was following the instructions for \"Comment 12\". I did find that just doing a \"git annex copy -t externalusb .\" then a \"git annex drop .\" from the root of my cloned and \"none trusted\" annexed repos to be more reliable, it just means I temporarily need a load of space to get myself out of my earlier mess. + +On testing this bug fix, I found a minor behavioural issue with [[git annex copy -f REMOTE . doesn't work as expected]] +"""]] From c737e86b62bc4afcb9be105b0304dc50fe88783b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 08:55:18 +0000 Subject: [PATCH 1394/2835] Added a comment --- .../comment_2_40920b88537b7715395808d8aa94bf03._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment diff --git a/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment new file mode 100644 index 0000000000..215619043a --- /dev/null +++ b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-03T08:55:18Z" + content=""" +Given that the softlinks contain all needed information (if the object exists, locally), an emergency way to get files \"out\" of git-annex would be nice. I am aware that one can script it, but a canonical way is always better, especially when things go south. +"""]] From 4881a1833a110137f1320643ac01b33af6ea5e4c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 08:56:48 +0000 Subject: [PATCH 1395/2835] Added a comment --- .../comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment diff --git a/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment b/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment new file mode 100644 index 0000000000..28305d3ac8 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-03T08:56:48Z" + content=""" +Pity. Mark as done/upstream (or similar) for house-keeping? +"""]] From e8796f439d4d9d043c87479d1c2680d92c0d1fc8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 08:58:20 +0000 Subject: [PATCH 1396/2835] --- doc/walkthrough/recover_data_from_lost+found.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/recover_data_from_lost+found.mdwn b/doc/walkthrough/recover_data_from_lost+found.mdwn index f101b3caf6..48ef2a1d73 100644 --- a/doc/walkthrough/recover_data_from_lost+found.mdwn +++ b/doc/walkthrough/recover_data_from_lost+found.mdwn @@ -14,6 +14,6 @@ First, check out the git repository again. Then, in the new checkout: The way that works is that when git-annex adds the same content that was in the repository before, all the old links to that content start working -again. This works particularly well if the SHA1 backend is used, but even +again. This works particularly well if the SHA* backends are used, but even with the default backend it will work pretty well, as long as fsck preserved the modification time of the files. From 0e0341ff02bcfa18e250a6124c8d78c1cc3f07dd Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 09:00:17 +0000 Subject: [PATCH 1397/2835] Added a comment --- ...comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment new file mode 100644 index 0000000000..c9b74d98f0 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-03T09:00:17Z" + content=""" +I did not. Thanks :) + +This still means that you can't re-inject a new version of a file unless you have the old one if you are using a SHA* backend, but that might be a corner case anyway. +"""]] From 51eb3db90a2011c736729a645c0e7191f4a44573 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 09:03:22 +0000 Subject: [PATCH 1398/2835] Added a comment --- .../comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment new file mode 100644 index 0000000000..ec0f88d13c --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 4" + date="2011-04-03T09:03:22Z" + content=""" +Thanks a lot. I tried various howtos around the net, but none of them worked; yours did. (I tried it in one of the copies of the broken repo which I keep around for obvious reasons). +"""]] From 25fb248277a7d970e301806d2bcbec71e987b20b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 12:39:50 +0000 Subject: [PATCH 1399/2835] --- ...getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn new file mode 100644 index 0000000000..f5d5a66bfc --- /dev/null +++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn @@ -0,0 +1,9 @@ +I'm not sure if this is my stupidity or if it's a bug, but + + git annex copy --force --to REMOTE . + +just zip's through really quickly and doesn't actually force a copy to a remote location. This is just following up on the [[git-annex directory hashing problems on osx]]. I want to just do a force copy of all my data to my portable disk to really make sure that the data is really there. I would similarly would want to make sure I can force a + + git annex copy --force --from REMOTE . + +to pull down files from a remote. From 7ee30bd1ac68c47c009c703ce967f1362af96302 Mon Sep 17 00:00:00 2001 From: gernot Date: Sun, 3 Apr 2011 15:35:53 +0000 Subject: [PATCH 1400/2835] Added a comment --- ...2_e14e84b770305893f2fc6e4938359f47._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment new file mode 100644 index 0000000000..4fc9647e81 --- /dev/null +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="gernot" + ip="213.168.117.192" + subject="comment 2" + date="2011-04-03T15:35:52Z" + content=""" +'git add .git-annex' didn't do anything. That's when I noticed that this +repository is on a case-insensitive HFS+ file system. + +So, if I get this right it's not a new bug, but similar to this situation: +[[git-annex_directory_hashing_problems_on_osx]] + +Assuming that it was the file system's fault, I went ahead and upgraded yet +another clone. That one (on an ext3 file system) had neither staged changes +nor left-over untracked files. Everything seems to just have fallen right into +place. Is that possible or still weird? + +"""]] From 5f3385ce3b8aca1c06981a6c9e0684232ad59be6 Mon Sep 17 00:00:00 2001 From: gernot Date: Sun, 3 Apr 2011 15:41:01 +0000 Subject: [PATCH 1401/2835] Added a comment --- ...15_37f1d669c1fa53ee371f781c7bb820ae._comment | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment new file mode 100644 index 0000000000..d722d546a3 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="gernot" + ip="213.168.117.192" + subject="comment 15" + date="2011-04-03T15:41:00Z" + content=""" +I also ran into problems on a case-insensitive HFS+ file system, it seems. I +tried following the instructions in comment 12: + + 1. Remove everything in .git-annex besides uuid.log and trust.log + 2. git annex fsck --fast + 3. Commit + +However, I still see upper and lower case directories in .git-annex. Did I +misunderstand that they should all be lower case now? + +"""]] From 862df8aa214da146fd93ddff69dfd3aeb6e4d7b6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 16:02:33 +0000 Subject: [PATCH 1402/2835] Added a comment --- ..._8a4ab1af59098f4950726cf53636c2b3._comment | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment new file mode 100644 index 0000000000..97eab78c91 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 16" + date="2011-04-03T16:02:33Z" + content=""" +I think the correct steps should be, make a backup first :) then ... + +1. git pull # update your clone, and commit everything so you don't lose anything +2. git annex fsck --fast # check the repo first, just in case +3. rm -rf .git-annex/?? # remove the old metadata +4. git annex fsck --fast # get git annex to regenerate it all +5. push your changes out to your other repos, you will need to make sure git-annex is updated everywhere if there are remotes in your setup. + +I eventually migrated all of my own annex'd repos and I no longer have the old hashed directories but the new ones in the form + + .git/annex/aaa/bbb/foo.log + +I did lose some tracking information but not data (as far as I can see for now), but that was quickly fixed by pushing and pulling to my bare repo which tracks most of my data. + +I also found that it worked a bit more reliably for me on the copies of repos that were located on case sensitive filesystems, but I guess that was expected. +"""]] From 58493c064ba49158c7209adddea3558e4fa3ddd0 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 16:05:39 +0000 Subject: [PATCH 1403/2835] Added a comment --- .../comment_3_ec04e306c96fd20ab912aea54a8340aa._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment new file mode 100644 index 0000000000..99095c1569 --- /dev/null +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2011-04-03T16:05:39Z" + content=""" +Yes you seem to have come across the same bug that I had initially reported :P +"""]] From c90344908dfdab933e92d2008784298087aaaaa5 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 16:06:34 +0000 Subject: [PATCH 1404/2835] Added a comment --- ...comment_4_b1f818b85c3540591c48e7ba8560d070._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment new file mode 100644 index 0000000000..45d3d8bac4 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-04-03T16:06:34Z" + content=""" +Nice work on the bisection. It's obviously a compiler bug. Having two test cases that differ in only as trivial and innocous a commit as 828a84ba3341d4b7a84292d8b9002a8095dd2382 might help a GHC developer track it down. + +We should probably forward this as a GHC bug. I hope you can find a different version or build of GHC to build git-annex with. +"""]] From 623d612ebca0d4a231652918155c794afa43162b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 16:13:02 +0000 Subject: [PATCH 1405/2835] --- doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn diff --git a/doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn b/doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn new file mode 100644 index 0000000000..6299899e4f --- /dev/null +++ b/doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn @@ -0,0 +1,5 @@ +Given that git/config will have information on remotes and maybe costs, it might be a good idea to do a simple round robin selection of remotes to download files where the costs are the same. + +This of course assumes that we like the idea of "parallel" launching and running of curl/rsync processes... + +This wish item is probably only useful for the paranoid people who store more than 1 copy of their data. From 216ad1a4d3434306cf3338217fbcf5cbe6e1c8d6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 12:18:38 -0400 Subject: [PATCH 1406/2835] Clear up short option confusion between --from and --force (-f is now --from, and there is no short option for --force). --- Options.hs | 2 +- debian/changelog | 2 ++ ..._annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Options.hs b/Options.hs index 10c3714e41..31b73e25a8 100644 --- a/Options.hs +++ b/Options.hs @@ -20,7 +20,7 @@ type Option = OptDescr (Annex ()) commonOptions :: [Option] commonOptions = - [ Option ['f'] ["force"] (NoArg (setforce True)) + [ Option [] ["force"] (NoArg (setforce True)) "allow actions that may lose annexed data" , Option ['F'] ["fast"] (NoArg (setfast True)) "avoid slow operations" diff --git a/debian/changelog b/debian/changelog index e504bd8f60..71ea599cd9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,8 @@ git-annex (0.20110402) UNRELEASED; urgency=low * Unused files on remotes, particulary special remotes, can now be identified and dropped, by using "--from remote" with git annex unused and git annex dropunused. + * Clear up short option confusion between --from and --force (-f is now + --from, and there is no short option for --force). -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 diff --git a/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn index b57471e5f8..3bda451499 100644 --- a/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn +++ b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn @@ -12,3 +12,7 @@ which gives me I would have expected that since *--to* is the same as *-t* and *--from* is the same as *-f* as the in program documentation suggests. But *-f* clashes with the force command, I would suggest that the short form of *--force* be changed to *-F* and possibly rename the *Fast* commands to *Quick* and use *-Q* as the short form of the *Quick* operations. I didn't try the *-f* option with the move command, but it probably suffers from the same issue. It's probably better to avoid clashing short forms of command options. I guess this issue is just a documentation issue and a minor interface change if needed and not a bug of git-annex, but a quirk. + +> Yeah, -f needs to be from; -F was already --fast. I have made --force not +> have any short option abbreviation, I think it's entirely reasonable to +> avoid fat-fingering an option that can lose data. [[done]] --[[Joey]] From c93ce86b008ef7d93180b64ded12443e6a0c115f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 12:21:41 -0400 Subject: [PATCH 1407/2835] close --- doc/bugs/Displayed_copy_speed_is_wrong.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/bugs/Displayed_copy_speed_is_wrong.mdwn b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn index c1209c73c0..cf3b31cf48 100644 --- a/doc/bugs/Displayed_copy_speed_is_wrong.mdwn +++ b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn @@ -3,3 +3,6 @@ When copying data to my remote, I regularly see speeds in excess of 100 MB/s on 2073939 100% 176.96MB/s 0:00:00 (xfer#1, to-check=0/1) This is definitely not correct. + +> Closing, as rsync does this to show you when it's making your life +> faster than it would be w/o rsync. [[done]] --[[Joey]] From ee313074ed65145692acff1402e6262c2ea4b437 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 12:27:13 -0400 Subject: [PATCH 1408/2835] close --- ...de_left_untracked_.git-annex__47____42___directories.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn index ab8b255a0f..7fdbc3ca4e 100644 --- a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn @@ -9,3 +9,8 @@ wrong. At the time I had git-annex version 0.20110329 and I've been using the SHA1 backend since version 1. + +> Yes, I agree with Jimmy, it's the same bug. So I'll be closing this one. +> Please keep us informed how the workaround committed to git-annex +> yesterday for the case insensativity issue works out. [[dup|done]] +> --[[Joey]] From 5c2e08c3f6211720a5be0e288744fd13f8f16d36 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 16:39:35 +0000 Subject: [PATCH 1409/2835] Added a comment --- .../comment_1_460335b0e59ad03871c524f1fe812357._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment diff --git a/doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment b/doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment new file mode 100644 index 0000000000..6a5fd3d530 --- /dev/null +++ b/doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T16:39:35Z" + content=""" +I dunno about parrallel downloads -- eek! -- but there is at least room for improvement of what \"git annex get\" does when there are multiple remotes that have a file, and the one it decides to use is not available, or very slow, or whatever. +"""]] From 199e897b15aef307073d49199b561344f17a62e2 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 16:49:01 +0000 Subject: [PATCH 1410/2835] Added a comment --- ...ent_1_3deb2c31cad37a49896f00d600253ee3._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment new file mode 100644 index 0000000000..d2692f26f0 --- /dev/null +++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T16:49:01Z" + content=""" +How remote is REMOTE? If it's a directory on the same computer, then git-annex copy --to is actually quickly checking that each file is present on the remote, and when it is, skipping copying it again. + +If the remote is ssh, git-annex copy talks to the remote to see if it has the file. This makes copy --to slow, as Rich [[complained_before|forum/batch_check_on_remote_when_using_copy]]. :) + +So, copy --to does not trust location tracking information (unless --fast is specified), which means that it should be doing exactly what you want it to do in your situation -- transferring every file that is really not present in the destination repository already. + +Neither does copy --from, by the way. It always checks if each file is present in the current repository's annex before trying to download it. +"""]] From 3c0835e542070da7dd4b401e54d38a4c4961639b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 16:53:51 +0000 Subject: [PATCH 1411/2835] Added a comment --- .../comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment new file mode 100644 index 0000000000..f7feac67cf --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 17" + date="2011-04-03T16:53:51Z" + content=""" +@gernot step 0 is to upgrade git-annex to current git, on all systems where you use it, in case that wasn't clear. + +"""]] From 54b903cb54adb6b0cac4308c5974342f0bea1e67 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 16:59:48 +0000 Subject: [PATCH 1412/2835] Added a comment --- ...mment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment new file mode 100644 index 0000000000..1079303197 --- /dev/null +++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2011-04-03T16:59:47Z" + content=""" +Remote as in \"another physical machine\". I assumed that + + git annex copy --force --to REMOTE . + +would have not trusted the contents in the current directory (or the remote that is being copied to) and then just go off and re-download/upload all the files and overwrite what is already there. I expected the combination of *--force* and copy *--to* that it would not bother to check if the files are there or not and just copy it regardless of the outcome. +"""]] From 8345a90e66474b9458c9d11d4b66ef01bdf1cfdd Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 17:12:36 +0000 Subject: [PATCH 1413/2835] Added a comment --- .../comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment new file mode 100644 index 0000000000..c3df214988 --- /dev/null +++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2011-04-03T17:12:35Z" + content=""" +On second thought maybe the current behaviour is better than what I am suggesting that the force command should do. I guess it's better to be safe than sorry. +"""]] From 83acc9ba52ecba85180355a8c08311bd4826ed0f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 14:34:00 -0400 Subject: [PATCH 1414/2835] encryption design document --- ...e_same_key_for_encryption_and_hashing.mdwn | 3 + doc/design.mdwn | 4 + doc/design/encryption.mdwn | 108 ++++++++++++++++++ doc/special_remotes/Amazon_S3.mdwn | 17 +-- 4 files changed, 116 insertions(+), 16 deletions(-) create mode 100644 doc/design.mdwn create mode 100644 doc/design/encryption.mdwn diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn index 0ec66652e2..1980a8f444 100644 --- a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn @@ -3,3 +3,6 @@ While using HMAC instead of "plain" hash functions is inherently more secure, it Also, ttbomk, HMAC needs two keys, not one. Are you re-using the same key twice? Compability for old buckets and support for different ones can be maintained by introducing a new option and simply copying over the encryption key's identifier into this new option should it be missing. + +> See [[design/encryption]]. I don't think this bug needs to be kept +> open. [[done]] --[[Joey]] diff --git a/doc/design.mdwn b/doc/design.mdwn new file mode 100644 index 0000000000..dc66d5c80a --- /dev/null +++ b/doc/design.mdwn @@ -0,0 +1,4 @@ +git-annex's high-level design is mostly inherent in the data that it +stores in git, and alongside git. See [[internals]] for details. + +See [[encryption]] for design of encryption elements. diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn new file mode 100644 index 0000000000..003336dd3e --- /dev/null +++ b/doc/design/encryption.mdwn @@ -0,0 +1,108 @@ +git-annex mostly does not use encryption. Anyone with access to a git +repository can see all the filenames in it, its history, and can access +any annexed file contents. + +Encryption is needed when using [[special_remotes]] like Amazon S3, where +file content is sent to an untrusted party who does not have access to the +git repository. + +Such an encrypted remote uses strong encryption on the contents of files, +as well as the filenames. The size of the encrypted files, and access +patterns of the data, should be the only clues to what type of is stored in +such a remote. + +## encryption backends + +It makes sense to support multiple encryption backends. So, there +should be a way to tell what backend is responsible for a given filename +in an encrypted remote. (And since special remotes can also store files +unencrypted, differentiate from those as well.) + +At a high level, an encryption backend needs to support these operations: + +* Given a key/value backend key, produce and return an encrypted key. + + The same naming scheme git-annex uses for keys in regular key/value + [[backends]] can be used. So a filename for a key might be + "GPG-s12345--armoureddatahere" + +* Given a streaming source of file content, encrypt it, and send it in + a stream to an action that consumes the encrypted content. + +* Given a streaming source of encrypted content, decrypt it, and send + it in a stream to an anction that consumes the decrypted content. + +* Initialize itself. + +* Clean up. + +* Configure an encryption key to use. + +The rest of this page will describe a single encryption backend using GPG. +Probably only one will be needed, but who knows? Maybe that backend will +turn out badly designed, or some other encryptor needed. Designing +with more than one encryption backend in mind helps future-proofing. + +## encryption key management + +[[!template id=note text=""" +The basis of this scheme was originally developed by Lars Wirzenius et al +[for Obnam](http://braawi.org/obnam/encryption/). +"""]] + +Data is encrypted by gpg, using a symmetric cipher. The passphrase of the +cipher is itself checked into your git repository, encrypted using one or +more gpg public keys. This scheme allows new gpg private keys to be given +access to content that has already been stored in the remote. + +Different encrypted remotes need to be able to each use different ciphers. +There does not seem to be a benefit to allowing multiple cipers to be +used within a single remote, and it would add a lot of complexity. +Instead, if you want a new cipher, create a new S3 bucket, or whatever. +There does not seem to be much benefit to using the same cipher for +two different enrypted remotes. + +So, the encrypted cipher could just be stored with the rest of a remote's +configuration in `.git-annex/remotes.log` (see [[internals]]). When `git +annex intiremote` makes a remote, it can generate a random symmetric +cipher, and encrypt it with the specified gpg key. To allow another gpg +public key access, update the encrypted cipher to be encrypted to both gpg +keys. + +## filename enumeration + +If the names of files are encrypted, this makes it harder for +git-annex (let alone untrusted third parties!) to get a list +of the files that are stored on a given enrypted remote. This has been +a concern, and it has been considered to use a hash like HMAC, rather +than gpg encrypting filenames, to make it easier. (For git-annex, but +possibly also for attackers!) But, does git-annex really ever need to do +such an enumeration? + +Apparently not. `git annex unused --from remote` can now check for +unused data that is stored on a remote, and it does so based only on +location log data for the remote. This assumes that the location log is +kept accurately. + +What about `git annex fsck --from remote`? Such a command should be able to, +for each file in the repository, contact the encrypted remote to check +if it has the file. This can be done without enumeration, although it will +mean running gpg once per file fscked, to get the encrypted filename. + +### risks + +A risk of this scheme is that, once the symmetric cipher has been obtained, it +allows full access to all the encrypted content. This scheme does not allow +revoking a given gpg key access to the cipher, since anyone with such a key +could have already decrypted the cipher and stored a copy. + +If git-annex stores the decrypted symmetric cipher in memory, then there +is a risk that it could be intercepted from there by an attacker. Gpg +amelorates these type of risks by using locked memory. + +This design does not support obfuscating the size of files by chunking +them, as that would have added a lot of complexity, for dubious benefits. +If the untrusted party running the encrypted remote wants to know file sizes, +they could correlate chunks that are accessed together. Enctypting data +changes the original file size enough to avoid it being used as a direct +fingerprint at least. diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn index 384110d1df..2cf23187d1 100644 --- a/doc/special_remotes/Amazon_S3.mdwn +++ b/doc/special_remotes/Amazon_S3.mdwn @@ -37,19 +37,4 @@ only be used for public data. ** Encryption is not yet supported. ** -When encryption is enabled, all files stored in the bucket are -encrypted with gpg. Additionally, the filenames themselves are encrypted -(using HMAC). The size of the encrypted files, and -access patterns of the data, should be the only clues to what type of -data you are storing in S3. - -[[!template id=note text=""" -This scheme was originally developed by Lars Wirzenius et al -[for Obnam](http://braawi.org/obnam/encryption/). -"""]] -The data stored in S3 is encrypted by gpg with a symmetric cipher. The -passphrase of the cipher is itself checked into your git repository, -encrypted using one or more gpg public keys. This scheme allows new private -keys to be given access to a bucket's content, after the bucket is created -and is in use. The symmetric cipher is also hashed together with filenames -used in the bucket, in order to obfuscate the filenames. +See [[design/encryption]]. From dbe41e667bba1096de8d60b75f932efcbf674f85 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 14:43:38 -0400 Subject: [PATCH 1415/2835] update --- doc/design/encryption.mdwn | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 003336dd3e..43d8119e3b 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -20,6 +20,13 @@ unencrypted, differentiate from those as well.) At a high level, an encryption backend needs to support these operations: +* Create a new encrypted cipher, or update the cipher. Some input + parameters will specifiy things like the gpg public keys that + can access the cipher. + +* Initialize an instance of the encryption backend, that will use a + specified encrypted cipher. + * Given a key/value backend key, produce and return an encrypted key. The same naming scheme git-annex uses for keys in regular key/value @@ -32,8 +39,6 @@ At a high level, an encryption backend needs to support these operations: * Given a streaming source of encrypted content, decrypt it, and send it in a stream to an anction that consumes the decrypted content. -* Initialize itself. - * Clean up. * Configure an encryption key to use. From 8c9d9eb8af88035a05378214e86b679fce091acf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 14:47:43 -0400 Subject: [PATCH 1416/2835] update --- doc/design/encryption.mdwn | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 43d8119e3b..c9b1bdb5dc 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -11,6 +11,8 @@ as well as the filenames. The size of the encrypted files, and access patterns of the data, should be the only clues to what type of is stored in such a remote. +[[!toc]] + ## encryption backends It makes sense to support multiple encryption backends. So, there @@ -94,7 +96,7 @@ for each file in the repository, contact the encrypted remote to check if it has the file. This can be done without enumeration, although it will mean running gpg once per file fscked, to get the encrypted filename. -### risks +## risks A risk of this scheme is that, once the symmetric cipher has been obtained, it allows full access to all the encrypted content. This scheme does not allow @@ -108,6 +110,6 @@ amelorates these type of risks by using locked memory. This design does not support obfuscating the size of files by chunking them, as that would have added a lot of complexity, for dubious benefits. If the untrusted party running the encrypted remote wants to know file sizes, -they could correlate chunks that are accessed together. Enctypting data +they could correlate chunks that are accessed together. Encrypting data changes the original file size enough to avoid it being used as a direct fingerprint at least. From 0d1f2023340dd30e81bc003144a37e0fe03c333b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 14:53:12 -0400 Subject: [PATCH 1417/2835] update --- doc/design/encryption.mdwn | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index c9b1bdb5dc..72a7ad286e 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -78,13 +78,10 @@ keys. ## filename enumeration -If the names of files are encrypted, this makes it harder for -git-annex (let alone untrusted third parties!) to get a list -of the files that are stored on a given enrypted remote. This has been -a concern, and it has been considered to use a hash like HMAC, rather -than gpg encrypting filenames, to make it easier. (For git-annex, but -possibly also for attackers!) But, does git-annex really ever need to do -such an enumeration? +If the names of files are encrypted or securely hashed, or whatever is +chosen, this makes it harder for git-annex (let alone untrusted third parties!) +to get a list of the files that are stored on a given enrypted remote. +But, does git-annex really ever need to do such an enumeration? Apparently not. `git annex unused --from remote` can now check for unused data that is stored on a remote, and it does so based only on From 9c4285406b465d324f7b48d81e48614c27bc48ff Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 19:40:13 +0000 Subject: [PATCH 1418/2835] Added link to design --- doc/index.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index cb2f485c92..ea0c830bc6 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -10,6 +10,7 @@ To get a feel for it, see the [[walkthrough]]. * [[bugs]] * [[todo]] * [[forum]] +* [[design]] * [[comments]] * [[contact]] * Flattr this From fef63846b9e990c2b3d9c7f3e88af5a05fa3b543 Mon Sep 17 00:00:00 2001 From: gernot Date: Sun, 3 Apr 2011 19:46:17 +0000 Subject: [PATCH 1419/2835] Added a comment --- ...nt_18_db64c91dd1322a0ab168190686db494f._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment new file mode 100644 index 0000000000..550558ec16 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="gernot" + ip="213.168.117.192" + subject="comment 18" + date="2011-04-03T19:46:16Z" + content=""" +Joey, sorry, I got it wrong. I thought upgrading git didn't help and you +adjusted things in git-annex instead. + +Anyway, can I get around upgrading on all hosts by reformatting the drive to +case-sensitive HFS+? Or will I have to upgrade git (currently version 1.7.2.5) +eventually anyway? + +"""]] From 261b1e6310885fcad3b50c8cd7240ccdc5ed54a9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 15:51:24 -0400 Subject: [PATCH 1420/2835] update --- doc/design/encryption.mdwn | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 72a7ad286e..0242aabebb 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -102,8 +102,11 @@ could have already decrypted the cipher and stored a copy. If git-annex stores the decrypted symmetric cipher in memory, then there is a risk that it could be intercepted from there by an attacker. Gpg -amelorates these type of risks by using locked memory. - +amelorates these type of risks by using locked memory. For git-annex, note +that an attacker with local machine access can tell at least all the +filenames and metadata of files stored in the encrypted remote anyway, +and can access whatever content is stored locally. + This design does not support obfuscating the size of files by chunking them, as that would have added a lot of complexity, for dubious benefits. If the untrusted party running the encrypted remote wants to know file sizes, From 3e0119a89f8968128f5436a68171279e5c6e2295 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 15:53:40 -0400 Subject: [PATCH 1421/2835] move --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index ea0c830bc6..b5880823e1 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -10,7 +10,6 @@ To get a feel for it, see the [[walkthrough]]. * [[bugs]] * [[todo]] * [[forum]] -* [[design]] * [[comments]] * [[contact]] * Flattr this @@ -46,6 +45,7 @@ files with git. * [[key-value backends|backends]] for data storage * [[internals]] * [[bare_repositories]] +* [[design]] * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. From 6fd8efbc3cc754f574c022f325e8a442fe5cb02c Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 19:53:44 +0000 Subject: [PATCH 1422/2835] Added a comment --- .../comment_19_ff555c271637af065203ca99c9eeaf89._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment new file mode 100644 index 0000000000..2676b35897 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 19" + date="2011-04-03T19:53:44Z" + content=""" +Git does not need to be upgraded. Git-annex needs to be upgraded to git rev 616e6f8a840ef4d99632d12a2e7ea15c3cfb1805 or newer, on all machines. +"""]] From 218c58f3c9d08e981a4dbade8aa9c9acc0facaf1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 20:03:14 +0000 Subject: [PATCH 1423/2835] Added a comment --- ...mment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment diff --git a/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment b/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment new file mode 100644 index 0000000000..f2ecc46d0a --- /dev/null +++ b/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-04-03T20:03:14Z" + content=""" +New encryption keys could be used for different directories/files/patterns/times/whatever. One could then encrypt this new key for the public keys of other people/machines and push them out along with the actual data. This would allow some level of access restriction or future revocation. git-annex would need to keep track of which files can be decrypted with which keys. I am undecided if that information needs to be encrypted or not. + +Encrypted object files should be checksummed in encrypted form so that it's possible to verify integrity without knowing any keys. Same goes for encrypted keys, etc. + +Chunking files in this context seems like needless overkill. This might make sense to store a DVD image on CDs or similar, at some point. But not for encryption, imo. Coming up with sane chunk sizes for all use cases is literally impossible and as you pointed out, correlation by the remote admin is trivial. +"""]] From b73accb4831adab09d4059d05e68b40fac7fa482 Mon Sep 17 00:00:00 2001 From: "http://dieter-be.myopenid.com/" Date: Sun, 3 Apr 2011 20:30:22 +0000 Subject: [PATCH 1424/2835] Added a comment: symlinks --- ...mment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment diff --git a/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment b/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment new file mode 100644 index 0000000000..3a223e1c7b --- /dev/null +++ b/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://dieter-be.myopenid.com/" + nickname="dieter" + subject="symlinks" + date="2011-04-03T20:30:21Z" + content=""" +> (Sadly, it cannot create a symlink, as git still wants to write the file afterwards. +> So the nice current behavior of unavailable files being clearly missing due to dangling symlinks, would be lost when using smudge/clean filters. (Contact git developers to get an interface to do this?) + +Have you checked what the smudge filter sees when the input is a symlink? Because git supports tracking symlinks, so it should also support pushing symlinks through a smudge filter, right? +Either way: yes, contact the git devs, one can only ask and hope. And if you can demonstrate the awesomeness of git-annex they might get more 1interested :) +"""]] From 6ee8c9ecaa7b6a3567b3d225db041c3832b4f977 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 22:59:46 +0000 Subject: [PATCH 1425/2835] --- ...t_--_same_as_get__44___but_for_defaults.mdwn | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn diff --git a/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn new file mode 100644 index 0000000000..9cd56749e8 --- /dev/null +++ b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn @@ -0,0 +1,17 @@ +I am running centralized git-annex exclusively. + +Similar to + + git annex get + +I'd like to have a + + git annex put + +which would put all files on the default remote(s). + +My main reason for not wanting to use copy --to is that I need to specify the remote's name in this case which makes writing a wrapper unnecessarily hard. Also, this would allow + + mr push + +to do the right thing all by itself. From 81ae895aeb054bc9efbad15732046b182cda9d19 Mon Sep 17 00:00:00 2001 From: fmarier Date: Mon, 4 Apr 2011 07:40:43 +0000 Subject: [PATCH 1426/2835] Added a comment: Exporting to a FAT filesystem? --- ...mment_1_04bcc4795d431e8cb32293aab29bbfe2._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment diff --git a/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment b/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment new file mode 100644 index 0000000000..510e449842 --- /dev/null +++ b/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="fmarier" + ip="121.73.248.43" + subject="Exporting to a FAT filesystem?" + date="2011-04-04T07:40:41Z" + content=""" +I'm using git-annex to keep my music in sync between all of my different machines. What I'd love to be able to do is to also keep it in sync with my iRiver player. Unfortunately, the firmware, Rockbox, doesn't support ext3, so I'm stuck with a FAT filesystem. + +I can see how the design of git-annex makes it rather difficult to get rid of the symlinks, so how about taking a different approach: something like a \"git annex export DEST\" which would take a destination (not a git remote) and rsync the content over to there as regular files. + +Maybe \"git annex sync DEST\" or \"git annex rsync DEST\" would be better names if we want to convey the idea that the destination will be made to look like the source repo, including performing the necessary deletions. +"""]] From 19782240b9c493cff0121487a103a7bfc3d325b9 Mon Sep 17 00:00:00 2001 From: fmarier Date: Mon, 4 Apr 2011 07:43:08 +0000 Subject: [PATCH 1427/2835] --- doc/users/fmarier.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/users/fmarier.mdwn diff --git a/doc/users/fmarier.mdwn b/doc/users/fmarier.mdwn new file mode 100644 index 0000000000..ecf3426978 --- /dev/null +++ b/doc/users/fmarier.mdwn @@ -0,0 +1,6 @@ +# François Marier + +Free Software and Debian Developer. Lead developer of [Libravatar](http://www.libravatar.org) + +* [Blog](http://feeding.cloud.geek.nz) +* [Identica](http://identi.ca/fmarier) / [Twitter](http://twitter.com/fmarier) From 8527643324102de36bb43785a14cf0a5020004e7 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 4 Apr 2011 10:19:02 +0000 Subject: [PATCH 1428/2835] --- ...or_bug:_errors_are_not_verbose_enough.mdwn | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn diff --git a/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn b/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn new file mode 100644 index 0000000000..8def2e8c3f --- /dev/null +++ b/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn @@ -0,0 +1,24 @@ +Current: + + % git annex status + git-annex: unknown command + +Better: + + % git annex status + git-annex: status: unknown command + +Current: + + % git annex fsck + [...] + git-annex: 18 failed + +Better: + + % git annex fsck + [...] + git-annex: fsck: 18 failed + + +etc pp. From 27eaab346903aa26c7639014dd26520b8d8da95c Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 4 Apr 2011 18:13:46 +0000 Subject: [PATCH 1429/2835] Added a comment --- ...comment_1_d5413c8acce308505e4e2bec82fb1261._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment diff --git a/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment new file mode 100644 index 0000000000..fe1d5520f4 --- /dev/null +++ b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-04T18:13:46Z" + content=""" +This begs the question: What is the default remote? It's probably *not* the same repository that git's master branch is tracking (ie, origin/master). It seems there would have to be an annex.defaultremote setting. + +BTW, mr can easily be configured on a per-repo basis so that \"mr push\" copies to somewhere: `push = git push; git annex push wherever` +"""]] From 8528a39f77477dd96c71ce92b87395920c5b7e2b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 4 Apr 2011 18:20:45 +0000 Subject: [PATCH 1430/2835] Added a comment --- ...ent_2_bb4a97ebadb5c53809fc78431eabd7c8._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment diff --git a/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment b/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment new file mode 100644 index 0000000000..7618c9a7b6 --- /dev/null +++ b/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-04T18:20:45Z" + content=""" +Hey @fmarier. Well, this bug report is closed because you can already get rid of the symlinks. Just put a bare git repo on your fat filesystem, and use git-annex copy --to/--from there. + +Now, that puts all the files that are on the device in .git/annex/objects/xx/yy/blah.mp3 -- how well rockbox would support that I don't know. And if it tries to modify or delete those files, git annex also can't help you manage those changes. + +Another recent option is the [[special_remotes/directory]] special remote type, which again uses \"xx/yy/blah.mp3\" and can't track changes made to the files. This could perhaps be extended in the direction you suggest, although trying to fit this into the special remote infrastructure might not be a good fit really. + +The most likely way this has to get dealt with is really by using [[todo/smudge]] filters, which would eliminate the symlinks and allow copying a non-bare git repo onto vfat. +"""]] From 9a8861c08f4856594952a4a68f7024d67f958187 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 4 Apr 2011 20:45:31 +0000 Subject: [PATCH 1431/2835] Added a comment --- ...mment_2_0aa227c85d34dfff4e94febca44abea8._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment diff --git a/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment new file mode 100644 index 0000000000..3090b575b7 --- /dev/null +++ b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-04T20:45:30Z" + content=""" +In my case, the remotes are the same, but adding a new option could make sense. + +And while I can tell mr what to do explicitly, I would prefer if it did the right thing all by itself. Having to change configs in two separate places is less than ideal. + +I am not sure what you mean by `git annex push` as that does not exist. Did you mean copy? +"""]] From 708ae036819b9efe69e31cfacea78f8933779885 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 4 Apr 2011 23:15:05 +0000 Subject: [PATCH 1432/2835] --- ...stormning:_git_annex_push___38___pull.mdwn | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/forum/bainstormning:_git_annex_push___38___pull.mdwn diff --git a/doc/forum/bainstormning:_git_annex_push___38___pull.mdwn b/doc/forum/bainstormning:_git_annex_push___38___pull.mdwn new file mode 100644 index 0000000000..a2d320e352 --- /dev/null +++ b/doc/forum/bainstormning:_git_annex_push___38___pull.mdwn @@ -0,0 +1,21 @@ +Wouldn't it make sense to offer + + git annex pull + +which would basically do + + git pull + git annex get + +and + + git annex push + +which would do + + git annex commit . + git annex put # (the proposed "send to default annex" command) + git commit -a -m "$HOST $(date +%F-%H-%M-%S)" # or similar + git push + +Resulting in commands that are totally analogous to git push & pull: Sync all data from/to a remote. From 5dea16889e42042a0841662d9e3b2c7be2ca4923 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM" Date: Tue, 5 Apr 2011 04:37:52 +0000 Subject: [PATCH 1433/2835] --- doc/forum/Problems_with_large_numbers_of_files.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files.mdwn diff --git a/doc/forum/Problems_with_large_numbers_of_files.mdwn b/doc/forum/Problems_with_large_numbers_of_files.mdwn new file mode 100644 index 0000000000..943b979d6e --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files.mdwn @@ -0,0 +1,8 @@ +I'm trying to use git-annex to archive scientific data. I'm often dealing with large numbers of files, sometimes 10k or more. When I try to git-annex add these files I get this error: + + +Stack space overflow: current size 8388608 bytes. +Use `+RTS -Ksize' to increase it. + + +This is with the latest version of git-annex and a current version of git on OS 10.6.7. After this error occurs, I am unable to un-annex the files and I'm forced to recover from a backup. From 43dac0b4c96e04144e125430d7bbabc80dc4b2d8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM" Date: Tue, 5 Apr 2011 04:38:18 +0000 Subject: [PATCH 1434/2835] --- doc/forum/Problems_with_large_numbers_of_files.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/forum/Problems_with_large_numbers_of_files.mdwn b/doc/forum/Problems_with_large_numbers_of_files.mdwn index 943b979d6e..1dbddd3e28 100644 --- a/doc/forum/Problems_with_large_numbers_of_files.mdwn +++ b/doc/forum/Problems_with_large_numbers_of_files.mdwn @@ -1,8 +1,8 @@ I'm trying to use git-annex to archive scientific data. I'm often dealing with large numbers of files, sometimes 10k or more. When I try to git-annex add these files I get this error: -Stack space overflow: current size 8388608 bytes. -Use `+RTS -Ksize' to increase it. + Stack space overflow: current size 8388608 bytes. + Use `+RTS -Ksize' to increase it. This is with the latest version of git-annex and a current version of git on OS 10.6.7. After this error occurs, I am unable to un-annex the files and I'm forced to recover from a backup. From ef67b19c808d373783b9d74bdc61e69dff3c88ca Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Tue, 5 Apr 2011 07:27:47 +0000 Subject: [PATCH 1435/2835] Added a comment --- ..._08791cb78b982087c2a07316fe3ed46c._comment | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment new file mode 100644 index 0000000000..94043a7001 --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2011-04-05T07:27:46Z" + content=""" +Heh, cool, I was thinking throwing about 28million files at git-annex. Let me know how it goes, I suspect you have just run into a default limits OSX problem. + +You probably just need to up some system limits (you will need to read the error messages that first appear) then do something like + +
+# this is really for the run time, you can set these settings in /etc/sysctl.conf
+sudo sysctl -w kern.maxproc=2048
+sudo sysctl -w kern.maxprocperuid=1024
+
+# tell launchd about having higher limits
+sudo echo \"limit maxfiles 1024 unlimited\" >> /etc/launchd.conf
+sudo echo \"limit maxproc 1024 2048\" >> /etc/launchd.conf
+
+ +There are other system limits which you can check by doing a \"ulimit -a\", once you make the above changes, you will need to reboot to make the changes take affect. I am unsure if the above will help as it is an example of what I did on 10.6.6 a few months ago to fix some forking issues. From the error you got you will probably need to increase the stacksize to something bigger or even make it unlimited if you feel lucky, the default stacksize on OSX is 8192, try making it say 10times that size first and see what happens. +"""]] From 24e2c13387179d3ca1ed2dd50f5d3fbafbc8f32e Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 5 Apr 2011 17:46:03 +0000 Subject: [PATCH 1436/2835] Added a comment --- ..._0392a11219463e40c53bae73c8188b69._comment | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment new file mode 100644 index 0000000000..8ea5531f43 --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-05T17:46:03Z" + content=""" +This message comes from ghc's runtime memory manager. Apparently your ghc defaults to limiting the stack to 80 mb. +Mine seems to limit it slightly higher -- I have seen haskell programs successfully grow as large as 350 mb, although generally not intentionally. :) + +Here's how to adjust the limit at runtime, obviously you'd want a larger number: + +
+# git-annex +RTS -K100 -RTS find
+Stack space overflow: current size 100 bytes.
+Use `+RTS -Ksize -RTS' to increase it.
+
+ +I've tried to avoid git-annex using quantities of memory that scale with the number of files in the repo, and I think in general successfully -- I run it on 32 mb and 128 mb machines, FWIW. There are some tricky cases, and haskell makes it easy to accidentally write code that uses much more memory than would be expected. + +One well known case is `git annex unused`, which *has* to build a structure of every annexed file. I have been considering using a bloom filter or something to avoid that. + +Another possible case is when running a command like `git annex add`, and passing it a lot of files/directories. Some code tries to preserve the order of your input after passing it through `git ls-files` (which destroys ordering), and to do so it needs to buffer both the input and the result in ram. + +It's possible to build git-annex with memory profiling and generate some quite helpful profiling data. Edit the Makefile and add this to GHCFLAGS: `-prof -auto-all -caf-all -fforce-recomp` then when running git-annex, add the parameters: `+RTS -p -RTS` , and look for the git-annex.prof file. +"""]] From 094983a2bdbb7cbf2aea96addd3afe1ddedc6c69 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 5 Apr 2011 14:00:51 -0400 Subject: [PATCH 1437/2835] support PROFILE=1 to enable profiling --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6a1531a3c8..24f1b1e1f9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ PREFIX=/usr -GHCFLAGS=-O2 -Wall -ignore-package monads-fd +IGNORE=-ignore-package monads-fd +GHCFLAGS=-O2 -Wall -ignore-package $(IGNORE) +ifdef PROFILE +GHCFLAGS=-prof -auto-all -caf-all -fforce-recomp $(IGNORE) +endif GHCMAKE=ghc $(GHCFLAGS) --make bins=git-annex git-annex-shell From 2bd955bc65f0fdd2f48a5514bcf343d880295068 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 5 Apr 2011 18:02:05 +0000 Subject: [PATCH 1438/2835] Added a comment --- .../comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment new file mode 100644 index 0000000000..8e4101e37e --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-05T18:02:05Z" + content=""" +Oh, you'll need profiling builds of various haskell libraries to build with profiling support. If that's not easily accomplished, if you could show me the form of the command you're running, and also how git annex unannex fails, that would be helpful for investigating. +"""]] From 237e96bc6a3504525e82eadb8fc9c5eea6e94d66 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 5 Apr 2011 18:05:00 +0000 Subject: [PATCH 1439/2835] Added a comment --- ...omment_1_3a0bf74b51586354b7a91f8b43472376._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment diff --git a/doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment b/doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment new file mode 100644 index 0000000000..3d69e8f290 --- /dev/null +++ b/doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-05T18:05:00Z" + content=""" +Maybe, otoh, part of the point of git-annex is that the data may be too large to pull down all of it. + +I find mr useful as a policy layer over top of git-annex, so \"mr update\" can pull down appropriate quantities of data from +appropriate locations. +"""]] From a301a38d9969febdea3a4f3d3eb2d98077d3d66f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 5 Apr 2011 14:29:44 -0400 Subject: [PATCH 1440/2835] redundancy --- doc/design/encryption.mdwn | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 0242aabebb..f8f8656a74 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -43,8 +43,6 @@ At a high level, an encryption backend needs to support these operations: * Clean up. -* Configure an encryption key to use. - The rest of this page will describe a single encryption backend using GPG. Probably only one will be needed, but who knows? Maybe that backend will turn out badly designed, or some other encryptor needed. Designing From 08a23997dd5068218e7fd05bfb23cf52dd6299b0 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 5 Apr 2011 18:41:49 +0000 Subject: [PATCH 1441/2835] Added a comment --- ...comment_2_a610b3d056a059899178859a3a821ea5._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment diff --git a/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment b/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment new file mode 100644 index 0000000000..d5461e23c0 --- /dev/null +++ b/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-05T18:41:49Z" + content=""" +I see no use case for verifying encrypted object files w/o access to the encryption key. And possible use cases for not allowing anyone to verify your data. + +If there are to be multiple encryption keys usable within a single encrypted remote, than they would need to be given some kind of name (a since symmetric key is used, there is no pubkey to provide a name), and the name encoded in the files stored in the remote. While certainly doable I'm not sold that adding a layer of indirection is worthwhile. It only seems it would be worthwhile if setting up a new encrypted remote was expensive to do. Perhaps that could be the case for some type of remote other than S3 buckets. +"""]] From 421df56d6c2932043afb78147eada388127ea91f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Tue, 5 Apr 2011 20:09:53 +0000 Subject: [PATCH 1442/2835] rename forum/bainstormning:_git_annex_push___38___pull.mdwn to forum/bainstorming:_git_annex_push___38___pull.mdwn --- ...___pull.mdwn => bainstorming:_git_annex_push___38___pull.mdwn} | 0 .../comment_1_3a0bf74b51586354b7a91f8b43472376._comment | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename doc/forum/{bainstormning:_git_annex_push___38___pull.mdwn => bainstorming:_git_annex_push___38___pull.mdwn} (100%) rename doc/forum/{bainstormning:_git_annex_push___38___pull => bainstorming:_git_annex_push___38___pull}/comment_1_3a0bf74b51586354b7a91f8b43472376._comment (100%) diff --git a/doc/forum/bainstormning:_git_annex_push___38___pull.mdwn b/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn similarity index 100% rename from doc/forum/bainstormning:_git_annex_push___38___pull.mdwn rename to doc/forum/bainstorming:_git_annex_push___38___pull.mdwn diff --git a/doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment b/doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment similarity index 100% rename from doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment rename to doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment From b4d6c52b712fb1ffa4de91842e9be9380b2015cb Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Tue, 5 Apr 2011 20:52:53 +0000 Subject: [PATCH 1443/2835] Added a comment --- ...ent_2_b02ca09914e788393c01196686f95831._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment diff --git a/doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment b/doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment new file mode 100644 index 0000000000..e0ecc1a819 --- /dev/null +++ b/doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-05T20:52:52Z" + content=""" +No-so-subtle sarcasm taken and acknowledged :) + +Arguably, git-annex should know about any local limits and not have them implemented via mr from the outside. I guess my concern boils down to having git-annex do the right thing all by itself with minimal user interaction. And while I really do appreciate the flexibility of chaining commands, I am a firm believer in exposing the common use cases as easily as possible. + +And yes, I am fully aware that not all annexes are created equal. Point in case, I would never use git annex pull on my laptop, but I would git annex push extensively. + + +"""]] From fbec3aa7512d6c77e655c118692950eeb3e789ab Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM" Date: Tue, 5 Apr 2011 21:14:12 +0000 Subject: [PATCH 1444/2835] Added a comment --- ..._7cb65d013e72bd2b7e90452079d42ac9._comment | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment new file mode 100644 index 0000000000..bac9fd7cad --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM" + nickname="Justin" + subject="comment 4" + date="2011-04-05T21:14:12Z" + content=""" +@joey + +OK, I'll try increasing the stack size and see if that helps. + +For reference, I was running: + +git annex add . + +on a directory containing about 100k files spread over many nested subdirectories. I actually have more than a dozen projects like this that I plan to keep in git annex, possibly in separate repositories if necessary. I could probably tar the data and then archive that, but I like the idea of being able to see the structure of my data even though the contents of the files are on a different machine. + +After the crash, running: + +git annex unannex + +does nothing and returns instantly. What exactly is 'git annex add' doing? I know that it's moving files into the key-value store and adding symlinks, but I don't know what else it does. + +--Justin + + + +If + +"""]] From 711d48f32a205ad2023489f131e9a3b70080e900 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Tue, 5 Apr 2011 23:24:18 +0000 Subject: [PATCH 1445/2835] Added a comment --- ...mment_3_cca186a9536cd3f6e86994631b14231c._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment diff --git a/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment b/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment new file mode 100644 index 0000000000..d3c483fdf3 --- /dev/null +++ b/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 3" + date="2011-04-05T23:24:17Z" + content=""" +Assuming you're storing your encrypted annex with me and I with you, our regular cron jobs to verify all data will catch corruption in each other's annexes. + +Checksums of the encrypted objects could be optional, mitigating any potential attack scenarios. + +It's not only about the cost of setting up new remotes. It would also be a way to keep data in one annex while making it accessible only in a subset of them. For example, I might need some private letters at work, but I don't want my work machine to be able to access them all. +"""]] From 788fcc2bb498b582b5c6f73ff38ca200bdd18c05 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 6 Apr 2011 21:41:13 -0400 Subject: [PATCH 1446/2835] typo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 24f1b1e1f9..13adfe0a2d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr IGNORE=-ignore-package monads-fd -GHCFLAGS=-O2 -Wall -ignore-package $(IGNORE) +GHCFLAGS=-O2 -Wall $(IGNORE) ifdef PROFILE GHCFLAGS=-prof -auto-all -caf-all -fforce-recomp $(IGNORE) endif From c1bbe434224cce9a1afab0f358c6df83fa014e9c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 6 Apr 2011 21:53:06 -0400 Subject: [PATCH 1447/2835] Add build depend on perlmagick so docs are consistently built. Closes: #621410 --- debian/changelog | 2 ++ debian/control | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 71ea599cd9..87ad9d01b5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,8 @@ git-annex (0.20110402) UNRELEASED; urgency=low and git annex dropunused. * Clear up short option confusion between --from and --force (-f is now --from, and there is no short option for --force). + * Add build depend on perlmagick so docs are consistently built. + Closes: #621410 -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 diff --git a/debian/control b/debian/control index 4b31a295f9..37e6220437 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync, git | git-core +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync, git | git-core, perlmagick Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From ab0e03498f8a52bdf140587a4c7e13b8fe949ce7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 6 Apr 2011 21:57:22 -0400 Subject: [PATCH 1448/2835] Add doc-base file. Closes: #621408 --- debian/changelog | 1 + debian/doc-base | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 debian/doc-base diff --git a/debian/changelog b/debian/changelog index 87ad9d01b5..6ccb9eac96 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,7 @@ git-annex (0.20110402) UNRELEASED; urgency=low --from, and there is no short option for --force). * Add build depend on perlmagick so docs are consistently built. Closes: #621410 + * Add doc-base file. Closes: #621408 -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 diff --git a/debian/doc-base b/debian/doc-base new file mode 100644 index 0000000000..f71a233333 --- /dev/null +++ b/debian/doc-base @@ -0,0 +1,9 @@ +Document: git-annex +Title: git-annex documentation +Author: Joey Hess +Abstract: All the documentation from git-annex's website. +Section: File Management + +Format: HTML +Index: /usr/share/doc/git-annex/html/index.html +Files: /usr/share/doc/git-annex/html/*.html From a4d37c4550b911567d0dfb29ed5ba89f619065a7 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Thu, 7 Apr 2011 08:03:12 +0000 Subject: [PATCH 1449/2835] --- .../sparse_git_checkouts_with_annex.mdwn | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex.mdwn diff --git a/doc/forum/sparse_git_checkouts_with_annex.mdwn b/doc/forum/sparse_git_checkouts_with_annex.mdwn new file mode 100644 index 0000000000..32e6e4bbc4 --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex.mdwn @@ -0,0 +1,28 @@ +I checked in my music collection into git annex (about 25000 files) and i'm really impressed by the performance of git annex (after i've done an git-repack). Now i'm also moving my movies into the same git-annex, but i have the following layout of my disk drives: + +* small raid-1 for important stuff (music, documents), which is also backupped (aka: raid) +* big bulk data store (aka: media) + +In the git-annex the following layout of files is used: + +* documents/ <- on raid +* music/ <- on raid +* videos/ <- on media + +Now i didn't simply clone the raid-annex to media, but did an sparse-checkout (possible since version 1.7.0) + +* raid: .git-annex/, documents/ and music +* media: .git-annex/, videos/ + +As you can see i have to checkout the .git-annex directory with the file-logs twice which slows down git operations. Everything else works fine until now. git-annex doesn't have any problem, that only a part of the symlinks are present, which is really great. Is there a possibility to sparse checkout the .git-annex directory also? Perhaps splitting the log files in .git-annex/ into N subfolders, corresponding to the toplevel subfolders, like this? + +* Before: + $ ls .git-annex + 00 01 02.... +* After: + $ ls .git-annex + documents/ music/ videos/ + $ ls .git-annex/documents + 00 01 02.... + +This would make it possible to checkout only the part of the log files which i'm interested in. From eca9914be1213f8110afef35e647ce14ae22bfb7 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Thu, 7 Apr 2011 08:04:39 +0000 Subject: [PATCH 1450/2835] (sorry for noise, had to format the code blocks) --- .../sparse_git_checkouts_with_annex.mdwn | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/doc/forum/sparse_git_checkouts_with_annex.mdwn b/doc/forum/sparse_git_checkouts_with_annex.mdwn index 32e6e4bbc4..97d2f445d3 100644 --- a/doc/forum/sparse_git_checkouts_with_annex.mdwn +++ b/doc/forum/sparse_git_checkouts_with_annex.mdwn @@ -16,13 +16,16 @@ Now i didn't simply clone the raid-annex to media, but did an sparse-checkout (p As you can see i have to checkout the .git-annex directory with the file-logs twice which slows down git operations. Everything else works fine until now. git-annex doesn't have any problem, that only a part of the symlinks are present, which is really great. Is there a possibility to sparse checkout the .git-annex directory also? Perhaps splitting the log files in .git-annex/ into N subfolders, corresponding to the toplevel subfolders, like this? -* Before: - $ ls .git-annex - 00 01 02.... -* After: - $ ls .git-annex - documents/ music/ videos/ - $ ls .git-annex/documents - 00 01 02.... +Before: + + $ ls .git-annex + 00 01 02.... + +After: + + $ ls .git-annex + documents/ music/ videos/ + $ ls .git-annex/documents + 00 01 02.... This would make it possible to checkout only the part of the log files which i'm interested in. From 7634f92e83f8a789df1d660ed95a08d5c136f70f Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Thu, 7 Apr 2011 13:44:38 +0000 Subject: [PATCH 1451/2835] Added a comment: Reported the issue to GHC --- ...ment_5_67406dd8d9bd4944202353508468c907._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment new file mode 100644 index 0000000000..bffa9bb868 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Reported the issue to GHC" + date="2011-04-07T13:44:36Z" + content=""" +Finally got around to [report the issue to GHC tracker](http://hackage.haskell.org/trac/ghc/ticket/5085#comment:7). + +Looks quite alike (at least to the haskell-illiterate person like me) to a highest-priority issue that's hanging right at the top of the list. +There are other similar reports, but they seem to be either related to PowerPC Macs, closed as invalid or due to needinfo inactivity. + +Guess any further discussion belongs there, unless ghc developers will bounce it back. +Thanks a lot for your help, Joey, and for sharing a great thing that git-annex is. +"""]] From 00f1c720eddf95ce21e6d2c35623a82e97ed604c Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 16:32:04 +0000 Subject: [PATCH 1452/2835] Added a comment --- ...mment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment new file mode 100644 index 0000000000..7adf4fc4d6 --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-07T16:32:04Z" + content=""" +That's awesome, I had not heard of git sparse checkouts before. + +It does not make sense to tie the log files to the directory of the corresponding files, as then the logs would have to move when the files are moved, which would be a PITA and likely make merging log file changes very complex. Also, of course, multiple files in different locations can point at the same content, which has the same log file. And, to cap it off, git-annex can need to access the log file for a given key without having the slightest idea what file in the repository might point to it, and it would be very expensive to scan the whole repository to find out what that file is in order to lookup the filename of the log file. + +The most likely change in git-annex that will make this better is in [[this_todo_item|todo/branching]] -- but it's unknown how to do it yet. +"""]] From 7e76c60a0c6e89c88fcfdaed2fcac6f3c2368d7b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 16:33:30 +0000 Subject: [PATCH 1453/2835] Added a comment --- .../comment_2_e357db3ccc4079f07a291843975535eb._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment new file mode 100644 index 0000000000..d8088a2d82 --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-07T16:33:30Z" + content=""" +BTW, git-annex unused *will* have a problem that not all the symlinks are present. It will suggest dropping content belonging to the excluded symlinks. +"""]] From 079e57adf77de3601d9fe0df63aa472317889c11 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 12:33:48 -0400 Subject: [PATCH 1454/2835] current thoughts --- doc/todo/branching.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index b7441c6e4e..06bd50beed 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -105,3 +105,13 @@ too. The problem would then be that any locationlog lookup would need to look in all other branches (any branch could have more current info after all), which could get expensive. + +## way outside the box approach + +Another approach I have been mulling over is keeping the log file +branch checked out in .git-annex/logs/ -- this would be a checkout of a git +repository inside a git repository, using "git fake bare" techniques. This +would solve the merge problem, since git auto merge could be used. It would +still mean all the log files are on-disk, which annoys some. It would +require some tighter integration with git, so that after a pull, the log +repo is updated with the data pulled. --[[Joey]] From df65f0c77d335644a64abe81d554eedfbb157b5d Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 16:41:00 +0000 Subject: [PATCH 1455/2835] Added a comment --- ...ent_5_86a42ee3173a5d38f803e64b79496ab3._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment new file mode 100644 index 0000000000..7dcccef2e5 --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 5" + date="2011-04-07T16:41:00Z" + content=""" +I think what is happening with \"git annex unannex\" is that \"git annex add\" crashes before it can \"git add\" the symlinks. unannex only looks at files that \"git ls-files\" shows, and so files that are not added to git are not seen. So, this can be recovered from by looking at git status and manually adding the symlinks to git, and then unannex. + +That also suggests that \"git annex add .\" has done something before crashing. That's consistent with you passing it < 2 parameters; it's not just running out of memory trying to expand and preserve order of its parameters (like it might if you ran \"git annex add experiment-1/ experiment-2/\") + +I'm pretty sure I know where the space leak is now. git-annex builds up a queue of git commands, so that it can run git a minimum number of times. Currently, this queue is only flushed at the end. I had been meaning to work on having it flush the queue periodically to avoid it growing without bounds, and I will prioritize doing that. + +(The only other thing that \"git annex add\" does is record location log information.) +"""]] From bc51387e6dd426f46f9ab0ef23e6e3eefe7a4417 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 13:59:31 -0400 Subject: [PATCH 1456/2835] Periodically flush git command queue, to avoid boating memory usage too much. Since the queue is flushed in between subcommand actions being run, there should be no issues with actions that expect to queue up some stuff and have it run after they do other stuff. So I didn't have to audit for such assumptions. --- Annex.hs | 29 +-------------------------- AnnexQueue.hs | 47 ++++++++++++++++++++++++++++++++++++++++++++ CmdLine.hs | 13 ++++++------ Command/Add.hs | 4 ++-- Command/Fix.hs | 4 ++-- Command/FromKey.hs | 4 ++-- Command/Move.hs | 3 ++- Command/PreCommit.hs | 3 ++- Command/Unannex.hs | 3 ++- Content.hs | 3 ++- GitQueue.hs | 29 +++++++++++++++++++++------ Remote/Git.hs | 3 ++- Upgrade/V1.hs | 13 ++++++------ debian/changelog | 2 ++ 14 files changed, 101 insertions(+), 59 deletions(-) create mode 100644 AnnexQueue.hs diff --git a/Annex.hs b/Annex.hs index 2723c6a008..f4e5d599d0 100644 --- a/Annex.hs +++ b/Annex.hs @@ -13,10 +13,7 @@ module Annex ( eval, getState, changeState, - gitRepo, - queue, - queueRun, - queueRunAt, + gitRepo ) where import Control.Monad.State @@ -25,7 +22,6 @@ import qualified GitRepo as Git import qualified GitQueue import qualified BackendClass import qualified RemoteClass -import Utility -- git-annex's monad type Annex = StateT AnnexState IO @@ -93,26 +89,3 @@ changeState a = do {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo gitRepo = getState repo - -{- Adds a git command to the queue. -} -queue :: String -> [CommandParam] -> FilePath -> Annex () -queue command params file = do - state <- get - let q = repoqueue state - put state { repoqueue = GitQueue.add q command params file } - -{- Runs (and empties) the queue. -} -queueRun :: Annex () -queueRun = do - state <- get - let q = repoqueue state - g <- gitRepo - liftIO $ GitQueue.run g q - put state { repoqueue = GitQueue.empty } - -{- Runs the queue if the specified number of items have been queued. -} -queueRunAt :: Integer -> Annex () -queueRunAt n = do - state <- get - let q = repoqueue state - when (GitQueue.size q >= n) queueRun diff --git a/AnnexQueue.hs b/AnnexQueue.hs new file mode 100644 index 0000000000..58e77a6e85 --- /dev/null +++ b/AnnexQueue.hs @@ -0,0 +1,47 @@ +{- git-annex command queue + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module AnnexQueue ( + add, + flush, + flushWhenFull +) where + +import Control.Monad.State (liftIO) +import Control.Monad (when, unless) + +import Annex +import Messages +import qualified GitQueue +import Utility + +{- Adds a git command to the queue, possibly running previously queued + - actions if enough have accumulated. -} +add :: String -> [CommandParam] -> FilePath -> Annex () +add command params file = do + q <- getState repoqueue + store $ GitQueue.add q command params file + +{- Runs the queue if it is full. Should be called periodically. -} +flushWhenFull :: Annex () +flushWhenFull = do + q <- getState repoqueue + when (GitQueue.full q) $ flush False + +{- Runs (and empties) the queue. -} +flush :: Bool -> Annex () +flush silent = do + q <- getState repoqueue + unless (0 == GitQueue.size q) $ do + unless silent $ + showSideAction "Recording state in git..." + g <- gitRepo + q' <- liftIO $ GitQueue.flush g q + store q' + +store :: GitQueue.Queue -> Annex () +store q = changeState $ \s -> s { repoqueue = q } diff --git a/CmdLine.hs b/CmdLine.hs index de03d96ed4..684ebf979a 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -14,11 +14,11 @@ module CmdLine ( import System.IO.Error (try) import System.Console.GetOpt import Control.Monad.State (liftIO) -import Control.Monad (when, unless) +import Control.Monad (when) import qualified Annex +import qualified AnnexQueue import qualified GitRepo as Git -import qualified GitQueue import Types import Command import BackendList @@ -81,7 +81,9 @@ tryRun :: Annex.AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions tryRun' :: Annex.AnnexState -> Integer -> [Annex Bool] -> IO () tryRun' state errnum (a:as) = do - result <- try $ Annex.run state a + result <- try $ Annex.run state $ do + AnnexQueue.flushWhenFull + a case result of Left err -> do Annex.eval state $ showErr err @@ -100,10 +102,7 @@ startup = do {- Cleanup actions. -} shutdown :: Annex Bool shutdown = do - q <- Annex.getState Annex.repoqueue - unless (0 == GitQueue.size q) $ do - showSideAction "Recording state in git..." - Annex.queueRun + AnnexQueue.flush False liftIO $ Git.reap diff --git a/Command/Add.hs b/Command/Add.hs index da98bffa4f..b532ab045d 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import System.Posix.Files import Command -import qualified Annex +import qualified AnnexQueue import qualified Backend import LocationLog import Types @@ -60,5 +60,5 @@ cleanup file key = do let mtime = modificationTime s liftIO $ touch file (TimeSpec mtime) False - Annex.queue "add" [Param "--"] file + AnnexQueue.add "add" [Param "--"] file return True diff --git a/Command/Fix.hs b/Command/Fix.hs index 513e07a310..d898ce517d 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -12,7 +12,7 @@ import System.Posix.Files import System.Directory import Command -import qualified Annex +import qualified AnnexQueue import Utility import Content import Messages @@ -44,5 +44,5 @@ perform file link = do cleanup :: FilePath -> CommandCleanup cleanup file = do - Annex.queue "add" [Param "--"] file + AnnexQueue.add "add" [Param "--"] file return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 8c1a1028fe..eadaa13e1f 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -13,7 +13,7 @@ import System.Directory import Control.Monad (unless) import Command -import qualified Annex +import qualified AnnexQueue import Utility import qualified Backend import Content @@ -46,5 +46,5 @@ perform file = do cleanup :: FilePath -> CommandCleanup cleanup file = do - Annex.queue "add" [Param "--"] file + AnnexQueue.add "add" [Param "--"] file return True diff --git a/Command/Move.hs b/Command/Move.hs index 951695d66e..e5e78d2495 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -12,6 +12,7 @@ import Control.Monad.State (liftIO) import Command import qualified Command.Drop import qualified Annex +import qualified AnnexQueue import LocationLog import Types import Content @@ -59,7 +60,7 @@ remoteHasKey remote key present = do g <- Annex.gitRepo let remoteuuid = Remote.uuid remote logfile <- liftIO $ logChange g key remoteuuid status - Annex.queue "add" [Param "--"] logfile + AnnexQueue.add "add" [Param "--"] logfile where status = if present then ValuePresent else ValueMissing diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 727a637285..1db40f75fa 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -11,6 +11,7 @@ import Control.Monad.State (liftIO) import Command import qualified Annex +import qualified AnnexQueue import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix @@ -42,5 +43,5 @@ cleanup file = do -- stage the symlink g <- Annex.gitRepo liftIO $ Git.run g "reset" [Params "-q --", File file] - Annex.queueRun + AnnexQueue.flush True return True diff --git a/Command/Unannex.hs b/Command/Unannex.hs index b0ce31ceed..94db500c68 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -13,6 +13,7 @@ import System.Directory import Command import qualified Annex +import qualified AnnexQueue import Utility import qualified Backend import LocationLog @@ -68,6 +69,6 @@ cleanup file key = do -- Commit staged changes at end to avoid confusing the -- pre-commit hook if this file is later added back to -- git as a normal, non-annexed file. - Annex.queue "commit" [Params "-a -m", Param "content removed from git annex"] "-a" + AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] "-a" return True diff --git a/Content.hs b/Content.hs index ba265c9307..f63c02311f 100644 --- a/Content.hs +++ b/Content.hs @@ -36,6 +36,7 @@ import LocationLog import UUID import qualified GitRepo as Git import qualified Annex +import qualified AnnexQueue import Utility import StatFS import Key @@ -72,7 +73,7 @@ logStatus key status = do unless (Git.repoIsLocalBare g) $ do u <- getUUID g logfile <- liftIO $ logChange g key u status - Annex.queue "add" [Param "--"] logfile + AnnexQueue.add "add" [Param "--"] logfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into diff --git a/GitQueue.hs b/GitQueue.hs index dfe2976da1..480027fa0c 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -10,7 +10,8 @@ module GitQueue ( empty, add, size, - run + full, + flush ) where import qualified Data.Map as M @@ -32,9 +33,21 @@ data Action = Action { {- A queue of actions to perform (in any order) on a git repository, - with lists of files to perform them on. This allows coalescing - similar git commands. -} -data Queue = Queue Integer (M.Map Action [FilePath]) +data Queue = Queue Int (M.Map Action [FilePath]) deriving (Show, Eq) +{- A recommended maximum size for the queue, after which it should be + - run. + - + - 10240 is semi-arbitrary. If we assume git filenames are between 10 and + - 255 characters long, then the queue will build up between 100kb and + - 2550kb long commands. The max command line length on linux is somewhere + - above 20k, so this is a fairly good balance -- the queue will buffer + - only a few megabytes of stuff and a minimal number of commands will be + - run by xargs. -} +maxSize :: Int +maxSize = 10240 + {- Constructor for empty queue. -} empty :: Queue empty = Queue 0 M.empty @@ -47,14 +60,18 @@ add (Queue n m) subcommand params file = Queue (n + 1) m' m' = M.insertWith' (++) action [file] m {- Number of items in a queue. -} -size :: Queue -> Integer +size :: Queue -> Int size (Queue n _) = n +{- Is a queue large enough that it should be flushed? -} +full :: Queue -> Bool +full (Queue n _) = n > maxSize + {- Runs a queue on a git repository. -} -run :: Git.Repo -> Queue -> IO () -run repo (Queue _ m) = do +flush :: Git.Repo -> Queue -> IO Queue +flush repo (Queue _ m) = do forM_ (M.toList m) $ uncurry $ runAction repo - return () + return empty {- Runs an Action on a list of files in a git repository. - diff --git a/Remote/Git.hs b/Remote/Git.hs index a458455109..2936beaf7d 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -19,6 +19,7 @@ import RemoteClass import Types import qualified GitRepo as Git import qualified Annex +import qualified AnnexQueue import Locations import UUID import Utility @@ -150,7 +151,7 @@ copyToRemote r key Annex.eval a $ do ok <- Content.getViaTmp key $ \f -> liftIO $ copyFile keysrc f - Annex.queueRun + AnnexQueue.flush True return ok | Git.repoIsSsh r = do g <- Annex.gitRepo diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 4ce2612d60..9278bce603 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -24,6 +24,7 @@ import Types import Locations import LocationLog import qualified Annex +import qualified AnnexQueue import qualified GitRepo as Git import Backend import Messages @@ -68,7 +69,7 @@ upgrade = do updateSymlinks moveLocationLogs - Annex.queueRun + AnnexQueue.flush True setVersion -- add new line to auto-merge hashed location logs @@ -106,8 +107,7 @@ updateSymlinks = do link <- calcGitLink f k liftIO $ removeFile f liftIO $ createSymbolicLink link f - Annex.queue "add" [Param "--"] f - Annex.queueRunAt 10240 + AnnexQueue.add "add" [Param "--"] f moveLocationLogs :: Annex () moveLocationLogs = do @@ -137,10 +137,9 @@ moveLocationLogs = do old <- liftIO $ readLog f new <- liftIO $ readLog dest liftIO $ writeLog dest (old++new) - Annex.queue "add" [Param "--"] dest - Annex.queue "add" [Param "--"] f - Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f - Annex.queueRunAt 10240 + AnnexQueue.add "add" [Param "--"] dest + AnnexQueue.add "add" [Param "--"] f + AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] f oldlog2key :: FilePath -> Maybe (FilePath, Key) oldlog2key l = diff --git a/debian/changelog b/debian/changelog index 6ccb9eac96..fdc740cb85 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,6 +12,8 @@ git-annex (0.20110402) UNRELEASED; urgency=low * Add build depend on perlmagick so docs are consistently built. Closes: #621410 * Add doc-base file. Closes: #621408 + * Periodically flush git command queue, to avoid boating memory usage + too much. -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 From d804062f8df64219a781b06f27e609464d6e3159 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 18:09:13 +0000 Subject: [PATCH 1457/2835] Added a comment --- ...omment_6_4551274288383c9cc27cbf85b122d307._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment new file mode 100644 index 0000000000..fff8f7cdde --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 6" + date="2011-04-07T18:09:13Z" + content=""" +I've committed the queue flush improvements, so it will buffer up to 10240 git actions, and then flush the queue. + +There may be other memory leaks at scale (besides the two I mentioned earlier), but this seems promising. I'm well into running `git annex add` on a half million files and it's using 18 mb ram and has flushed the queue several times. This run +will fail due to running out of inodes for the log files, not due to memory. :) +"""]] From 17e974d32d956fd99416da706e7ae008ed7af308 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 18:13:46 +0000 Subject: [PATCH 1458/2835] Comment moderation --- ...omment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment | 11 +++++++++++ ...omment_3_2082f4d708a584a1403cc1d4d005fb56._comment | 10 ++++++++++ 2 files changed, 21 insertions(+) create mode 100644 doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment create mode 100644 doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment diff --git a/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment b/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment new file mode 100644 index 0000000000..f3db75c2f6 --- /dev/null +++ b/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="fmarier" + subject="comment 3" + date="2011-04-05T10:00:21Z" + content=""" +Thanks for the reply @joey. + +While it would certainly be possible for a bare repo to exist on my iRiver, the problem is that the music player uses the filesystem to organize files into directories like \"Artist/Album/Track.ogg\". So replacing that with \"..../xx/yy/Track.ogg\" would make it fairly difficult to browse my music collection and select the album/track I want to listen to :) + +So unless I have the files physically organized like the symlinks, then it's probably not going to work very for that particular workflow. Smudge filters are interesting though. In the meantime, I'll look into rsyncing from another box which has the right filesystem layout onto my iRiver directly. +"""]] diff --git a/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment new file mode 100644 index 0000000000..01dc7813ff --- /dev/null +++ b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-04-04T10:28:01Z" + content=""" +Going one step further, a --min-copy could put all files so that numcopies is satisfied. --all could push to all available ones. + +To take everything another step further, if it was possible to group remotes, one could act on the groups. \"all\" would be an obvious choice for a group that always exists, everything else would be set up by the user. +"""]] From 4ea0b7c28850eb703562cd9dc84a02c49b5fda00 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 14:45:10 -0400 Subject: [PATCH 1459/2835] add --- doc/todo/git-annex_unused_eats_memory.mdwn | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 doc/todo/git-annex_unused_eats_memory.mdwn diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn new file mode 100644 index 0000000000..6ce7140045 --- /dev/null +++ b/doc/todo/git-annex_unused_eats_memory.mdwn @@ -0,0 +1,25 @@ +`git-annex unused` has to compare large sets of data +(all keys with content present in the repository, +with all keys used by files in the repository), and so +uses more memory than git-annex typically needs; around +60-80 mb when run in a repository with 80 thousand files. + +I would like to reduce this. One idea is to use a bloom filter. +For example, construct a bloom filter of all keys used by files in +the repository. Then for each key with content present, check if it's +in the bloom filter. Since there can be false negatives, this might +miss finding some unused keys. The probability/size of filter +could be tunable. + +Another way might be to scan the git log for files that got removed +or changed what key they pointed to. Correlate with keys with content +currently present in the repository (possibly using a bloom filter again), +and that would yield a shortlist of keys that are probably not used. +Then scan thru all files in the repo to make sure that none point to keys +on the shortlist. + +---- + +`git annex unused --from remote` is much worse, using hundreds of mb of +memory. It has not been profiled at all yet, and can probably be improved +somewhat by fixing whatever memory leak it (probably) has. From 135d75f2b9c0f77b837da4243dea32a97d4088a5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 15:00:06 -0400 Subject: [PATCH 1460/2835] avoid list traverse on queue I wanted to use M.insertWith' (\_ l -> file:l) action [] m , but the order of the parameters and which to ignore is not clear, and seems unsafe to rely on. --- GitQueue.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/GitQueue.hs b/GitQueue.hs index 480027fa0c..be0fcfc4af 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -57,7 +57,11 @@ add :: Queue -> String -> [CommandParam] -> FilePath -> Queue add (Queue n m) subcommand params file = Queue (n + 1) m' where action = Action subcommand params - m' = M.insertWith' (++) action [file] m + -- There are probably few items in the map, but there + -- can be a lot of files per item. So, optimise adding + -- files. + m' = M.insertWith' const action files m + files = file:(M.findWithDefault [] action m) {- Number of items in a queue. -} size :: Queue -> Int From 36894655f34eb46b386f708b889f05ffd269f0cf Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 19:59:30 +0000 Subject: [PATCH 1461/2835] Added a comment --- ...comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment diff --git a/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment b/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment new file mode 100644 index 0000000000..14eb1acac1 --- /dev/null +++ b/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-04-07T19:59:30Z" + content=""" +@Richard the easy way to deal with that scenario is to set up a remote that work can access, and only put in it files work should be able to see. Needing to specify which key a file should be encrypted to when putting it in a remote that supported multiple keys would add another level of complexity which that avoids. + +Of course, the right approach is probably to have a separate repository for work. If you don't trust it with seeing file contents, you probably also don't trust it with the contents of your git repository. +"""]] From bd1bbc21fac4b36975d03638d15e9aab915927cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 16:05:30 -0400 Subject: [PATCH 1462/2835] update --- doc/design/encryption.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index f8f8656a74..8a8f38108e 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -61,8 +61,8 @@ more gpg public keys. This scheme allows new gpg private keys to be given access to content that has already been stored in the remote. Different encrypted remotes need to be able to each use different ciphers. -There does not seem to be a benefit to allowing multiple cipers to be -used within a single remote, and it would add a lot of complexity. +Allowing multiple cipers to be used within a single remote would add a lot +of complexity, so is not planned to be supported. Instead, if you want a new cipher, create a new S3 bucket, or whatever. There does not seem to be much benefit to using the same cipher for two different enrypted remotes. From f5b2d650bb75dc7ca2f77dae59fb1ab7f7405e03 Mon Sep 17 00:00:00 2001 From: Fraser Tweedale Date: Fri, 8 Apr 2011 10:08:11 +1000 Subject: [PATCH 1463/2835] recognise differently-named shaN programs --- Backend/SHA.hs | 18 ++++++++++-------- TestConfig.hs | 14 +++++++++++++- configure.hs | 9 +++++---- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 0ec555ce3f..42b5efeff0 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -35,7 +35,7 @@ backends = catMaybes $ map genBackend [1, 256, 512, 224, 384] genBackend :: SHASize -> Maybe (Backend Annex) genBackend size - | supported size = Just b + | shaCommand size /= "" = Just b | otherwise = Nothing where b = Backend.File.backend @@ -43,12 +43,14 @@ genBackend size , getKey = keyValue size , fsckKey = Backend.File.checkKey $ checkKeyChecksum size } - supported 1 = SysConfig.sha1sum - supported 256 = SysConfig.sha256sum - supported 224 = SysConfig.sha224sum - supported 384 = SysConfig.sha384sum - supported 512 = SysConfig.sha512sum - supported _ = False + +shaCommand :: SHASize -> String +shaCommand 1 = SysConfig.sha1 +shaCommand 256 = SysConfig.sha256 +shaCommand 224 = SysConfig.sha224 +shaCommand 384 = SysConfig.sha384 +shaCommand 512 = SysConfig.sha512 +shaCommand _ = "" shaName :: SHASize -> String shaName size = "SHA" ++ show size @@ -63,7 +65,7 @@ shaN size file = do then error $ command ++ " parse error" else return $ head bits where - command = "sha" ++ (show size) ++ "sum" + command = shaCommand size {- A key is a checksum of its contents. -} keyValue :: SHASize -> FilePath -> Annex (Maybe Key) diff --git a/TestConfig.hs b/TestConfig.hs index 5e59681ddf..d1560b660f 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -72,13 +72,25 @@ selectCmd k cmds = search cmds where search [] = do testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show cmds + error $ "* need one of these commands, but none are available: " ++ show (map (head . words) cmds) search (c:cs) = do ret <- system $ quiet c if (ret == ExitSuccess) then return $ Config k (StringConfig c) else search cs +whichCmd :: ConfigKey -> [String] -> Test +whichCmd k cmds = search cmds + where + search [] = do + testEnd $ Config k (StringConfig "") + return $ Config k (StringConfig "") + search (c:cs) = do + ret <- system $ quiet c + if (ret == ExitSuccess) + then return $ Config k (StringConfig $ head $ words c) + else search cs + quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" diff --git a/configure.hs b/configure.hs index f8cd577e99..c0e3d8106e 100644 --- a/configure.hs +++ b/configure.hs @@ -20,10 +20,11 @@ tests = [ shaTestCases :: [Int] -> [TestCase] shaTestCases l = map make l - where - make n = - let cmd = "sha" ++ show n ++ "sum" - in TestCase cmd $ requireCmd cmd (cmd ++ " "sha" ++ show n ++ x ++ " Date: Fri, 8 Apr 2011 01:13:06 +0000 Subject: [PATCH 1464/2835] --- doc/forum/wishlist:_git_annex_status.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_status.mdwn diff --git a/doc/forum/wishlist:_git_annex_status.mdwn b/doc/forum/wishlist:_git_annex_status.mdwn new file mode 100644 index 0000000000..add865410e --- /dev/null +++ b/doc/forum/wishlist:_git_annex_status.mdwn @@ -0,0 +1,19 @@ +Ideally, it would look similar to this. And yes, I put "put" in there ;) + + non-annex % git annex status + git annex status: error: not a git annex repository + annex % git annex status + annex object storage version: A + annex backend engine: {WORM,SHA512,...} + Estimated local annex size: B MiB + Estimated total annex size: C MiB + Files without file size information in local annex: D + Files without file size information in total annex: E + Last fsck: datetime + Last git pull: datetime - $annex_name + Last git push: datetime - $annex_name + Last git annex get: datetime - $annex_name + Last git annex put: datetime - $annex_name + annex % + +Datetime could be ISO's YYYY-MM-DDThh:mm:ss or, personal preference, YYYY-MM-DD--hh-mm-ss. I prefer the latter as it's DNS-, tag- and filename-safe which is why I am using it for everything. In a perfect world, ISO would standardize YYYY-MM-DD-T-hh-mm-ss-Z[-SSSSSSSS][--$timezone], but meh. From b889543507959c0a311bc8387f78ec4c0e93f7a8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 21:47:56 -0400 Subject: [PATCH 1465/2835] let's use Maybe String for commands that may not be avilable --- Backend/SHA.hs | 12 ++++++------ TestConfig.hs | 18 +++++++++++++----- debian/changelog | 1 + 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 42b5efeff0..d9aeb72aa4 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -27,7 +27,7 @@ import qualified SysConfig import Key type SHASize = Int - + backends :: [Backend Annex] -- order is slightly significant; want sha1 first ,and more general -- sizes earlier @@ -35,8 +35,8 @@ backends = catMaybes $ map genBackend [1, 256, 512, 224, 384] genBackend :: SHASize -> Maybe (Backend Annex) genBackend size - | shaCommand size /= "" = Just b - | otherwise = Nothing + | shaCommand size == Nothing = Nothing + | otherwise = Just b where b = Backend.File.backend { name = shaName size @@ -44,13 +44,13 @@ genBackend size , fsckKey = Backend.File.checkKey $ checkKeyChecksum size } -shaCommand :: SHASize -> String +shaCommand :: SHASize -> Maybe String shaCommand 1 = SysConfig.sha1 shaCommand 256 = SysConfig.sha256 shaCommand 224 = SysConfig.sha224 shaCommand 384 = SysConfig.sha384 shaCommand 512 = SysConfig.sha512 -shaCommand _ = "" +shaCommand _ = Nothing shaName :: SHASize -> String shaName size = "SHA" ++ show size @@ -65,7 +65,7 @@ shaN size file = do then error $ command ++ " parse error" else return $ head bits where - command = shaCommand size + command = fromJust $ shaCommand size {- A key is a checksum of its contents. -} keyValue :: SHASize -> FilePath -> Annex (Maybe Key) diff --git a/TestConfig.hs b/TestConfig.hs index d1560b660f..f6ca2afcd8 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -7,7 +7,10 @@ import System.Cmd import System.Exit type ConfigKey = String -data ConfigValue = BoolConfig Bool | StringConfig String +data ConfigValue = + BoolConfig Bool | + StringConfig String | + MaybeStringConfig (Maybe String) data Config = Config ConfigKey ConfigValue type Test = IO Config @@ -17,15 +20,17 @@ data TestCase = TestCase TestName Test instance Show ConfigValue where show (BoolConfig b) = show b show (StringConfig s) = show s + show (MaybeStringConfig s) = show s instance Show Config where - show (Config key value) = unlines + show (Config key value) = unlines [ key ++ " :: " ++ valuetype value , key ++ " = " ++ show value ] where valuetype (BoolConfig _) = "Bool" valuetype (StringConfig _) = "String" + valuetype (MaybeStringConfig _) = "Maybe String" writeSysConfig :: [Config] -> IO () writeSysConfig config = writeFile "SysConfig.hs" body @@ -83,12 +88,13 @@ whichCmd :: ConfigKey -> [String] -> Test whichCmd k cmds = search cmds where search [] = do - testEnd $ Config k (StringConfig "") - return $ Config k (StringConfig "") + let r = Config k (MaybeStringConfig Nothing) + testEnd r + return r search (c:cs) = do ret <- system $ quiet c if (ret == ExitSuccess) - then return $ Config k (StringConfig $ head $ words c) + then return $ Config k (MaybeStringConfig $ Just $ head $ words c) else search cs quiet :: String -> String @@ -103,3 +109,5 @@ testEnd :: Config -> IO () testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes" testEnd (Config _ (BoolConfig False)) = putStrLn $ " no" testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s +testEnd (Config _ (MaybeStringConfig (Just s))) = putStrLn $ " " ++ s +testEnd (Config _ (MaybeStringConfig Nothing)) = putStrLn $ " not available" diff --git a/debian/changelog b/debian/changelog index fdc740cb85..3072ac476a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -14,6 +14,7 @@ git-annex (0.20110402) UNRELEASED; urgency=low * Add doc-base file. Closes: #621408 * Periodically flush git command queue, to avoid boating memory usage too much. + * Support "sha1" and "sha512" commands on FreeBSD. Thanks, Fraser Tweedale -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 From e2404ca40933ccdc046d08f7040b2d86e32d5f88 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 22:03:31 -0400 Subject: [PATCH 1466/2835] refactor away whichCmd and some other cleanup --- TestConfig.hs | 36 ++++++++++++++++-------------------- configure.hs | 6 +++--- debian/changelog | 3 ++- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/TestConfig.hs b/TestConfig.hs index f6ca2afcd8..9b2759e199 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -72,30 +72,26 @@ testCmd k cmdline = do {- Ensures that one of a set of commands is available by running each in - turn. The Config is set to the first one found. -} -selectCmd :: ConfigKey -> [String] -> Test -selectCmd k cmds = search cmds +selectCmd :: Bool -> ConfigKey -> [String] -> String -> Test +selectCmd required k cmds param = search cmds where - search [] = do - testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show (map (head . words) cmds) + search [] = failure search (c:cs) = do - ret <- system $ quiet c + ret <- system $ quiet c ++ " " ++ param if (ret == ExitSuccess) - then return $ Config k (StringConfig c) - else search cs - -whichCmd :: ConfigKey -> [String] -> Test -whichCmd k cmds = search cmds - where - search [] = do - let r = Config k (MaybeStringConfig Nothing) - testEnd r - return r - search (c:cs) = do - ret <- system $ quiet c - if (ret == ExitSuccess) - then return $ Config k (MaybeStringConfig $ Just $ head $ words c) + then success c else search cs + success c + | required == True = return $ Config k (StringConfig c) + | otherwise = return $ Config k (MaybeStringConfig $ Just c) + failure + | required == True = do + testEnd $ Config k (BoolConfig False) + error $ "* need one of these commands, but none are available: " ++ show cmds + | otherwise = do + let r = Config k (MaybeStringConfig Nothing) + testEnd r + return r quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" diff --git a/configure.hs b/configure.hs index c0e3d8106e..0661813aef 100644 --- a/configure.hs +++ b/configure.hs @@ -11,7 +11,7 @@ tests = [ , testCp "cp_a" "-a" , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] + , TestCase "uuid generator" $ selectCmd True "uuid" ["uuid", "uuidgen"] "" , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" @@ -22,9 +22,9 @@ shaTestCases :: [Int] -> [TestCase] shaTestCases l = map make l where make n = let - cmds = map (\x -> "sha" ++ show n ++ x ++ " "sha" ++ show n ++ x) ["", "sum"] key = "sha" ++ show n - in TestCase key $ whichCmd key cmds + in TestCase key $ selectCmd False key cmds " Sat, 02 Apr 2011 13:45:54 -0400 From a77c34d2b4795a5d41f82a78e999ce33f43d8862 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 00:12:00 -0400 Subject: [PATCH 1467/2835] refactor --- TestConfig.hs | 30 ++++++++++++++++-------------- configure.hs | 8 ++++---- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/TestConfig.hs b/TestConfig.hs index 9b2759e199..bab297003a 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -72,26 +72,28 @@ testCmd k cmdline = do {- Ensures that one of a set of commands is available by running each in - turn. The Config is set to the first one found. -} -selectCmd :: Bool -> ConfigKey -> [String] -> String -> Test -selectCmd required k cmds param = search cmds +selectCmd :: ConfigKey -> [String] -> String -> Test +selectCmd k = searchCmd + (\match -> return $ Config k $ StringConfig match) + (\cmds -> do + testEnd $ Config k $ BoolConfig False + error $ "* need one of these commands, but none are available: " ++ show cmds + ) + +maybeSelectCmd :: ConfigKey -> [String] -> String -> Test +maybeSelectCmd k = searchCmd + (\match -> return $ Config k $ MaybeStringConfig $ Just match) + (\_ -> return $ Config k $ MaybeStringConfig Nothing) + +searchCmd :: (String -> Test) -> ([String] -> Test) -> [String] -> String -> Test +searchCmd success failure cmds param = search cmds where - search [] = failure + search [] = failure cmds search (c:cs) = do ret <- system $ quiet c ++ " " ++ param if (ret == ExitSuccess) then success c else search cs - success c - | required == True = return $ Config k (StringConfig c) - | otherwise = return $ Config k (MaybeStringConfig $ Just c) - failure - | required == True = do - testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show cmds - | otherwise = do - let r = Config k (MaybeStringConfig Nothing) - testEnd r - return r quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" diff --git a/configure.hs b/configure.hs index 0661813aef..d340f937d4 100644 --- a/configure.hs +++ b/configure.hs @@ -6,12 +6,12 @@ import Data.List import TestConfig tests :: [TestCase] -tests = [ - TestCase "version" $ getVersion +tests = + [ TestCase "version" $ getVersion , testCp "cp_a" "-a" , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid generator" $ selectCmd True "uuid" ["uuid", "uuidgen"] "" + , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] "" , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" @@ -24,7 +24,7 @@ shaTestCases l = map make l let cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"] key = "sha" ++ show n - in TestCase key $ selectCmd False key cmds " Date: Fri, 8 Apr 2011 07:23:09 +0000 Subject: [PATCH 1468/2835] Added a comment --- .../comment_1_994bfd12c5d82e08040d6116915c5090._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment diff --git a/doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment b/doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment new file mode 100644 index 0000000000..7b5e7bd449 --- /dev/null +++ b/doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2011-04-08T07:23:08Z" + content=""" ++1 for this feature, I've been longing for something like this other than rolling my own perl/shell scripts to parse the outputs of \"git annex whereis .\" to see how many files are on my machine or not. +"""]] From 2aae340e98341a4c6681ff8ec722117f8155aca3 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Fri, 8 Apr 2011 07:31:04 +0000 Subject: [PATCH 1469/2835] Added a comment --- .../comment_3_fcfafca994194d57dccf5319c7c9e646._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment new file mode 100644 index 0000000000..1b849ef891 --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 3" + date="2011-04-08T07:31:03Z" + content=""" +So perhaps checking if git-status (or similar) complains about missing files is a possible solution for this? +"""]] From 275f1bd5c5a72c9a0f8fe0e21282278a6adbf6d9 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Fri, 8 Apr 2011 07:54:38 +0000 Subject: [PATCH 1470/2835] Added a comment --- ..._04dc14880f31eee2b6d767d4d4258c5a._comment | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment new file mode 100644 index 0000000000..9280fc51da --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 4" + date="2011-04-08T07:54:37Z" + content=""" +And something else i've done is, that i symlinked the video/ directory from the media annex to the normal raid annex + + ln -s ~/media/annex/video ~/annex + +And it's working out great. + + ~annex $ git annex whereis video/series/episode1.avi + whereis video/series/episode1.avi(1 copy) + f210b45a-60d3-11e0-b593-3318d96f2520 -- Trantor - Media + ok + +I really like this, perhaps it is a good idea to store all log files in every repo, but maybe there is a possibilitiy to to pack multiple log files into one single file, where not only the time, the present bit and the annex-repository is stored, but also the file key. I don't know if this format would also be merged correctly by the union merge driver. + +"""]] From 784f47d2822a9f0e4cfa507cfbd01f99f4da60e6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Fri, 8 Apr 2011 12:19:05 +0000 Subject: [PATCH 1471/2835] --- doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn new file mode 100644 index 0000000000..608704d9d7 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn @@ -0,0 +1 @@ +I have files with very long filenames on an xfs at home. On my laptop the annex should have been checked out on an encfs, but there filenames can't be as long as on the xfs. So perhaps it would be good to limit the keysize to a sane substring of the filename e.g. use only the first 120 characters. From 72e6bed8bab48a77aa3b0211dcf801d21a7b886a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 8 Apr 2011 17:14:25 +0000 Subject: [PATCH 1472/2835] Added a comment --- ...comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment new file mode 100644 index 0000000000..41d3afb3eb --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-08T17:14:25Z" + content=""" +Seems like you probably have files in git with nearly as long filenames as the key files. Course, you can rename those yourself. + +This couldn't be changed directly in WORM without some ugly transition, but it would be possible to implement it as a WORM100 or so. OTOH, if you're going to git annex migrate, you might as well use SHA1. +"""]] From fbd77e02177463efa1f546ec78e158de5475e793 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawn5vmWzuHG-guyTurgCKCyIOl-uRTWpFyw" Date: Fri, 8 Apr 2011 17:49:27 +0000 Subject: [PATCH 1473/2835] --- doc/not.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/not.mdwn b/doc/not.mdwn index 1d38b848c6..fea8b3e969 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -33,7 +33,7 @@ * git-annex is also not [boar](http://code.google.com/p/boar/), although it shares many of its goals and characteristics. Boar implements - its own version control system, rather than simply embarcing and + its own version control system, rather than simply embracing and extending git. And while boar supports distributed clones of a repository, it does not support keeping different files in different clones of the same repository, which git-annex does, and is an important feature for From f3cf20d22a5c27b83138c4ee062edb7532fecbb3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 14:56:57 -0400 Subject: [PATCH 1474/2835] document bup special remotes --- doc/git-annex.mdwn | 10 ++++++++-- doc/not.mdwn | 6 ++---- doc/special_remotes.mdwn | 1 + doc/special_remotes/bup.mdwn | 34 ++++++++++++++++++++++++++++++++++ doc/walkthrough.mdwn | 1 + doc/walkthrough/using_bup.mdwn | 18 ++++++++++++++++++ 6 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 doc/special_remotes/bup.mdwn create mode 100644 doc/walkthrough/using_bup.mdwn diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 7d0fb3e792..3514002a49 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -402,9 +402,15 @@ Here are all the supported configuration settings. to or from this remote. For example, to force ipv6, and limit the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" -* `annex.ssh-options`, `annex.rsync-options` +* `remote..annex-bup-split-options` - Default ssh and rsync options to use if a remote does not have + Options to pass to bup split when storing content in this remote. + For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k" + (There is no corresponding option for bup join.) + +* `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options` + + Default ssh, rsync, and bup options to use if a remote does not have specific options. * `annex.diskreserve` diff --git a/doc/not.mdwn b/doc/not.mdwn index 1d38b848c6..8d5dbd0b52 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -2,10 +2,8 @@ * git-annex is not a backup system. It may be a useful component of an [[archival|use_case/bob]] system, or a way to deliver files to a backup - system. - - For a backup system that uses git, take a look at - [bup](http://github.com/apenwarr/bup). + system. For a backup system that uses git and that git-annex supports + storing data in, see [[special_remotes/bup]]. * git-annex is not unison, but if you're finding unison's checksumming too slow, or its strict mirroring of everything to both places too diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index f4d479aa9c..a33d3f6127 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -7,6 +7,7 @@ types of remotes. These can be used just like any normal remote by git-annex. They cannot be used by other git commands though. * [[Amazon_S3]] +* [[bup]] * [[directory]] ## Unused content on special remotes diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn new file mode 100644 index 0000000000..09eadd0085 --- /dev/null +++ b/doc/special_remotes/bup.mdwn @@ -0,0 +1,34 @@ +This special remote type stores file contents in a +[bup](http://github.com/apenwarr/bup) repository. By using git-annex +in the front-end, and bup as a remote, you get an easy git-style +interface to large files, and easy backups of the file contents using git. + +See [[walkthrough/using_bup]] for usage examples. + +## configuration + +These parameters can be passed to `git annex initremote` to configure bup: + +* `encryption` - Required. Either "none" to disable encryption, + or a value that can be looked up (using gpg -k) to find a gpg encryption + key that will be given access to the remote. Note that additional gpg + keys can be given access to a remote by rerunning initremote with + the new key id. + +* `remote` - Required. This is passed to `bup` as the `--remote` + to use to store data. `bup init` will be run to create the + repository. Example: "remote=example.com:/big/mybup" + +Options to pass to `bup split` when sending content to bup can also +be specified, by using `git config annex.bup-split-options`. This +can be used to, for example, limit its bandwidth. + +## data security + +When encryption=none, there is **no** protection against your data being read +by anyone who can access the bup remote. However, bup does transfer data +using ssh, and if you trust the security of the remote, that's fine. + +** Encryption is not yet supported. ** + +See [[design/encryption]]. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 53f0be6bb4..c648807494 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -15,6 +15,7 @@ A walkthrough of the basic features of git-annex. using_ssh_remotes moving_file_content_between_repositories using_Amazon_S3 + using_bup using_the_URL_backend using_the_SHA1_backend migrating_data_to_a_new_backend diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn new file mode 100644 index 0000000000..1a506c2811 --- /dev/null +++ b/doc/walkthrough/using_bup.mdwn @@ -0,0 +1,18 @@ +Another [[special_remote|special_remotes]] that git-annex can use is +a [[special_remotes/bup]] repository. Bup stores large file contents +in a git repository of its own, with deduplication. Combined with +git-annex, you can have git on both the frontend and the backend. + +Here's how to create a bup remote, and describe it. + + # git annex initremote mybup type=bup encryption=none remote=example.com/big/mybup + initremote bup (init) ok + # git annex describe mybup "my bup repository at example.com" + describe mybup ok + +Now the remote can be used like any other remote. + + # git annex move my_cool_big_file --to mybup + move my_cool_big_file (to mybup...) ok + +See [[special_remotes/bup]] for details. From 44c65f40b7f67ee5d53769c6e5fc87f2c7849425 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 16:44:43 -0400 Subject: [PATCH 1475/2835] bup is now supported as a special type of remote. --- Remote.hs | 2 + Remote/Bup.hs | 133 +++++++++++++++++++++++++++++++++ configure.hs | 1 + debian/changelog | 1 + debian/control | 2 +- doc/walkthrough/using_bup.mdwn | 8 +- 6 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 Remote/Bup.hs diff --git a/Remote.hs b/Remote.hs index 26097da747..bb661c5a90 100644 --- a/Remote.hs +++ b/Remote.hs @@ -46,12 +46,14 @@ import Config import qualified Remote.Git import qualified Remote.S3 +import qualified Remote.Bup import qualified Remote.Directory remoteTypes :: [RemoteType Annex] remoteTypes = [ Remote.Git.remote , Remote.S3.remote + , Remote.Bup.remote , Remote.Directory.remote ] diff --git a/Remote/Bup.hs b/Remote/Bup.hs new file mode 100644 index 0000000000..ef34e2c635 --- /dev/null +++ b/Remote/Bup.hs @@ -0,0 +1,133 @@ +{- Using bup as a remote. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Bup (remote) where + +import IO +import Control.Exception.Extensible (IOException) +import qualified Data.Map as M +import Control.Monad (unless) +import Control.Monad.State (liftIO) +import System.Process +import System.Exit + +import RemoteClass +import Types +import qualified GitRepo as Git +import qualified Annex +import UUID +import Locations +import LocationLog +import Config +import Utility +import Messages +import Remote.Special + +remote :: RemoteType Annex +remote = RemoteType { + typename = "bup", + enumerate = findSpecialRemotes "bupremote", + generate = gen, + setup = bupSetup +} + +gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen r u c = do + cst <- remoteCost r expensiveRemoteCost + bupremote <- getConfig r "bupremote" (error "missing bupremote") + return $ this cst bupremote + where + this cst bupremote = Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store r bupremote, + retrieveKeyFile = retrieve bupremote, + removeKey = remove, + hasKey = checkPresent u, + hasKeyCheap = True, + config = c + } + +bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) +bupSetup u c = do + -- verify configuration is sane + let bupremote = case M.lookup "remote" c of + Nothing -> error "Specify remote=" + Just r -> r + case M.lookup "encryption" c of + Nothing -> error "Specify encryption=key or encryption=none" + Just "none" -> return () + Just _ -> error "encryption keys not yet supported" + + -- bup init will create the repository. + -- (If the repository already exists, bup init again appears safe.) + showNote "bup init" + ok <- bup "init" bupremote [] + unless ok $ error "bup init failed" + + -- The bup remote is stored in git config, as well as this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c "bupremote" bupremote + + return $ M.delete "directory" c + +bupParams :: String -> String -> [CommandParam] -> [CommandParam] +bupParams command bupremote params = + (Param command) : [Param "-r", Param bupremote] ++ params + +bup :: String -> String -> [CommandParam] -> Annex Bool +bup command bupremote params = do + showProgress -- make way for bup output + liftIO $ boolSystem "bup" $ bupParams command bupremote params + +store :: Git.Repo -> String -> Key -> Annex Bool +store r bupremote k = do + g <- Annex.gitRepo + let src = gitAnnexLocation g k + o <- getConfig r "bup-split-options" "" + let os = map Param $ words o + bup "split" bupremote $ os ++ [Param "-n", Param (show k), File src] + +retrieve :: String -> Key -> FilePath -> Annex Bool +retrieve bupremote k f = do + let params = bupParams "join" bupremote [Param $ show k] + ret <- liftIO $ try $ do + -- pipe bup's stdout directly to file + tofile <- openFile f WriteMode + p <- runProcess "bup" (toCommand params) + Nothing Nothing Nothing (Just tofile) Nothing + r <- waitForProcess p + case r of + ExitSuccess -> return True + _ -> return False + case ret of + Right r -> return r + Left e -> return False + +remove :: Key -> Annex Bool +remove _ = do + warning "content cannot be removed from bup remote" + return False + +{- Bup does not provide a way to tell if a given dataset is present + - in a bup repository. One way it to check if the git repository has + - a branch matching the name (as created by bup split -n). + - + - However, git-annex's ususal reasons for checking if a remote really + - has a key also don't really apply in the case of bup, since, short + - of deleting bup's git repository, data cannot be removed from it. + - + - So, trust git-annex's location log; if it says a bup repository has + - content, assume it's right. + -} +checkPresent :: UUID -> Key -> Annex (Either IOException Bool) +checkPresent u k = do + g <- Annex.gitRepo + liftIO $ try $ do + uuids <- keyLocations g k + return $ u `elem` uuids diff --git a/configure.hs b/configure.hs index d340f937d4..4ab3052395 100644 --- a/configure.hs +++ b/configure.hs @@ -15,6 +15,7 @@ tests = , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" + , TestCase "bup" $ testCmd "bup" "bup --version >/dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath ] ++ shaTestCases [1, 256, 512, 224, 384] diff --git a/debian/changelog b/debian/changelog index 7f104be101..91c0c8f4b3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,6 @@ git-annex (0.20110402) UNRELEASED; urgency=low + * bup is now supported as a special type of remote. * Use lowercase hash directories for locationlog files, to avoid some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still diff --git a/debian/control b/debian/control index 37e6220437..15155b9b43 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: git-annex Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync -Suggests: graphviz +Suggests: graphviz, bup Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 1a506c2811..7e1562d12e 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -6,13 +6,17 @@ git-annex, you can have git on both the frontend and the backend. Here's how to create a bup remote, and describe it. # git annex initremote mybup type=bup encryption=none remote=example.com/big/mybup - initremote bup (init) ok + initremote bup (bup init) + Initialized empty Git repository in /big/mybup/ + ok # git annex describe mybup "my bup repository at example.com" describe mybup ok Now the remote can be used like any other remote. # git annex move my_cool_big_file --to mybup - move my_cool_big_file (to mybup...) ok + move my_cool_big_file (to mybup...) + Receiving index from server: 1100/1100, done. + ok See [[special_remotes/bup]] for details. From f2324c941567bb5ac67835fd3c3a82109bbac4be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 16:51:49 -0400 Subject: [PATCH 1476/2835] bupdates --- doc/special_remotes/bup.mdwn | 6 ++++++ doc/walkthrough/using_bup.mdwn | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 09eadd0085..950b64e9c0 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -5,6 +5,12 @@ interface to large files, and easy backups of the file contents using git. See [[walkthrough/using_bup]] for usage examples. +Each individual key is stored in a bup remote using `bup split`, with +a git branch named the same as the key name. Content is retrieved from +bup using `bup join`. All other bup operations are up to you -- consider +running `bup fsck --generate` in a cron job to generate recovery blocks, +for example; or clone bup's git repository to further back it up. + ## configuration These parameters can be passed to `git annex initremote` to configure bup: diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 7e1562d12e..12aa610372 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -19,4 +19,13 @@ Now the remote can be used like any other remote. Receiving index from server: 1100/1100, done. ok +Note that, unlike other remotes, bup does not really support removing +content from its git repositories. This is a feature. :) + + # git annex move my_cool_big_file --from mybup + move my_cool_big_file + content cannot be removed from bup remote + failed + git-annex: 1 failed + See [[special_remotes/bup]] for details. From 80b6c6d4bee93def8c398353ea45cb65155806aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 16:54:53 -0400 Subject: [PATCH 1477/2835] tweak --- doc/walkthrough/using_bup.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 12aa610372..e84ff613b7 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -23,7 +23,7 @@ Note that, unlike other remotes, bup does not really support removing content from its git repositories. This is a feature. :) # git annex move my_cool_big_file --from mybup - move my_cool_big_file + move my_cool_big_file (from mybup...) content cannot be removed from bup remote failed git-annex: 1 failed From 1861a065b28160586811c88b571e4ce80b0e806c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 16:59:13 -0400 Subject: [PATCH 1478/2835] close --- doc/todo/add_a_git_backend.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index 34322d9811..d84e9b69b5 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -12,3 +12,5 @@ use all of git-annex's features everywhere else. >> of the git backend using bup. --[[Joey]] >>> Very much so. Generally speaking, having one or more versioned storage back-ends with current data in the local annexes sounds incredibly useful. Still being able to get at old data in via the back-end and/or making offline backups of the full history are excellent use cases. -- RichiH + +[[done]], the bup special remote type is written! --[[Joey]] From c90db9c93fbecc491498509ded7f3ab8249b8094 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 8 Apr 2011 20:59:37 +0000 Subject: [PATCH 1479/2835] Added a comment --- .../comment_5_3459f0b41d818c23c8fb33edb89df634._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment new file mode 100644 index 0000000000..a1300f2e64 --- /dev/null +++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 5" + date="2011-04-08T20:59:37Z" + content=""" +My estimates were pretty close -- the new bup special remote type took 133 lines of code, and 2 hours to write. A testament to the flexibility of the special remote infrastructure. :) +"""]] From f66b6f6360a448adc3e2ad501bd1e7673ebe0192 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:07:44 -0400 Subject: [PATCH 1480/2835] move esoteric stuff from walkthrough to cheatsheet --- doc/cheatsheet.mdwn | 14 ++++++++++++++ doc/walkthrough.mdwn | 15 +++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 doc/cheatsheet.mdwn diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn new file mode 100644 index 0000000000..4287756a6c --- /dev/null +++ b/doc/cheatsheet.mdwn @@ -0,0 +1,14 @@ +A suppliment to the [[walkthrough]]. + +[[!toc]] + +[[!inline feeds=no show=0 template=walkthrough pagenames=""" + walkthrough/using_Amazon_S3 + walkthrough/using_bup + walkthrough/using_the_URL_backend + walkthrough/using_the_SHA1_backend + walkthrough/migrating_data_to_a_new_backend + walkthrough/untrusted_repositories + walkthrough/what_to_do_when_you_lose_a_repository + walkthrough/recover_data_from_lost+found +"""]] diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index c648807494..ae305cb1be 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -14,16 +14,15 @@ A walkthrough of the basic features of git-annex. modifying_annexed_files using_ssh_remotes moving_file_content_between_repositories - using_Amazon_S3 - using_bup - using_the_URL_backend - using_the_SHA1_backend - migrating_data_to_a_new_backend unused_data fsck:_verifying_your_data fsck:_when_things_go_wrong backups - untrusted_repositories - what_to_do_when_you_lose_a_repository - recover_data_from_lost+found """]] + +---- + +So ends the walkthrough. By now you should be able to use git-annex. + +Want more? See the [[cheatsheet]] for info about all of git-annex's hidden +features. From dacf53dacc63c20f09ccf5089b543d7b17e4a99e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:09:50 -0400 Subject: [PATCH 1481/2835] reorg --- doc/walkthrough.mdwn | 8 +------- doc/walkthrough/more.mdwn | 4 ++++ 2 files changed, 5 insertions(+), 7 deletions(-) create mode 100644 doc/walkthrough/more.mdwn diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index ae305cb1be..eaae6b455c 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -18,11 +18,5 @@ A walkthrough of the basic features of git-annex. fsck:_verifying_your_data fsck:_when_things_go_wrong backups + more """]] - ----- - -So ends the walkthrough. By now you should be able to use git-annex. - -Want more? See the [[cheatsheet]] for info about all of git-annex's hidden -features. diff --git a/doc/walkthrough/more.mdwn b/doc/walkthrough/more.mdwn new file mode 100644 index 0000000000..1eaf9009f6 --- /dev/null +++ b/doc/walkthrough/more.mdwn @@ -0,0 +1,4 @@ +So ends the walkthrough. By now you should be able to use git-annex. + +Want more? See the [[cheatsheet]] for info about all of git-annex's hidden +features. From f3953d5d4f22932822a7c5dbd5085c03c5232b21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:12:29 -0400 Subject: [PATCH 1482/2835] reword --- doc/walkthrough/using_the_URL_backend.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough/using_the_URL_backend.mdwn b/doc/walkthrough/using_the_URL_backend.mdwn index 585fd0668a..50da4dad89 100644 --- a/doc/walkthrough/using_the_URL_backend.mdwn +++ b/doc/walkthrough/using_the_URL_backend.mdwn @@ -1,5 +1,5 @@ -git-annex has multiple key-value [[backends]]. So far this walkthrough has -demonstrated the default, WORM (Write Once, Read Many) backend. +git-annex has multiple key-value [[backends]]. So far you have been using +the default, WORM (Write Once, Read Many) backend. Another handy backend is the URL backend, which can fetch file's content from remote URLs. Here's how to set up some files in your repository From 280b9677d60c0f9c9b83063fa7edea72e0774421 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:13:50 -0400 Subject: [PATCH 1483/2835] expand --- doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn index 1159b22171..16a55b37b3 100644 --- a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn +++ b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn @@ -15,4 +15,5 @@ To remind yourself later what happened, you can change its description, too: This retains the [[location_tracking]] information for the repository. Maybe you'll find the drive later. Maybe that's impossible. Either way, -this lets git-annex tell you why a file is no longer accessible. +this lets git-annex tell you why a file is no longer accessible, and +it avoids it relying on that drive to hold any content. From 9de1f5d9cfec3492ebee55fabc64f36a595d5be5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:15:14 -0400 Subject: [PATCH 1484/2835] typo --- doc/walkthrough/using_bup.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index e84ff613b7..7ba9d1fce9 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -5,7 +5,7 @@ git-annex, you can have git on both the frontend and the backend. Here's how to create a bup remote, and describe it. - # git annex initremote mybup type=bup encryption=none remote=example.com/big/mybup + # git annex initremote mybup type=bup encryption=none remote=example.com:/big/mybup initremote bup (bup init) Initialized empty Git repository in /big/mybup/ ok From 5bca5733fc674ed8244b5589855dde5c92df2934 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 8 Apr 2011 21:51:17 +0000 Subject: [PATCH 1485/2835] typo --- doc/design/encryption.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 8a8f38108e..90b722b9b8 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -61,7 +61,7 @@ more gpg public keys. This scheme allows new gpg private keys to be given access to content that has already been stored in the remote. Different encrypted remotes need to be able to each use different ciphers. -Allowing multiple cipers to be used within a single remote would add a lot +Allowing multiple ciphers to be used within a single remote would add a lot of complexity, so is not planned to be supported. Instead, if you want a new cipher, create a new S3 bucket, or whatever. There does not seem to be much benefit to using the same cipher for From 59ffe8e12713b66b1efa303a6ca7ba51d8dee535 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 8 Apr 2011 21:55:37 +0000 Subject: [PATCH 1486/2835] Added a comment --- .../comment_7_d18cf944352f8303799c86f2c0354e8e._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment new file mode 100644 index 0000000000..7d2ad5eba7 --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 7" + date="2011-04-08T21:55:36Z" + content=""" +http://xfs.org/index.php/XFS_FAQ#Q:_Performance:_mkfs.xfs_-n_size.3D64k_option +"""]] From bed6cc80b9bffeab677fc1d2124a2d76a230e031 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 8 Apr 2011 22:02:41 +0000 Subject: [PATCH 1487/2835] Added a comment --- .../comment_2_fe735d728878d889ccd34ec12b3a7dea._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment new file mode 100644 index 0000000000..d00191f9d8 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-08T22:02:41Z" + content=""" +What if your files have the same prefix and it happens to be 100 chars long? This can not be solved within WORM, but as Joey pointed out, SHA* exists. +"""]] From 939c88787ad785a9d00e376b8417ab6a5c40b339 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 8 Apr 2011 22:05:26 +0000 Subject: [PATCH 1488/2835] --- doc/todo/add_a_git_backend.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index d84e9b69b5..2b224710ed 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -14,3 +14,5 @@ use all of git-annex's features everywhere else. >>> Very much so. Generally speaking, having one or more versioned storage back-ends with current data in the local annexes sounds incredibly useful. Still being able to get at old data in via the back-end and/or making offline backups of the full history are excellent use cases. -- RichiH [[done]], the bup special remote type is written! --[[Joey]] + +> Yay! -- RichiH From 5d7ef05d95dc2a8e926a87cf552b119457231db0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 20:44:52 -0400 Subject: [PATCH 1489/2835] local bup remotes --- doc/special_remotes/bup.mdwn | 4 ++-- doc/walkthrough/using_bup.mdwn | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 950b64e9c0..4872072a03 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -22,8 +22,8 @@ These parameters can be passed to `git annex initremote` to configure bup: the new key id. * `remote` - Required. This is passed to `bup` as the `--remote` - to use to store data. `bup init` will be run to create the - repository. Example: "remote=example.com:/big/mybup" + to use to store data. To create the repository,`bup init` will be run. + Example: "remote=example.com:/big/mybup" or "remote=/big/mybup" Options to pass to `bup split` when sending content to bup can also be specified, by using `git config annex.bup-split-options`. This diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 7ba9d1fce9..4ce189480f 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -5,6 +5,12 @@ git-annex, you can have git on both the frontend and the backend. Here's how to create a bup remote, and describe it. +[[!template id=note test=""" +Instead of specifying a remote system, you could choose to make a bup +remote that is only accessible on the current system, by passing +"remote=/big/mybup". +"""] + # git annex initremote mybup type=bup encryption=none remote=example.com:/big/mybup initremote bup (bup init) Initialized empty Git repository in /big/mybup/ From 1bfd8d659ae23c3f8996974deda59a277a857b63 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 20:46:40 -0400 Subject: [PATCH 1490/2835] typo --- doc/walkthrough/using_bup.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 4ce189480f..4ee2bbf088 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -9,7 +9,7 @@ Here's how to create a bup remote, and describe it. Instead of specifying a remote system, you could choose to make a bup remote that is only accessible on the current system, by passing "remote=/big/mybup". -"""] +"""]] # git annex initremote mybup type=bup encryption=none remote=example.com:/big/mybup initremote bup (bup init) From 3650f42bcfb631e1ef6d53be725b72cbbfc249cd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 20:47:23 -0400 Subject: [PATCH 1491/2835] typo --- doc/walkthrough/using_bup.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 4ee2bbf088..c3200dd320 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -5,7 +5,7 @@ git-annex, you can have git on both the frontend and the backend. Here's how to create a bup remote, and describe it. -[[!template id=note test=""" +[[!template id=note text=""" Instead of specifying a remote system, you could choose to make a bup remote that is only accessible on the current system, by passing "remote=/big/mybup". From c253d07a82f1ffac4aa256c09a6fd7dcbc780923 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 20:55:22 -0400 Subject: [PATCH 1492/2835] minor --- Remote/Bup.hs | 2 +- Remote/Git.hs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index ef34e2c635..5fbe9a8b5d 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -107,7 +107,7 @@ retrieve bupremote k f = do _ -> return False case ret of Right r -> return r - Left e -> return False + Left _ -> return False remove :: Key -> Annex Bool remove _ = do diff --git a/Remote/Git.hs b/Remote/Git.hs index 2936beaf7d..c315d457d6 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -57,9 +57,7 @@ gen r u _ = do u' <- getUUID r' - let defcst = if not $ Git.repoIsUrl r - then cheapRemoteCost - else expensiveRemoteCost + let defcst = if cheap then cheapRemoteCost else expensiveRemoteCost cst <- remoteCost r' defcst return $ Remote { @@ -70,7 +68,7 @@ gen r u _ = do retrieveKeyFile = copyFromRemote r', removeKey = dropKey r', hasKey = inAnnex r', - hasKeyCheap = not (Git.repoIsUrl r'), + hasKeyCheap = cheap, config = Nothing } From 1bfd3922c06e6a6982a6ec78ea21432652e471d9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 21:37:59 -0400 Subject: [PATCH 1493/2835] set cost for local bup repos to cheap --- Remote/Bup.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 5fbe9a8b5d..6a7609aad9 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -37,8 +37,10 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do - cst <- remoteCost r expensiveRemoteCost bupremote <- getConfig r "bupremote" (error "missing bupremote") + let local = ':' `notElem` bupremote + cst <- remoteCost r (if local then cheapRemoteCost else expensiveRemoteCost) + return $ this cst bupremote where this cst bupremote = Remote { From c5174f0cb8a234dbd3656f108194a4e3dec6fec5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 23:08:21 -0400 Subject: [PATCH 1494/2835] make local bup repos a bit more expensive than local git repos does have to run bup and reassemble files, after all --- Config.hs | 2 ++ Remote/Bup.hs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Config.hs b/Config.hs index 17a1fa9856..53f1a455fd 100644 --- a/Config.hs +++ b/Config.hs @@ -51,6 +51,8 @@ remoteCost r def = do cheapRemoteCost :: Int cheapRemoteCost = 100 +semiCheapRemoteCost :: Int +semiCheapRemoteCost = 150 expensiveRemoteCost :: Int expensiveRemoteCost = 200 diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 6a7609aad9..8d92792e19 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -39,7 +39,7 @@ gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do bupremote <- getConfig r "bupremote" (error "missing bupremote") let local = ':' `notElem` bupremote - cst <- remoteCost r (if local then cheapRemoteCost else expensiveRemoteCost) + cst <- remoteCost r (if local then semiCheapRemoteCost else expensiveRemoteCost) return $ this cst bupremote where From fdf1c648129042c4d61fedb93830a6d00e06ac21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 11:13:01 -0400 Subject: [PATCH 1495/2835] notes --- doc/todo/branching.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index 06bd50beed..79f278480e 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -115,3 +115,14 @@ would solve the merge problem, since git auto merge could be used. It would still mean all the log files are on-disk, which annoys some. It would require some tighter integration with git, so that after a pull, the log repo is updated with the data pulled. --[[Joey]] + +## notes + +Another approach could be to use git-notes. It supports merging branches +of notes, with union merge strategy (a hook would have to do this after +a pull, it's not done automatically). + +Problem: Notes are usually attached to git +objects, and there are no git objects corresponding to git-annex keys. + +Problem: Notes are not normally copied when cloning. From e7d30fe3da0530bce6e8498ecb9020bbbabccf43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 11:41:10 -0400 Subject: [PATCH 1496/2835] mention how to use default bup repo --- doc/special_remotes/bup.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 4872072a03..19f2320f8c 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -24,6 +24,7 @@ These parameters can be passed to `git annex initremote` to configure bup: * `remote` - Required. This is passed to `bup` as the `--remote` to use to store data. To create the repository,`bup init` will be run. Example: "remote=example.com:/big/mybup" or "remote=/big/mybup" + (To use the default `~/.bup` repository on the local host, specify "remote=") Options to pass to `bup split` when sending content to bup can also be specified, by using `git config annex.bup-split-options`. This From 141e55ff11394e2f162397957c96c02ad3f0bd37 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 12:34:49 -0400 Subject: [PATCH 1497/2835] store annex.uuid in bup repos --- GitRepo.hs | 6 ------ Remote/Bup.hs | 57 ++++++++++++++++++++++++++++++++++++++++++++++++--- Utility.hs | 9 ++++++++ 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 1b14e4a636..543ad801a4 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -573,12 +573,6 @@ repoAbsPath d = do h <- myHomeDir return $ h d' -myHomeDir :: IO FilePath -myHomeDir = do - uid <- getEffectiveUserID - u <- getUserEntryForID uid - return $ homeDirectory u - expandTilde :: FilePath -> IO FilePath expandTilde = expandt True where diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 8d92792e19..d43b03a92d 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -10,10 +10,12 @@ module Remote.Bup (remote) where import IO import Control.Exception.Extensible (IOException) import qualified Data.Map as M -import Control.Monad (unless) +import Control.Monad (unless, when) import Control.Monad.State (liftIO) import System.Process import System.Exit +import System.FilePath +import Data.List.Utils import RemoteClass import Types @@ -26,6 +28,7 @@ import Config import Utility import Messages import Remote.Special +import Ssh remote :: RemoteType Annex remote = RemoteType { @@ -38,8 +41,7 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do bupremote <- getConfig r "bupremote" (error "missing bupremote") - let local = ':' `notElem` bupremote - cst <- remoteCost r (if local then semiCheapRemoteCost else expensiveRemoteCost) + cst <- remoteCost r (if bupLocal bupremote then semiCheapRemoteCost else expensiveRemoteCost) return $ this cst bupremote where @@ -72,6 +74,8 @@ bupSetup u c = do ok <- bup "init" bupremote [] unless ok $ error "bup init failed" + storeBupUUID u bupremote + -- The bup remote is stored in git config, as well as this remote's -- persistant state, so it can vary between hosts. gitConfigSpecialRemote u c "bupremote" bupremote @@ -133,3 +137,50 @@ checkPresent u k = do liftIO $ try $ do uuids <- keyLocations g k return $ u `elem` uuids + +{- Store UUID in the annex.uuid setting of the bup repository. -} +storeBupUUID :: UUID -> FilePath -> Annex () +storeBupUUID u bupremote = do + r <- liftIO $ bup2GitRemote bupremote + if Git.repoIsUrl r + then do + showNote "storing uuid" + let dir = shellEscape (Git.workTree r) + sshparams <- sshToRepo r + [Param $ "cd " ++ dir ++ + " && git config annex.uuid " ++ u] + ok <- liftIO $ boolSystem "ssh" sshparams + unless ok $ do error "ssh failed" + else liftIO $ do + r' <- Git.configRead r + let olduuid = Git.configGet r' "annex.uuid" "" + when (olduuid == "") $ + Git.run r' "config" [Param "annex.uuid", Param u] + +{- Converts a bup remote path spec into a Git.Repo. There are some + - differences in path representation between git and bup. -} +bup2GitRemote :: FilePath -> IO Git.Repo +bup2GitRemote "" = do + -- bup -r "" operates on ~/.bup + h <- myHomeDir + Git.repoFromAbsPath $ h ".bup" +bup2GitRemote r + | bupLocal r = + if r !! 0 == '/' + then Git.repoFromAbsPath r + else error "please specify an absolute path" + | otherwise = Git.repoFromUrl $ "ssh://" ++ host ++ slash dir + where + bits = split ":" r + host = bits !! 0 + dir = join ":" $ drop 1 bits + -- "host:~user/dir" is not supported specially by bup; + -- "host:dir" is relative to the home directory; + -- "host:" goes in ~/.bup + slash d + | d == "" = "/~/.bup" + | d !! 0 == '/' = d + | otherwise = "/~/" ++ d + +bupLocal :: FilePath -> Bool +bupLocal = notElem ':' diff --git a/Utility.hs b/Utility.hs index 72f5c50638..1c6b4d21e6 100644 --- a/Utility.hs +++ b/Utility.hs @@ -23,6 +23,7 @@ module Utility ( safeWriteFile, dirContains, dirContents, + myHomeDir, prop_idempotent_shellEscape, prop_idempotent_shellEscape_multiword, @@ -36,6 +37,7 @@ import System.Posix.Process import System.Posix.Signals import System.Posix.Files import System.Posix.Types +import System.Posix.User import Data.String.Utils import System.Path import System.FilePath @@ -247,3 +249,10 @@ dirContents d = do notcruft "." = False notcruft ".." = False notcruft _ = True + +{- Current user's home directory. -} +myHomeDir :: IO FilePath +myHomeDir = do + uid <- getEffectiveUserID + u <- getUserEntryForID uid + return $ homeDirectory u From c739c7d7872557ca0c5d3518fb0a3bdbade871eb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 12:41:17 -0400 Subject: [PATCH 1498/2835] change name of buprepo Instead of remote=, use buprepo= Anyone already using bup will need to re-run git annex initremote. --- Remote/Bup.hs | 54 +++++++++++++++++++--------------- doc/special_remotes/bup.mdwn | 6 ++-- doc/walkthrough/using_bup.mdwn | 4 +-- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index d43b03a92d..b6d08d89bc 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -33,26 +33,27 @@ import Ssh remote :: RemoteType Annex remote = RemoteType { typename = "bup", - enumerate = findSpecialRemotes "bupremote", + enumerate = findSpecialRemotes "buprepo", generate = gen, setup = bupSetup } gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do - bupremote <- getConfig r "bupremote" (error "missing bupremote") - cst <- remoteCost r (if bupLocal bupremote then semiCheapRemoteCost else expensiveRemoteCost) + buprepo <- getConfig r "buprepo" (error "missing buprepo") + cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost) +-- u' <- getBupUUID r u - return $ this cst bupremote + return $ this cst buprepo u' where - this cst bupremote = Remote { - uuid = u, + this cst buprepo u' = Remote { + uuid = u', cost = cst, name = Git.repoDescribe r, - storeKey = store r bupremote, - retrieveKeyFile = retrieve bupremote, + storeKey = store r buprepo, + retrieveKeyFile = retrieve buprepo, removeKey = remove, - hasKey = checkPresent u, + hasKey = checkPresent u', hasKeyCheap = True, config = c } @@ -60,7 +61,7 @@ gen r u c = do bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) bupSetup u c = do -- verify configuration is sane - let bupremote = case M.lookup "remote" c of + let buprepo = case M.lookup "remote" c of Nothing -> error "Specify remote=" Just r -> r case M.lookup "encryption" c of @@ -71,37 +72,37 @@ bupSetup u c = do -- bup init will create the repository. -- (If the repository already exists, bup init again appears safe.) showNote "bup init" - ok <- bup "init" bupremote [] + ok <- bup "init" buprepo [] unless ok $ error "bup init failed" - storeBupUUID u bupremote + storeBupUUID u buprepo - -- The bup remote is stored in git config, as well as this remote's + -- The buprepo is stored in git config, as well as this repo's -- persistant state, so it can vary between hosts. - gitConfigSpecialRemote u c "bupremote" bupremote + gitConfigSpecialRemote u c "buprepo" buprepo return $ M.delete "directory" c bupParams :: String -> String -> [CommandParam] -> [CommandParam] -bupParams command bupremote params = - (Param command) : [Param "-r", Param bupremote] ++ params +bupParams command buprepo params = + (Param command) : [Param "-r", Param buprepo] ++ params bup :: String -> String -> [CommandParam] -> Annex Bool -bup command bupremote params = do +bup command buprepo params = do showProgress -- make way for bup output - liftIO $ boolSystem "bup" $ bupParams command bupremote params + liftIO $ boolSystem "bup" $ bupParams command buprepo params store :: Git.Repo -> String -> Key -> Annex Bool -store r bupremote k = do +store r buprepo k = do g <- Annex.gitRepo let src = gitAnnexLocation g k o <- getConfig r "bup-split-options" "" let os = map Param $ words o - bup "split" bupremote $ os ++ [Param "-n", Param (show k), File src] + bup "split" buprepo $ os ++ [Param "-n", Param (show k), File src] retrieve :: String -> Key -> FilePath -> Annex Bool -retrieve bupremote k f = do - let params = bupParams "join" bupremote [Param $ show k] +retrieve buprepo k f = do + let params = bupParams "join" buprepo [Param $ show k] ret <- liftIO $ try $ do -- pipe bup's stdout directly to file tofile <- openFile f WriteMode @@ -140,8 +141,8 @@ checkPresent u k = do {- Store UUID in the annex.uuid setting of the bup repository. -} storeBupUUID :: UUID -> FilePath -> Annex () -storeBupUUID u bupremote = do - r <- liftIO $ bup2GitRemote bupremote +storeBupUUID u buprepo = do + r <- liftIO $ bup2GitRemote buprepo if Git.repoIsUrl r then do showNote "storing uuid" @@ -157,6 +158,11 @@ storeBupUUID u bupremote = do when (olduuid == "") $ Git.run r' "config" [Param "annex.uuid", Param u] +{- Allow for bup repositories on removable media by checking + - local bup repositories -} +--getBupUUID :: UUID -> FilePath -> Annex () +--getBupUUID u buprepo = do + {- Converts a bup remote path spec into a Git.Repo. There are some - differences in path representation between git and bup. -} bup2GitRemote :: FilePath -> IO Git.Repo diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 19f2320f8c..c74bdaf7e2 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -21,10 +21,10 @@ These parameters can be passed to `git annex initremote` to configure bup: keys can be given access to a remote by rerunning initremote with the new key id. -* `remote` - Required. This is passed to `bup` as the `--remote` +* `buprepo` - Required. This is passed to `bup` as the `--remote` to use to store data. To create the repository,`bup init` will be run. - Example: "remote=example.com:/big/mybup" or "remote=/big/mybup" - (To use the default `~/.bup` repository on the local host, specify "remote=") + Example: "buprepo=example.com:/big/mybup" or "buprepo=/big/mybup" + (To use the default `~/.bup` repository on the local host, specify "buprepo=") Options to pass to `bup split` when sending content to bup can also be specified, by using `git config annex.bup-split-options`. This diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index c3200dd320..3a6a8776aa 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -8,10 +8,10 @@ Here's how to create a bup remote, and describe it. [[!template id=note text=""" Instead of specifying a remote system, you could choose to make a bup remote that is only accessible on the current system, by passing -"remote=/big/mybup". +"buprepo=/big/mybup". """]] - # git annex initremote mybup type=bup encryption=none remote=example.com:/big/mybup + # git annex initremote mybup type=bup encryption=none buprepo=example.com:/big/mybup initremote bup (bup init) Initialized empty Git repository in /big/mybup/ ok From f808a8335064d22fac1f9b7ee6c32a6680dd6cbc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 12:45:30 -0400 Subject: [PATCH 1499/2835] more buprepo fixes --- Remote/Bup.hs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index b6d08d89bc..fdc3919e23 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -42,7 +42,7 @@ gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do buprepo <- getConfig r "buprepo" (error "missing buprepo") cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost) --- u' <- getBupUUID r u + u' <- getBupUUID buprepo u return $ this cst buprepo u' where @@ -61,8 +61,8 @@ gen r u c = do bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) bupSetup u c = do -- verify configuration is sane - let buprepo = case M.lookup "remote" c of - Nothing -> error "Specify remote=" + let buprepo = case M.lookup "buprepo" c of + Nothing -> error "Specify buprepo=" Just r -> r case M.lookup "encryption" c of Nothing -> error "Specify encryption=key or encryption=none" @@ -160,8 +160,9 @@ storeBupUUID u buprepo = do {- Allow for bup repositories on removable media by checking - local bup repositories -} ---getBupUUID :: UUID -> FilePath -> Annex () ---getBupUUID u buprepo = do +getBupUUID :: FilePath -> UUID -> Annex UUID +getBupUUID buprepo u = do + return u -- TODO {- Converts a bup remote path spec into a Git.Repo. There are some - differences in path representation between git and bup. -} From 54286c993dab913515b12ae8d87c2944ea11e6b2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 12:59:18 -0400 Subject: [PATCH 1500/2835] support bup repositories on removable media --- Remote/Bup.hs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index fdc3919e23..dc653631d5 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -159,10 +159,23 @@ storeBupUUID u buprepo = do Git.run r' "config" [Param "annex.uuid", Param u] {- Allow for bup repositories on removable media by checking - - local bup repositories -} + - local bup repositories to see if they are available, and getting their + - uuid (which may be different from the stored uuid for the bup remote). + - + - If a bup repository is not available, returns a dummy uuid of "". + - This will cause checkPresent to indicate nothing from the bup remote + - is known to be present. + -} getBupUUID :: FilePath -> UUID -> Annex UUID -getBupUUID buprepo u = do - return u -- TODO +getBupUUID buprepo u = liftIO $ do + r <- bup2GitRemote buprepo + if Git.repoIsUrl r + then return u + else do + ret <- try $ Git.configRead r + case ret of + Right r' -> return $ Git.configGet r' "annex.uuid" "" + Left _ -> return "" {- Converts a bup remote path spec into a Git.Repo. There are some - differences in path representation between git and bup. -} From 40a7ed859c55df12d5da99e596171d27bfcb5ee2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 13:10:01 -0400 Subject: [PATCH 1501/2835] link --- doc/index.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index b5880823e1..224d25d2cc 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -43,8 +43,9 @@ files with git. * [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage -* [[internals]] +* [[special_remotes]] (including [[special_remotes/Amazon_S3]] and [[special_remotes/bup]]) * [[bare_repositories]] +* [[internals]] * [[design]] * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. From cdea1a6b3a5310287479e0ee580dccbf352822bb Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 9 Apr 2011 17:15:01 +0000 Subject: [PATCH 1502/2835] Comment moderation --- .../comment_1_d8e34fc2bc4e5cf761574608f970d496._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment diff --git a/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment b/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment new file mode 100644 index 0000000000..4aceb3abd3 --- /dev/null +++ b/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 1" + date="2011-04-08T12:41:43Z" + content=""" +I also think, that fetching keys via rsync can be done by one rsync process, when the keys are fetched from one host. This would avoid establishing a new TCP connection for every file. +"""]] From 1e7ad2ee7c63c02c8f6b9df465576c8245bba8a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 14:15:38 -0400 Subject: [PATCH 1503/2835] reformat to work around man page conversion bugs --- doc/git-annex-shell.mdwn | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index b32114a74d..060d23ca18 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -17,6 +17,8 @@ user's restricted login shell. # COMMANDS +Any command not listed below is passed through to git-shell. + * configlist directory This outputs a subset of the git configuration, in the same form as @@ -40,16 +42,14 @@ user's restricted login shell. This runs rsync in server mode to transfer out the content of a key. +## OPTIONS + +Same as git-annex or git-shell, depending on the command being run. + Note that the directory parameter should be an absolute path, otherwise it is assumed to be relative to the user's home directory. Also the first "/~/" or "/~user/" is expanded to the specified home directory. -Any other command is passed through to git-shell. - -# OPTIONS - -Same as git-annex or git-shell, depending on the command being run. - # SEE ALSO [[git-annex]](1) From 8ad901a647a9c7cf179dc2dd73d121adc43a28fb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 14:26:32 -0400 Subject: [PATCH 1504/2835] refactor --- Command/Map.hs | 3 +-- Remote/Git.hs | 36 +----------------------------------- Ssh.hs | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Command/Map.hs b/Command/Map.hs index dc3acb56e4..2325c87e14 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -16,7 +16,6 @@ import Data.List.Utils import Command import qualified Annex import qualified GitRepo as Git -import qualified Remote.Git import Messages import Types import Utility @@ -203,7 +202,7 @@ tryScan r Git.hConfigRead r configlist = - Remote.Git.onRemote r (pipedconfig, Nothing) "configlist" [] + onRemote r (pipedconfig, Nothing) "configlist" [] manualconfiglist = do let sshcmd = "cd " ++ shellEscape(Git.workTree r) ++ " && " ++ diff --git a/Remote/Git.hs b/Remote/Git.hs index c315d457d6..7724df79af 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -5,10 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Remote.Git ( - remote, - onRemote -) where +module Remote.Git (remote) where import Control.Exception.Extensible import Control.Monad.State (liftIO) @@ -194,34 +191,3 @@ rsyncParams r sending key file = do -- goes, so the source/dest parameter can be a dummy value, -- that just enables remote rsync mode. dummy = Param ":" - -{- Uses a supplied function to run a git-annex-shell command on a remote. - - - - Or, if the remote does not support running remote commands, returns - - a specified error value. -} -onRemote - :: Git.Repo - -> (FilePath -> [CommandParam] -> IO a, a) - -> String - -> [CommandParam] - -> Annex a -onRemote r (with, errorval) command params = do - s <- git_annex_shell r command params - case s of - Just (c, ps) -> liftIO $ with c ps - Nothing -> return errorval - -{- Generates parameters to run a git-annex-shell command on a remote. -} -git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam])) -git_annex_shell r command params - | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) - | Git.repoIsSsh r = do - sshparams <- sshToRepo r [Param sshcmd] - return $ Just ("ssh", sshparams) - | otherwise = return Nothing - where - dir = Git.workTree r - shellcmd = "git-annex-shell" - shellopts = (Param command):(File dir):params - sshcmd = shellcmd ++ " " ++ - unwords (map shellEscape $ toCommand shellopts) diff --git a/Ssh.hs b/Ssh.hs index 6d01a56423..0cf2919c25 100644 --- a/Ssh.hs +++ b/Ssh.hs @@ -7,6 +7,8 @@ module Ssh where +import Control.Monad.State (liftIO) + import qualified GitRepo as Git import Utility import Types @@ -24,3 +26,36 @@ sshToRepo repo sshcmd = do Just p -> [Param "-p", Param (show p)] let sshhost = Param $ Git.urlHostUser repo return $ sshoptions ++ sshport ++ [sshhost] ++ sshcmd + +{- Generates parameters to run a git-annex-shell command on a remote + - repository. -} +git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam])) +git_annex_shell r command params + | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) + | Git.repoIsSsh r = do + sshparams <- sshToRepo r [Param sshcmd] + return $ Just ("ssh", sshparams) + | otherwise = return Nothing + where + dir = Git.workTree r + shellcmd = "git-annex-shell" + shellopts = (Param command):(File dir):params + sshcmd = shellcmd ++ " " ++ + unwords (map shellEscape $ toCommand shellopts) + +{- Uses a supplied function (such as boolSystem) to run a git-annex-shell + - command on a remote. + - + - Or, if the remote does not support running remote commands, returns + - a specified error value. -} +onRemote + :: Git.Repo + -> (FilePath -> [CommandParam] -> IO a, a) + -> String + -> [CommandParam] + -> Annex a +onRemote r (with, errorval) command params = do + s <- git_annex_shell r command params + case s of + Just (c, ps) -> liftIO $ with c ps + Nothing -> return errorval From ede234136b38e2039f9f056a6c05b10c65a07b51 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 14:40:14 -0400 Subject: [PATCH 1505/2835] bup and git-annex-shell Looked at having git-annex shell support running bup server.. Then looked at how bup runs bup server over ssh, and fled away, screaming.. --- doc/special_remotes/bup.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index c74bdaf7e2..90b84e9f47 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -30,6 +30,11 @@ Options to pass to `bup split` when sending content to bup can also be specified, by using `git config annex.bup-split-options`. This can be used to, for example, limit its bandwidth. +## notes + +[[git-annex-shell]] does not support bup, due to the wacky way that bup +starts its server. So, to use bup, you need full shell access to the server. + ## data security When encryption=none, there is **no** protection against your data being read From 66950189fcfc9d005c5c2d13ce2060a815362b6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 15:36:54 -0400 Subject: [PATCH 1506/2835] actually check that bup has keys I don't trust the location log, even for bup. Too many things could go wrong. --- Remote/Bup.hs | 84 +++++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index dc653631d5..916afeb406 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -23,13 +23,14 @@ import qualified GitRepo as Git import qualified Annex import UUID import Locations -import LocationLog import Config import Utility import Messages import Remote.Special import Ssh +type BupRepo = String + remote :: RemoteType Annex remote = RemoteType { typename = "bup", @@ -42,18 +43,19 @@ gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do buprepo <- getConfig r "buprepo" (error "missing buprepo") cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost) - u' <- getBupUUID buprepo u + bupr <- liftIO $ bup2GitRemote buprepo + (u', bupr') <- getBupUUID bupr u - return $ this cst buprepo u' + return $ this cst buprepo u' bupr' where - this cst buprepo u' = Remote { + this cst buprepo u' bupr = Remote { uuid = u', cost = cst, name = Git.repoDescribe r, storeKey = store r buprepo, retrieveKeyFile = retrieve buprepo, removeKey = remove, - hasKey = checkPresent u', + hasKey = checkPresent r bupr, hasKeyCheap = True, config = c } @@ -83,16 +85,16 @@ bupSetup u c = do return $ M.delete "directory" c -bupParams :: String -> String -> [CommandParam] -> [CommandParam] +bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam] bupParams command buprepo params = (Param command) : [Param "-r", Param buprepo] ++ params -bup :: String -> String -> [CommandParam] -> Annex Bool +bup :: String -> BupRepo -> [CommandParam] -> Annex Bool bup command buprepo params = do showProgress -- make way for bup output liftIO $ boolSystem "bup" $ bupParams command buprepo params -store :: Git.Repo -> String -> Key -> Annex Bool +store :: Git.Repo -> BupRepo -> Key -> Annex Bool store r buprepo k = do g <- Annex.gitRepo let src = gitAnnexLocation g k @@ -100,7 +102,7 @@ store r buprepo k = do let os = map Param $ words o bup "split" buprepo $ os ++ [Param "-n", Param (show k), File src] -retrieve :: String -> Key -> FilePath -> Annex Bool +retrieve :: BupRepo -> Key -> FilePath -> Annex Bool retrieve buprepo k f = do let params = bupParams "join" buprepo [Param $ show k] ret <- liftIO $ try $ do @@ -124,33 +126,28 @@ remove _ = do {- Bup does not provide a way to tell if a given dataset is present - in a bup repository. One way it to check if the git repository has - a branch matching the name (as created by bup split -n). - - - - However, git-annex's ususal reasons for checking if a remote really - - has a key also don't really apply in the case of bup, since, short - - of deleting bup's git repository, data cannot be removed from it. - - - - So, trust git-annex's location log; if it says a bup repository has - - content, assume it's right. -} -checkPresent :: UUID -> Key -> Annex (Either IOException Bool) -checkPresent u k = do - g <- Annex.gitRepo - liftIO $ try $ do - uuids <- keyLocations g k - return $ u `elem` uuids +checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either IOException Bool) +checkPresent r bupr k + | Git.repoIsUrl bupr = do + showNote ("checking " ++ Git.repoDescribe r ++ "...") + ok <- onBupRemote bupr boolSystem "git" params + return $ Right ok + | otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine bupr params + where + params = + [ Params "show-ref --quiet --verify" + , Param $ "refs/heads/" ++ show k] {- Store UUID in the annex.uuid setting of the bup repository. -} -storeBupUUID :: UUID -> FilePath -> Annex () +storeBupUUID :: UUID -> BupRepo -> Annex () storeBupUUID u buprepo = do r <- liftIO $ bup2GitRemote buprepo if Git.repoIsUrl r then do showNote "storing uuid" - let dir = shellEscape (Git.workTree r) - sshparams <- sshToRepo r - [Param $ "cd " ++ dir ++ - " && git config annex.uuid " ++ u] - ok <- liftIO $ boolSystem "ssh" sshparams + ok <- onBupRemote r boolSystem "git" + [Params $ "config annex.uuid " ++ u] unless ok $ do error "ssh failed" else liftIO $ do r' <- Git.configRead r @@ -158,6 +155,13 @@ storeBupUUID u buprepo = do when (olduuid == "") $ Git.run r' "config" [Param "annex.uuid", Param u] +onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a +onBupRemote r a command params = do + let dir = shellEscape (Git.workTree r) + sshparams <- sshToRepo r [Param $ + "cd " ++ dir ++ " && " ++ (unwords $ command : toCommand params)] + liftIO $ a "ssh" sshparams + {- Allow for bup repositories on removable media by checking - local bup repositories to see if they are available, and getting their - uuid (which may be different from the stored uuid for the bup remote). @@ -165,21 +169,21 @@ storeBupUUID u buprepo = do - If a bup repository is not available, returns a dummy uuid of "". - This will cause checkPresent to indicate nothing from the bup remote - is known to be present. + - + - Also, returns a version of the repo with config read, if it is local. -} -getBupUUID :: FilePath -> UUID -> Annex UUID -getBupUUID buprepo u = liftIO $ do - r <- bup2GitRemote buprepo - if Git.repoIsUrl r - then return u - else do - ret <- try $ Git.configRead r - case ret of - Right r' -> return $ Git.configGet r' "annex.uuid" "" - Left _ -> return "" +getBupUUID :: Git.Repo -> UUID -> Annex (UUID, Git.Repo) +getBupUUID r u + | Git.repoIsUrl r = return (u, r) + | otherwise = liftIO $ do + ret <- try $ Git.configRead r + case ret of + Right r' -> return (Git.configGet r' "annex.uuid" "", r') + Left _ -> return ("", r) {- Converts a bup remote path spec into a Git.Repo. There are some - differences in path representation between git and bup. -} -bup2GitRemote :: FilePath -> IO Git.Repo +bup2GitRemote :: BupRepo -> IO Git.Repo bup2GitRemote "" = do -- bup -r "" operates on ~/.bup h <- myHomeDir @@ -202,5 +206,5 @@ bup2GitRemote r | d !! 0 == '/' = d | otherwise = "/~/" ++ d -bupLocal :: FilePath -> Bool +bupLocal :: BupRepo -> Bool bupLocal = notElem ':' From dbea472f06b806248a6501163eccf4279c058f75 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 15:57:45 -0400 Subject: [PATCH 1507/2835] update --- doc/git-annex.mdwn | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3514002a49..3e91e7ad92 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -134,7 +134,7 @@ Many git-annex commands will stage changes for later `git commit` by you. * initremote name [param=value ...] - Sets up a [[special_remote|special_remotes]]. The remote's + Sets up a special remote. The remote's configuration is specified by the parameters. If a remote with the specified name has already been configured, its configuration is modified by any values specified. In either case, the remote will be @@ -244,19 +244,19 @@ Many git-annex commands will stage changes for later `git commit` by you. * trust [repository ...] - Records that a repository is [[trusted|trust]] to not unexpectedly lose + Records that a repository is trusted to not unexpectedly lose content. Use with care. To trust the current repository, use "." * untrust [repository ...] - Records that a repository is [[not trusted|trust]] and could lose content + Records that a repository is not trusted and could lose content at any time. * semitrust [repository ...] - Returns a repository to the default [[semi trusted|trust]] state. + Returns a repository to the default semi trusted state. * fromkey file @@ -384,7 +384,7 @@ Here are all the supported configuration settings. by the --from and --to options.) This is, for example, useful if the remote is located somewhere - without [[git-annex-shell]]. (For example, if it's on GitHub). + without git-annex-shell. (For example, if it's on GitHub). Or, it could be used if the network connection between two repositories is too slow to be used normally. @@ -427,7 +427,23 @@ Here are all the supported configuration settings. Automatically maintained, and used to automate upgrades between versions. - +* `remote..buprepo` + + Used by bup special remotes, this configures + the location of the bup repository to use. Normally this is automaticaly + set up by `git annex initremote`, but you can change it if needed. + +* `remote..directory` + + Used by directory special remotes, this configures + the location of the directory where annexed files are stored for this + remote. Normally this is automaticaly set up by `git annex initremote`, + but you can change it if needed. + +* `remote..s3` + + Used to identify Amazon S3 special remotes. + Normally this is automaticaly set up by `git annex initremote`. # CONFIGURATION VIA .gitattributes From c4bdc59da013edb5723447bcdc9bfa7241cba337 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 16:04:12 -0400 Subject: [PATCH 1508/2835] forwarded --- ...vacuate__40__static__41__:_strange_closure_type_30799.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn index 87a0435208..f9a61a8590 100644 --- a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn @@ -69,3 +69,7 @@ Thanks! > > Probably your best bet will be changing to a different version or build of > GHC.. --[[Joey]] + +--- + +forwarded to GHC upstream; closing [[done]] --[[Joey]] From de14252f780fb0259dbd9e6f1fa71c1f092a2135 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 16:08:07 -0400 Subject: [PATCH 1509/2835] rainy day bug maintenance Sitting on the porch, enjoying a thunderstorm, what else to do? --- doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 3 +++ doc/bugs/annex_add_in_annex.mdwn | 4 ++++ doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn | 7 +++---- .../touch.hsc_has_problems_on_non-linux_based_systems.mdwn | 2 ++ 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 doc/bugs/annex_add_in_annex.mdwn diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn index f90eb5ae3c..f256b0ed1f 100644 --- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn +++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn @@ -24,3 +24,6 @@ I can create an annex remote named 'test:/test'. git itself does not allow colon >>> be displayed when there is no configured remote corresponding to the >>> repository. So this is not a bug unless some documentation of that is >>> unclear. --[[Joey]] + +>>>> Nobody spoke up to say it's unclear, so closing as PEBKAC :) +>>>> [[done]] --[[Joey]] diff --git a/doc/bugs/annex_add_in_annex.mdwn b/doc/bugs/annex_add_in_annex.mdwn new file mode 100644 index 0000000000..80084cec12 --- /dev/null +++ b/doc/bugs/annex_add_in_annex.mdwn @@ -0,0 +1,4 @@ +I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories. + +> There is a guard against `git annex add .git-annex/foo`, but it doesn't +> notice `cd .git-annex; git annex add foo`. --[[Joey]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn index e5937fe83f..db6a35293c 100644 --- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn @@ -95,7 +95,6 @@ new file mode 100644 index 0000000..e69de29 -Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories. - -> There is a guard against `git annex add .git-annex/foo`, but it doesn't -> notice `cd .git-annex; git annex add foo`. --[[Joey]] +> Closing this bug, as it seems I have dealt with it adequately now. +> [[done]] +> --[[Joey]] diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn index 20c16e3cfa..c447f75f0c 100644 --- a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn @@ -15,3 +15,5 @@ make: *** [Touch.hs] Error 1 I dug around the OSX documentation and fcntl.h header file and it seems that UTIME_OMIT, UTIME_NOW, AT_FDCWD and AT_SYMLINK_NOFOLLOW aren't defined (at least on OSX). I suspect the BSD's in general will have problems compiling git-annex. + +[[!meta title="annexed symlink mtime matching code is disabled on non-linux systems; needs testing"] From ad7f87880e0aca651162e8960b398f6c8d52e7a9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 16:12:32 -0400 Subject: [PATCH 1510/2835] move wishlists to todo --- ...a___34__git_annex__34___command_that_will_skip_duplicates.mdwn | 0 .../comment_1_fd213310ee548d8726791d2b02237fde._comment | 0 .../comment_2_4394bde1c6fd44acae649baffe802775._comment | 0 .../wishlist:___34__git_annex_add__34___multiple_processes.mdwn | 0 .../comment_1_85b14478411a33e6186a64bd41f0910d._comment | 0 .../comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment | 0 .../comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment | 0 doc/{bugs => todo}/wishlist:_support_for_more_ssh_urls_.mdwn | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename doc/{bugs => todo}/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn (100%) rename doc/{bugs => todo}/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment (100%) rename doc/{bugs => todo}/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment (100%) rename doc/{bugs => todo}/wishlist:___34__git_annex_add__34___multiple_processes.mdwn (100%) rename doc/{bugs => todo}/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment (100%) rename doc/{bugs => todo}/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment (100%) rename doc/{bugs => todo}/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment (100%) rename doc/{bugs => todo}/wishlist:_support_for_more_ssh_urls_.mdwn (100%) diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn similarity index 100% rename from doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn rename to doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment similarity index 100% rename from doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment rename to doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment similarity index 100% rename from doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment rename to doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn similarity index 100% rename from doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn rename to doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment similarity index 100% rename from doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment rename to doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment similarity index 100% rename from doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment rename to doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment similarity index 100% rename from doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment rename to doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn similarity index 100% rename from doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn rename to doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn From 74e877357d1f03827dc1fac438bf8ed8af0d47ad Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 9 Apr 2011 20:11:59 +0000 Subject: [PATCH 1511/2835] Added a comment --- ...comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment new file mode 100644 index 0000000000..d9c291b178 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-09T20:11:59Z" + content=""" +I wouldn't say it's completly impossible for a WORM100 to work. It would just have the contract that the pair of mtime+100chars has to be unique for each unique piece of data. + +But, I have yet to be convinced there's any point, since SHA1 exists. +"""]] From 29022a4143888bcae9b35cba427568f5da33c7da Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 9 Apr 2011 23:43:15 +0000 Subject: [PATCH 1512/2835] --- doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn index f256b0ed1f..722dac50b8 100644 --- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn +++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn @@ -27,3 +27,5 @@ I can create an annex remote named 'test:/test'. git itself does not allow colon >>>> Nobody spoke up to say it's unclear, so closing as PEBKAC :) >>>> [[done]] --[[Joey]] + +>>>>> I still think git-annex should follow the same rules as git in this regard, but if your design decision is different, I won't try to argue the point :) -- RichiH From 1880563b02b2101fd5d6cc23a2b02c169e91a506 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 9 Apr 2011 23:45:29 +0000 Subject: [PATCH 1513/2835] Added a comment --- .../comment_4_8f7ba9372463863dda5aae13205861bf._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment new file mode 100644 index 0000000000..5c08cad6e0 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 4" + date="2011-04-09T23:45:28Z" + content=""" +mtime+100chars can still get collisions and a _lot_ easier than even SHA1. This introduces more problems that it solves, imo. +"""]] From da05f77f7a0873cad2a380461c6e46832624fd3d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" Date: Wed, 13 Apr 2011 17:53:27 +0000 Subject: [PATCH 1514/2835] Added a comment --- .../comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment diff --git a/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment b/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment new file mode 100644 index 0000000000..d6bba93651 --- /dev/null +++ b/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="comment 4" + date="2011-04-13T17:53:26Z" + content=""" +.1 cents: Having IRC would be really nice for seeking quick help. E.g. like I was trying to do now, google lead me to this page. +"""]] From 279a8549fcbfd7945b4b87c62f85bfb9058c1ac2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Apr 2011 16:00:17 -0400 Subject: [PATCH 1515/2835] analized and closed --- doc/bugs/weird_local_clone_confuses.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/weird_local_clone_confuses.mdwn b/doc/bugs/weird_local_clone_confuses.mdwn index d209dd80de..371f6d9a5e 100644 --- a/doc/bugs/weird_local_clone_confuses.mdwn +++ b/doc/bugs/weird_local_clone_confuses.mdwn @@ -11,4 +11,8 @@ just sees a directory w/o a .git subdir, and gives up. --- Just tested, and the new support for bare repositories didn't solve this. +(Because config.bare is not set.) + +I think this is not something git-annex should go out of its way to +support. [[done]] --[[Joey]] From 268f2dc710ad433f3ea622ff1422a86250312571 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 13 Apr 2011 22:21:47 +0000 Subject: [PATCH 1516/2835] --- doc/forum/bainstorming:_git_annex_push___38___pull.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn b/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn index a2d320e352..8a6c552b80 100644 --- a/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn +++ b/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn @@ -19,3 +19,10 @@ which would do git push Resulting in commands that are totally analogous to git push & pull: Sync all data from/to a remote. + +> Update: + +This is useful: + + git config [--global] alias.annex-push '!git pull && git annex add . && git annex copy . --to $REMOTE --fast --quiet && git commit -a -m "$HOST $(date +%F--%H-%M-%S-%Z)" && git push' + From a9ecebedf144c2e2190baab0f895624a144d6a36 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 13 Apr 2011 22:38:55 +0000 Subject: [PATCH 1517/2835] --- doc/forum/wishlist:alias_system.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/wishlist:alias_system.mdwn diff --git a/doc/forum/wishlist:alias_system.mdwn b/doc/forum/wishlist:alias_system.mdwn new file mode 100644 index 0000000000..1f5012966e --- /dev/null +++ b/doc/forum/wishlist:alias_system.mdwn @@ -0,0 +1 @@ +To implement things like my custom `git annex-push` without the dash, i.e. `git annex push`, an alias system for git-annex would be nice. From 35a4039166a72192e49a1b0f9ddbe95ef518c162 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Thu, 14 Apr 2011 11:25:00 +0000 Subject: [PATCH 1518/2835] Added a comment --- .../comment_5_404b723a681eb93fee015cea8024b6bc._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment diff --git a/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment b/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment new file mode 100644 index 0000000000..042dcc1f38 --- /dev/null +++ b/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 5" + date="2011-04-14T11:24:59Z" + content=""" +I would also like an git-annex channel. Would be #git-annex@OFTC ok? +"""]] From f7018e47e48cc61ef6e84adcff89f892cee2c8db Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Apr 2011 15:09:30 -0400 Subject: [PATCH 1519/2835] typo --- doc/design/encryption.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 90b722b9b8..915eee1a13 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -39,7 +39,7 @@ At a high level, an encryption backend needs to support these operations: a stream to an action that consumes the encrypted content. * Given a streaming source of encrypted content, decrypt it, and send - it in a stream to an anction that consumes the decrypted content. + it in a stream to an action that consumes the decrypted content. * Clean up. From 1e84dab4c8def55699fc1b673bd0abd0f5dc4aee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Apr 2011 15:09:36 -0400 Subject: [PATCH 1520/2835] RemoteConfig type --- Command/InitRemote.hs | 12 ++++++------ Remote.hs | 22 +++++++++++----------- Remote/Bup.hs | 4 ++-- Remote/Directory.hs | 4 ++-- Remote/Git.hs | 2 +- Remote/S3real.hs | 8 ++++---- Remote/Special.hs | 3 ++- RemoteClass.hs | 8 +++++--- 8 files changed, 33 insertions(+), 30 deletions(-) diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs index 39ec366539..4c2fc3a078 100644 --- a/Command/InitRemote.hs +++ b/Command/InitRemote.hs @@ -44,14 +44,14 @@ start params = notBareRepo $ do where ws = words params name = head ws - config = Remote.keyValToMap $ tail ws + config = Remote.keyValToConfig $ tail ws -perform :: RemoteClass.RemoteType Annex -> UUID -> M.Map String String -> CommandPerform +perform :: RemoteClass.RemoteType Annex -> UUID -> RemoteClass.RemoteConfig -> CommandPerform perform t u c = do c' <- RemoteClass.setup t u c return $ Just $ cleanup u c' -cleanup :: UUID -> M.Map String String -> CommandCleanup +cleanup :: UUID -> RemoteClass.RemoteConfig -> CommandCleanup cleanup u c = do Remote.configSet u c g <- Annex.gitRepo @@ -65,7 +65,7 @@ cleanup u c = do return True {- Look up existing remote's UUID and config by name, or generate a new one -} -findByName :: String -> Annex (UUID, M.Map String String) +findByName :: String -> Annex (UUID, RemoteClass.RemoteConfig) findByName name = do m <- Remote.readRemoteLog case findByName' name m of @@ -74,7 +74,7 @@ findByName name = do uuid <- liftIO $ genUUID return $ (uuid, M.insert nameKey name M.empty) -findByName' :: String -> M.Map UUID (M.Map String String) -> Maybe (UUID, M.Map String String) +findByName' :: String -> M.Map UUID RemoteClass.RemoteConfig -> Maybe (UUID, RemoteClass.RemoteConfig) findByName' n m = if null matches then Nothing else Just $ head matches where matches = filter (matching . snd) $ M.toList m @@ -85,7 +85,7 @@ findByName' n m = if null matches then Nothing else Just $ head matches | otherwise -> False {- find the specified remote type -} -findType :: M.Map String String -> Annex (RemoteClass.RemoteType Annex) +findType :: RemoteClass.RemoteConfig -> Annex (RemoteClass.RemoteType Annex) findType config = case M.lookup typeKey config of Nothing -> error "Specify the type of remote with type=" diff --git a/Remote.hs b/Remote.hs index bb661c5a90..8d2ab0399a 100644 --- a/Remote.hs +++ b/Remote.hs @@ -25,7 +25,7 @@ module Remote ( remoteLog, readRemoteLog, configSet, - keyValToMap + keyValToConfig ) where import Control.Monad.State (liftIO) @@ -137,17 +137,17 @@ remoteLog = do return $ gitStateDir g ++ "remote.log" {- Adds or updates a remote's config in the log. -} -configSet :: UUID -> M.Map String String -> Annex () +configSet :: UUID -> RemoteConfig -> Annex () configSet u c = do m <- readRemoteLog l <- remoteLog liftIO $ safeWriteFile l $ unlines $ sort $ map toline $ M.toList $ M.insert u c m where - toline (u', c') = u' ++ " " ++ (unwords $ mapToKeyVal c') + toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c') {- Map of remotes by uuid containing key/value config maps. -} -readRemoteLog :: Annex (M.Map UUID (M.Map String String)) +readRemoteLog :: Annex (M.Map UUID RemoteConfig) readRemoteLog = do l <- remoteLog s <- liftIO $ catch (readFile l) ignoreerror @@ -155,7 +155,7 @@ readRemoteLog = do where ignoreerror _ = return "" -remoteLogParse :: String -> M.Map UUID (M.Map String String) +remoteLogParse :: String -> M.Map UUID RemoteConfig remoteLogParse s = M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s where @@ -165,18 +165,18 @@ remoteLogParse s = where w = words l u = w !! 0 - c = keyValToMap $ tail w + c = keyValToConfig $ tail w -{- Given Strings like "key=value", generates a Map. -} -keyValToMap :: [String] -> M.Map String String -keyValToMap ws = M.fromList $ map (/=/) ws +{- Given Strings like "key=value", generates a RemoteConfig. -} +keyValToConfig :: [String] -> RemoteConfig +keyValToConfig ws = M.fromList $ map (/=/) ws where (/=/) s = (k, v) where k = takeWhile (/= '=') s v = drop (1 + length k) s -mapToKeyVal :: M.Map String String -> [String] -mapToKeyVal m = map toword $ sort $ M.toList m +configToKeyVal :: M.Map String String -> [String] +configToKeyVal m = map toword $ sort $ M.toList m where toword (k, v) = k ++ "=" ++ v diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 916afeb406..66c78970c9 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -39,7 +39,7 @@ remote = RemoteType { setup = bupSetup } -gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do buprepo <- getConfig r "buprepo" (error "missing buprepo") cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost) @@ -60,7 +60,7 @@ gen r u c = do config = c } -bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) +bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig bupSetup u c = do -- verify configuration is sane let buprepo = case M.lookup "buprepo" c of diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 0d3478b79d..2313f79a05 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -35,7 +35,7 @@ remote = RemoteType { setup = directorySetup } -gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u _ = do dir <- getConfig r "directory" (error "missing directory") cst <- remoteCost r cheapRemoteCost @@ -51,7 +51,7 @@ gen r u _ = do config = Nothing } -directorySetup :: UUID -> M.Map String String -> Annex (M.Map String String) +directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig directorySetup u c = do -- verify configuration is sane let dir = case M.lookup "directory" c of diff --git a/Remote/Git.hs b/Remote/Git.hs index 7724df79af..bab452a331 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -40,7 +40,7 @@ list = do g <- Annex.gitRepo return $ Git.remotes g -gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u _ = do {- It's assumed to be cheap to read the config of non-URL remotes, - so this is done each time git-annex is run. Conversely, diff --git a/Remote/S3real.hs b/Remote/S3real.hs index bb82d54e0e..af4e48048a 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -37,7 +37,7 @@ remote = RemoteType { setup = s3Setup } -gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do cst <- remoteCost r expensiveRemoteCost return $ this cst @@ -54,14 +54,14 @@ gen r u c = do config = c } -s3ConnectionRequired :: M.Map String String -> Annex AWSConnection +s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection s3ConnectionRequired c = do conn <- s3Connection c case conn of Nothing -> error "Cannot connect to S3" Just conn' -> return conn' -s3Connection :: M.Map String String -> Annex (Maybe AWSConnection) +s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) s3Connection c = do ak <- getEnvKey "AWS_ACCESS_KEY_ID" sk <- getEnvKey "AWS_SECRET_ACCESS_KEY" @@ -78,7 +78,7 @@ s3Connection c = do _ -> error $ "bad S3 port value: " ++ s getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") -s3Setup :: UUID -> M.Map String String -> Annex (M.Map String String) +s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane case M.lookup "encryption" c of diff --git a/Remote/Special.hs b/Remote/Special.hs index b5d5a137fe..53ac2c6eed 100644 --- a/Remote/Special.hs +++ b/Remote/Special.hs @@ -13,6 +13,7 @@ import Data.String.Utils import Control.Monad.State (liftIO) import Types +import RemoteClass import qualified GitRepo as Git import qualified Annex import UUID @@ -32,7 +33,7 @@ findSpecialRemotes s = do match k _ = startswith "remote." k && endswith (".annex-"++s) k {- Sets up configuration for a special remote in .git/config. -} -gitConfigSpecialRemote :: UUID -> M.Map String String -> String -> String -> Annex () +gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex () gitConfigSpecialRemote u c k v = do g <- Annex.gitRepo liftIO $ do diff --git a/RemoteClass.hs b/RemoteClass.hs index 8055c16b06..f954e4ff8f 100644 --- a/RemoteClass.hs +++ b/RemoteClass.hs @@ -15,6 +15,8 @@ import Data.Map as M import qualified GitRepo as Git import Key +type RemoteConfig = M.Map String String + {- There are different types of remotes. -} data RemoteType a = RemoteType { -- human visible type name @@ -22,9 +24,9 @@ data RemoteType a = RemoteType { -- enumerates remotes of this type enumerate :: a [Git.Repo], -- generates a remote of this type - generate :: Git.Repo -> String -> Maybe (M.Map String String) -> a (Remote a), + generate :: Git.Repo -> String -> Maybe RemoteConfig -> a (Remote a), -- initializes or changes a remote - setup :: String -> M.Map String String -> a (M.Map String String) + setup :: String -> RemoteConfig -> a RemoteConfig } {- An individual remote. -} @@ -48,7 +50,7 @@ data Remote a = Remote { -- operation. hasKeyCheap :: Bool, -- a Remote can have a persistent configuration store - config :: Maybe (M.Map String String) + config :: Maybe RemoteConfig } instance Show (Remote a) where From 452c63035af391e109ad37c0722c2f32cbf77428 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 15 Apr 2011 19:32:09 +0000 Subject: [PATCH 1521/2835] Added a comment --- .../comment_6_0d87d0e26461494b1d7f8a701a924729._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment diff --git a/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment b/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment new file mode 100644 index 0000000000..8dfd0f8203 --- /dev/null +++ b/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 6" + date="2011-04-15T19:32:08Z" + content=""" +We seem to be using #vcs-home @ OFTC for now. madduck is fine with it and joeyh pokes his head in there, as well. I just added a CIA bot to #vcs-home and this comment is a test if pushing works. -- RichiH +"""]] From 467020e56015d5760b1aa580f8cb2a2c5f91a18c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 15 Apr 2011 19:40:30 +0000 Subject: [PATCH 1522/2835] --- ...from_the_website__39__s_repo__44___not_your_personal_one.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn diff --git a/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn b/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn new file mode 100644 index 0000000000..6926e3cca2 --- /dev/null +++ b/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn @@ -0,0 +1 @@ +I just added a CIA bot to #vcs-home and tracking commits immediately would be nice. -- RichiH From 480d780297dac12576a90c25cca5cb989e1a1e4f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Apr 2011 18:18:39 -0400 Subject: [PATCH 1523/2835] add --- Crypto.hs | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 Crypto.hs diff --git a/Crypto.hs b/Crypto.hs new file mode 100644 index 0000000000..4ea43838a2 --- /dev/null +++ b/Crypto.hs @@ -0,0 +1,111 @@ +{- git-annex crypto + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Crypto ( + genCipher, + updateCipher, + storeCipher, + extractCipher, + decryptCipher, + encryptKey, + encryptContent, + decryptContent +) where + +import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.Map as M +import System.IO +import System.Cmd.Utils + +import Types +import RemoteClass +import Utility + +data Cipher = Cipher String -- XXX ideally, this would be a locked memory region +data EncryptedCipher = EncryptedCipher String + deriving Show + +{- Creates a new Cipher, encrypted as specified in the remote's configuration -} +genCipher :: RemoteConfig -> IO EncryptedCipher +genCipher config = do + random <- genrandom + encryptCipher config $ Cipher random + where + genrandom = gpgPipeRead + [ Params "--armor --gen-random" + , Param $ show randomquality + , Param $ show ciphersize + ] + randomquality = 1 -- 1 is /dev/urandom; 2 is /dev/random + ciphersize = 1024 + +{- Updates an existing Cipher, re-encrypting it as specified in the + - remote's configuration -} +updateCipher :: RemoteConfig -> EncryptedCipher -> IO EncryptedCipher +updateCipher config encipher = do + cipher <- decryptCipher config encipher + encryptCipher config cipher + +{- Stores an EncryptedCipher in a remote's configuration. -} +storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig +storeCipher config (EncryptedCipher c) = M.insert "cipher" c config + +{- Extracts an EncryptedCipher from a remote's configuration. -} +extractCipher :: RemoteConfig -> EncryptedCipher +extractCipher config = case M.lookup "cipher" config of + Just c -> EncryptedCipher c + Nothing -> error "missing cipher in remote config" + +{- Encryptes a Cipher as specified by a remote's configuration. -} +encryptCipher :: RemoteConfig -> Cipher -> IO EncryptedCipher +encryptCipher config (Cipher c) = do + encipher <- gpgPipeBoth (encrypt++recipient) c + return $ EncryptedCipher encipher + where + encrypt = + [ Params "--encrypt --armor" + , Params "--trust-model always" + ] + recipient = case M.lookup "encryption" config of + Nothing -> [ Param "--default-recipient-self" ] + Just r -> + -- Force gpg to only encrypt to the specified + -- recipients, not configured defaults. + [ Params "--no-encrypt-to --no-default-recipient" + , Param "--recipient" + , Param r + ] + +{- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} +decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher +decryptCipher = error "TODO" + +{- Genetates an encrypted form of a Key. The enctyption does not need to be + - reversable, nor does it need to be the same type of encryption used + - on content. -} +encryptKey :: Cipher -> Key -> IO Key +encryptKey = error "TODO" + +{- Streams content, encrypting. -} +encryptContent :: Cipher -> L.ByteString -> IO L.ByteString +encryptContent = error "TODO" + +{- Streams encrypted content, decrypting. -} +decryptContent :: Cipher -> L.ByteString -> IO L.ByteString +decryptContent = error "TODO" + + +gpgParams :: [CommandParam] -> [String] +gpgParams params = ["--batch", "--quiet"] ++ toCommand params + +gpgPipeRead :: [CommandParam] -> IO String +gpgPipeRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict + +gpgPipeBoth :: [CommandParam] -> String -> IO String +gpgPipeBoth params input = do + (_, s) <- pipeBoth "gpg" (gpgParams params) input + return s From 7fdf20f577f63f8437c63d7d83e70d34de89269f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 13:25:27 -0400 Subject: [PATCH 1524/2835] encryption key management working Encrypted remotes don't yet encrypt data, but git annex initremote can be used to generate a cipher and add additional gpg keys that can use it. --- Crypto.hs | 120 ++++++++++++++++++++++++++++++-------------- Remote/Bup.hs | 12 ++--- Remote/Encrypted.hs | 31 ++++++++++++ Remote/S3real.hs | 8 ++- 4 files changed, 120 insertions(+), 51 deletions(-) create mode 100644 Remote/Encrypted.hs diff --git a/Crypto.hs b/Crypto.hs index 4ea43838a2..f32d429c3b 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -1,4 +1,7 @@ {- git-annex crypto + - + - Currently using gpg; could later be modified to support different + - crypto backends if neccessary. - - Copyright 2011 Joey Hess - @@ -18,71 +21,91 @@ module Crypto ( import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M -import System.IO +import qualified Codec.Binary.Base64 as B64 import System.Cmd.Utils +import Data.String.Utils +import Data.List +import Data.Bits.Utils import Types import RemoteClass import Utility data Cipher = Cipher String -- XXX ideally, this would be a locked memory region -data EncryptedCipher = EncryptedCipher String - deriving Show + +data EncryptedCipher = EncryptedCipher String KeyIds + +data KeyIds = KeyIds [String] + +instance Show KeyIds where + show (KeyIds ks) = join "," ks + +instance Read KeyIds where + readsPrec _ s = [(KeyIds (split "," s), "")] {- Creates a new Cipher, encrypted as specified in the remote's configuration -} genCipher :: RemoteConfig -> IO EncryptedCipher -genCipher config = do +genCipher c = do + ks <- configKeyIds c random <- genrandom - encryptCipher config $ Cipher random + encryptCipher (Cipher random) ks where genrandom = gpgPipeRead - [ Params "--armor --gen-random" + [ Params "--gen-random" , Param $ show randomquality , Param $ show ciphersize ] - randomquality = 1 -- 1 is /dev/urandom; 2 is /dev/random - ciphersize = 1024 + randomquality = 1 :: Int -- 1 is /dev/urandom; 2 is /dev/random + ciphersize = 1024 :: Int -{- Updates an existing Cipher, re-encrypting it as specified in the - - remote's configuration -} +{- Updates an existing Cipher, re-encrypting it to add KeyIds specified in + - the remote's configuration. -} updateCipher :: RemoteConfig -> EncryptedCipher -> IO EncryptedCipher -updateCipher config encipher = do - cipher <- decryptCipher config encipher - encryptCipher config cipher +updateCipher c encipher@(EncryptedCipher _ ks) = do + ks' <- configKeyIds c + cipher <- decryptCipher c encipher + encryptCipher cipher (combine ks ks') + where + combine (KeyIds a) (KeyIds b) = KeyIds $ a ++ b {- Stores an EncryptedCipher in a remote's configuration. -} storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig -storeCipher config (EncryptedCipher c) = M.insert "cipher" c config +storeCipher c (EncryptedCipher t ks) = + M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (show ks) c + where + toB64 = B64.encode . s2w8 {- Extracts an EncryptedCipher from a remote's configuration. -} -extractCipher :: RemoteConfig -> EncryptedCipher -extractCipher config = case M.lookup "cipher" config of - Just c -> EncryptedCipher c - Nothing -> error "missing cipher in remote config" - -{- Encryptes a Cipher as specified by a remote's configuration. -} -encryptCipher :: RemoteConfig -> Cipher -> IO EncryptedCipher -encryptCipher config (Cipher c) = do - encipher <- gpgPipeBoth (encrypt++recipient) c - return $ EncryptedCipher encipher +extractCipher :: RemoteConfig -> Maybe EncryptedCipher +extractCipher c = + case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of + (Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (read ks) + _ -> Nothing where - encrypt = - [ Params "--encrypt --armor" - , Params "--trust-model always" - ] - recipient = case M.lookup "encryption" config of - Nothing -> [ Param "--default-recipient-self" ] - Just r -> - -- Force gpg to only encrypt to the specified - -- recipients, not configured defaults. - [ Params "--no-encrypt-to --no-default-recipient" - , Param "--recipient" - , Param r - ] + fromB64 s = case B64.decode s of + Nothing -> error "bad base64 encoded cipher in remote config" + Just ws -> w82s ws + +{- Encrypts a Cipher to the specified KeyIds. -} +encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher +encryptCipher (Cipher c) (KeyIds ks) = do + let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids + encipher <- gpgPipeBoth (encrypt++recipients ks') c + return $ EncryptedCipher encipher (KeyIds ks') + where + encrypt = [ Params "--encrypt" ] + recipients l = + -- Force gpg to only encrypt to the specified + -- recipients, not configured defaults. + [ Params "--no-encrypt-to --no-default-recipient"] ++ + (concat $ map (\k -> [Param "--recipient", Param k]) l) {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher -decryptCipher = error "TODO" +decryptCipher _ (EncryptedCipher encipher _) = + return . Cipher =<< gpgPipeBoth decrypt encipher + where + decrypt = [ Params "--decrypt" ] {- Genetates an encrypted form of a Key. The enctyption does not need to be - reversable, nor does it need to be the same type of encryption used @@ -100,7 +123,10 @@ decryptContent = error "TODO" gpgParams :: [CommandParam] -> [String] -gpgParams params = ["--batch", "--quiet"] ++ toCommand params +gpgParams params = + -- avoid console IO, and be quiet, even about checking the trustdb + ["--batch", "--quiet", "--trust-model", "always"] ++ + toCommand params gpgPipeRead :: [CommandParam] -> IO String gpgPipeRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict @@ -109,3 +135,19 @@ gpgPipeBoth :: [CommandParam] -> String -> IO String gpgPipeBoth params input = do (_, s) <- pipeBoth "gpg" (gpgParams params) input return s + +configKeyIds :: RemoteConfig -> IO KeyIds +configKeyIds c = do + let k = configGet c "encryption" + s <- gpgPipeRead [Params "--with-colons --list-public-keys", Param k] + return $ KeyIds $ parseWithColons s + where + parseWithColons s = map keyIdField $ filter pubKey $ lines s + pubKey = isPrefixOf "pub:" + keyIdField s = (split ":" s) !! 4 + +configGet :: RemoteConfig -> String -> String +configGet c key = + case M.lookup key c of + Just v -> v + Nothing -> error $ "missing " ++ key ++ " in remote config" diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 66c78970c9..b4403bb03e 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -26,8 +26,9 @@ import Locations import Config import Utility import Messages -import Remote.Special import Ssh +import Remote.Special +import Remote.Encrypted type BupRepo = String @@ -66,10 +67,7 @@ bupSetup u c = do let buprepo = case M.lookup "buprepo" c of Nothing -> error "Specify buprepo=" Just r -> r - case M.lookup "encryption" c of - Nothing -> error "Specify encryption=key or encryption=none" - Just "none" -> return () - Just _ -> error "encryption keys not yet supported" + c' <- encryptionSetup c -- bup init will create the repository. -- (If the repository already exists, bup init again appears safe.) @@ -81,9 +79,9 @@ bupSetup u c = do -- The buprepo is stored in git config, as well as this repo's -- persistant state, so it can vary between hosts. - gitConfigSpecialRemote u c "buprepo" buprepo + gitConfigSpecialRemote u c' "buprepo" buprepo - return $ M.delete "directory" c + return c' bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam] bupParams command buprepo params = diff --git a/Remote/Encrypted.hs b/Remote/Encrypted.hs new file mode 100644 index 0000000000..ae40446209 --- /dev/null +++ b/Remote/Encrypted.hs @@ -0,0 +1,31 @@ +{- common functions for encrypted remotes + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Encrypted where + +import qualified Data.Map as M +import Control.Monad.State (liftIO) + +import Types +import RemoteClass +import Crypto + +{- Encryption setup for a remote. The user must specify whether to use + - an encryption key, or not encrypt. An encrypted cipher is created, or is + - updated to be accessible to an additional encryption key. -} +encryptionSetup :: RemoteConfig -> Annex RemoteConfig +encryptionSetup c = + case (M.lookup "encryption" c, extractCipher c) of + (Nothing, Nothing) -> error "Specify encryption=key or encryption=none" + (Just "none", _) -> return c + (Nothing, Just _) -> return c + (Just _, Nothing) -> use $ genCipher c + (Just _, Just v) -> use $ updateCipher c v + where + use a = do + cipher <- liftIO a + return $ M.delete "encryption" $ storeCipher c cipher diff --git a/Remote/S3real.hs b/Remote/S3real.hs index af4e48048a..0f6327f575 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -28,6 +28,7 @@ import Messages import Locations import Config import Remote.Special +import Remote.Encrypted remote :: RemoteType Annex remote = RemoteType { @@ -81,11 +82,8 @@ s3Connection c = do s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane - case M.lookup "encryption" c of - Nothing -> error "Specify encryption=key or encryption=none" - Just "none" -> return () - Just _ -> error "encryption keys not yet supported" - let fullconfig = M.union c defaults + c' <- encryptionSetup c + let fullconfig = M.union c' defaults -- check bucket location to see if the bucket exists, and create it let datacenter = fromJust $ M.lookup "datacenter" fullconfig From 669851454cd3032d2097842f7b6027b3464da032 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 16:26:47 -0400 Subject: [PATCH 1525/2835] crypto library almost complete Piping data through gpg with symmetric cipher is working. Only Key encryption is not done. --- Crypto.hs | 111 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index f32d429c3b..2e20dddb10 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -15,8 +15,8 @@ module Crypto ( extractCipher, decryptCipher, encryptKey, - encryptContent, - decryptContent + withEncryptedContent, + withDecryptedContent, ) where import qualified Data.ByteString.Lazy.Char8 as L @@ -26,8 +26,14 @@ import System.Cmd.Utils import Data.String.Utils import Data.List import Data.Bits.Utils +import System.IO +import System.Posix.IO +import System.Posix.Types +import Control.Concurrent +import Control.Exception import Types +import Key import RemoteClass import Utility @@ -50,13 +56,16 @@ genCipher c = do random <- genrandom encryptCipher (Cipher random) ks where - genrandom = gpgPipeRead - [ Params "--gen-random" + genrandom = gpgRead + -- Armor the random data, to avoid newlines, + -- since gpg only reads ciphers up to the first + -- newline. + [ Params "--gen-random --armor" , Param $ show randomquality , Param $ show ciphersize ] randomquality = 1 :: Int -- 1 is /dev/urandom; 2 is /dev/random - ciphersize = 1024 :: Int + ciphersize = 256 :: Int {- Updates an existing Cipher, re-encrypting it to add KeyIds specified in - the remote's configuration. -} @@ -72,8 +81,6 @@ updateCipher c encipher@(EncryptedCipher _ ks) = do storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig storeCipher c (EncryptedCipher t ks) = M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (show ks) c - where - toB64 = B64.encode . s2w8 {- Extracts an EncryptedCipher from a remote's configuration. -} extractCipher :: RemoteConfig -> Maybe EncryptedCipher @@ -81,16 +88,12 @@ extractCipher c = case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of (Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (read ks) _ -> Nothing - where - fromB64 s = case B64.decode s of - Nothing -> error "bad base64 encoded cipher in remote config" - Just ws -> w82s ws {- Encrypts a Cipher to the specified KeyIds. -} encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher encryptCipher (Cipher c) (KeyIds ks) = do let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids - encipher <- gpgPipeBoth (encrypt++recipients ks') c + encipher <- gpgPipeStrict (encrypt++recipients ks') c return $ EncryptedCipher encipher (KeyIds ks') where encrypt = [ Params "--encrypt" ] @@ -103,43 +106,80 @@ encryptCipher (Cipher c) (KeyIds ks) = do {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher decryptCipher _ (EncryptedCipher encipher _) = - return . Cipher =<< gpgPipeBoth decrypt encipher + return . Cipher =<< gpgPipeStrict decrypt encipher where - decrypt = [ Params "--decrypt" ] + decrypt = [ Param "--decrypt" ] -{- Genetates an encrypted form of a Key. The enctyption does not need to be +{- Generates an encrypted form of a Key. The encryption does not need to be - reversable, nor does it need to be the same type of encryption used - - on content. -} + - on content. It does need to be repeatable. -} encryptKey :: Cipher -> Key -> IO Key -encryptKey = error "TODO" +encryptKey c k = + return Key { + -- FIXME: should use HMAC with the cipher; I don't + -- have Data.Crypto in Debian yet though. + keyName = show k, + keyBackendName = "INSECURE", + keySize = Nothing, -- size and mtime omitted + keyMtime = Nothing -- to avoid leaking data + } -{- Streams content, encrypting. -} -encryptContent :: Cipher -> L.ByteString -> IO L.ByteString -encryptContent = error "TODO" +{- Streams encrypted content to an action. -} +withEncryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +withEncryptedContent = gpgCipher [Params "--symmetric --force-mdc"] -{- Streams encrypted content, decrypting. -} -decryptContent :: Cipher -> L.ByteString -> IO L.ByteString -decryptContent = error "TODO" +{- Streams decrypted content to an action. -} +withDecryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +withDecryptedContent = gpgCipher [Param "--decrypt"] gpgParams :: [CommandParam] -> [String] gpgParams params = - -- avoid console IO, and be quiet, even about checking the trustdb + -- avoid prompting, and be quiet, even about checking the trustdb ["--batch", "--quiet", "--trust-model", "always"] ++ toCommand params -gpgPipeRead :: [CommandParam] -> IO String -gpgPipeRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict +gpgRead :: [CommandParam] -> IO String +gpgRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict -gpgPipeBoth :: [CommandParam] -> String -> IO String -gpgPipeBoth params input = do - (_, s) <- pipeBoth "gpg" (gpgParams params) input - return s +gpgPipeStrict :: [CommandParam] -> String -> IO String +gpgPipeStrict params input = do + (_, output) <- pipeBoth "gpg" (gpgParams params) input + return output + +gpgPipeBytes :: [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString) +gpgPipeBytes params input = do + (pid, fromh, toh) <- hPipeBoth "gpg" (gpgParams params) + _ <- forkIO $ finally (L.hPut toh input) (hClose toh) + output <- L.hGetContents fromh + return (pid, output) + +{- Runs gpg with a cipher and some parameters, feeding it an input, + - and piping its output lazily to an action. -} +gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +gpgCipher params (Cipher c) input a = do + -- pipe the passphrase into gpg on a fd + (frompipe, topipe) <- createPipe + toh <- fdToHandle topipe + let Fd fromno = frompipe + _ <- forkIO $ do + hPutStrLn toh c + hClose toh + let passphrase = [Param "--passphrase-fd", Param $ show fromno] + (pid, output) <- gpgPipeBytes (passphrase ++ params) input + + ret <- a output + + -- cleanup + forceSuccess pid + closeFd frompipe + + return ret configKeyIds :: RemoteConfig -> IO KeyIds configKeyIds c = do let k = configGet c "encryption" - s <- gpgPipeRead [Params "--with-colons --list-public-keys", Param k] + s <- gpgRead [Params "--with-colons --list-public-keys", Param k] return $ KeyIds $ parseWithColons s where parseWithColons s = map keyIdField $ filter pubKey $ lines s @@ -151,3 +191,12 @@ configGet c key = case M.lookup key c of Just v -> v Nothing -> error $ "missing " ++ key ++ " in remote config" + +toB64 :: String -> String +toB64 = B64.encode . s2w8 + +fromB64 :: String -> String +fromB64 s = + case B64.decode s of + Nothing -> error "bad base64 encoded data" + Just ws -> w82s ws From 5efd41327045f8da55c972b7391309c99dee5afc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 16:29:28 -0400 Subject: [PATCH 1526/2835] add encryption support to directory special remotes --- Remote/Directory.hs | 6 ++++-- doc/special_remotes/directory.mdwn | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 2313f79a05..bb1ef60e49 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -26,6 +26,7 @@ import Config import Content import Utility import Remote.Special +import Remote.Encrypted remote :: RemoteType Annex remote = RemoteType { @@ -59,11 +60,12 @@ directorySetup u c = do Just d -> d e <- liftIO $ doesDirectoryExist dir when (not e) $ error $ "Directory does not exist: " ++ dir + c' <- encryptionSetup c -- The directory is stored in git config, not in this remote's -- persistant state, so it can vary between hosts. - gitConfigSpecialRemote u c "directory" dir - return $ M.delete "directory" c + gitConfigSpecialRemote u c' "directory" dir + return $ M.delete "directory" c' dirKey :: FilePath -> Key -> FilePath dirKey d k = d hashDirMixed k f f diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn index daa0b74128..18d30e3110 100644 --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -7,4 +7,4 @@ the drive's mountpoint as a directory remote. Setup example: - # git annex initremote usbdrive directory=/media/usbdrive/ + # git annex initremote usbdrive directory=/media/usbdrive/ encryption=none From 9fe7e6be7064d9c47e6c6fd4f1b3a70da604727d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 16:41:46 -0400 Subject: [PATCH 1527/2835] add cipher field to AnnexState --- Annex.hs | 5 ++++- Crypto.hs | 13 +------------ CryptoTypes.hs | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 CryptoTypes.hs diff --git a/Annex.hs b/Annex.hs index f4e5d599d0..9086db9bf7 100644 --- a/Annex.hs +++ b/Annex.hs @@ -22,6 +22,7 @@ import qualified GitRepo as Git import qualified GitQueue import qualified BackendClass import qualified RemoteClass +import qualified CryptoTypes -- git-annex's monad type Annex = StateT AnnexState IO @@ -41,7 +42,8 @@ data AnnexState = AnnexState , toremote :: Maybe String , fromremote :: Maybe String , exclude :: [String] - } deriving (Show) + , cipher :: Maybe CryptoTypes.Cipher + } newState :: Git.Repo -> [BackendClass.Backend Annex] -> AnnexState newState gitrepo allbackends = AnnexState @@ -58,6 +60,7 @@ newState gitrepo allbackends = AnnexState , toremote = Nothing , fromremote = Nothing , exclude = [] + , cipher = Nothing } {- Create and returns an Annex state object for the specified git repo. -} diff --git a/Crypto.hs b/Crypto.hs index 2e20dddb10..337aedff65 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -36,18 +36,7 @@ import Types import Key import RemoteClass import Utility - -data Cipher = Cipher String -- XXX ideally, this would be a locked memory region - -data EncryptedCipher = EncryptedCipher String KeyIds - -data KeyIds = KeyIds [String] - -instance Show KeyIds where - show (KeyIds ks) = join "," ks - -instance Read KeyIds where - readsPrec _ s = [(KeyIds (split "," s), "")] +import CryptoTypes {- Creates a new Cipher, encrypted as specified in the remote's configuration -} genCipher :: RemoteConfig -> IO EncryptedCipher diff --git a/CryptoTypes.hs b/CryptoTypes.hs new file mode 100644 index 0000000000..944a9d34e0 --- /dev/null +++ b/CryptoTypes.hs @@ -0,0 +1,22 @@ +{- git-annex crypto types + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CryptoTypes where + +import Data.String.Utils + +data Cipher = Cipher String -- XXX ideally, this would be a locked memory region + +data EncryptedCipher = EncryptedCipher String KeyIds + +data KeyIds = KeyIds [String] + +instance Show KeyIds where + show (KeyIds ks) = join "," ks + +instance Read KeyIds where + readsPrec _ s = [(KeyIds (split "," s), "")] From 4f9fafa02354d275d6fa83ff42ada4ebd1bc83d8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 18:22:52 -0400 Subject: [PATCH 1528/2835] full encryption support for directory special remotes --- Crypto.hs | 7 ++++++- Remote/Directory.hs | 48 ++++++++++++++++++++++++++++++++------------- Remote/Encrypted.hs | 43 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 15 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 337aedff65..9f404c1b17 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -9,6 +9,8 @@ -} module Crypto ( + Cipher, + EncryptedCipher, genCipher, updateCipher, storeCipher, @@ -133,7 +135,10 @@ gpgRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict gpgPipeStrict :: [CommandParam] -> String -> IO String gpgPipeStrict params input = do - (_, output) <- pipeBoth "gpg" (gpgParams params) input + (pid, fromh, toh) <- hPipeBoth "gpg" (gpgParams params) + _ <- forkIO $ finally (hPutStr toh input) (hClose toh) + output <- hGetContentsStrict fromh + forceSuccess pid return output gpgPipeBytes :: [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString) diff --git a/Remote/Directory.hs b/Remote/Directory.hs index bb1ef60e49..5ea0a1e6b3 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -7,6 +7,7 @@ module Remote.Directory (remote) where +import qualified Data.ByteString.Lazy.Char8 as L import IO import Control.Exception.Extensible (IOException) import qualified Data.Map as M @@ -27,6 +28,7 @@ import Content import Utility import Remote.Special import Remote.Encrypted +import Crypto remote :: RemoteType Annex remote = RemoteType { @@ -37,17 +39,17 @@ remote = RemoteType { } gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) -gen r u _ = do +gen r u c = do dir <- getConfig r "directory" (error "missing directory") cst <- remoteCost r cheapRemoteCost return $ Remote { uuid = u, cost = cst, name = Git.repoDescribe r, - storeKey = store dir, - retrieveKeyFile = retrieve dir, - removeKey = remove dir, - hasKey = checkPresent dir, + storeKey = storeKeyEncrypted c $ store dir, + retrieveKeyFile = retrieveKeyFileEncrypted c $ retrieve dir, + removeKey = removeKeyEncrypted c $ remove dir, + hasKey = hasKeyEncrypted c $ checkPresent dir, hasKeyCheap = True, config = Nothing } @@ -72,25 +74,43 @@ dirKey d k = d hashDirMixed k f f where f = keyFile k -store :: FilePath -> Key -> Annex Bool -store d k = do +store :: FilePath -> Key -> Maybe (Cipher, Key) -> Annex Bool +store d k c = do g <- Annex.gitRepo - let src = gitAnnexLocation g k + let src = gitAnnexLocation g k liftIO $ catch (copy src) (const $ return False) where - dest = dirKey d k - dir = parentDir dest - copy src = do + copy src = case c of + Just (cipher, enckey) -> do + content <- L.readFile src + let dest = dirKey d enckey + prep dest + withEncryptedContent cipher content $ \s -> do + L.writeFile dest s + cleanup True dest + _ -> do + let dest = dirKey d k + prep dest + ok <- copyFile src dest + cleanup ok dest + prep dest = liftIO $ do + let dir = parentDir dest createDirectoryIfMissing True dir allowWrite dir - ok <- copyFile src dest + cleanup ok dest = do when ok $ do + let dir = parentDir dest preventWrite dest preventWrite dir return ok -retrieve :: FilePath -> Key -> FilePath -> Annex Bool -retrieve d k f = liftIO $ copyFile (dirKey d k) f +retrieve :: FilePath -> Key -> FilePath -> Maybe (Cipher, Key) -> Annex Bool +retrieve d k f Nothing = liftIO $ copyFile (dirKey d k) f +retrieve d k f (Just (cipher, enckey)) = + liftIO $ flip catch (const $ return False) $ do + content <- L.readFile (dirKey d enckey) + withDecryptedContent cipher content $ L.writeFile f + return True remove :: FilePath -> Key -> Annex Bool remove d k = liftIO $ catch del (const $ return False) diff --git a/Remote/Encrypted.hs b/Remote/Encrypted.hs index ae40446209..2a0fb13bca 100644 --- a/Remote/Encrypted.hs +++ b/Remote/Encrypted.hs @@ -13,6 +13,8 @@ import Control.Monad.State (liftIO) import Types import RemoteClass import Crypto +import qualified Annex +import Messages {- Encryption setup for a remote. The user must specify whether to use - an encryption key, or not encrypt. An encrypted cipher is created, or is @@ -29,3 +31,44 @@ encryptionSetup c = use a = do cipher <- liftIO a return $ M.delete "encryption" $ storeCipher c cipher + +{- Helpers that can be applied to a Remote's normal actions to + - add crypto support. -} +storeKeyEncrypted :: Maybe RemoteConfig -> (Key -> Maybe (Cipher, Key) -> Annex a) -> Key -> Annex a +storeKeyEncrypted c a k = a k =<< cipherKey c k +retrieveKeyFileEncrypted :: Maybe RemoteConfig -> (Key -> FilePath -> Maybe (Cipher, Key) -> Annex a) -> Key -> FilePath -> Annex a +retrieveKeyFileEncrypted c a k f = a k f =<< cipherKey c k +removeKeyEncrypted :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a +removeKeyEncrypted = withEncryptedKey +hasKeyEncrypted :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a +hasKeyEncrypted = withEncryptedKey + +{- Gets encryption Cipher, and encrypted version of Key. + - + - The decrypted Cipher is cached in the Annex state. -} +cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key)) +cipherKey Nothing _ = return Nothing +cipherKey (Just c) k = do + cache <- Annex.getState Annex.cipher + case cache of + Just cipher -> ret cipher + Nothing -> case extractCipher c of + Nothing -> return Nothing + Just encipher -> do + showNote "getting encryption key" + cipher <- liftIO $ decryptCipher c encipher + Annex.changeState (\s -> s { Annex.cipher = Just cipher }) + ret cipher + where + ret cipher = do + k' <- liftIO $ encryptKey cipher k + return $ Just (cipher, k') + +{- Passes the encrypted version of the key to the action when encryption + - is enabled, and the non-encrypted version otherwise. -} +withEncryptedKey :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a +withEncryptedKey c a k = do + v <- cipherKey c k + case v of + Nothing -> a k + Just (_, k') -> a k' From 1247bfeaa7356e766d3ea09fa50bd300650f78af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 19:13:05 -0400 Subject: [PATCH 1529/2835] gpg recommended --- configure.hs | 1 + debian/changelog | 2 ++ debian/control | 2 +- doc/install.mdwn | 10 ++++------ 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/configure.hs b/configure.hs index 4ab3052395..c81aa17e69 100644 --- a/configure.hs +++ b/configure.hs @@ -16,6 +16,7 @@ tests = , TestCase "rsync" $ requireCmd "rsync" "rsync --version >/dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" , TestCase "bup" $ testCmd "bup" "bup --version >/dev/null" + , TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath ] ++ shaTestCases [1, 256, 512, 224, 384] diff --git a/debian/changelog b/debian/changelog index 91c0c8f4b3..04a6896e32 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.20110402) UNRELEASED; urgency=low * bup is now supported as a special type of remote. + * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted + using GPG for privacy. * Use lowercase hash directories for locationlog files, to avoid some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still diff --git a/debian/control b/debian/control index 15155b9b43..42c79c91d8 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: git-annex Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync -Suggests: graphviz, bup +Suggests: graphviz, bup, gnupg Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/install.mdwn b/doc/install.mdwn index 70ab8e30b6..746352cb8a 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -13,19 +13,17 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * utf8-string: +* TestPack +* QuickCheck 2 * hS3 (optional, but recommended) * `uuid`: - (or uuidgen from util-linux) + (or `uuidgen` from util-linux) * `xargs`: * `rsync`: * `curl` : (optional, but recommended) * `sha1sum`: (optional, but recommended) +* `gpg`: (optional; needed for encryption) * [Ikiwiki](http://ikiwiki.info) is needed to build the documentation, but that will be skipped if it is not installed. Then just [[download]] git-annex and run: `make; make install` - -Additionally, to run the test suite (via `make test`), you will need: - -* `TestPack` -* `QuickCheck` 2 From d2e74efdb2e5b819d5c56f167291b006badd94cb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 19:30:31 -0400 Subject: [PATCH 1530/2835] document encryption --- doc/design/encryption.mdwn | 37 ++-------------------------- doc/encryption.mdwn | 35 ++++++++++++++++++++++++++ doc/special_remotes/Amazon_S3.mdwn | 15 +++-------- doc/special_remotes/bup.mdwn | 15 +++-------- doc/special_remotes/directory.mdwn | 6 ++--- doc/walkthrough/using_Amazon_S3.mdwn | 12 +++++---- 6 files changed, 53 insertions(+), 67 deletions(-) create mode 100644 doc/encryption.mdwn diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 915eee1a13..5a4bc8bbdf 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -1,15 +1,5 @@ -git-annex mostly does not use encryption. Anyone with access to a git -repository can see all the filenames in it, its history, and can access -any annexed file contents. - -Encryption is needed when using [[special_remotes]] like Amazon S3, where -file content is sent to an untrusted party who does not have access to the -git repository. - -Such an encrypted remote uses strong encryption on the contents of files, -as well as the filenames. The size of the encrypted files, and access -patterns of the data, should be the only clues to what type of is stored in -such a remote. +This was the design doc for [[encryption]] and is preserved for +the curious. [[!toc]] @@ -20,29 +10,6 @@ should be a way to tell what backend is responsible for a given filename in an encrypted remote. (And since special remotes can also store files unencrypted, differentiate from those as well.) -At a high level, an encryption backend needs to support these operations: - -* Create a new encrypted cipher, or update the cipher. Some input - parameters will specifiy things like the gpg public keys that - can access the cipher. - -* Initialize an instance of the encryption backend, that will use a - specified encrypted cipher. - -* Given a key/value backend key, produce and return an encrypted key. - - The same naming scheme git-annex uses for keys in regular key/value - [[backends]] can be used. So a filename for a key might be - "GPG-s12345--armoureddatahere" - -* Given a streaming source of file content, encrypt it, and send it in - a stream to an action that consumes the encrypted content. - -* Given a streaming source of encrypted content, decrypt it, and send - it in a stream to an action that consumes the decrypted content. - -* Clean up. - The rest of this page will describe a single encryption backend using GPG. Probably only one will be needed, but who knows? Maybe that backend will turn out badly designed, or some other encryptor needed. Designing diff --git a/doc/encryption.mdwn b/doc/encryption.mdwn new file mode 100644 index 0000000000..0f83bb7f90 --- /dev/null +++ b/doc/encryption.mdwn @@ -0,0 +1,35 @@ +git-annex mostly does not use encryption. Anyone with access to a git +repository can see all the filenames in it, its history, and can access +any annexed file contents. + +Encryption is needed when using [[special_remotes]] like Amazon S3, where +file content is sent to an untrusted party who does not have access to the +git repository. + +Such an encrypted remote uses strong GPG encryption on the contents of files, +as well as HMAC hashing of the filenames. The size of the encrypted files, +and access patterns of the data, should be the only clues to what is +stored in such a remote. + +You should decide whether to use encryption with a special remote before +any data is stored in it. So, `git annex initremote` requires you +to specify "encryption=none" when first setting up a remote in order +to disable encryption. + +If you want to use encryption, run `git annex initremote` with +"encryption=USERID". The value will be passed to `gpg` to find encryption keys. +Typically, you will say "encryption=2512E3C7" to use a specific gpg key. +Or, you might say "encryption=joey@kitenet.net" to search for matching keys. + +The [[encryption_design|design/encryption]] allows additional encryption keys +to be added on to a special remote later. Once a key is added, it is able +to access content that has already been stored in the special remote. +To add a new key, just run `git annex initremote` again, specifying the +new encryption key: + + git annex initremote myremote encryption=788A3F4C + +Note that once a key has been given access to a remote, it's not +possible to revoke that access, short of deleting the remote. See +[[encryption_design|design/encryption]] for other security risks +associated with encryption. diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn index 2cf23187d1..87cde32993 100644 --- a/doc/special_remotes/Amazon_S3.mdwn +++ b/doc/special_remotes/Amazon_S3.mdwn @@ -9,11 +9,12 @@ See [[walkthrough/using_Amazon_S3]] for usage examples. A number of parameters can be passed to `git annex initremote` to configure the S3 remote. -* `encryption` - Required. Either "none" to disable encryption, +* `encryption` - Required. Either "none" to disable encryption + (not recommended), or a value that can be looked up (using gpg -k) to find a gpg encryption key that will be given access to the remote. Note that additional gpg keys can be given access to a remote by rerunning initremote with - the new key id. + the new key id. See [[encryption]]. * `datacenter` - Defaults to "US". Other values include "EU", "us-west-1", and "ap-southeast-1". @@ -28,13 +29,3 @@ the S3 remote. * `bucket` - S3 requires that buckets have a globally unique name, so by default, a bucket name is chosen based on the remote name and UUID. This can be specified to pick a bucket name. - -## data security - -When encryption=none, there is **no** protection against your data being read -as it is sent to/from S3, or by Amazon when it is stored in S3. This should -only be used for public data. - -** Encryption is not yet supported. ** - -See [[design/encryption]]. diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 90b84e9f47..5bc1fb7a2e 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -15,11 +15,12 @@ for example; or clone bup's git repository to further back it up. These parameters can be passed to `git annex initremote` to configure bup: -* `encryption` - Required. Either "none" to disable encryption, +* `encryption` - Required. Either "none" to disable encryption of content + stored in bup (ssh will still be used to transport it securely), or a value that can be looked up (using gpg -k) to find a gpg encryption key that will be given access to the remote. Note that additional gpg keys can be given access to a remote by rerunning initremote with - the new key id. + the new key id. See [[encryption]]. * `buprepo` - Required. This is passed to `bup` as the `--remote` to use to store data. To create the repository,`bup init` will be run. @@ -34,13 +35,3 @@ can be used to, for example, limit its bandwidth. [[git-annex-shell]] does not support bup, due to the wacky way that bup starts its server. So, to use bup, you need full shell access to the server. - -## data security - -When encryption=none, there is **no** protection against your data being read -by anyone who can access the bup remote. However, bup does transfer data -using ssh, and if you trust the security of the remote, that's fine. - -** Encryption is not yet supported. ** - -See [[design/encryption]]. diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn index 18d30e3110..8006c44fc9 100644 --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -1,8 +1,8 @@ This special remote type stores file contents in directory. -One use case for this would be if you have a removable drive, that you -cannot put a git repository on for some reason, and you want to use it -to sneakernet files between systems. Just set up both systems to use +One use case for this would be if you have a removable drive that +you want to use it to sneakernet files between systems (possibly with +[[encrypted|encryption]] contents). Just set up both systems to use the drive's mountpoint as a directory remote. Setup example: diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index b8eb7da530..7f84f2ac2a 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -2,17 +2,19 @@ git-annex extends git's usual remotes with some [[special_remotes]], that are not git repositories. This way you can set up a remote using say, Amazon S3, and use git-annex to transfer files into the cloud. -**Note that encrypted buckets are not (yet) supported. Data sent to S3 -is without encryption susceptible to snooping.** - First, export your S3 credentials: # export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91" # export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t" -Next, create the S3 remote, and describe it. +Now, create a gpg key, if you don't already have one. This will be used +to encrypt everything stored in S3, for your privacy. Once you have +a gpg key, run `gpg --list-secret-keys` to look up its key id, something +like "2512E3C7" - # git annex initremote mys3 type=S3 encryption=none +Next, create the S3 remote, and describe it. + + # git annex initremote mys3 type=S3 encryption=2512E3C7 initremote mys3 (checking bucket) (creating bucket in US) ok # git annex describe mys3 "at Amazon's US datacenter" describe mys3 ok From 98e3817466130209d88d5061be9a590cdd609e78 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 19:45:59 -0400 Subject: [PATCH 1531/2835] don't let users change encryption type once remote is initted --- Remote/Encrypted.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Remote/Encrypted.hs b/Remote/Encrypted.hs index 2a0fb13bca..0ff2833b37 100644 --- a/Remote/Encrypted.hs +++ b/Remote/Encrypted.hs @@ -23,7 +23,8 @@ encryptionSetup :: RemoteConfig -> Annex RemoteConfig encryptionSetup c = case (M.lookup "encryption" c, extractCipher c) of (Nothing, Nothing) -> error "Specify encryption=key or encryption=none" - (Just "none", _) -> return c + (Just "none", Nothing) -> return c + (Just "none", Just _) -> error "Cannot change encryption type of existing remote." (Nothing, Just _) -> return c (Just _, Nothing) -> use $ genCipher c (Just _, Just v) -> use $ updateCipher c v From 991efddfa1333839885c9bc5490ff79d7dfc046c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 21:41:14 -0400 Subject: [PATCH 1532/2835] refactor --- Remote/Directory.hs | 99 ++++++++++++++++++++++++--------------------- Remote/Encrypted.hs | 52 +++++++++++++++--------- 2 files changed, 86 insertions(+), 65 deletions(-) diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 5ea0a1e6b3..2d31d12b2d 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -42,17 +42,20 @@ gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do dir <- getConfig r "directory" (error "missing directory") cst <- remoteCost r cheapRemoteCost - return $ Remote { - uuid = u, - cost = cst, - name = Git.repoDescribe r, - storeKey = storeKeyEncrypted c $ store dir, - retrieveKeyFile = retrieveKeyFileEncrypted c $ retrieve dir, - removeKey = removeKeyEncrypted c $ remove dir, - hasKey = hasKeyEncrypted c $ checkPresent dir, - hasKeyCheap = True, - config = Nothing - } + return $ encryptedRemote c + (storeEncrypted dir) + (retrieveEncrypted dir) + Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store dir, + retrieveKeyFile = retrieve dir, + removeKey = remove dir, + hasKey = checkPresent dir, + hasKeyCheap = True, + config = Nothing + } directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig directorySetup u c = do @@ -74,43 +77,47 @@ dirKey d k = d hashDirMixed k f f where f = keyFile k -store :: FilePath -> Key -> Maybe (Cipher, Key) -> Annex Bool -store d k c = do +store :: FilePath -> Key -> Annex Bool +store d k = do g <- Annex.gitRepo - let src = gitAnnexLocation g k - liftIO $ catch (copy src) (const $ return False) - where - copy src = case c of - Just (cipher, enckey) -> do - content <- L.readFile src - let dest = dirKey d enckey - prep dest - withEncryptedContent cipher content $ \s -> do - L.writeFile dest s - cleanup True dest - _ -> do - let dest = dirKey d k - prep dest - ok <- copyFile src dest - cleanup ok dest - prep dest = liftIO $ do - let dir = parentDir dest - createDirectoryIfMissing True dir - allowWrite dir - cleanup ok dest = do - when ok $ do - let dir = parentDir dest - preventWrite dest - preventWrite dir - return ok + let src = gitAnnexLocation g k + let dest = dirKey d k + liftIO $ catch (storeHelper dest $ copyFile src dest) (const $ return False) -retrieve :: FilePath -> Key -> FilePath -> Maybe (Cipher, Key) -> Annex Bool -retrieve d k f Nothing = liftIO $ copyFile (dirKey d k) f -retrieve d k f (Just (cipher, enckey)) = - liftIO $ flip catch (const $ return False) $ do - content <- L.readFile (dirKey d enckey) - withDecryptedContent cipher content $ L.writeFile f - return True +storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted d (cipher, enck) k = do + g <- Annex.gitRepo + let src = gitAnnexLocation g k + let dest = dirKey d enck + liftIO $ catch (storeHelper dest $ encrypt src dest) (const $ return False) + where + encrypt src dest = do + content <- L.readFile src + withEncryptedContent cipher content $ L.writeFile dest + return True + +storeHelper :: FilePath -> IO Bool -> IO Bool +storeHelper dest a = do + let dir = parentDir dest + createDirectoryIfMissing True dir + allowWrite dir + ok <- a + when ok $ do + preventWrite dest + preventWrite dir + return ok + +retrieve :: FilePath -> Key -> FilePath -> Annex Bool +retrieve d k f = liftIO $ copyFile (dirKey d k) f + +retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted d (cipher, enck) f = + liftIO $ catch decrypt (const $ return False) + where + decrypt = do + content <- L.readFile (dirKey d enck) + withDecryptedContent cipher content $ L.writeFile f + return True remove :: FilePath -> Key -> Annex Bool remove d k = liftIO $ catch del (const $ return False) diff --git a/Remote/Encrypted.hs b/Remote/Encrypted.hs index 0ff2833b37..255b41d730 100644 --- a/Remote/Encrypted.hs +++ b/Remote/Encrypted.hs @@ -33,16 +33,39 @@ encryptionSetup c = cipher <- liftIO a return $ M.delete "encryption" $ storeCipher c cipher -{- Helpers that can be applied to a Remote's normal actions to - - add crypto support. -} -storeKeyEncrypted :: Maybe RemoteConfig -> (Key -> Maybe (Cipher, Key) -> Annex a) -> Key -> Annex a -storeKeyEncrypted c a k = a k =<< cipherKey c k -retrieveKeyFileEncrypted :: Maybe RemoteConfig -> (Key -> FilePath -> Maybe (Cipher, Key) -> Annex a) -> Key -> FilePath -> Annex a -retrieveKeyFileEncrypted c a k f = a k f =<< cipherKey c k -removeKeyEncrypted :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a -removeKeyEncrypted = withEncryptedKey -hasKeyEncrypted :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a -hasKeyEncrypted = withEncryptedKey +{- Modifies a Remote to support encryption. + - + - Two additional functions must be provided by the remote, + - to support storing and retrieving encrypted content. -} +encryptedRemote + :: Maybe RemoteConfig + -> ((Cipher, Key) -> Key -> Annex Bool) + -> ((Cipher, Key) -> FilePath -> Annex Bool) + -> Remote Annex + -> Remote Annex +encryptedRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = + r { + storeKey = store, + retrieveKeyFile = retrieve, + removeKey = withkey $ removeKey r, + hasKey = withkey $ hasKey r + } + where + store k = do + v <- cipherKey c k + case v of + Nothing -> (storeKey r) k + Just x -> storeKeyEncrypted x k + retrieve k f = do + v <- cipherKey c k + case v of + Nothing -> (retrieveKeyFile r) k f + Just x -> retrieveKeyFileEncrypted x f + withkey a k = do + v <- cipherKey c k + case v of + Nothing -> a k + Just (_, k') -> a k' {- Gets encryption Cipher, and encrypted version of Key. - @@ -64,12 +87,3 @@ cipherKey (Just c) k = do ret cipher = do k' <- liftIO $ encryptKey cipher k return $ Just (cipher, k') - -{- Passes the encrypted version of the key to the action when encryption - - is enabled, and the non-encrypted version otherwise. -} -withEncryptedKey :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a -withEncryptedKey c a k = do - v <- cipherKey c k - case v of - Nothing -> a k - Just (_, k') -> a k' From 480cc353c46d88c55b252fbb6c5dc4feff08995c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 23:01:29 -0400 Subject: [PATCH 1533/2835] incomplete and buggy encryption support for bup Some kind of laziness issue that I don't want to debug right now, and decryption is not implemented. --- Remote/Bup.hs | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index b4403bb03e..6f4c9278e8 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -7,6 +7,7 @@ module Remote.Bup (remote) where +import qualified Data.ByteString.Lazy.Char8 as L import IO import Control.Exception.Extensible (IOException) import qualified Data.Map as M @@ -16,6 +17,7 @@ import System.Process import System.Exit import System.FilePath import Data.List.Utils +import System.Cmd.Utils import RemoteClass import Types @@ -29,6 +31,7 @@ import Messages import Ssh import Remote.Special import Remote.Encrypted +import Crypto type BupRepo = String @@ -47,16 +50,17 @@ gen r u c = do bupr <- liftIO $ bup2GitRemote buprepo (u', bupr') <- getBupUUID bupr u - return $ this cst buprepo u' bupr' - where - this cst buprepo u' bupr = Remote { + return $ encryptedRemote c + (storeEncrypted r buprepo) + (retrieveEncrypted buprepo) + Remote { uuid = u', cost = cst, name = Git.repoDescribe r, storeKey = store r buprepo, retrieveKeyFile = retrieve buprepo, removeKey = remove, - hasKey = checkPresent r bupr, + hasKey = checkPresent r bupr', hasKeyCheap = True, config = c } @@ -92,13 +96,34 @@ bup command buprepo params = do showProgress -- make way for bup output liftIO $ boolSystem "bup" $ bupParams command buprepo params +bupSplitParams :: Git.Repo -> BupRepo -> Key -> CommandParam -> Annex [CommandParam] +bupSplitParams r buprepo k src = do + o <- getConfig r "bup-split-options" "" + let os = map Param $ words o + showProgress -- make way for bup output + return $ bupParams "split" buprepo + (os ++ [Param "-n", Param (show k), src]) + store :: Git.Repo -> BupRepo -> Key -> Annex Bool store r buprepo k = do g <- Annex.gitRepo let src = gitAnnexLocation g k - o <- getConfig r "bup-split-options" "" - let os = map Param $ words o - bup "split" buprepo $ os ++ [Param "-n", Param (show k), File src] + params <- bupSplitParams r buprepo k (File src) + liftIO $ boolSystem "bup" params + +storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted r buprepo (cipher, enck) k = do + g <- Annex.gitRepo + let src = gitAnnexLocation g k + params <- bupSplitParams r buprepo enck (Param "-") + liftIO $ flip catch (const $ return False) $ do + content <- L.readFile src + -- FIXME hangs after a while + (pid, h) <- hPipeTo "bup" (toCommand params) + withEncryptedContent cipher content $ L.hPut h + hClose h + forceSuccess pid + return True retrieve :: BupRepo -> Key -> FilePath -> Annex Bool retrieve buprepo k f = do @@ -116,6 +141,10 @@ retrieve buprepo k f = do Right r -> return r Left _ -> return False +retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted bupreoo (cipher, enck) f = do + error "TODO" + remove :: Key -> Annex Bool remove _ = do warning "content cannot be removed from bup remote" From d82898841581bab7785a0010f49e21c5eec5b51b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 23:02:09 -0400 Subject: [PATCH 1534/2835] proper encrypted keys For HMAC, using the Data.Digest.Pure.SHA library. I have been avoiding this library for checksumming generally, since it's (probably) not as fast as external utilities, but it's fine to use it for HMAC. --- Crypto.hs | 17 +++++++++-------- doc/install.mdwn | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 9f404c1b17..25d9a11573 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -24,6 +24,8 @@ module Crypto ( import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import qualified Codec.Binary.Base64 as B64 +import Data.ByteString.Lazy.UTF8 (fromString) +import Data.Digest.Pure.SHA import System.Cmd.Utils import Data.String.Utils import Data.List @@ -105,12 +107,11 @@ decryptCipher _ (EncryptedCipher encipher _) = - reversable, nor does it need to be the same type of encryption used - on content. It does need to be repeatable. -} encryptKey :: Cipher -> Key -> IO Key -encryptKey c k = +encryptKey (Cipher c) k = return Key { - -- FIXME: should use HMAC with the cipher; I don't - -- have Data.Crypto in Debian yet though. - keyName = show k, - keyBackendName = "INSECURE", + keyName = showDigest $ + hmacSha1 (fromString $ show k) (fromString c), + keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data } @@ -154,12 +155,12 @@ gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) gpgCipher params (Cipher c) input a = do -- pipe the passphrase into gpg on a fd (frompipe, topipe) <- createPipe - toh <- fdToHandle topipe - let Fd fromno = frompipe _ <- forkIO $ do + toh <- fdToHandle topipe hPutStrLn toh c hClose toh - let passphrase = [Param "--passphrase-fd", Param $ show fromno] + let Fd passphrasefd = frompipe + let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd] (pid, output) <- gpgPipeBytes (passphrase ++ params) input ret <- a output diff --git a/doc/install.mdwn b/doc/install.mdwn index 746352cb8a..7a0c6020ba 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -13,6 +13,7 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * utf8-string: +* SHA: * TestPack * QuickCheck 2 * hS3 (optional, but recommended) From 11da36e48fb0a9de35b8b386a0c4156b6dfd0ead Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 23:05:26 -0400 Subject: [PATCH 1535/2835] build dep update --- debian/control | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 42c79c91d8..ae0ca0c598 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,18 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync, git | git-core, perlmagick +Build-Depends: + debhelper (>= 7.0.50), + ghc6, + libghc6-missingh-dev, + libghc6-pcre-light-dev, + libghc6-testpack-dev, + libghc6-sha-dev + ikiwiki, + perlmagick, + git | git-core, + uuid, + rsync, Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex @@ -10,7 +21,11 @@ Homepage: http://git-annex.branchable.com/ Package: git-annex Architecture: any Section: utils -Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync +Depends: ${misc:Depends}, ${shlibs:Depends}, + git | git-core, + uuid, + rsync, + openssh-client Suggests: graphviz, bup, gnupg Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file From d996637fd68430b4236d2899c49827cbf457471f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 00:34:38 -0400 Subject: [PATCH 1536/2835] fix stall while storing encrypted data in bup Forking a new process rather than relying on a thread to feed gpg. The feeder thread was stalling, probably when the main thread got to the point it was wait()ing on the gpg to exit. --- Crypto.hs | 42 ++++++++++++++++++++++++++++-------------- Remote/Bup.hs | 31 +++++++++++++------------------ 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 25d9a11573..4ec186ea22 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -17,6 +17,7 @@ module Crypto ( extractCipher, decryptCipher, encryptKey, + withEncryptedContentHandle, withEncryptedContent, withDecryptedContent, ) where @@ -33,8 +34,10 @@ import Data.Bits.Utils import System.IO import System.Posix.IO import System.Posix.Types +import System.Posix.Process import Control.Concurrent import Control.Exception +import System.Exit import Types import Key @@ -116,6 +119,11 @@ encryptKey (Cipher c) k = keyMtime = Nothing -- to avoid leaking data } +{- Runs an action passing it a handle from which it can + - stream encrypted content. -} +withEncryptedContentHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withEncryptedContentHandle = gpgCipherHandle [Params "--symmetric --force-mdc"] + {- Streams encrypted content to an action. -} withEncryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a withEncryptedContent = gpgCipher [Params "--symmetric --force-mdc"] @@ -142,17 +150,10 @@ gpgPipeStrict params input = do forceSuccess pid return output -gpgPipeBytes :: [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString) -gpgPipeBytes params input = do - (pid, fromh, toh) <- hPipeBoth "gpg" (gpgParams params) - _ <- forkIO $ finally (L.hPut toh input) (hClose toh) - output <- L.hGetContents fromh - return (pid, output) - {- Runs gpg with a cipher and some parameters, feeding it an input, - - and piping its output lazily to an action. -} -gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a -gpgCipher params (Cipher c) input a = do + - and passing a handle to its output to an action. -} +gpgCipherHandle :: [CommandParam] -> Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +gpgCipherHandle params (Cipher c) input a = do -- pipe the passphrase into gpg on a fd (frompipe, topipe) <- createPipe _ <- forkIO $ do @@ -161,16 +162,29 @@ gpgCipher params (Cipher c) input a = do hClose toh let Fd passphrasefd = frompipe let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd] - (pid, output) <- gpgPipeBytes (passphrase ++ params) input - - ret <- a output + + (pid, fromh, toh) <- hPipeBoth "gpg" $ + gpgParams $ passphrase ++ params + _ <- forkProcess $ do + L.hPut toh input + hClose toh + exitSuccess + hClose toh + ret <- a fromh -- cleanup forceSuccess pid closeFd frompipe - return ret +{- Runs gpg with a cipher and some parameters, feeding it an input, + - and piping its output lazily to an action. -} +gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +gpgCipher params c input a = do + gpgCipherHandle params c input $ \h -> do + content <- L.hGetContents h + a content + configKeyIds :: RemoteConfig -> IO KeyIds configKeyIds c = do let k = configGet c "encryption" diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 6f4c9278e8..771212372f 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -17,7 +17,6 @@ import System.Process import System.Exit import System.FilePath import Data.List.Utils -import System.Cmd.Utils import RemoteClass import Types @@ -96,6 +95,15 @@ bup command buprepo params = do showProgress -- make way for bup output liftIO $ boolSystem "bup" $ bupParams command buprepo params +pipeBup :: [CommandParam] -> Maybe Handle -> Maybe Handle -> IO Bool +pipeBup params inh outh = do + p <- runProcess "bup" (toCommand params) + Nothing Nothing inh outh Nothing + ok <- waitForProcess p + case ok of + ExitSuccess -> return True + _ -> return False + bupSplitParams :: Git.Repo -> BupRepo -> Key -> CommandParam -> Annex [CommandParam] bupSplitParams r buprepo k src = do o <- getConfig r "bup-split-options" "" @@ -118,28 +126,15 @@ storeEncrypted r buprepo (cipher, enck) k = do params <- bupSplitParams r buprepo enck (Param "-") liftIO $ flip catch (const $ return False) $ do content <- L.readFile src - -- FIXME hangs after a while - (pid, h) <- hPipeTo "bup" (toCommand params) - withEncryptedContent cipher content $ L.hPut h - hClose h - forceSuccess pid - return True + withEncryptedContentHandle cipher content $ \h -> do + pipeBup params (Just h) Nothing retrieve :: BupRepo -> Key -> FilePath -> Annex Bool retrieve buprepo k f = do let params = bupParams "join" buprepo [Param $ show k] - ret <- liftIO $ try $ do - -- pipe bup's stdout directly to file + liftIO $ flip catch (const $ return False) $ do tofile <- openFile f WriteMode - p <- runProcess "bup" (toCommand params) - Nothing Nothing Nothing (Just tofile) Nothing - r <- waitForProcess p - case r of - ExitSuccess -> return True - _ -> return False - case ret of - Right r -> return r - Left _ -> return False + pipeBup params Nothing (Just tofile) retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool retrieveEncrypted bupreoo (cipher, enck) f = do From b6b04642c8d513aaa75b924e1ef8480fa39f3109 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 00:40:23 -0400 Subject: [PATCH 1537/2835] rename --- Remote/Bup.hs | 4 ++-- Remote/Directory.hs | 4 ++-- Remote/{Encrypted.hs => Encryptable.hs} | 8 ++++---- Remote/S3real.hs | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) rename Remote/{Encrypted.hs => Encryptable.hs} (93%) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 771212372f..698d1b188a 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -29,7 +29,7 @@ import Utility import Messages import Ssh import Remote.Special -import Remote.Encrypted +import Remote.Encryptable import Crypto type BupRepo = String @@ -49,7 +49,7 @@ gen r u c = do bupr <- liftIO $ bup2GitRemote buprepo (u', bupr') <- getBupUUID bupr u - return $ encryptedRemote c + return $ encryptableRemote c (storeEncrypted r buprepo) (retrieveEncrypted buprepo) Remote { diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 2d31d12b2d..a84a1f45a7 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -27,7 +27,7 @@ import Config import Content import Utility import Remote.Special -import Remote.Encrypted +import Remote.Encryptable import Crypto remote :: RemoteType Annex @@ -42,7 +42,7 @@ gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do dir <- getConfig r "directory" (error "missing directory") cst <- remoteCost r cheapRemoteCost - return $ encryptedRemote c + return $ encryptableRemote c (storeEncrypted dir) (retrieveEncrypted dir) Remote { diff --git a/Remote/Encrypted.hs b/Remote/Encryptable.hs similarity index 93% rename from Remote/Encrypted.hs rename to Remote/Encryptable.hs index 255b41d730..a9a7472fb5 100644 --- a/Remote/Encrypted.hs +++ b/Remote/Encryptable.hs @@ -1,11 +1,11 @@ -{- common functions for encrypted remotes +{- common functions for encryptable remotes - - Copyright 2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module Remote.Encrypted where +module Remote.Encryptable where import qualified Data.Map as M import Control.Monad.State (liftIO) @@ -37,13 +37,13 @@ encryptionSetup c = - - Two additional functions must be provided by the remote, - to support storing and retrieving encrypted content. -} -encryptedRemote +encryptableRemote :: Maybe RemoteConfig -> ((Cipher, Key) -> Key -> Annex Bool) -> ((Cipher, Key) -> FilePath -> Annex Bool) -> Remote Annex -> Remote Annex -encryptedRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = +encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = r { storeKey = store, retrieveKeyFile = retrieve, diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 0f6327f575..1fa387d68e 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -28,7 +28,7 @@ import Messages import Locations import Config import Remote.Special -import Remote.Encrypted +import Remote.Encryptable remote :: RemoteType Annex remote = RemoteType { From 9606409b9dd5218b0540418170ec86e9e1cec038 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 00:57:11 -0400 Subject: [PATCH 1538/2835] bup encryption support 100% working --- Remote/Bup.hs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 698d1b188a..0edb33ba8a 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -17,6 +17,7 @@ import System.Process import System.Exit import System.FilePath import Data.List.Utils +import System.Cmd.Utils import RemoteClass import Types @@ -124,7 +125,7 @@ storeEncrypted r buprepo (cipher, enck) k = do g <- Annex.gitRepo let src = gitAnnexLocation g k params <- bupSplitParams r buprepo enck (Param "-") - liftIO $ flip catch (const $ return False) $ do + liftIO $ catchBool $ do content <- L.readFile src withEncryptedContentHandle cipher content $ \h -> do pipeBup params (Just h) Nothing @@ -132,13 +133,19 @@ storeEncrypted r buprepo (cipher, enck) k = do retrieve :: BupRepo -> Key -> FilePath -> Annex Bool retrieve buprepo k f = do let params = bupParams "join" buprepo [Param $ show k] - liftIO $ flip catch (const $ return False) $ do + liftIO $ catchBool $ do tofile <- openFile f WriteMode pipeBup params Nothing (Just tofile) retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool -retrieveEncrypted bupreoo (cipher, enck) f = do - error "TODO" +retrieveEncrypted buprepo (cipher, enck) f = do + let params = bupParams "join" buprepo [Param $ show enck] + liftIO $ catchBool $ do + (pid, h) <- hPipeFrom "bup" $ toCommand params + content <- L.hGetContents h + withDecryptedContent cipher content $ L.writeFile f + forceSuccess pid + return True remove :: Key -> Annex Bool remove _ = do From 89fab6c7b8955ef26e653d539f7be3b70129c15e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 00:57:29 -0400 Subject: [PATCH 1539/2835] refactor --- Remote/Directory.hs | 25 +++++++++++-------------- Utility.hs | 5 +++++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Remote/Directory.hs b/Remote/Directory.hs index a84a1f45a7..d9bee80c3f 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -82,14 +82,14 @@ store d k = do g <- Annex.gitRepo let src = gitAnnexLocation g k let dest = dirKey d k - liftIO $ catch (storeHelper dest $ copyFile src dest) (const $ return False) + liftIO $ catchBool $ storeHelper dest $ copyFile src dest storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted d (cipher, enck) k = do g <- Annex.gitRepo let src = gitAnnexLocation g k let dest = dirKey d enck - liftIO $ catch (storeHelper dest $ encrypt src dest) (const $ return False) + liftIO $ catchBool $ storeHelper dest $ encrypt src dest where encrypt src dest = do content <- L.readFile src @@ -112,23 +112,20 @@ retrieve d k f = liftIO $ copyFile (dirKey d k) f retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool retrieveEncrypted d (cipher, enck) f = - liftIO $ catch decrypt (const $ return False) - where - decrypt = do - content <- L.readFile (dirKey d enck) - withDecryptedContent cipher content $ L.writeFile f - return True + liftIO $ catchBool $ do + content <- L.readFile (dirKey d enck) + withDecryptedContent cipher content $ L.writeFile f + return True remove :: FilePath -> Key -> Annex Bool -remove d k = liftIO $ catch del (const $ return False) +remove d k = liftIO $ catchBool $ do + allowWrite dir + removeFile file + removeDirectory dir + return True where file = dirKey d k dir = parentDir file - del = do - allowWrite dir - removeFile file - removeDirectory dir - return True checkPresent :: FilePath -> Key -> Annex (Either IOException Bool) checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k) diff --git a/Utility.hs b/Utility.hs index 1c6b4d21e6..5639a8799a 100644 --- a/Utility.hs +++ b/Utility.hs @@ -24,6 +24,7 @@ module Utility ( dirContains, dirContents, myHomeDir, + catchBool, prop_idempotent_shellEscape, prop_idempotent_shellEscape_multiword, @@ -256,3 +257,7 @@ myHomeDir = do uid <- getEffectiveUserID u <- getUserEntryForID uid return $ homeDirectory u + +{- Catches IO errors and returns a Bool -} +catchBool :: IO Bool -> IO Bool +catchBool = flip catch (const $ return False) From 50cfcdf54b828fbeab532b712e00063ae9e82581 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 01:13:21 -0400 Subject: [PATCH 1540/2835] make encrypted remotes have slightly higher costs --- Config.hs | 17 ++++++++++++++++- Remote/Encryptable.hs | 4 +++- test.hs | 2 ++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Config.hs b/Config.hs index 53f1a455fd..a324427d44 100644 --- a/Config.hs +++ b/Config.hs @@ -52,10 +52,25 @@ remoteCost r def = do cheapRemoteCost :: Int cheapRemoteCost = 100 semiCheapRemoteCost :: Int -semiCheapRemoteCost = 150 +semiCheapRemoteCost = 110 expensiveRemoteCost :: Int expensiveRemoteCost = 200 +{- Adjust's a remote's cost to reflect it being encrypted. -} +encryptedRemoteCostAdj :: Int +encryptedRemoteCostAdj = 50 + +{- Make sure the remote cost numbers work out. -} +prop_cost_sane :: Bool +prop_cost_sane = False `notElem` + [ expensiveRemoteCost > 0 + , cheapRemoteCost < semiCheapRemoteCost + , semiCheapRemoteCost < expensiveRemoteCost + , cheapRemoteCost + encryptedRemoteCostAdj > semiCheapRemoteCost + , cheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost + , semiCheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost + ] + {- Checks if a repo should be ignored, based either on annex-ignore - setting, or on command-line options. Allows command-line to override - annex-ignore. -} diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index a9a7472fb5..aa7c2a5691 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -15,6 +15,7 @@ import RemoteClass import Crypto import qualified Annex import Messages +import Config {- Encryption setup for a remote. The user must specify whether to use - an encryption key, or not encrypt. An encrypted cipher is created, or is @@ -48,7 +49,8 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = storeKey = store, retrieveKeyFile = retrieve, removeKey = withkey $ removeKey r, - hasKey = withkey $ hasKey r + hasKey = withkey $ hasKey r, + cost = cost r + encryptedRemoteCostAdj } where store k = do diff --git a/test.hs b/test.hs index 30ebe6e593..cdec4ea614 100644 --- a/test.hs +++ b/test.hs @@ -39,6 +39,7 @@ import qualified Remote import qualified Content import qualified Command.DropUnused import qualified Key +import qualified Config main :: IO () main = do @@ -61,6 +62,7 @@ quickcheck = TestLabel "quickcheck" $ TestList , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics , qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics + , qctest "prop_cost_sane" Config.prop_cost_sane ] blackbox :: Test From 4d136e1ef5a3c06bbc8e10a5aa7ac20e17a39c4f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 01:34:28 -0400 Subject: [PATCH 1541/2835] use different parts of cipher for hmac and gpg Per bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing It may be paranoid to worry about the cipher being recovered from hmac keys, but yes.. let's be paranoid. --- Crypto.hs | 35 ++++++++++++++----- ...e_same_key_for_encryption_and_hashing.mdwn | 4 +-- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 4ec186ea22..e42a21e282 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -45,6 +45,24 @@ import RemoteClass import Utility import CryptoTypes +{- The first half of a Cipher is used for HMAC; the remainder + - is used as the GPG symmetric encryption passphrase. + - + - 256 is enough for gpg's symetric cipher; unlike weaker public key + - crypto, the key does not need to be too large. + -} +cipherHalf :: Int +cipherHalf = 256 + +cipherSize :: Int +cipherSize = cipherHalf * 2 + +cipherPassphrase :: Cipher -> String +cipherPassphrase (Cipher c) = drop cipherHalf c + +cipherHmac :: Cipher -> String +cipherHmac (Cipher c) = take cipherHalf c + {- Creates a new Cipher, encrypted as specified in the remote's configuration -} genCipher :: RemoteConfig -> IO EncryptedCipher genCipher c = do @@ -58,10 +76,10 @@ genCipher c = do -- newline. [ Params "--gen-random --armor" , Param $ show randomquality - , Param $ show ciphersize + , Param $ show cipherSize ] - randomquality = 1 :: Int -- 1 is /dev/urandom; 2 is /dev/random - ciphersize = 256 :: Int + -- 1 is /dev/urandom; 2 is /dev/random + randomquality = 1 :: Int {- Updates an existing Cipher, re-encrypting it to add KeyIds specified in - the remote's configuration. -} @@ -110,10 +128,11 @@ decryptCipher _ (EncryptedCipher encipher _) = - reversable, nor does it need to be the same type of encryption used - on content. It does need to be repeatable. -} encryptKey :: Cipher -> Key -> IO Key -encryptKey (Cipher c) k = +encryptKey c k = return Key { - keyName = showDigest $ - hmacSha1 (fromString $ show k) (fromString c), + keyName = showDigest $ hmacSha1 + (fromString $ show k) + (fromString $ cipherHmac c), keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data @@ -153,12 +172,12 @@ gpgPipeStrict params input = do {- Runs gpg with a cipher and some parameters, feeding it an input, - and passing a handle to its output to an action. -} gpgCipherHandle :: [CommandParam] -> Cipher -> L.ByteString -> (Handle -> IO a) -> IO a -gpgCipherHandle params (Cipher c) input a = do +gpgCipherHandle params c input a = do -- pipe the passphrase into gpg on a fd (frompipe, topipe) <- createPipe _ <- forkIO $ do toh <- fdToHandle topipe - hPutStrLn toh c + hPutStrLn toh $ cipherPassphrase c hClose toh let Fd passphrasefd = frompipe let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd] diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn index 1980a8f444..9fc31fa485 100644 --- a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn @@ -4,5 +4,5 @@ Also, ttbomk, HMAC needs two keys, not one. Are you re-using the same key twice? Compability for old buckets and support for different ones can be maintained by introducing a new option and simply copying over the encryption key's identifier into this new option should it be missing. -> See [[design/encryption]]. I don't think this bug needs to be kept -> open. [[done]] --[[Joey]] +> Bug was filed prematurely, but was a good bit of paranoia, and gpg and +> hmac are given different secret keys [[done]] --[[Joey]] From 67cced26dc3407a749f01010515e2d2827af2a10 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:01:34 -0400 Subject: [PATCH 1542/2835] S3 crypto support Untested, I will need to dust off my S3 keys, and plug the modem back in that was unplugged last night due to very low battery bank power. But it compiles, so it's probably perfect. :) --- Remote/S3real.hs | 50 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 1fa387d68e..b88b22037f 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -29,6 +29,7 @@ import Locations import Config import Remote.Special import Remote.Encryptable +import Crypto remote :: RemoteType Annex remote = RemoteType { @@ -41,16 +42,22 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do cst <- remoteCost r expensiveRemoteCost - return $ this cst + return $ gen' r u c cst +gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex +gen' r u c cst = + encryptableRemote c + (storeEncrypted this) + (retrieveEncrypted this) + this where - this cst = Remote { + this = Remote { uuid = u, cost = cst, name = Git.repoDescribe r, - storeKey = store (this cst), - retrieveKeyFile = retrieve (this cst), - removeKey = remove (this cst), - hasKey = checkPresent (this cst), + storeKey = store this, + retrieveKeyFile = retrieve this, + removeKey = remove this, + hasKey = checkPresent this, hasKeyCheap = False, config = c } @@ -139,9 +146,21 @@ checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do noconn = Left $ error "S3 not configured" store :: Remote Annex -> Key -> Annex Bool -store r k = s3Action r False $ \(conn, bucket) -> do +store r k = storeHelper r k =<< lazyKeyContent k + +storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted r (cipher, enck) k = do + content <- lazyKeyContent k + content' <- liftIO $ withEncryptedContent cipher content return + storeHelper r enck content' + +lazyKeyContent :: Key -> Annex L.ByteString +lazyKeyContent k = do g <- Annex.gitRepo - content <- liftIO $ L.readFile $ gitAnnexLocation g k + liftIO $ L.readFile $ gitAnnexLocation g k + +storeHelper :: Remote Annex -> Key -> L.ByteString -> Annex Bool +storeHelper r k content = s3Action r False $ \(conn, bucket) -> do let object = setStorageClass storageclass $ bucketKey bucket k content res <- liftIO $ sendObject conn object case res of @@ -156,16 +175,25 @@ store r k = s3Action r False $ \(conn, bucket) -> do _ -> STANDARD retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool -retrieve r k f = s3Action r False $ \(conn, bucket) -> do +retrieve = retrieveHelper (return . obj_data) + +retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted r (cipher, enck) f = retrieveHelper decrypt r enck f + where + decrypt o = withDecryptedContent cipher (obj_data o) return + +retrieveHelper :: (S3Object -> IO L.ByteString) -> Remote Annex -> Key -> FilePath -> Annex Bool +retrieveHelper a r k f = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ getObject conn $ bucketKey bucket k L.empty case res of Right o -> do - liftIO $ L.writeFile f (obj_data o) + content <- liftIO $ a o + liftIO $ L.writeFile f content return True Left e -> do warning $ prettyReqError e return False - + remove :: Remote Annex -> Key -> Annex Bool remove r k = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty From 3d2a0f68b53dd6045558c232f5e5d860dde3ff91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:08:11 -0400 Subject: [PATCH 1543/2835] note --- doc/special_remotes/bup.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 5bc1fb7a2e..e59ff240de 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -3,6 +3,10 @@ This special remote type stores file contents in a in the front-end, and bup as a remote, you get an easy git-style interface to large files, and easy backups of the file contents using git. +This is particularly well suited to collaboration on projects involving +large files, since both the git-annex and bup repositories can be +accessed like any other git repository. + See [[walkthrough/using_bup]] for usage examples. Each individual key is stored in a bup remote using `bup split`, with From eafb51959225d871b94136b7b02c5cf7752d49be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:09:33 -0400 Subject: [PATCH 1544/2835] hmacSha1 wants the secret key as first parameter I was offline last night and going by function signatures, and unable to tell which was which. Note sure it matters to HMAC which comes first; better safe than sorry. --- Crypto.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index e42a21e282..12c70ef048 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -131,8 +131,8 @@ encryptKey :: Cipher -> Key -> IO Key encryptKey c k = return Key { keyName = showDigest $ hmacSha1 - (fromString $ show k) - (fromString $ cipherHmac c), + (fromString $ cipherHmac c) + (fromString $ show k), keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data From 80981600a0994ba36d3867d5ada55b5ea5a3d9cb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:13:54 -0400 Subject: [PATCH 1545/2835] looked up HMAC block size details --- Crypto.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Crypto.hs b/Crypto.hs index 12c70ef048..25a87c3ad9 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -48,6 +48,9 @@ import CryptoTypes {- The first half of a Cipher is used for HMAC; the remainder - is used as the GPG symmetric encryption passphrase. - + - HMAC SHA1 needs only 64 bytes. The remainder is for expansion, + - perhaps to HMAC SHA512, which needs 128 bytes (ideally). + - - 256 is enough for gpg's symetric cipher; unlike weaker public key - crypto, the key does not need to be too large. -} From 83423211a21d061b4f0d62c925dae7aa4cc62f98 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:27:24 -0400 Subject: [PATCH 1546/2835] design wrapup --- doc/design/encryption.mdwn | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 5a4bc8bbdf..b30e01cdda 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -22,8 +22,8 @@ The basis of this scheme was originally developed by Lars Wirzenius et al [for Obnam](http://braawi.org/obnam/encryption/). """]] -Data is encrypted by gpg, using a symmetric cipher. The passphrase of the -cipher is itself checked into your git repository, encrypted using one or +Data is encrypted by gpg, using a symmetric cipher. +The cipher is itself checked into your git repository, encrypted using one or more gpg public keys. This scheme allows new gpg private keys to be given access to content that has already been stored in the remote. @@ -58,6 +58,33 @@ for each file in the repository, contact the encrypted remote to check if it has the file. This can be done without enumeration, although it will mean running gpg once per file fscked, to get the encrypted filename. +So, the files stored in the remote should be encrypted. But, it needs +to be a repeatable encryption, so they cannot just be gpg encrypted, +that would yeild a new name each time. Instead, HMAC is used. Any hash +could be used with HMAC; currently SHA1 is used. + +It was suggested that it might not be wise to use the same cipher for both +gpg and HMAC. Being paranoid, it's best not to tie the security of one +to the security of the other. So, the encrypted cipher described above is +actually split in two; half is used for HMAC, and half for gpg. + +---- + +Does the HMAC cipher need to be gpg encrypted? Imagine if it were +stored in plainext in the git repository. Anyone who can access +the git repository already knows the actual filenames, and typically also +the content hashes of annexed content. Having access to the HMAC cipher +could perhaps be said to only let them verify that data they already +know. + +While this seems a pretty persuasive argument, I'm not 100% convinced, and +anyway, most times that the HMAC cipher is needed, the gpg cipher is also +needed. Keeping the HMAC cipher encrypted does slow down two things: +dropping content from encrypted remotes, and checking if encrypted remotes +really have content. If it's later determined to be safe to not encrypt the +HMAC cipher, the current design allows changing that, even for existing +remotes. + ## risks A risk of this scheme is that, once the symmetric cipher has been obtained, it From f486768b169ec392448af75dc3a5f90de9d5a353 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:31:56 -0400 Subject: [PATCH 1547/2835] tweak wording --- Remote/Encryptable.hs | 2 +- doc/walkthrough/using_Amazon_S3.mdwn | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index aa7c2a5691..9e4f58ed40 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -81,7 +81,7 @@ cipherKey (Just c) k = do Nothing -> case extractCipher c of Nothing -> return Nothing Just encipher -> do - showNote "getting encryption key" + showNote "unlocking" cipher <- liftIO $ decryptCipher c encipher Annex.changeState (\s -> s { Annex.cipher = Just cipher }) ret cipher diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index 7f84f2ac2a..0a13900b8d 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -30,8 +30,8 @@ repository use the same S3 remote is easy: Now the remote can be used like any other remote. # git annex copy my_cool_big_file --to mys3 - copy my_cool_big_file (to mys3...) ok + copy my_cool_big_file (unlocking) (checking mys3...) (to mys3...) ok # git annex move video/hackity_hack_and_kaxxt.mov --to mys3 - move video/hackity_hack_and_kaxxt.mov (to mys3...) ok + move video/hackity_hack_and_kaxxt.mov (unlocking) (checking mys3...) (to mys3...) ok See [[special_remotes/Amazon_S3]] for details. From 808040d72a9a3322096a2bb2448da4265c62a751 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:38:28 -0400 Subject: [PATCH 1548/2835] update to mention encryption --- doc/use_case/Alice.mdwn | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 80280580e3..24ed7c2a92 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -2,10 +2,13 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server -out there on the net. She stores data in the Cloud. All these things can -have different files on them, but Alice no longer has to deal with the -tedious process of keeping them manually in sync, or remembering where -she put a file. +out there on the net. She stores data, encrypted in the Cloud. + +All these things can have different files on them, but Alice no longer +has to deal with the tedious process of keeping them manually in sync, +or remembering where she put a file. git-annex manages all these data +sources as if they were git remotes. +[[more about special remotes|special_remotes]] When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her From d93e2f52c337f1d8016dbd887d69a5eec74e5a08 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:41:52 -0400 Subject: [PATCH 1549/2835] update --- doc/not.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index 009e1a79d7..2827dd12d0 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -5,6 +5,12 @@ system. For a backup system that uses git and that git-annex supports storing data in, see [[special_remotes/bup]]. +* git-annex is not a filesystem or DropBox clone. But there + is a FUSE filesystem built on top of git-annex, called + [ShareBox](https://github.com/chmduquesne/sharebox), and there is + interest in making it easy to use and covering some of the use + cases supported by DropBox. + * git-annex is not unison, but if you're finding unison's checksumming too slow, or its strict mirroring of everything to both places too limiting, then git-annex could be a useful alternative. From dcbe94b3a5c132470635bca522527c3c6c586948 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:44:32 -0400 Subject: [PATCH 1550/2835] layout --- doc/use_case/Alice.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 24ed7c2a92..cdd3ea546d 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -7,7 +7,7 @@ out there on the net. She stores data, encrypted in the Cloud. All these things can have different files on them, but Alice no longer has to deal with the tedious process of keeping them manually in sync, or remembering where she put a file. git-annex manages all these data -sources as if they were git remotes. +sources as if they were git remotes. [[more about special remotes|special_remotes]] When she has 1 bar on her cell, Alice queues up interesting files on her From 416c5e38e7e8966dbf232692c0c5ab6b6430cea5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:45:34 -0400 Subject: [PATCH 1551/2835] link --- doc/index.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index 224d25d2cc..eb6307f327 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -44,6 +44,7 @@ files with git. * [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage * [[special_remotes]] (including [[special_remotes/Amazon_S3]] and [[special_remotes/bup]]) +* [[encryption]] * [[bare_repositories]] * [[internals]] * [[design]] From db348896c3164d468bb137eaba77a9990aa2d6ac Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 17 Apr 2011 16:02:22 +0000 Subject: [PATCH 1552/2835] --- .../S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn index 9fc31fa485..2c0037c903 100644 --- a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn @@ -6,3 +6,5 @@ Compability for old buckets and support for different ones can be maintained by > Bug was filed prematurely, but was a good bit of paranoia, and gpg and > hmac are given different secret keys [[done]] --[[Joey]] + +>> Thanks :) -- RIchiH From dd207994bc8026f22bb366e654a3945d5e995f87 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 12:36:12 -0400 Subject: [PATCH 1553/2835] reword again On second thought, "unlocking" is confusable with git-annex unlock. --- Remote/Encryptable.hs | 2 +- doc/walkthrough/using_Amazon_S3.mdwn | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index 9e4f58ed40..11df2673cf 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -81,7 +81,7 @@ cipherKey (Just c) k = do Nothing -> case extractCipher c of Nothing -> return Nothing Just encipher -> do - showNote "unlocking" + showNote "gpg" cipher <- liftIO $ decryptCipher c encipher Annex.changeState (\s -> s { Annex.cipher = Just cipher }) ret cipher diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index 0a13900b8d..a0ce951a84 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -14,24 +14,24 @@ like "2512E3C7" Next, create the S3 remote, and describe it. - # git annex initremote mys3 type=S3 encryption=2512E3C7 - initremote mys3 (checking bucket) (creating bucket in US) ok - # git annex describe mys3 "at Amazon's US datacenter" - describe mys3 ok + # git annex initremote cloud type=S3 encryption=2512E3C7 + initremote cloud (checking bucket) (creating bucket in US) ok + # git annex describe cloud "at Amazon's US datacenter" + describe cloud ok The configuration for the S3 remote is stored in git. So to make another repository use the same S3 remote is easy: # cd /media/usb/annex # git pull laptop master - # git annex initremote mys3 - initremote mys3 (checking bucket) ok + # git annex initremote cloud + initremote cloud (checking bucket) ok Now the remote can be used like any other remote. - # git annex copy my_cool_big_file --to mys3 - copy my_cool_big_file (unlocking) (checking mys3...) (to mys3...) ok - # git annex move video/hackity_hack_and_kaxxt.mov --to mys3 - move video/hackity_hack_and_kaxxt.mov (unlocking) (checking mys3...) (to mys3...) ok + # git annex copy my_cool_big_file --to cloud + copy my_cool_big_file (gpg) (checking cloud...) (to cloud...) ok + # git annex move video/hackity_hack_and_kaxxt.mov --to cloud + move video/hackity_hack_and_kaxxt.mov (gpg) (checking cloud...) (to cloud...) ok See [[special_remotes/Amazon_S3]] for details. From 36f048979ff897bc631f499f6f2fe63803e0c1fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 12:43:36 -0400 Subject: [PATCH 1554/2835] releasing version 0.20110417 --- debian/changelog | 4 ++-- debian/control | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 04a6896e32..750f467d58 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110402) UNRELEASED; urgency=low +git-annex (0.20110417) unstable; urgency=low * bup is now supported as a special type of remote. * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted @@ -20,7 +20,7 @@ git-annex (0.20110402) UNRELEASED; urgency=low * Support "sha1" and "sha512" commands on FreeBSD, and allow building if any/all SHA commands are not available. Thanks, Fraser Tweedale - -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 + -- Joey Hess Sun, 17 Apr 2011 12:00:24 -0400 git-annex (0.20110401) experimental; urgency=low diff --git a/debian/control b/debian/control index ae0ca0c598..ebc2487c5e 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Build-Depends: libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, - libghc6-sha-dev + libghc6-sha-dev, ikiwiki, perlmagick, git | git-core, From 8e53d0032e174fa613230568fe8c1f6347b75735 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 12:44:29 -0400 Subject: [PATCH 1555/2835] add news item for git-annex 0.20110417 --- doc/news/version_0.20110316.mdwn | 21 --------------------- doc/news/version_0.20110417.mdwn | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 doc/news/version_0.20110316.mdwn create mode 100644 doc/news/version_0.20110417.mdwn diff --git a/doc/news/version_0.20110316.mdwn b/doc/news/version_0.20110316.mdwn deleted file mode 100644 index 968722c0e6..0000000000 --- a/doc/news/version_0.20110316.mdwn +++ /dev/null @@ -1,21 +0,0 @@ -This version reorganises the layout of git-annex's files in your repository. -There is an upgrade process to convert a repository from the old git-annex -to this version. While git-annex will attempt to transparently handle -upgrades, you may want to drive the upgrade process by hand. -See [[upgrades]] for details. - -git-annex 0.20110316 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * New repository format, annex.version=2. - * The first time git-annex is run in an old format repository, it - will automatically upgrade it to the new format, staging all - necessary changes to git. Also added a "git annex upgrade" command. - * Colons are now avoided in filenames, so bare clones of git repos - can be put on USB thumb drives formatted with vFAT or similar - filesystems. - * Added two levels of hashing to object directory and .git-annex logs, - to improve scalability with enormous numbers of annexed - objects. (With one hundred million annexed objects, each - directory would contain fewer than 1024 files.) - * The setkey, fromkey, and dropkey subcommands have changed how - the key is specified. --backend is no longer used with these."""]] diff --git a/doc/news/version_0.20110417.mdwn b/doc/news/version_0.20110417.mdwn new file mode 100644 index 0000000000..7e28ea2138 --- /dev/null +++ b/doc/news/version_0.20110417.mdwn @@ -0,0 +1,21 @@ +git-annex 0.20110417 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * bup is now supported as a special type of remote. + * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted + using GPG for privacy. + * Use lowercase hash directories for locationlog files, to avoid + some issues with git on OSX with the mixed-case directories. + No migration is needed; the old mixed case hash directories are still + read; new information is written to the new directories. + * Unused files on remotes, particulary special remotes, can now be + identified and dropped, by using "--from remote" with git annex unused + and git annex dropunused. + * Clear up short option confusion between --from and --force (-f is now + --from, and there is no short option for --force). + * Add build depend on perlmagick so docs are consistently built. + Closes: #[621410](http://bugs.debian.org/621410) + * Add doc-base file. Closes: #[621408](http://bugs.debian.org/621408) + * Periodically flush git command queue, to avoid boating memory usage + too much. + * Support "sha1" and "sha512" commands on FreeBSD, and allow building + if any/all SHA commands are not available. Thanks, Fraser Tweedale"""]] \ No newline at end of file From d9690a9b5d6e706abe41fd76800ce9c526ad0b4e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 13:11:38 -0400 Subject: [PATCH 1556/2835] cleanup --- Crypto.hs | 29 +++++++++++++++-------------- Remote/Bup.hs | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 25a87c3ad9..bbe8a6f4ce 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -17,7 +17,8 @@ module Crypto ( extractCipher, decryptCipher, encryptKey, - withEncryptedContentHandle, + withEncryptedHandle, + withDecryptedHandle, withEncryptedContent, withDecryptedContent, ) where @@ -141,19 +142,27 @@ encryptKey c k = keyMtime = Nothing -- to avoid leaking data } -{- Runs an action passing it a handle from which it can +{- Runs an action, passing it a handle from which it can - stream encrypted content. -} -withEncryptedContentHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a -withEncryptedContentHandle = gpgCipherHandle [Params "--symmetric --force-mdc"] +withEncryptedHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"] + +{- Runs an action, passing it a handle from which it can + - stream decrypted content. -} +withDecryptedHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withDecryptedHandle = gpgCipherHandle [Param "--decrypt"] {- Streams encrypted content to an action. -} withEncryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a -withEncryptedContent = gpgCipher [Params "--symmetric --force-mdc"] +withEncryptedContent = pass withEncryptedHandle {- Streams decrypted content to an action. -} withDecryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a -withDecryptedContent = gpgCipher [Param "--decrypt"] +withDecryptedContent = pass withDecryptedHandle +pass :: (Cipher -> L.ByteString -> (Handle -> IO a) -> IO a) + -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +pass to c i a = to c i $ \h -> a =<< L.hGetContents h gpgParams :: [CommandParam] -> [String] gpgParams params = @@ -199,14 +208,6 @@ gpgCipherHandle params c input a = do closeFd frompipe return ret -{- Runs gpg with a cipher and some parameters, feeding it an input, - - and piping its output lazily to an action. -} -gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a -gpgCipher params c input a = do - gpgCipherHandle params c input $ \h -> do - content <- L.hGetContents h - a content - configKeyIds :: RemoteConfig -> IO KeyIds configKeyIds c = do let k = configGet c "encryption" diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 0edb33ba8a..16e1bbdcb5 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -127,7 +127,7 @@ storeEncrypted r buprepo (cipher, enck) k = do params <- bupSplitParams r buprepo enck (Param "-") liftIO $ catchBool $ do content <- L.readFile src - withEncryptedContentHandle cipher content $ \h -> do + withEncryptedHandle cipher content $ \h -> do pipeBup params (Just h) Nothing retrieve :: BupRepo -> Key -> FilePath -> Annex Bool From 7aa668f4b488cc29fe0722f8f01071540ed56434 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 14:30:22 -0400 Subject: [PATCH 1557/2835] Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. --- Crypto.hs | 2 +- debian/changelog | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Crypto.hs b/Crypto.hs index bbe8a6f4ce..6b5d1218a3 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -167,7 +167,7 @@ pass to c i a = to c i $ \h -> a =<< L.hGetContents h gpgParams :: [CommandParam] -> [String] gpgParams params = -- avoid prompting, and be quiet, even about checking the trustdb - ["--batch", "--quiet", "--trust-model", "always"] ++ + ["--quiet", "--trust-model", "always"] ++ toCommand params gpgRead :: [CommandParam] -> IO String diff --git a/debian/changelog b/debian/changelog index 750f467d58..c837da8763 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (0.20110418) UNRELEASED; urgency=low + + * Don't run gpg in batch mode, so it can prompt for passphrase when + there is no agent. + + -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 + git-annex (0.20110417) unstable; urgency=low * bup is now supported as a special type of remote. From a91a51fc03a68f2a5ede6df5182471f6ebfcc037 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 14:41:24 -0400 Subject: [PATCH 1558/2835] Add missing build dep on dataenc. --- debian/changelog | 1 + debian/control | 2 ++ doc/install.mdwn | 1 + 3 files changed, 4 insertions(+) diff --git a/debian/changelog b/debian/changelog index c837da8763..aa00d3fca3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.20110418) UNRELEASED; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. + * Add missing build dep on dataenc. -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 diff --git a/debian/control b/debian/control index ebc2487c5e..00740b7403 100644 --- a/debian/control +++ b/debian/control @@ -8,6 +8,7 @@ Build-Depends: libghc6-pcre-light-dev, libghc6-testpack-dev, libghc6-sha-dev, + libghc6-dataenc-dev, ikiwiki, perlmagick, git | git-core, @@ -40,3 +41,4 @@ Description: manage files with git, without checking their contents into git versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. + diff --git a/doc/install.mdwn b/doc/install.mdwn index 7a0c6020ba..3d15eac604 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -14,6 +14,7 @@ To build and use git-annex, you will need: * pcre-light: * utf8-string: * SHA: +* dataenc: * TestPack * QuickCheck 2 * hS3 (optional, but recommended) From 1d943233185dc17466a985d0bd408389ee27de7b Mon Sep 17 00:00:00 2001 From: praet Date: Sun, 17 Apr 2011 20:22:24 +0000 Subject: [PATCH 1559/2835] --- doc/forum/wishlist:_command_options_changes.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes.mdwn diff --git a/doc/forum/wishlist:_command_options_changes.mdwn b/doc/forum/wishlist:_command_options_changes.mdwn new file mode 100644 index 0000000000..1d1b7087b8 --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes.mdwn @@ -0,0 +1,14 @@ +Some suggestions for changes to command options: + + * --verbose: + * add alternate: -v + + * --from: + * replace with: -s $SOURCE || --source=$SOURCE + + * --to: + * replace with: -d $DESTINATION || --destination=$DESTINATION + + * --force: + * re-add alternate: -f (was removed in v0.20110417) + From f5845ce97b67d98495e42dec35e9821a561d5884 Mon Sep 17 00:00:00 2001 From: praet Date: Sun, 17 Apr 2011 20:26:56 +0000 Subject: [PATCH 1560/2835] --- doc/forum/wishlist:_command_options_changes.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/wishlist:_command_options_changes.mdwn b/doc/forum/wishlist:_command_options_changes.mdwn index 1d1b7087b8..250fd75c81 100644 --- a/doc/forum/wishlist:_command_options_changes.mdwn +++ b/doc/forum/wishlist:_command_options_changes.mdwn @@ -10,5 +10,5 @@ Some suggestions for changes to command options: * replace with: -d $DESTINATION || --destination=$DESTINATION * --force: - * re-add alternate: -f (was removed in v0.20110417) + * add alternate: -F (-f was removed in v0.20110417) From 0c856a99254b323cc5a8840a522092014f77cc38 Mon Sep 17 00:00:00 2001 From: praet Date: Sun, 17 Apr 2011 20:29:34 +0000 Subject: [PATCH 1561/2835] --- doc/forum/wishlist:_command_options_changes.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/forum/wishlist:_command_options_changes.mdwn b/doc/forum/wishlist:_command_options_changes.mdwn index 250fd75c81..4509f815d7 100644 --- a/doc/forum/wishlist:_command_options_changes.mdwn +++ b/doc/forum/wishlist:_command_options_changes.mdwn @@ -10,5 +10,7 @@ Some suggestions for changes to command options: * replace with: -d $DESTINATION || --destination=$DESTINATION * --force: - * add alternate: -F (-f was removed in v0.20110417) + * add alternate: -F + * "-f" was removed in v0.20110417 + * since it forces unsafe operations, should be capitalized to reduce chance of accidental usage. From 86f7feb2784221459d4015b0d70f537dbf2e9bb7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 18:18:27 -0400 Subject: [PATCH 1562/2835] initremote: show gpg keys --- Crypto.hs | 8 ++++++++ Remote/Encryptable.hs | 7 ++++--- doc/walkthrough/using_Amazon_S3.mdwn | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 6b5d1218a3..1617f5aadb 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -13,6 +13,7 @@ module Crypto ( EncryptedCipher, genCipher, updateCipher, + describeCipher, storeCipher, extractCipher, decryptCipher, @@ -95,6 +96,13 @@ updateCipher c encipher@(EncryptedCipher _ ks) = do where combine (KeyIds a) (KeyIds b) = KeyIds $ a ++ b +describeCipher :: EncryptedCipher -> String +describeCipher (EncryptedCipher _ (KeyIds ks)) = + "with gpg " ++ keys ks ++ " " ++ unwords ks + where + keys [_] = "key" + keys _ = "keys" + {- Stores an EncryptedCipher in a remote's configuration. -} storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig storeCipher c (EncryptedCipher t ks) = diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index 11df2673cf..493ff12143 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -27,11 +27,12 @@ encryptionSetup c = (Just "none", Nothing) -> return c (Just "none", Just _) -> error "Cannot change encryption type of existing remote." (Nothing, Just _) -> return c - (Just _, Nothing) -> use $ genCipher c - (Just _, Just v) -> use $ updateCipher c v + (Just _, Nothing) -> use "encryption setup" $ genCipher c + (Just _, Just v) -> use "encryption updated" $ updateCipher c v where - use a = do + use m a = do cipher <- liftIO a + showNote $ m ++ " " ++ describeCipher cipher return $ M.delete "encryption" $ storeCipher c cipher {- Modifies a Remote to support encryption. diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index a0ce951a84..c842583546 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -15,7 +15,7 @@ like "2512E3C7" Next, create the S3 remote, and describe it. # git annex initremote cloud type=S3 encryption=2512E3C7 - initremote cloud (checking bucket) (creating bucket in US) ok + initremote cloud (encryption setup with gpg key C910D9222512E3C7) (checking bucket) (creating bucket in US) ok # git annex describe cloud "at Amazon's US datacenter" describe cloud ok From 684ad747100ccf5023415ea5e6996bc0e0d97583 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 17 Apr 2011 23:46:38 +0000 Subject: [PATCH 1563/2835] Added a comment --- ..._1_bfba72a696789bf21b2435dea15f967a._comment | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment b/doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment new file mode 100644 index 0000000000..0ab113211e --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-04-17T23:46:37Z" + content=""" +--to and --from seem to have different semantics than --source and --destination. Subtle, but still different. + +That being said, I am not sure --from and --to are needed at all. Calling the local repo . and all remotes by their name, they are arguably redundant and removing them would make the syntax a lot prettier; mv and cp don't need them, either. + +I am not sure changing syntax at this point is considered good style though personally, I wouldn't mind adapting and would actually prefer it over using --to and --from. + +-v and -q would be nice. + + +Richard +"""]] From 4cbd71b05771479061c3b1a029dc4aabe748d1fb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 13:40:02 -0400 Subject: [PATCH 1564/2835] enable gpg batch mode when GPG_AGENT_INFO is set --- Crypto.hs | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 1617f5aadb..1f4493b948 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -38,8 +38,9 @@ import System.Posix.IO import System.Posix.Types import System.Posix.Process import Control.Concurrent -import Control.Exception +import Control.Exception (finally) import System.Exit +import System.Environment import Types import Key @@ -172,18 +173,26 @@ pass :: (Cipher -> L.ByteString -> (Handle -> IO a) -> IO a) -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a pass to c i a = to c i $ \h -> a =<< L.hGetContents h -gpgParams :: [CommandParam] -> [String] -gpgParams params = - -- avoid prompting, and be quiet, even about checking the trustdb - ["--quiet", "--trust-model", "always"] ++ - toCommand params +gpgParams :: [CommandParam] -> IO [String] +gpgParams params = do + -- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous + -- gpg output about password prompts. + e <- catch (getEnv "GPG_AGENT_INFO") (const $ return "") + let batch = if null e then [] else ["--batch"] + return $ batch ++ defaults ++ toCommand params + where + -- be quiet, even about checking the trustdb + defaults = ["--quiet", "--trust-model", "always"] gpgRead :: [CommandParam] -> IO String -gpgRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict +gpgRead params = do + params' <- gpgParams params + pOpen ReadFromPipe "gpg" params' hGetContentsStrict gpgPipeStrict :: [CommandParam] -> String -> IO String gpgPipeStrict params input = do - (pid, fromh, toh) <- hPipeBoth "gpg" (gpgParams params) + params' <- gpgParams params + (pid, fromh, toh) <- hPipeBoth "gpg" params' _ <- forkIO $ finally (hPutStr toh input) (hClose toh) output <- hGetContentsStrict fromh forceSuccess pid @@ -202,8 +211,8 @@ gpgCipherHandle params c input a = do let Fd passphrasefd = frompipe let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd] - (pid, fromh, toh) <- hPipeBoth "gpg" $ - gpgParams $ passphrase ++ params + params' <- gpgParams $ passphrase ++ params + (pid, fromh, toh) <- hPipeBoth "gpg" params' _ <- forkProcess $ do L.hPut toh input hClose toh From 1687fecd33ff73a71b2084532e9731796758047a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 13:45:32 -0400 Subject: [PATCH 1565/2835] bug --- doc/bugs/encrypted_S3_stalls.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/encrypted_S3_stalls.mdwn diff --git a/doc/bugs/encrypted_S3_stalls.mdwn b/doc/bugs/encrypted_S3_stalls.mdwn new file mode 100644 index 0000000000..c4484b9c4f --- /dev/null +++ b/doc/bugs/encrypted_S3_stalls.mdwn @@ -0,0 +1,7 @@ +Sending large-ish (few megabytes) files to encrypted S3 remotes stalls out. +It works for the tiny files I was using to test while developing it, on +dialup. + +There was a similar issue with bup, which I fixed by forking a process +rather than using a thread to do some IO. Probably need the same here. +--[[Joey]] From a441e08da1e6305f36db782ec9eda44f213ffa29 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 14:45:19 -0400 Subject: [PATCH 1566/2835] Fix stalls in S3 when transferring encrypted data. Stalls were caused by code that did approximatly: content' <- liftIO $ withEncryptedContent cipher content return store content' The return evaluated without actually reading content from S3, and so the cleanup code began waiting on gpg to exit before gpg could send all its data. Fixing it involved moving the `store` type action into the IO monad: liftIO $ withEncryptedContent cipher content store Which was a bit of a pain to do, thank you type system, but avoids the problem as now the whole content is consumed, and stored, before cleanup. --- Crypto.hs | 5 +- Remote/S3real.hs | 76 +++++++++++++++++-------------- debian/changelog | 1 + doc/bugs/encrypted_S3_stalls.mdwn | 2 + 4 files changed, 50 insertions(+), 34 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 1f4493b948..41f6b999ba 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -199,7 +199,10 @@ gpgPipeStrict params input = do return output {- Runs gpg with a cipher and some parameters, feeding it an input, - - and passing a handle to its output to an action. -} + - and passing a handle to its output to an action. + - + - Note that to avoid deadlock with the cleanup stage, + - the action must fully consume gpg's input before returning. -} gpgCipherHandle :: [CommandParam] -> Cipher -> L.ByteString -> (Handle -> IO a) -> IO a gpgCipherHandle params c input a = do -- pipe the passphrase into gpg on a fd diff --git a/Remote/S3real.hs b/Remote/S3real.hs index b88b22037f..fe68a7f5b7 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -100,13 +100,13 @@ s3Setup u c = do loc <- liftIO $ getBucketLocation conn bucket case loc of Right _ -> return () - Left err@(NetworkError _) -> error $ prettyReqError err + Left err@(NetworkError _) -> s3Error err Left (AWSError _ _) -> do showNote $ "creating bucket in " ++ datacenter res <- liftIO $ createBucketIn conn bucket datacenter case res of Right _ -> return () - Left err -> error $ prettyReqError err + Left err -> s3Error err gitConfigSpecialRemote u fullconfig "s3" "true" return fullconfig @@ -141,33 +141,32 @@ checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do case res of Right _ -> return $ Right True Left (AWSError _ _) -> return $ Right False - Left e -> return $ Left (error $ prettyReqError e) + Left e -> return $ Left (s3Error e) where noconn = Left $ error "S3 not configured" store :: Remote Annex -> Key -> Annex Bool -store r k = storeHelper r k =<< lazyKeyContent k +store r k = s3Action r False $ \(conn, bucket) -> do + content <- lazyKeyContent k + res <- liftIO $ storeHelper (conn, bucket) r k content + s3Bool res storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool -storeEncrypted r (cipher, enck) k = do +storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do content <- lazyKeyContent k - content' <- liftIO $ withEncryptedContent cipher content return - storeHelper r enck content' + res <- liftIO $ withEncryptedContent cipher content $ \s -> do + storeHelper (conn, bucket) r enck s + s3Bool res lazyKeyContent :: Key -> Annex L.ByteString lazyKeyContent k = do g <- Annex.gitRepo liftIO $ L.readFile $ gitAnnexLocation g k -storeHelper :: Remote Annex -> Key -> L.ByteString -> Annex Bool -storeHelper r k content = s3Action r False $ \(conn, bucket) -> do +storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> L.ByteString -> IO (AWSResult ()) +storeHelper (conn, bucket) r k content = do let object = setStorageClass storageclass $ bucketKey bucket k content - res <- liftIO $ sendObject conn object - case res of - Right _ -> return True - Left e -> do - warning $ prettyReqError e - return False + sendObject conn object where storageclass = case fromJust $ M.lookup "storageclass" $ fromJust $ config r of @@ -175,30 +174,41 @@ storeHelper r k content = s3Action r False $ \(conn, bucket) -> do _ -> STANDARD retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool -retrieve = retrieveHelper (return . obj_data) - -retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool -retrieveEncrypted r (cipher, enck) f = retrieveHelper decrypt r enck f - where - decrypt o = withDecryptedContent cipher (obj_data o) return - -retrieveHelper :: (S3Object -> IO L.ByteString) -> Remote Annex -> Key -> FilePath -> Annex Bool -retrieveHelper a r k f = s3Action r False $ \(conn, bucket) -> do +retrieve r k f = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ getObject conn $ bucketKey bucket k L.empty case res of Right o -> do - content <- liftIO $ a o - liftIO $ L.writeFile f content + liftIO $ L.writeFile f $ obj_data o return True - Left e -> do - warning $ prettyReqError e - return False - + Left e -> s3Warning e + +retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do + res <- liftIO $ getObject conn $ bucketKey bucket enck L.empty + case res of + Right o -> liftIO $ + withDecryptedContent cipher (obj_data o) $ \content -> do + L.writeFile f content + return True + Left e -> s3Warning e + remove :: Remote Annex -> Key -> Annex Bool remove r k = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty case res of Right _ -> return True - Left e -> do - warning $ prettyReqError e - return False + Left e -> s3Warning e + +s3Warning :: ReqError -> Annex Bool +s3Warning e = do + warning $ prettyReqError e + return False + +s3Error :: ReqError -> a +s3Error e = error $ prettyReqError e + +s3Bool :: AWSResult () -> Annex Bool +s3Bool res = do + case res of + Right _ -> return True + Left e -> s3Warning e diff --git a/debian/changelog b/debian/changelog index aa00d3fca3..60ccace7af 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.20110418) UNRELEASED; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. * Add missing build dep on dataenc. + * Fix stalls in S3 when transferring encrypted data. -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 diff --git a/doc/bugs/encrypted_S3_stalls.mdwn b/doc/bugs/encrypted_S3_stalls.mdwn index c4484b9c4f..109e6e793a 100644 --- a/doc/bugs/encrypted_S3_stalls.mdwn +++ b/doc/bugs/encrypted_S3_stalls.mdwn @@ -5,3 +5,5 @@ dialup. There was a similar issue with bup, which I fixed by forking a process rather than using a thread to do some IO. Probably need the same here. --[[Joey]] + +[[done]] --[[Joey]] From b1274b637863ebb4e14d39ca2cf00a27c9d1f142 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 14:50:09 -0400 Subject: [PATCH 1567/2835] refactor --- Remote/S3real.hs | 98 ++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index fe68a7f5b7..5d8435932b 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -62,30 +62,6 @@ gen' r u c cst = config = c } -s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection -s3ConnectionRequired c = do - conn <- s3Connection c - case conn of - Nothing -> error "Cannot connect to S3" - Just conn' -> return conn' - -s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) -s3Connection c = do - ak <- getEnvKey "AWS_ACCESS_KEY_ID" - sk <- getEnvKey "AWS_SECRET_ACCESS_KEY" - if (null ak || null sk) - then do - warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3" - return Nothing - else return $ Just $ AWSConnection host port ak sk - where - host = fromJust $ (M.lookup "host" c) - port = let s = fromJust $ (M.lookup "port" c) in - case reads s of - [(p, _)] -> p - _ -> error $ "bad S3 port value: " ++ s - getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") - s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane @@ -121,30 +97,6 @@ s3Setup u c = do , ("bucket", bucket) ] -s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a -s3Action r noconn action = do - when (config r == Nothing) $ - error $ "Missing configuration for special remote " ++ name r - let bucket = M.lookup "bucket" $ fromJust $ config r - conn <- s3Connection (fromJust $ config r) - case (bucket, conn) of - (Just b, Just c) -> action (c, b) - _ -> return noconn - -bucketKey :: String -> Key -> L.ByteString -> S3Object -bucketKey bucket k content = S3Object bucket (show k) "" [] content - -checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool) -checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do - showNote ("checking " ++ name r ++ "...") - res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty - case res of - Right _ -> return $ Right True - Left (AWSError _ _) -> return $ Right False - Left e -> return $ Left (s3Error e) - where - noconn = Left $ error "S3 not configured" - store :: Remote Annex -> Key -> Annex Bool store r k = s3Action r False $ \(conn, bucket) -> do content <- lazyKeyContent k @@ -195,9 +147,18 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do remove :: Remote Annex -> Key -> Annex Bool remove r k = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty + s3Bool res + +checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool) +checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do + showNote ("checking " ++ name r ++ "...") + res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty case res of - Right _ -> return True - Left e -> s3Warning e + Right _ -> return $ Right True + Left (AWSError _ _) -> return $ Right False + Left e -> return $ Left (s3Error e) + where + noconn = Left $ error "S3 not configured" s3Warning :: ReqError -> Annex Bool s3Warning e = do @@ -212,3 +173,40 @@ s3Bool res = do case res of Right _ -> return True Left e -> s3Warning e + +s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection +s3ConnectionRequired c = do + conn <- s3Connection c + case conn of + Nothing -> error "Cannot connect to S3" + Just conn' -> return conn' + +s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) +s3Connection c = do + ak <- getEnvKey "AWS_ACCESS_KEY_ID" + sk <- getEnvKey "AWS_SECRET_ACCESS_KEY" + if (null ak || null sk) + then do + warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3" + return Nothing + else return $ Just $ AWSConnection host port ak sk + where + host = fromJust $ (M.lookup "host" c) + port = let s = fromJust $ (M.lookup "port" c) in + case reads s of + [(p, _)] -> p + _ -> error $ "bad S3 port value: " ++ s + getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") + +s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a +s3Action r noconn action = do + when (config r == Nothing) $ + error $ "Missing configuration for special remote " ++ name r + let bucket = M.lookup "bucket" $ fromJust $ config r + conn <- s3Connection (fromJust $ config r) + case (bucket, conn) of + (Just b, Just c) -> action (c, b) + _ -> return noconn + +bucketKey :: String -> Key -> L.ByteString -> S3Object +bucketKey bucket k content = S3Object bucket (show k) "" [] content From 5985acdfad8a6791f0b2fc54a1e116cee9c12479 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 15:26:50 -0400 Subject: [PATCH 1568/2835] bup: Avoid memory leak when transferring encrypted data. This was a most surprising leak. It occurred in the process that is forked off to feed data to gpg. That process was passed a lazy ByteString of input, and ghc seemed to not GC the ByteString as it was lazily read and consumed, so memory slowly leaked as the file was read and passed through gpg to bup. To fix it, I simply changed the feeder to take an IO action that returns the lazy bytestring, and fed the result directly to hPut. AFAICS, this should change nothing WRT buffering. But somehow it makes ghc's GC do the right thing. Probably I triggered some weakness in ghc's GC (version 6.12.1). (Note that S3 still has this leak, and others too. Fixing it will involve another dance with the type system.) Update: One theory I have is that this has something to do with the forking of the feeder process. Perhaps, when the ByteString is produced before the fork, ghc decides it need to hold a pointer to the start of it, for some reason -- maybe it doesn't realize that it is only used in the forked process. --- Crypto.hs | 20 ++++++++++---------- Remote/Bup.hs | 6 ++---- Remote/Directory.hs | 6 ++---- Remote/S3real.hs | 4 ++-- debian/changelog | 3 ++- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 41f6b999ba..478d837615 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -153,24 +153,24 @@ encryptKey c k = {- Runs an action, passing it a handle from which it can - stream encrypted content. -} -withEncryptedHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withEncryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"] {- Runs an action, passing it a handle from which it can - stream decrypted content. -} -withDecryptedHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withDecryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a withDecryptedHandle = gpgCipherHandle [Param "--decrypt"] {- Streams encrypted content to an action. -} -withEncryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +withEncryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a withEncryptedContent = pass withEncryptedHandle {- Streams decrypted content to an action. -} -withDecryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +withDecryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a withDecryptedContent = pass withDecryptedHandle -pass :: (Cipher -> L.ByteString -> (Handle -> IO a) -> IO a) - -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +pass :: (Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a) + -> Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a pass to c i a = to c i $ \h -> a =<< L.hGetContents h gpgParams :: [CommandParam] -> IO [String] @@ -203,8 +203,8 @@ gpgPipeStrict params input = do - - Note that to avoid deadlock with the cleanup stage, - the action must fully consume gpg's input before returning. -} -gpgCipherHandle :: [CommandParam] -> Cipher -> L.ByteString -> (Handle -> IO a) -> IO a -gpgCipherHandle params c input a = do +gpgCipherHandle :: [CommandParam] -> Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a +gpgCipherHandle params c a b = do -- pipe the passphrase into gpg on a fd (frompipe, topipe) <- createPipe _ <- forkIO $ do @@ -217,11 +217,11 @@ gpgCipherHandle params c input a = do params' <- gpgParams $ passphrase ++ params (pid, fromh, toh) <- hPipeBoth "gpg" params' _ <- forkProcess $ do - L.hPut toh input + L.hPut toh =<< a hClose toh exitSuccess hClose toh - ret <- a fromh + ret <- b fromh -- cleanup forceSuccess pid diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 16e1bbdcb5..6ae002c3b9 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -126,8 +126,7 @@ storeEncrypted r buprepo (cipher, enck) k = do let src = gitAnnexLocation g k params <- bupSplitParams r buprepo enck (Param "-") liftIO $ catchBool $ do - content <- L.readFile src - withEncryptedHandle cipher content $ \h -> do + withEncryptedHandle cipher (L.readFile src) $ \h -> do pipeBup params (Just h) Nothing retrieve :: BupRepo -> Key -> FilePath -> Annex Bool @@ -142,8 +141,7 @@ retrieveEncrypted buprepo (cipher, enck) f = do let params = bupParams "join" buprepo [Param $ show enck] liftIO $ catchBool $ do (pid, h) <- hPipeFrom "bup" $ toCommand params - content <- L.hGetContents h - withDecryptedContent cipher content $ L.writeFile f + withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f forceSuccess pid return True diff --git a/Remote/Directory.hs b/Remote/Directory.hs index d9bee80c3f..c680d61212 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -92,8 +92,7 @@ storeEncrypted d (cipher, enck) k = do liftIO $ catchBool $ storeHelper dest $ encrypt src dest where encrypt src dest = do - content <- L.readFile src - withEncryptedContent cipher content $ L.writeFile dest + withEncryptedContent cipher (L.readFile src) $ L.writeFile dest return True storeHelper :: FilePath -> IO Bool -> IO Bool @@ -113,8 +112,7 @@ retrieve d k f = liftIO $ copyFile (dirKey d k) f retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool retrieveEncrypted d (cipher, enck) f = liftIO $ catchBool $ do - content <- L.readFile (dirKey d enck) - withDecryptedContent cipher content $ L.writeFile f + withDecryptedContent cipher (L.readFile (dirKey d enck)) $ L.writeFile f return True remove :: FilePath -> Key -> Annex Bool diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 5d8435932b..f40deaf17e 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -106,7 +106,7 @@ store r k = s3Action r False $ \(conn, bucket) -> do storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do content <- lazyKeyContent k - res <- liftIO $ withEncryptedContent cipher content $ \s -> do + res <- liftIO $ withEncryptedContent cipher (return content) $ \s -> do storeHelper (conn, bucket) r enck s s3Bool res @@ -139,7 +139,7 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ getObject conn $ bucketKey bucket enck L.empty case res of Right o -> liftIO $ - withDecryptedContent cipher (obj_data o) $ \content -> do + withDecryptedContent cipher (return $ obj_data o) $ \content -> do L.writeFile f content return True Left e -> s3Warning e diff --git a/debian/changelog b/debian/changelog index 60ccace7af..4e9ea441d1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,8 @@ git-annex (0.20110418) UNRELEASED; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. * Add missing build dep on dataenc. - * Fix stalls in S3 when transferring encrypted data. + * S3: Fix stalls when transferring encrypted data. + * bup: Avoid memory leak when transferring encrypted data. -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 From 030c7a056b0795adf037464d608da5170638f4ac Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 16:03:27 -0400 Subject: [PATCH 1569/2835] update --- doc/bugs/S3_memory_leaks.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/S3_memory_leaks.mdwn diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn new file mode 100644 index 0000000000..d7cae2933a --- /dev/null +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -0,0 +1,13 @@ +S3 has two memory leaks. + +One only occurs with encryption. It was was fixed for bup, but +not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. + +The other occurs independant of encryption use. Copying a 100 mb +file to S3 causes an immediate sharp memory spike to 119 mb. +Copying the file back from S3 causes a slow memory increase toward 119 mb. +It's likely that this memory is used by the hS3 library, if it does not +construct the message to Amazon lazily. (And it may not be possible to +construct it lazily, if it includes checksum headers..) I have +emailed the hS3 author about this. +--[[Joey]] From ef6d265af410634ce720a5e190d9014d1b55a538 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 19 Apr 2011 20:13:10 +0000 Subject: [PATCH 1570/2835] Added a comment --- ..._f6a637c78c989382e3c22d41b7fb4cc2._comment | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment b/doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment new file mode 100644 index 0000000000..0072ae1d71 --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-19T20:13:10Z" + content=""" +Let's see.. + +* -v is already an alias for --verbose + +* I don't find --source and --destination as easy to type or as clear as --from or --to. + +* -F is fast, so it cannot be used for --force. And I have no desire to make it easy to mistype a short option and enable --force; it can lose data. + +@richard while it would be possible to support some syntax like \"git annex copy . remote\"; what is it supposed to do if there are local files named foo and bar, and a remotes named foo and bar? Does \"git annex copy foo bar\" copy file foo to remote bar, or file bar from remote foo? I chose to use --from/--to to specify remotes independant of files to avoid such +ambiguity, which plain old `cp` doesn't have since it's operating entirely on filesystem objects, not both filesystem objects and abstract remotes. + +Seems like nothing to do here. [[done]] --[[Joey]] +"""]] From 4837176897ae5ade15b23de4999c370d3ac2ef3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 16:31:35 -0400 Subject: [PATCH 1571/2835] update on memory leak Finished applying to S3 the change that fixed the memory leak in bup, but it didn't seem to help S3.. with encryption it still grows to 2x file size. --- Remote/S3real.hs | 13 +++++-------- doc/bugs/S3_memory_leaks.mdwn | 1 + 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index f40deaf17e..5095b40392 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -99,22 +99,19 @@ s3Setup u c = do store :: Remote Annex -> Key -> Annex Bool store r k = s3Action r False $ \(conn, bucket) -> do - content <- lazyKeyContent k + g <- Annex.gitRepo + content <- liftIO $ L.readFile $ gitAnnexLocation g k res <- liftIO $ storeHelper (conn, bucket) r k content s3Bool res storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do - content <- lazyKeyContent k - res <- liftIO $ withEncryptedContent cipher (return content) $ \s -> do + g <- Annex.gitRepo + let f = gitAnnexLocation g k + res <- liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> do storeHelper (conn, bucket) r enck s s3Bool res -lazyKeyContent :: Key -> Annex L.ByteString -lazyKeyContent k = do - g <- Annex.gitRepo - liftIO $ L.readFile $ gitAnnexLocation g k - storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> L.ByteString -> IO (AWSResult ()) storeHelper (conn, bucket) r k content = do let object = setStorageClass storageclass $ bucketKey bucket k content diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index d7cae2933a..f0522304c2 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -2,6 +2,7 @@ S3 has two memory leaks. One only occurs with encryption. It was was fixed for bup, but not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. +(The fix I used for bup doesn't seem to work with S3.) The other occurs independant of encryption use. Copying a 100 mb file to S3 causes an immediate sharp memory spike to 119 mb. From 936ad63cf47287549412fa0013bcc975c11026d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 17:56:56 -0400 Subject: [PATCH 1572/2835] heard from hS3 author --- doc/bugs/S3_memory_leaks.mdwn | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index f0522304c2..4182601253 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -1,14 +1,21 @@ S3 has two memory leaks. +## with encryption + One only occurs with encryption. It was was fixed for bup, but not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. (The fix I used for bup doesn't seem to work with S3.) +## always + The other occurs independant of encryption use. Copying a 100 mb file to S3 causes an immediate sharp memory spike to 119 mb. Copying the file back from S3 causes a slow memory increase toward 119 mb. It's likely that this memory is used by the hS3 library, if it does not construct the message to Amazon lazily. (And it may not be possible to -construct it lazily, if it includes checksum headers..) I have -emailed the hS3 author about this. +construct it lazily, if it includes checksum headers..) + +I have emailed the hS3 author about this. He wrote back quickly, seems +only getting the size of the file is causing it to be buffered, and a quick +fix should be forthcoming. --[[Joey]] From 143fc7b6923c6b6b39a175332eebbaa7645970c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 21:40:21 -0400 Subject: [PATCH 1573/2835] finalize release --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 4e9ea441d1..91ab3a4024 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110418) UNRELEASED; urgency=low +git-annex (0.20110419) unstable; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. @@ -6,7 +6,7 @@ git-annex (0.20110418) UNRELEASED; urgency=low * S3: Fix stalls when transferring encrypted data. * bup: Avoid memory leak when transferring encrypted data. - -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 + -- Joey Hess Tue, 19 Apr 2011 21:26:51 -0400 git-annex (0.20110417) unstable; urgency=low From 0c9896114e8ef125b04ff368efde82a88136bdcf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 22:26:40 -0400 Subject: [PATCH 1574/2835] add news item for git-annex 0.20110419 --- doc/news/version_0.20110320.mdwn | 9 --------- doc/news/version_0.20110419.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 9 deletions(-) delete mode 100644 doc/news/version_0.20110320.mdwn create mode 100644 doc/news/version_0.20110419.mdwn diff --git a/doc/news/version_0.20110320.mdwn b/doc/news/version_0.20110320.mdwn deleted file mode 100644 index 84475829c3..0000000000 --- a/doc/news/version_0.20110320.mdwn +++ /dev/null @@ -1,9 +0,0 @@ -git-annex 0.20110320 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix dropping of files using the URL backend. - * Fix support for remotes with '.' in their names. - * Add version command to show git-annex version as well as repository - version information. - * No longer auto-upgrade to repository format 2, to avoid accidental - upgrades, etc. Use git-annex upgrade when you're ready to run this - version."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110419.mdwn b/doc/news/version_0.20110419.mdwn new file mode 100644 index 0000000000..2506ad0a87 --- /dev/null +++ b/doc/news/version_0.20110419.mdwn @@ -0,0 +1,7 @@ +git-annex 0.20110419 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Don't run gpg in batch mode, so it can prompt for passphrase when + there is no agent. + * Add missing build dep on dataenc. + * S3: Fix stalls when transferring encrypted data. + * bup: Avoid memory leak when transferring encrypted data."""]] \ No newline at end of file From 130252d9382d1c52174ac2dc3ff72edad2d8befa Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 20 Apr 2011 21:30:13 +0000 Subject: [PATCH 1575/2835] Added a comment --- .../comment_3_bf1114533d2895804e531e76eb6b8095._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment b/doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment new file mode 100644 index 0000000000..9fcbae6d20 --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 3" + date="2011-04-20T21:28:06Z" + content=""" +Good point. scp fixes this by using a colon, but as colons aren't needed in git-annex remotes' names... -- RichiH +"""]] From 8d6b4f6f7d898f76cb82ac2dfee8a2bb12ed04bf Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 20 Apr 2011 21:33:17 +0000 Subject: [PATCH 1576/2835] Added a comment --- .../comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment b/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment new file mode 100644 index 0000000000..0fa5cfac51 --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 4" + date="2011-04-20T21:30:17Z" + content=""" +Good point. scp fixes this by using a colon, but as colons aren't needed in git-annex remotes' names... -- RichiH +"""]] From 156f6c7aa658d4ab3705bb1fc40d9eca659cfc03 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 20 Apr 2011 21:46:33 +0000 Subject: [PATCH 1577/2835] removed --- .../comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment b/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment deleted file mode 100644 index 0fa5cfac51..0000000000 --- a/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" - nickname="Richard" - subject="comment 4" - date="2011-04-20T21:30:17Z" - content=""" -Good point. scp fixes this by using a colon, but as colons aren't needed in git-annex remotes' names... -- RichiH -"""]] From e3651b1285311f547e167531ec2a14108f3d491c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 01:51:15 -0400 Subject: [PATCH 1578/2835] update --- doc/bugs/S3_memory_leaks.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index 4182601253..86e3dd1a42 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -17,5 +17,7 @@ construct it lazily, if it includes checksum headers..) I have emailed the hS3 author about this. He wrote back quickly, seems only getting the size of the file is causing it to be buffered, and a quick -fix should be forthcoming. +fix should be forthcoming. Update: 0.5.6 has been released which will +allow providing file size out of band to avoid buffering when uploading. +Downloading will take further work in hS3. --[[Joey]] From 43639f69f665fca39df675668f1e1f3138d310ef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 02:07:40 -0400 Subject: [PATCH 1579/2835] ghc7 * Update Debian build dependencies for ghc 7. * Debian package is now built with S3 support. Thanks Joachim Breitner for making this possible, also thanks Greg Heartsfield for working to improve the hS3 library for git-annex. Also hid a conflicting new symbol from Control.Monad.State --- Annex.hs | 2 +- Backend.hs | 2 +- debian/changelog | 9 +++++++++ debian/control | 11 ++++++----- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Annex.hs b/Annex.hs index 9086db9bf7..03f9130b6a 100644 --- a/Annex.hs +++ b/Annex.hs @@ -16,7 +16,7 @@ module Annex ( gitRepo ) where -import Control.Monad.State +import Control.Monad.State hiding (state) import qualified GitRepo as Git import qualified GitQueue diff --git a/Backend.hs b/Backend.hs index c0e93acc23..0edff2ab5e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -32,7 +32,7 @@ module Backend ( maybeLookupBackendName ) where -import Control.Monad.State +import Control.Monad.State hiding (state) import System.IO.Error (try) import System.FilePath import System.Posix.Files diff --git a/debian/changelog b/debian/changelog index 91ab3a4024..cdb530c247 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +git-annex (0.20110420) UNRELEASED; urgency=low + + * Update Debian build dependencies for ghc 7. + * Debian package is now built with S3 support. Thanks Joachim Breitner for + making this possible, also thanks Greg Heartsfield for working to improve + the hS3 library for git-annex. + + -- Joey Hess Thu, 21 Apr 2011 02:00:00 -0400 + git-annex (0.20110419) unstable; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when diff --git a/debian/control b/debian/control index 00740b7403..7aba047173 100644 --- a/debian/control +++ b/debian/control @@ -4,11 +4,12 @@ Priority: optional Build-Depends: debhelper (>= 7.0.50), ghc6, - libghc6-missingh-dev, - libghc6-pcre-light-dev, - libghc6-testpack-dev, - libghc6-sha-dev, - libghc6-dataenc-dev, + libghc-missingh-dev, + libghc-pcre-light-dev, + libghc-testpack-dev, + libghc-sha-dev, + libghc-dataenc-dev, + libghc-hs3-dev, ikiwiki, perlmagick, git | git-core, From d8329731c617b278967304389c300f2c832db28d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 09:58:32 -0400 Subject: [PATCH 1580/2835] missing build dep --- debian/control | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 7aba047173..444d712611 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,8 @@ Build-Depends: libghc-testpack-dev, libghc-sha-dev, libghc-dataenc-dev, - libghc-hs3-dev, + libghc-utf8-string-dev, + libghc-hs3-dev (>= 0.5.6), ikiwiki, perlmagick, git | git-core, From 6fcd3e1ef77d3dc99da30cdf2b82489d4bd3d7df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 10:31:54 -0400 Subject: [PATCH 1581/2835] fix S3 upload buffering problem Provide file size to new version of hS3. --- Remote/S3real.hs | 45 ++++++++++++++++++++++++----------- debian/changelog | 9 ++++--- debian/control | 1 - doc/bugs/S3_memory_leaks.mdwn | 18 +++++--------- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 5095b40392..2b0234dc27 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -10,7 +10,7 @@ module Remote.S3 (remote) where import Control.Exception.Extensible (IOException) import Network.AWS.AWSConnection import Network.AWS.S3Object -import Network.AWS.S3Bucket +import Network.AWS.S3Bucket hiding (size) import Network.AWS.AWSResult import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M @@ -18,6 +18,8 @@ import Data.Maybe import Control.Monad (when) import Control.Monad.State (liftIO) import System.Environment +import System.Posix.Files +import System.Directory import RemoteClass import Types @@ -30,6 +32,7 @@ import Config import Remote.Special import Remote.Encryptable import Crypto +import Key remote :: RemoteType Annex remote = RemoteType { @@ -100,21 +103,35 @@ s3Setup u c = do store :: Remote Annex -> Key -> Annex Bool store r k = s3Action r False $ \(conn, bucket) -> do g <- Annex.gitRepo - content <- liftIO $ L.readFile $ gitAnnexLocation g k - res <- liftIO $ storeHelper (conn, bucket) r k content + res <- liftIO $ storeHelper (conn, bucket) r k $ gitAnnexLocation g k s3Bool res storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do g <- Annex.gitRepo let f = gitAnnexLocation g k - res <- liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> do - storeHelper (conn, bucket) r enck s + -- To get file size of the encrypted content, have to use a temp file. + -- (An alternative would be chunking to to a constant size.) + let tmp = gitAnnexTmpLocation g enck + liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s + res <- liftIO $ storeHelper (conn, bucket) r enck tmp + tmp_exists <- liftIO $ doesFileExist tmp + when tmp_exists $ liftIO $ removeFile tmp s3Bool res -storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> L.ByteString -> IO (AWSResult ()) -storeHelper (conn, bucket) r k content = do - let object = setStorageClass storageclass $ bucketKey bucket k content +storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> FilePath -> IO (AWSResult ()) +storeHelper (conn, bucket) r k file = do + content <- liftIO $ L.readFile file + -- size is provided to S3 so the whole content does not need to be + -- buffered to calculate it + size <- case keySize k of + Just s -> return $ fromIntegral s + Nothing -> do + s <- liftIO $ getFileStatus file + return $ fileSize s + let object = setStorageClass storageclass $ + S3Object bucket (show k) "" + [("Content-Length",(show size))] content sendObject conn object where storageclass = @@ -124,7 +141,7 @@ storeHelper (conn, bucket) r k content = do retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool retrieve r k f = s3Action r False $ \(conn, bucket) -> do - res <- liftIO $ getObject conn $ bucketKey bucket k L.empty + res <- liftIO $ getObject conn $ bucketKey bucket k case res of Right o -> do liftIO $ L.writeFile f $ obj_data o @@ -133,7 +150,7 @@ retrieve r k f = s3Action r False $ \(conn, bucket) -> do retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do - res <- liftIO $ getObject conn $ bucketKey bucket enck L.empty + res <- liftIO $ getObject conn $ bucketKey bucket enck case res of Right o -> liftIO $ withDecryptedContent cipher (return $ obj_data o) $ \content -> do @@ -143,13 +160,13 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do remove :: Remote Annex -> Key -> Annex Bool remove r k = s3Action r False $ \(conn, bucket) -> do - res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty + res <- liftIO $ deleteObject conn $ bucketKey bucket k s3Bool res checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool) checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do showNote ("checking " ++ name r ++ "...") - res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty + res <- liftIO $ getObjectInfo conn $ bucketKey bucket k case res of Right _ -> return $ Right True Left (AWSError _ _) -> return $ Right False @@ -205,5 +222,5 @@ s3Action r noconn action = do (Just b, Just c) -> action (c, b) _ -> return noconn -bucketKey :: String -> Key -> L.ByteString -> S3Object -bucketKey bucket k content = S3Object bucket (show k) "" [] content +bucketKey :: String -> Key -> S3Object +bucketKey bucket k = S3Object bucket (show k) "" [] L.empty diff --git a/debian/changelog b/debian/changelog index cdb530c247..30a3e7cc5a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,12 @@ git-annex (0.20110420) UNRELEASED; urgency=low * Update Debian build dependencies for ghc 7. - * Debian package is now built with S3 support. Thanks Joachim Breitner for - making this possible, also thanks Greg Heartsfield for working to improve - the hS3 library for git-annex. + * Debian package is now built with S3 support. + Thanks Joachim Breitner for making this possible. + * No longer needs to buffer entire files when sending them to S3. + (However, getting files from S3 still requires buffering.) + Thanks Greg Heartsfield for ongoing work to improve the hS3 library + for git-annex. -- Joey Hess Thu, 21 Apr 2011 02:00:00 -0400 diff --git a/debian/control b/debian/control index 444d712611..be2e359110 100644 --- a/debian/control +++ b/debian/control @@ -43,4 +43,3 @@ Description: manage files with git, without checking their contents into git versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. - diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index 86e3dd1a42..bfb3e1ec58 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -1,4 +1,4 @@ -S3 has two memory leaks. +S3 has memory leaks ## with encryption @@ -8,16 +8,10 @@ not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. ## always -The other occurs independant of encryption use. Copying a 100 mb -file to S3 causes an immediate sharp memory spike to 119 mb. Copying the file back from S3 causes a slow memory increase toward 119 mb. -It's likely that this memory is used by the hS3 library, if it does not -construct the message to Amazon lazily. (And it may not be possible to -construct it lazily, if it includes checksum headers..) -I have emailed the hS3 author about this. He wrote back quickly, seems -only getting the size of the file is causing it to be buffered, and a quick -fix should be forthcoming. Update: 0.5.6 has been released which will -allow providing file size out of band to avoid buffering when uploading. -Downloading will take further work in hS3. ---[[Joey]] +The author of hS3 is aware of the problem, and working on it. + +## fixed + +memory leak while uploading content to S3 From 45bdb2d4136f2faf61a3fde63477ad5d935583b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 10:53:29 -0400 Subject: [PATCH 1582/2835] ensure tmp dir exists --- Remote/S3real.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 2b0234dc27..07e33368ea 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -33,6 +33,7 @@ import Remote.Special import Remote.Encryptable import Crypto import Key +import Utility remote :: RemoteType Annex remote = RemoteType { @@ -113,6 +114,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do -- To get file size of the encrypted content, have to use a temp file. -- (An alternative would be chunking to to a constant size.) let tmp = gitAnnexTmpLocation g enck + liftIO $ createDirectoryIfMissing True (parentDir tmp) liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s res <- liftIO $ storeHelper (conn, bucket) r enck tmp tmp_exists <- liftIO $ doesFileExist tmp From 2467c567713321b061c3daf92df39d4e35226c7b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 11:06:29 -0400 Subject: [PATCH 1583/2835] update on S3 memory leaks The remaining leaks are in hS3. The leak with encryption was worked around by the use of the temp file. (And was probably originally caused by gpgCipherHandle sparking a thread which kept a reference to the start of the byte string.) --- debian/changelog | 3 +-- doc/bugs/S3_memory_leaks.mdwn | 15 +++------------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/debian/changelog b/debian/changelog index 30a3e7cc5a..7b0bdd103b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,8 +3,7 @@ git-annex (0.20110420) UNRELEASED; urgency=low * Update Debian build dependencies for ghc 7. * Debian package is now built with S3 support. Thanks Joachim Breitner for making this possible. - * No longer needs to buffer entire files when sending them to S3. - (However, getting files from S3 still requires buffering.) + * Somewhat improved memory usage of S3, still work to do. Thanks Greg Heartsfield for ongoing work to improve the hS3 library for git-annex. diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index bfb3e1ec58..d8b10b075c 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -1,17 +1,8 @@ S3 has memory leaks -## with encryption +Sending a file to S3 causes a slow memory increase toward the file size. -One only occurs with encryption. It was was fixed for bup, but -not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. -(The fix I used for bup doesn't seem to work with S3.) - -## always - -Copying the file back from S3 causes a slow memory increase toward 119 mb. +Copying the file back from S3 causes a slow memory increase toward the +file size. The author of hS3 is aware of the problem, and working on it. - -## fixed - -memory leak while uploading content to S3 From 82347fc5ab16823a2ad956e4cf44daaca710e48f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 11:23:44 -0400 Subject: [PATCH 1584/2835] seems -rtsopts is needed now for profiling --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 13adfe0a2d..a8eccc5bfd 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PREFIX=/usr IGNORE=-ignore-package monads-fd GHCFLAGS=-O2 -Wall $(IGNORE) ifdef PROFILE -GHCFLAGS=-prof -auto-all -caf-all -fforce-recomp $(IGNORE) +GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE) endif GHCMAKE=ghc $(GHCFLAGS) --make From dda812583f214dd9835c826b7b9bcc2e8685333e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 12:13:17 -0400 Subject: [PATCH 1585/2835] root caused --- doc/bugs/S3_memory_leaks.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index d8b10b075c..f612de3960 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -5,4 +5,6 @@ Sending a file to S3 causes a slow memory increase toward the file size. Copying the file back from S3 causes a slow memory increase toward the file size. -The author of hS3 is aware of the problem, and working on it. +The author of hS3 is aware of the problem, and working on it. I think I +have identified the root cause of the buffering; it's done by hS3 so it can +resend the data if S3 sends it a 307 redirect. --[[Joey]] From 6668a061a8bab5360a5af4e61ead823f9af93369 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 14:53:07 -0400 Subject: [PATCH 1586/2835] typo --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index be2e359110..7686389972 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: utils Priority: optional Build-Depends: debhelper (>= 7.0.50), - ghc6, + ghc, libghc-missingh-dev, libghc-pcre-light-dev, libghc-testpack-dev, From 24feee25c9ea92ef90e9ea44f50ec26e321a23a4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 15:11:51 -0400 Subject: [PATCH 1587/2835] releasing version 0.20110420 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7b0bdd103b..26e9e81721 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110420) UNRELEASED; urgency=low +git-annex (0.20110420) unstable; urgency=low * Update Debian build dependencies for ghc 7. * Debian package is now built with S3 support. @@ -7,7 +7,7 @@ git-annex (0.20110420) UNRELEASED; urgency=low Thanks Greg Heartsfield for ongoing work to improve the hS3 library for git-annex. - -- Joey Hess Thu, 21 Apr 2011 02:00:00 -0400 + -- Joey Hess Thu, 21 Apr 2011 15:00:48 -0400 git-annex (0.20110419) unstable; urgency=low From 66d951c3fd1a2aa19543d4148be8de734f54fd5c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 15:12:22 -0400 Subject: [PATCH 1588/2835] add news item for git-annex 0.20110420 --- doc/news/version_0.20110325.mdwn | 20 -------------------- doc/news/version_0.20110420.mdwn | 8 ++++++++ 2 files changed, 8 insertions(+), 20 deletions(-) delete mode 100644 doc/news/version_0.20110325.mdwn create mode 100644 doc/news/version_0.20110420.mdwn diff --git a/doc/news/version_0.20110325.mdwn b/doc/news/version_0.20110325.mdwn deleted file mode 100644 index d5a59afe65..0000000000 --- a/doc/news/version_0.20110325.mdwn +++ /dev/null @@ -1,20 +0,0 @@ -git-annex 0.20110325 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Free space checking is now done, for transfers of data for keys - that have free space metadata. (Notably, not for SHA* keys generated - with git-annex 0.2x or earlier.) The code is believed to work on - Linux, FreeBSD, and OSX; check compile-time messages to see if it - is not enabled for your OS. - * Add annex.diskreserve config setting, to control how much free space - to reserve for other purposes and avoid using (defaults to 1 mb). - * Add --fast flag, that can enable less expensive, but also less thorough - versions of some commands. - * fsck: In fast mode, avoid checking checksums. - * unused: In fast mode, just show all existing temp files as unused, - and avoid expensive scan for other unused content. - * migrate: Support migrating v1 SHA keys to v2 SHA keys with - size information that can be used for free space checking. - * Fix space leak in fsck and drop commands. - * migrate: Bugfix for case when migrating a file results in a key that - is already present in .git/annex/objects. - * dropunused: Significantly sped up; only read unused log file once."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110420.mdwn b/doc/news/version_0.20110420.mdwn new file mode 100644 index 0000000000..bb7fee219b --- /dev/null +++ b/doc/news/version_0.20110420.mdwn @@ -0,0 +1,8 @@ +git-annex 0.20110420 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Update Debian build dependencies for ghc 7. + * Debian package is now built with S3 support. + Thanks Joachim Breitner for making this possible. + * Somewhat improved memory usage of S3, still work to do. + Thanks Greg Heartsfield for ongoing work to improve the hS3 library + for git-annex."""]] \ No newline at end of file From 892593c5efacbc084d19af4b5d7164ededaea7ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 16:37:14 -0400 Subject: [PATCH 1589/2835] Use haskell Crypto library instead of haskell SHA library.a Since hS3 needs Crypto anyway, this actually reduces dependencies. --- Crypto.hs | 24 ++++++++++++++++-------- debian/changelog | 6 ++++++ debian/control | 2 +- doc/install.mdwn | 2 +- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 478d837615..ef7b49d8ff 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -27,20 +27,21 @@ module Crypto ( import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import qualified Codec.Binary.Base64 as B64 -import Data.ByteString.Lazy.UTF8 (fromString) -import Data.Digest.Pure.SHA -import System.Cmd.Utils import Data.String.Utils import Data.List import Data.Bits.Utils +import Data.HMAC +import Data.Array +import Codec.Utils +import System.Cmd.Utils import System.IO import System.Posix.IO import System.Posix.Types import System.Posix.Process -import Control.Concurrent -import Control.Exception (finally) import System.Exit import System.Environment +import Control.Concurrent +import Control.Exception (finally) import Types import Key @@ -143,9 +144,9 @@ decryptCipher _ (EncryptedCipher encipher _) = encryptKey :: Cipher -> Key -> IO Key encryptKey c k = return Key { - keyName = showDigest $ hmacSha1 - (fromString $ cipherHmac c) - (fromString $ show k), + keyName = showOctets $ hmac_sha1 + (s2w8 $ cipherHmac c) + (s2w8 $ show k), keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data @@ -252,3 +253,10 @@ fromB64 s = case B64.decode s of Nothing -> error "bad base64 encoded data" Just ws -> w82s ws + +showOctets :: [Octet] -> String +showOctets = concat . map hexChars + where + hexChars c = [arr ! (c `div` 16), arr ! (c `mod` 16)] + arr = listArray (0, 15) "0123456789abcdef" + diff --git a/debian/changelog b/debian/changelog index 26e9e81721..5955f29589 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.20110421) UNRELEASED; urgency=low + + * Use haskell Crypto library instead of haskell SHA library. + + -- Joey Hess Thu, 21 Apr 2011 16:35:27 -0400 + git-annex (0.20110420) unstable; urgency=low * Update Debian build dependencies for ghc 7. diff --git a/debian/control b/debian/control index 7686389972..97b04ea5b6 100644 --- a/debian/control +++ b/debian/control @@ -7,8 +7,8 @@ Build-Depends: libghc-missingh-dev, libghc-pcre-light-dev, libghc-testpack-dev, - libghc-sha-dev, libghc-dataenc-dev, + libghc-crypto-dev, libghc-utf8-string-dev, libghc-hs3-dev (>= 0.5.6), ikiwiki, diff --git a/doc/install.mdwn b/doc/install.mdwn index 3d15eac604..a6b86891d5 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -13,7 +13,7 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * utf8-string: -* SHA: +* crypto: * dataenc: * TestPack * QuickCheck 2 From b72de39ba469fb7f39be0728a10fe949619c7be0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 16:56:24 -0400 Subject: [PATCH 1590/2835] add test to ensure hmac remains stable --- Crypto.hs | 16 +++++++++++++--- test.hs | 2 ++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index ef7b49d8ff..4d6e38bac8 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -22,6 +22,8 @@ module Crypto ( withDecryptedHandle, withEncryptedContent, withDecryptedContent, + + prop_hmacWithCipher_sane ) where import qualified Data.ByteString.Lazy.Char8 as L @@ -144,9 +146,7 @@ decryptCipher _ (EncryptedCipher encipher _) = encryptKey :: Cipher -> Key -> IO Key encryptKey c k = return Key { - keyName = showOctets $ hmac_sha1 - (s2w8 $ cipherHmac c) - (s2w8 $ show k), + keyName = hmacWithCipher c (show k), keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data @@ -259,4 +259,14 @@ showOctets = concat . map hexChars where hexChars c = [arr ! (c `div` 16), arr ! (c `mod` 16)] arr = listArray (0, 15) "0123456789abcdef" + +hmacWithCipher :: Cipher -> String -> String +hmacWithCipher c = hmacWithCipher' (cipherHmac c) +hmacWithCipher' :: String -> String -> String +hmacWithCipher' c s = showOctets $ hmac_sha1 (s2w8 c) (s2w8 s) +{- Ensure that hmacWithCipher' returns the same thing forevermore. -} +prop_hmacWithCipher_sane :: Bool +prop_hmacWithCipher_sane = known_good == hmacWithCipher' "foo" "bar" + where + known_good = "46b4ec586117154dacd49d664e5d63fdc88efb51" diff --git a/test.hs b/test.hs index cdec4ea614..9304eee83f 100644 --- a/test.hs +++ b/test.hs @@ -40,6 +40,7 @@ import qualified Content import qualified Command.DropUnused import qualified Key import qualified Config +import qualified Crypto main :: IO () main = do @@ -63,6 +64,7 @@ quickcheck = TestLabel "quickcheck" $ TestList , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics , qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics , qctest "prop_cost_sane" Config.prop_cost_sane + , qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane ] blackbox :: Test From ca3f05fd6cf0f12dc29193a5c5b3fc01c31097dc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Fri, 22 Apr 2011 14:24:17 +0000 Subject: [PATCH 1591/2835] --- doc/forum/wishlist:_git-annex_replicate.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/wishlist:_git-annex_replicate.mdwn diff --git a/doc/forum/wishlist:_git-annex_replicate.mdwn b/doc/forum/wishlist:_git-annex_replicate.mdwn new file mode 100644 index 0000000000..0d926b3375 --- /dev/null +++ b/doc/forum/wishlist:_git-annex_replicate.mdwn @@ -0,0 +1,12 @@ +I'd like to be able to do something like the following: + + * Create encrypted git-annex remotes on a couple of semi-trusted machines - ones that have good connectivity, but non-redundant hardware + * set numcopies=3 + * run `git-annex replicate` and have git-annex run the appropriate copy commands to make sure every file is on at least 3 machines + +There would also likely be a `git annex rebalance` command which could be used if remotes were added or removed. If possible, it should copy files between servers directly, rather than proxy through a potentially slow client. + +There might be the need to have a 'replication_priority' option for each remote that configures which machines would be preferred. That way you could set your local server to a high priority to ensure that it is always 1 of the 3 machines used and files are distributed across 2 of the remaining remotes. Other than priority, other options that might help: + + * maxspace - A self imposed quota per remote machine. git-annex replicate should try to replicate files first to machines with more free space. maxspace would change the free space calculation to be `min(actual_free_space, maxspace - space_used_by_git_annex) + * bandwidth - when replication files, copies should be done between machines with the highest available bandwidth. ( I think this option could be useful for git-annex get in general) From 028b338c29fa26a3a821fd79172bc8a42a23387a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 22 Apr 2011 18:27:01 +0000 Subject: [PATCH 1592/2835] Added a comment --- ...comment_1_9926132ec6052760cdf28518a24e2358._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment diff --git a/doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment b/doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment new file mode 100644 index 0000000000..cec971ee3b --- /dev/null +++ b/doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-04-22T18:27:00Z" + content=""" +While having remotes redistribute introduces some obvious security concerns, I might use it. + +As remotes support a cost factor already, you can basically implement bandwidth through that. +"""]] From a03dc49bb21190bd2823df9d6b99b04158a955ee Mon Sep 17 00:00:00 2001 From: gernot Date: Sat, 23 Apr 2011 16:02:42 +0000 Subject: [PATCH 1593/2835] --- ...efine_remotes_that_must_have_all_files.mdwn | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn diff --git a/doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn b/doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn new file mode 100644 index 0000000000..156cfb0090 --- /dev/null +++ b/doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn @@ -0,0 +1,18 @@ +I would like to be able to name a few remotes that must retain *all* annexed +files. `git-annex fsck` should warn me if any files are missing from those +remotes, even if `annex.numcopies` has been satisfied by other remotes. + +I imagine this could also be useful for bup remotes, but I haven't actually +looked at those yet. + +Based on existing output, this is what a warning message could look like: + + fsck FILE + 3 of 3 trustworthy copies of FILE exist. + FILE is, however, still missing from these required remotes: + UUID -- Backup Drive 1 + UUID -- Backup Drive 2 + Back it up with git-annex copy. + Warning + +What do you think? From 65adb9240f45c265dc19d28976823186a8c4b7b0 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 23 Apr 2011 16:22:07 +0000 Subject: [PATCH 1594/2835] Added a comment --- ...mment_2_c43932f4194aba8fb2470b18e0817599._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment diff --git a/doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment b/doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment new file mode 100644 index 0000000000..9d50d15310 --- /dev/null +++ b/doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-23T16:22:07Z" + content=""" +Besides the cost values, annex.diskreserve was recently added. (But is not available for special remotes.) + +I have held off on adding high-level management stuff like this to git-annex, as it's hard to make it generic enough to cover use cases. + +A low-level way to accomplish this would be to have a way for `git annex get` and/or `copy` to skip files when `numcopies` is already satisfied. Then cron jobs could be used. +"""]] From 96a7b7926ed09aa264207907ea4e0a5e31a031cb Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 23 Apr 2011 16:27:13 +0000 Subject: [PATCH 1595/2835] Added a comment --- ...comment_1_cceccc1a1730ac688d712b81a44e31c3._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment diff --git a/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment b/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment new file mode 100644 index 0000000000..1f65fd982f --- /dev/null +++ b/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-23T16:27:13Z" + content=""" +Seems to have a scalability problem, what happens when such a repository becomes full? + +Another way to accomplish I think the same thing is to pick the repositories that you would include in such a set, and make all other repositories untrusted. And set numcopies as desired. Then git-annex will never remove files from the set of non-untrusted repositories, and fsck will warn if a file is present on only an untrusted repository. +"""]] From aa820623dc9ea648fb9fa8e9263557529155a7a9 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Sat, 23 Apr 2011 17:54:43 +0000 Subject: [PATCH 1596/2835] Added a comment --- ...comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment diff --git a/doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment b/doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment new file mode 100644 index 0000000000..e7eb06b3b1 --- /dev/null +++ b/doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 3" + date="2011-04-23T17:54:42Z" + content=""" +Hmm, so it seems there is almost a way to do this already. + +I think the one thing that isn't currently possible is to have 'plain' ssh remotes.. basically something just like the directory remote, but able to take a ssh user@host/path url. something like sshfs could be used to fake this, but for things like fsck you would want to do the sha1 calculations on the remote host. +"""]] From 9715f3132c5fa69e8edf2bc7c41c1a4e9c0602be Mon Sep 17 00:00:00 2001 From: gernot Date: Sun, 24 Apr 2011 11:20:06 +0000 Subject: [PATCH 1597/2835] Added a comment --- ...t_2_eec848fcf3979c03cbff2b7407c75a7a._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment diff --git a/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment b/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment new file mode 100644 index 0000000000..1855cdda01 --- /dev/null +++ b/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="gernot" + ip="87.79.209.169" + subject="comment 2" + date="2011-04-24T11:20:05Z" + content=""" +Right, I have thought about untrusting all but a few remotes to achieve +something similar before and I'm sure it would kind of work. It would be more +of an ugly workaround, however, because I would have to untrust remotes that +are, in reality, at least semi-trusted. That's why an extra option/attribute +for that kind of purpose/remote would be nice. + +Obviously I didn't see the scalability problem though. Good Point. Maybe I can +achieve the same thing by writing a log parsing script for myself? + +"""]] From 8512a4a1a1f5367249cdb12aab75ed5d1bb42c8a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 12:43:22 -0400 Subject: [PATCH 1598/2835] Remove testpack from build depends, as it is not available on all architectures. The test suite will not be run if it cannot be compiled. It may be possible later to split off the quickcheck using tests into a separate program and keep most of the tests using just hunit. --- Makefile | 7 +++++-- debian/changelog | 3 +++ debian/control | 1 - 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a8eccc5bfd..69fb518074 100644 --- a/Makefile +++ b/Makefile @@ -45,8 +45,11 @@ install: all fi test: $(bins) - $(GHCMAKE) test - ./test + if ! $(GHCMAKE) test; then \ + echo "** not running test suite" >&2; \ + else \ + ./test; \ + fi testcoverage: $(bins) rm -f test.tix test diff --git a/debian/changelog b/debian/changelog index 5955f29589..872277d0f5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,9 @@ git-annex (0.20110421) UNRELEASED; urgency=low * Use haskell Crypto library instead of haskell SHA library. + * Remove testpack from build depends, as it is not available + on all architectures. The test suite will not be run if it + cannot be compiled. -- Joey Hess Thu, 21 Apr 2011 16:35:27 -0400 diff --git a/debian/control b/debian/control index 97b04ea5b6..f9e6196aba 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,6 @@ Build-Depends: ghc, libghc-missingh-dev, libghc-pcre-light-dev, - libghc-testpack-dev, libghc-dataenc-dev, libghc-crypto-dev, libghc-utf8-string-dev, From b0b413c69f76bcfa46d01ff1623027707483c63c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 13:02:54 -0400 Subject: [PATCH 1599/2835] fix relative Not currently used, but was buggy. --- GitRepo.hs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 543ad801a4..3e177cf1be 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -59,7 +59,7 @@ module GitRepo ( prop_idempotent_deencode ) where -import Control.Monad (unless) +import Control.Monad (unless, when) import System.Directory import System.FilePath import System.Posix.Directory @@ -242,7 +242,11 @@ workTree Repo { location = Unknown } = undefined relative :: Repo -> FilePath -> IO FilePath relative repo@(Repo { location = Dir d }) file = do cwd <- getCurrentDirectory - return $ drop (length absrepo) (absfile cwd) + let file' = absfile cwd + let len = length absrepo + when (take len file' /= absrepo) $ + error $ file ++ " is not located inside git repository " ++ absrepo + return $ drop (length absrepo) file' where -- normalize both repo and file, so that repo -- will be substring of file @@ -251,7 +255,7 @@ relative repo@(Repo { location = Dir d }) file = do Nothing -> error $ "bad repo" ++ repoDescribe repo absfile c = case (secureAbsNormPath c file) of Just f -> f - Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo + Nothing -> file relative repo _ = assertLocal repo $ error "internal" {- Path of an URL repo. -} From e433c6f0bb2ee5f03217b85e3b677b961f5d391a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 13:36:39 -0400 Subject: [PATCH 1600/2835] generalized relPathDirTo functions --- Content.hs | 2 +- Utility.hs | 37 ++++++++++++++++--------------------- test.hs | 2 +- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/Content.hs b/Content.hs index f63c02311f..576eecb319 100644 --- a/Content.hs +++ b/Content.hs @@ -58,7 +58,7 @@ calcGitLink file key = do let absfile = case absNormPath cwd file of Just f -> f Nothing -> error $ "unable to normalize " ++ file - return $ relPathDirToDir (parentDir absfile) + return $ relPathDirToFile (parentDir absfile) (Git.workTree g) ".git" annexLocation key {- Updates the LocationLog when a key's presence changes. diff --git a/Utility.hs b/Utility.hs index 5639a8799a..13ebbfccba 100644 --- a/Utility.hs +++ b/Utility.hs @@ -13,8 +13,8 @@ module Utility ( parentDir, absPath, absPathFrom, - relPathCwdToDir, - relPathDirToDir, + relPathCwdToFile, + relPathDirToFile, boolSystem, shellEscape, shellUnEscape, @@ -29,7 +29,7 @@ module Utility ( prop_idempotent_shellEscape, prop_idempotent_shellEscape_multiword, prop_parentDir_basics, - prop_relPathDirToDir_basics + prop_relPathDirToFile_basics ) where import System.IO @@ -180,26 +180,21 @@ absPathFrom cwd file = Just f -> f Nothing -> error $ "unable to normalize " ++ file -{- Constructs a relative path from the CWD to a directory. +{- Constructs a relative path from the CWD to a file. - - For example, assuming CWD is /tmp/foo/bar: - - relPathCwdToDir "/tmp/foo" == "../" - - relPathCwdToDir "/tmp/foo/bar" == "" + - relPathCwdToFile "/tmp/foo" == ".." + - relPathCwdToFile "/tmp/foo/bar" == "" -} -relPathCwdToDir :: FilePath -> IO FilePath -relPathCwdToDir dir = liftM2 relPathDirToDir getCurrentDirectory (absPath dir) +relPathCwdToFile :: FilePath -> IO FilePath +relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f) -{- Constructs a relative path from one directory to another. +{- Constructs a relative path from a directory to a file. - - - Both directories must be absolute, and normalized (eg with absNormpath). - - - - The path will end with "/", unless it is empty. + - Both must be absolute, and normalized (eg with absNormpath). -} -relPathDirToDir :: FilePath -> FilePath -> FilePath -relPathDirToDir from to = - if not $ null path - then addTrailingPathSeparator path - else "" +relPathDirToFile :: FilePath -> FilePath -> FilePath +relPathDirToFile from to = path where s = [pathSeparator] pfrom = split s from @@ -211,12 +206,12 @@ relPathDirToDir from to = numcommon = length common path = join s $ dotdots ++ uncommon -prop_relPathDirToDir_basics :: FilePath -> FilePath -> Bool -prop_relPathDirToDir_basics from to +prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool +prop_relPathDirToFile_basics from to | from == to = null r - | otherwise = not (null r) && (last r == '/') + | otherwise = not (null r) where - r = relPathDirToDir from to + r = relPathDirToFile from to {- Removes a FileMode from a file. - For example, call with otherWriteMode to chmod o-w -} diff --git a/test.hs b/test.hs index 9304eee83f..7775fb8b5b 100644 --- a/test.hs +++ b/test.hs @@ -62,7 +62,7 @@ quickcheck = TestLabel "quickcheck" $ TestList , qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics - , qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics + , qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics , qctest "prop_cost_sane" Config.prop_cost_sane , qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane ] From 76911a446a7156ffb23679c6325fa8aab1edce13 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 14:54:24 -0400 Subject: [PATCH 1601/2835] Avoid using absolute paths when staging location log, as that can confuse git when a remote's path contains a symlink. Closes: #621386 This was a real PITA to fix, since location logs can be staged in both the current repo, as well as in local remote's repos, in which case the cwd will not be in the repo. And git add needs different params in both cases, when absolute paths are not used. In passing, git annex fsck now stages location log fixes. --- Command/Fsck.hs | 9 ++++----- Command/Move.hs | 8 +------- Content.hs | 16 +++++++++++++--- GitRepo.hs | 35 +++++++++++++++++++++++------------ debian/changelog | 2 ++ 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index bedb9fb992..20ef2c8083 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -64,11 +64,11 @@ verifyLocationLog key file = do case (present, u `elem` uuids) of (True, False) -> do - fix g u ValuePresent + fix u ValuePresent -- There is no data loss, so do not fail. return True (False, True) -> do - fix g u ValueMissing + fix u ValueMissing warning $ "** Based on the location log, " ++ file ++ "\n** was expected to be present, " ++ @@ -77,7 +77,6 @@ verifyLocationLog key file = do _ -> return True where - fix g u s = do + fix u s = do showNote "fixing location log" - _ <- liftIO $ logChange g key u s - return () + logStatusFor u key s diff --git a/Command/Move.hs b/Command/Move.hs index e5e78d2495..476bf866a0 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,19 +7,15 @@ module Command.Move where -import Control.Monad.State (liftIO) - import Command import qualified Command.Drop import qualified Annex -import qualified AnnexQueue import LocationLog import Types import Content import qualified Remote import UUID import Messages -import Utility command :: [Command] command = [repoCommand "move" paramPath seek @@ -57,10 +53,8 @@ showAction False file = showStart "copy" file - for bare repos. -} remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex () remoteHasKey remote key present = do - g <- Annex.gitRepo let remoteuuid = Remote.uuid remote - logfile <- liftIO $ logChange g key remoteuuid status - AnnexQueue.add "add" [Param "--"] logfile + logStatusFor remoteuuid key status where status = if present then ValuePresent else ValueMissing diff --git a/Content.hs b/Content.hs index 576eecb319..bf94562218 100644 --- a/Content.hs +++ b/Content.hs @@ -9,6 +9,7 @@ module Content ( inAnnex, calcGitLink, logStatus, + logStatusFor, getViaTmp, getViaTmpUnchecked, checkDiskSpace, @@ -61,7 +62,8 @@ calcGitLink file key = do return $ relPathDirToFile (parentDir absfile) (Git.workTree g) ".git" annexLocation key -{- Updates the LocationLog when a key's presence changes. +{- Updates the LocationLog when a key's presence changes in the current + - repository. - - Note that the LocationLog is not updated in bare repositories. - Operations that change a bare repository should be done from @@ -69,11 +71,19 @@ calcGitLink file key = do - updated instead. -} logStatus :: Key -> LogStatus -> Annex () logStatus key status = do + g <- Annex.gitRepo + u <- getUUID g + logStatusFor u key status + +{- Updates the LocationLog when a key's presence changes in a repository + - identified by UUID. -} +logStatusFor :: UUID -> Key -> LogStatus -> Annex () +logStatusFor u key status = do g <- Annex.gitRepo unless (Git.repoIsLocalBare g) $ do - u <- getUUID g logfile <- liftIO $ logChange g key u status - AnnexQueue.add "add" [Param "--"] logfile + rellogfile <- liftIO $ Git.workTreeFile g logfile + AnnexQueue.add "add" [Param "--"] rellogfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into diff --git a/GitRepo.hs b/GitRepo.hs index 3e177cf1be..2bf320eda2 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -21,8 +21,8 @@ module GitRepo ( repoDescribe, repoLocation, workTree, + workTreeFile, gitDir, - relative, urlPath, urlHost, urlPort, @@ -59,7 +59,7 @@ module GitRepo ( prop_idempotent_deencode ) where -import Control.Monad (unless, when) +import Control.Monad (unless) import System.Directory import System.FilePath import System.Posix.Directory @@ -236,27 +236,38 @@ workTree r@(Repo { location = Url _ }) = urlPath r workTree (Repo { location = Dir d }) = d workTree Repo { location = Unknown } = undefined -{- Given a relative or absolute filename in a repository, calculates the - - name to use to refer to the file relative to a git repository's top. - - This is the same form displayed and used by git. -} -relative :: Repo -> FilePath -> IO FilePath -relative repo@(Repo { location = Dir d }) file = do +{- Given a relative or absolute filename inside a git repository's + - workTree, calculates the name to use to refer to that file to git. + - + - This is complicated because the best choice can vary depending on + - whether the cwd is in a subdirectory of the git repository, or not. + - + - For example, when adding a file "/tmp/repo/foo", it's best to refer + - to it as "foo" if the cwd is outside the repository entirely + - (this avoids a gotcha with using the full path name when /tmp/repo + - is itself a symlink). But, if the cwd is "/tmp/repo/subdir", + - it's best to refer to "../foo". + -} +workTreeFile :: Repo -> FilePath -> IO FilePath +workTreeFile repo@(Repo { location = Dir d }) file = do cwd <- getCurrentDirectory let file' = absfile cwd - let len = length absrepo - when (take len file' /= absrepo) $ + unless (inrepo file') $ error $ file ++ " is not located inside git repository " ++ absrepo - return $ drop (length absrepo) file' + if (inrepo $ addTrailingPathSeparator cwd) + then return $ relPathDirToFile cwd file' + else return $ drop (length absrepo) file' where -- normalize both repo and file, so that repo -- will be substring of file absrepo = case (absNormPath "/" d) of - Just f -> f ++ "/" + Just f -> addTrailingPathSeparator f Nothing -> error $ "bad repo" ++ repoDescribe repo absfile c = case (secureAbsNormPath c file) of Just f -> f Nothing -> file -relative repo _ = assertLocal repo $ error "internal" + inrepo f = absrepo `isPrefixOf` f +workTreeFile repo _ = assertLocal repo $ error "internal" {- Path of an URL repo. -} urlPath :: Repo -> String diff --git a/debian/changelog b/debian/changelog index 872277d0f5..c6dfb1ff32 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,8 @@ git-annex (0.20110421) UNRELEASED; urgency=low * Remove testpack from build depends, as it is not available on all architectures. The test suite will not be run if it cannot be compiled. + * Avoid using absolute paths when staging location log, as that can + confuse git when a remote's path contains a symlink. Closes: #621386 -- Joey Hess Thu, 21 Apr 2011 16:35:27 -0400 From 3d3abab6797d49679bfcfa4d3bb919cfc7b5f811 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 15:28:41 -0400 Subject: [PATCH 1602/2835] move quickcheck Arbitrary declaration into test suite So git-annex can build w/o quickcheck installed. --- Key.hs | 13 ------------- test.hs | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Key.hs b/Key.hs index f52aea31b7..e1d8ee34d0 100644 --- a/Key.hs +++ b/Key.hs @@ -13,7 +13,6 @@ module Key ( prop_idempotent_key_read_show ) where -import Test.QuickCheck import Utility import System.Posix.Types @@ -71,17 +70,5 @@ readKey s = if key == Just stubKey then Nothing else key addfield 'm' k v = Just k { keyMtime = readMaybe v } addfield _ _ _ = Nothing --- for quickcheck -instance Arbitrary Key where - arbitrary = do - n <- arbitrary - b <- elements ['A'..'Z'] - return $ Key { - keyName = n, - keyBackendName = [b], - keySize = Nothing, - keyMtime = Nothing - } - prop_idempotent_key_read_show :: Key -> Bool prop_idempotent_key_read_show k = Just k == (readKey $ show k) diff --git a/test.hs b/test.hs index 7775fb8b5b..ab4766b428 100644 --- a/test.hs +++ b/test.hs @@ -7,6 +7,8 @@ import Test.HUnit import Test.HUnit.Tools +import Test.QuickCheck + import System.Directory import System.Posix.Directory (changeWorkingDirectory) import System.Posix.Files @@ -42,6 +44,18 @@ import qualified Key import qualified Config import qualified Crypto +-- for quickcheck +instance Arbitrary Key.Key where + arbitrary = do + n <- arbitrary + b <- elements ['A'..'Z'] + return $ Key.Key { + Key.keyName = n, + Key.keyBackendName = [b], + Key.keySize = Nothing, + Key.keyMtime = Nothing + } + main :: IO () main = do prepare From 7d71f8770bd4686c024071c0ca593ed8f16001e1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 16:02:57 -0400 Subject: [PATCH 1603/2835] releasing version 0.20110425 --- debian/changelog | 9 ++++----- debian/control | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debian/changelog b/debian/changelog index c6dfb1ff32..0ba7588f47 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,13 +1,12 @@ -git-annex (0.20110421) UNRELEASED; urgency=low +git-annex (0.20110425) unstable; urgency=low * Use haskell Crypto library instead of haskell SHA library. - * Remove testpack from build depends, as it is not available - on all architectures. The test suite will not be run if it - cannot be compiled. + * Remove testpack from build depends for non x86 architectures where it + is not available. The test suite will not be run if it cannot be compiled. * Avoid using absolute paths when staging location log, as that can confuse git when a remote's path contains a symlink. Closes: #621386 - -- Joey Hess Thu, 21 Apr 2011 16:35:27 -0400 + -- Joey Hess Mon, 25 Apr 2011 15:47:00 -0400 git-annex (0.20110420) unstable; urgency=low diff --git a/debian/control b/debian/control index f9e6196aba..c6b3e7d0d4 100644 --- a/debian/control +++ b/debian/control @@ -10,6 +10,7 @@ Build-Depends: libghc-crypto-dev, libghc-utf8-string-dev, libghc-hs3-dev (>= 0.5.6), + libghc-testpack-dev [any-i386 any-amd64], ikiwiki, perlmagick, git | git-core, From 4ea9579b42aff232090e8238e27d5eec3001bd69 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 16:03:26 -0400 Subject: [PATCH 1604/2835] add news item for git-annex 0.20110425 --- doc/news/version_0.20110328.mdwn | 11 ----------- doc/news/version_0.20110425.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 doc/news/version_0.20110328.mdwn create mode 100644 doc/news/version_0.20110425.mdwn diff --git a/doc/news/version_0.20110328.mdwn b/doc/news/version_0.20110328.mdwn deleted file mode 100644 index 512ce4647a..0000000000 --- a/doc/news/version_0.20110328.mdwn +++ /dev/null @@ -1,11 +0,0 @@ -git-annex 0.20110328 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes") - * Generalized remotes handling, laying groundwork for remotes that are - not regular git remotes. (Think Amazon S3.) - * Provide a less expensive version of `git annex copy --to`, enabled - via --fast. This assumes that location tracking information is correct, - rather than contacting the remote for every file. - * Bugfix: Keys could be received into v1 annexes from v2 annexes, via - v1 git-annex-shell. This results in some oddly named keys in the v1 - annex. Recognise and fix those keys when upgrading, instead of crashing."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110425.mdwn b/doc/news/version_0.20110425.mdwn new file mode 100644 index 0000000000..8f5c6515a0 --- /dev/null +++ b/doc/news/version_0.20110425.mdwn @@ -0,0 +1,7 @@ +git-annex 0.20110425 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Use haskell Crypto library instead of haskell SHA library. + * Remove testpack from build depends for non x86 architectures where it + is not available. The test suite will not be run if it cannot be compiled. + * Avoid using absolute paths when staging location log, as that can + confuse git when a remote's path contains a symlink. Closes: #[621386](http://bugs.debian.org/621386)"""]] \ No newline at end of file From 70d4e7349bd7ed04a9cc8c1b309c1fdfb375a7af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 22:04:12 -0400 Subject: [PATCH 1605/2835] ensure tmp file is writable, so rsync can resume It's possible that rsync finishes transferring a file and sets its mode, but the file transfer to the annex then fails. When resuming, rsync would then not be able to write to the tmp file. --- Content.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Content.hs b/Content.hs index bf94562218..dd0ea5ca1f 100644 --- a/Content.hs +++ b/Content.hs @@ -103,6 +103,8 @@ getViaTmp key action = do checkDiskSpace' (fromIntegral $ fileSize stat) key else checkDiskSpace key + when e $ liftIO $ allowWrite tmp + getViaTmpUnchecked key action {- Like getViaTmp, but does not check that there is enough disk space From ebd6ea1abb80c617f2bf1b693a54ec8b653db623 Mon Sep 17 00:00:00 2001 From: gernot Date: Tue, 26 Apr 2011 13:06:58 +0000 Subject: [PATCH 1606/2835] --- doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn b/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn new file mode 100644 index 0000000000..7db19697c3 --- /dev/null +++ b/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn @@ -0,0 +1,5 @@ +The instructions for building git-annex on [[install/Debian]] stable don't seem to be valid anymore. + +1. `dpkg-checkbuilddeps` is looking for the wrong packages, e.g. libghc-missingh-dev instead of libghc6-missingh-dev. + +2. Not all dependencies are available in the Squeeze repositories anymore (at least not Crypto and hS3), if I am not mistaken. From 27774bdd56e05dabf98deeb09b78433fd374465c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 11:24:23 -0400 Subject: [PATCH 1607/2835] Revert "Use haskell Crypto library instead of haskell SHA library.a" This reverts commit 892593c5efacbc084d19af4b5d7164ededaea7ff. Conflicts: Crypto.hs debian/control --- Crypto.hs | 20 ++++++-------------- debian/changelog | 7 +++++++ debian/control | 2 +- doc/install.mdwn | 2 +- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 4d6e38bac8..20b999f65a 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -29,21 +29,20 @@ module Crypto ( import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import qualified Codec.Binary.Base64 as B64 +import Data.ByteString.Lazy.UTF8 (fromString) +import Data.Digest.Pure.SHA +import System.Cmd.Utils import Data.String.Utils import Data.List import Data.Bits.Utils -import Data.HMAC -import Data.Array -import Codec.Utils -import System.Cmd.Utils import System.IO import System.Posix.IO import System.Posix.Types import System.Posix.Process -import System.Exit -import System.Environment import Control.Concurrent import Control.Exception (finally) +import System.Exit +import System.Environment import Types import Key @@ -253,17 +252,10 @@ fromB64 s = case B64.decode s of Nothing -> error "bad base64 encoded data" Just ws -> w82s ws - -showOctets :: [Octet] -> String -showOctets = concat . map hexChars - where - hexChars c = [arr ! (c `div` 16), arr ! (c `mod` 16)] - arr = listArray (0, 15) "0123456789abcdef" - hmacWithCipher :: Cipher -> String -> String hmacWithCipher c = hmacWithCipher' (cipherHmac c) hmacWithCipher' :: String -> String -> String -hmacWithCipher' c s = showOctets $ hmac_sha1 (s2w8 c) (s2w8 s) +hmacWithCipher' c s = showDigest $ hmacSha1 (fromString c) (fromString s) {- Ensure that hmacWithCipher' returns the same thing forevermore. -} prop_hmacWithCipher_sane :: Bool diff --git a/debian/changelog b/debian/changelog index 0ba7588f47..ce5e651b06 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (0.20110426) UNRELEASED; urgency=low + + * Switch back to haskell SHA library, so git-annex remains buildable on + Debian stable. + + -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 + git-annex (0.20110425) unstable; urgency=low * Use haskell Crypto library instead of haskell SHA library. diff --git a/debian/control b/debian/control index c6b3e7d0d4..b75b35afbc 100644 --- a/debian/control +++ b/debian/control @@ -6,8 +6,8 @@ Build-Depends: ghc, libghc-missingh-dev, libghc-pcre-light-dev, + libghc-sha-dev, libghc-dataenc-dev, - libghc-crypto-dev, libghc-utf8-string-dev, libghc-hs3-dev (>= 0.5.6), libghc-testpack-dev [any-i386 any-amd64], diff --git a/doc/install.mdwn b/doc/install.mdwn index a6b86891d5..3d15eac604 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -13,7 +13,7 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * utf8-string: -* crypto: +* SHA: * dataenc: * TestPack * QuickCheck 2 From de35104b41409e0b776b544589af9ca7fd15d1df Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 26 Apr 2011 15:27:49 +0000 Subject: [PATCH 1608/2835] Added a comment --- .../comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment new file mode 100644 index 0000000000..e464c84da1 --- /dev/null +++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-26T15:27:49Z" + content=""" +I have updated the instructions. +"""]] From 5535f91d917d1c48e1d5590420af84929353bba7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 11:30:49 -0400 Subject: [PATCH 1609/2835] update instructions for stable --- doc/install/Debian.mdwn | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index 90c9658a9c..64cbf32414 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -2,8 +2,23 @@ If using Debian testing or unstable: sudo apt-get install git-annex +## on Debian stable + git-annex is not yet in a Debian stable release, but the source -can be built from Debian stable. Just [[download]] it from git, and -then in the `git-annex` directory, run `dpkg-checkbuilddeps` and install -the necessary packages. You can also run `sudo debian/rules binary` to build +can be built from Debian stable. + +First, install build dependencies. Package names have changed since +Debian stable, so this must be done by hand, rather than by using +`dpkg-checkbuilddeps`: + + sudo apt-get install debhelper ghc6 libghc-missingh-dev \ + libghc6-pcre-light-dev libghc6-sha-dev \ + libghc6-dataenc-dev libghc6-utf8-string-dev \ + libghc6-testpack-dev + +Then [[download]] git-annex it from git. + +In the `git-annex` directory, run `sudo debian/rules binary` to build a `git-annex.deb`. + +Note that S3 support will not be available when building on stable. From 920d736fa9626be88a193bd981c04f44b08ff143 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 11:33:59 -0400 Subject: [PATCH 1610/2835] update --- doc/install/Debian.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index 64cbf32414..6bed4825ba 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -2,10 +2,10 @@ If using Debian testing or unstable: sudo apt-get install git-annex -## on Debian stable +### Debian stable -git-annex is not yet in a Debian stable release, but the source -can be built from Debian stable. +git-annex is not yet in a Debian stable release, and nobody has backported it +(yet?), but the source can be built from Debian stable. First, install build dependencies. Package names have changed since Debian stable, so this must be done by hand, rather than by using From 7a3b45db29cad8efcf0754cb9eafd1ca9874d057 Mon Sep 17 00:00:00 2001 From: gernot Date: Tue, 26 Apr 2011 18:56:45 +0000 Subject: [PATCH 1611/2835] Added a comment --- ..._f6ff8306c946219dbe39bb8938a349ab._comment | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment new file mode 100644 index 0000000000..40f570610a --- /dev/null +++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="gernot" + ip="213.196.216.21" + subject="comment 2" + date="2011-04-26T18:56:44Z" + content=""" +Thanks for the update, Joey. I think you forgot to change libghc-missingh-dev to libghc6-missingh-dev for the copy & paste instructions though. + +Also, after having checked that I have everything installed I'm still getting this error: + + ... + [15 of 77] Compiling Annex ( Annex.hs, Annex.o ) + + Annex.hs:19:35: + Module `Control.Monad.State' does not export `state' + make[1]: *** [git-annex] Error 1 + make[1]: Leaving directory `/home/gernot/dev/git-annex' + dh_auto_build: make -j1 returned exit code 2 + make: *** [binary] Error 2 + +"""]] From 168f010fdff967e4be8cf36e26f63ca2d712d4e6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 19:39:30 -0400 Subject: [PATCH 1612/2835] typo --- doc/install/Debian.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index 6bed4825ba..e9b04dc557 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -11,7 +11,7 @@ First, install build dependencies. Package names have changed since Debian stable, so this must be done by hand, rather than by using `dpkg-checkbuilddeps`: - sudo apt-get install debhelper ghc6 libghc-missingh-dev \ + sudo apt-get install debhelper ghc6 libghc6-missingh-dev \ libghc6-pcre-light-dev libghc6-sha-dev \ libghc6-dataenc-dev libghc6-utf8-string-dev \ libghc6-testpack-dev From 89dc1e5de3b185f5f8871ccf9af9dfb2d4ebe268 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 26 Apr 2011 23:40:33 +0000 Subject: [PATCH 1613/2835] Added a comment --- .../comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment new file mode 100644 index 0000000000..8b41116431 --- /dev/null +++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-26T23:40:33Z" + content=""" +Both problems fixed. +"""]] From 33d23a4ef929f3affb86d7cd6c733101f052d624 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 19:42:40 -0400 Subject: [PATCH 1614/2835] Control.Monad.State import fix for debian stable It doesn't export `state` there, so hiding it fails. Just list explicitly what we use. --- Annex.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index 03f9130b6a..9915112a59 100644 --- a/Annex.hs +++ b/Annex.hs @@ -16,7 +16,8 @@ module Annex ( gitRepo ) where -import Control.Monad.State hiding (state) +import Control.Monad.State + (liftIO, StateT, runStateT, evalStateT, liftM, get, put) import qualified GitRepo as Git import qualified GitQueue From bb8e3c5b6dda5874ec214d7f6dcfe6f8445d622b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 19:59:01 -0400 Subject: [PATCH 1615/2835] more Control.Monad.State export fix --- Backend.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend.hs b/Backend.hs index 0edff2ab5e..aec87ce665 100644 --- a/Backend.hs +++ b/Backend.hs @@ -32,7 +32,7 @@ module Backend ( maybeLookupBackendName ) where -import Control.Monad.State hiding (state) +import Control.Monad.State (liftIO, when) import System.IO.Error (try) import System.FilePath import System.Posix.Files From 948691e89322e88ed2f83bfee4a18dcbdd2f696e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 20:08:36 -0400 Subject: [PATCH 1616/2835] exit nonzero when there were failure, not just errors ya, I need a test suite for my test suite --- test.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.hs b/test.hs index ab4766b428..2cb0415ab9 100644 --- a/test.hs +++ b/test.hs @@ -64,8 +64,8 @@ main = do propigate r propigate :: (Counts, Int) -> IO () -propigate (Counts { errors = e }, _) - | e > 0 = error "failed" +propigate (Counts { errors = e , failures = f }, _) + | e+f > 0 = error "failed" | otherwise = return () quickcheck :: Test From dbdcb67f791502f4f6cc26917c1252b998efa482 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 20:21:24 -0400 Subject: [PATCH 1617/2835] fix test suite when run by root --- test.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test.hs b/test.hs index 2cb0415ab9..4e0eb3fb2a 100644 --- a/test.hs +++ b/test.hs @@ -577,10 +577,14 @@ checkcontent f = do checkunwritable :: FilePath -> Assertion checkunwritable f = do - r <- try $ writeFile f $ content f - case r of - Left _ -> return () -- expected permission error - Right _ -> assertFailure $ "was able to modify annexed file's " ++ f ++ " content" + -- Look at permissions bits rather than trying to write or using + -- fileAccess because if run as root, any file can be modified + -- despite permissions. + s <- getFileStatus f + let mode = fileMode s + if (mode == mode `unionFileModes` ownerWriteMode) + then assertFailure $ "able to modify annexed file's " ++ f ++ " content" + else return () checkwritable :: FilePath -> Assertion checkwritable f = do @@ -640,7 +644,7 @@ runchecks (a:as) f = do annexed_notpresent :: FilePath -> Assertion annexed_notpresent = runchecks - [checklink, checkdangling, checkunwritable, notinlocationlog] + [checklink, checkdangling, notinlocationlog] annexed_present :: FilePath -> Assertion annexed_present = runchecks From 568cb52b6e55b594dd67c466464ae4a9583accb4 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 27 Apr 2011 14:27:27 +0000 Subject: [PATCH 1618/2835] sftp backend? --- doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn new file mode 100644 index 0000000000..becfe23c79 --- /dev/null +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn @@ -0,0 +1,5 @@ +i think it would be useful to have a fourth kind of [[special remote]]s that connects to a dumb storage using sftp or rsync. this can be emulated by using sshfs, but that means lots of round-trips through the system and is limited to platforms where sshfs is available. + +typical use cases are backups to storate shared between a group of people where each user only has limited access (sftp or rsync), when using [[bup]] is not an option. + +an alternative to implementing yet another special remote would be to have some kind of plugin system by which external programs can provide an interface to key-value stores (i'd implement the sftp backend myself, but haven't learned haskell yet). From 0d6860bb6fc02d14b3f5c773f0a2c4248463bb48 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Apr 2011 11:07:54 -0400 Subject: [PATCH 1619/2835] drastically simplify instructions for debian stable :) --- doc/install/Debian.mdwn | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index e9b04dc557..db1329b45b 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -1,24 +1,8 @@ If using Debian testing or unstable: - sudo apt-get install git-annex +* `sudo apt-get install git-annex` -### Debian stable +If using Debian 6.0 stable: -git-annex is not yet in a Debian stable release, and nobody has backported it -(yet?), but the source can be built from Debian stable. - -First, install build dependencies. Package names have changed since -Debian stable, so this must be done by hand, rather than by using -`dpkg-checkbuilddeps`: - - sudo apt-get install debhelper ghc6 libghc6-missingh-dev \ - libghc6-pcre-light-dev libghc6-sha-dev \ - libghc6-dataenc-dev libghc6-utf8-string-dev \ - libghc6-testpack-dev - -Then [[download]] git-annex it from git. - -In the `git-annex` directory, run `sudo debian/rules binary` to build -a `git-annex.deb`. - -Note that S3 support will not be available when building on stable. +* Follow the instructions to [enable backports](http://backports.debian.org/Instructions/). +* `sudo apt-get -t squeeze-backports install git-annex` From e68f128a9bf46c8f4ebe51fcb3b6f63955cadd2e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Apr 2011 20:06:07 -0400 Subject: [PATCH 1620/2835] rsync special remote Fully tested and working, including resuming and encryption. (Though not resuming when sending *with* encryption; gpg doesn't produce identical output each time.) Uses same layout as the directory special remote and the .git/annex/objects/ directory. --- Content.hs | 12 ++ Remote.hs | 2 + Remote/Git.hs | 2 +- Remote/Rsync.hs | 193 ++++++++++++++++++ Remote/S3real.hs | 20 +- Utility.hs | 5 +- debian/changelog | 2 + ...ist:_special_remote_for_sftp_or_rsync.mdwn | 22 +- doc/special_remotes.mdwn | 1 + doc/special_remotes/directory.mdwn | 2 +- doc/special_remotes/rsync.mdwn | 23 +++ 11 files changed, 265 insertions(+), 19 deletions(-) create mode 100644 Remote/Rsync.hs create mode 100644 doc/special_remotes/rsync.mdwn diff --git a/Content.hs b/Content.hs index dd0ea5ca1f..99770f553d 100644 --- a/Content.hs +++ b/Content.hs @@ -12,6 +12,7 @@ module Content ( logStatusFor, getViaTmp, getViaTmpUnchecked, + withTmp, checkDiskSpace, preventWrite, allowWrite, @@ -127,6 +128,17 @@ getViaTmpUnchecked key action = do -- to resume its transfer return False +{- Creates a temp file, runs an action on it, and cleans up the temp file. -} +withTmp :: Key -> (FilePath -> Annex a) -> Annex a +withTmp key action = do + g <- Annex.gitRepo + let tmp = gitAnnexTmpLocation g key + liftIO $ createDirectoryIfMissing True (parentDir tmp) + res <- action tmp + tmp_exists <- liftIO $ doesFileExist tmp + when tmp_exists $ liftIO $ removeFile tmp + return res + {- Checks that there is disk space available to store a given key, - throwing an error if not. -} checkDiskSpace :: Key -> Annex () diff --git a/Remote.hs b/Remote.hs index 8d2ab0399a..f47bea560b 100644 --- a/Remote.hs +++ b/Remote.hs @@ -48,6 +48,7 @@ import qualified Remote.Git import qualified Remote.S3 import qualified Remote.Bup import qualified Remote.Directory +import qualified Remote.Rsync remoteTypes :: [RemoteType Annex] remoteTypes = @@ -55,6 +56,7 @@ remoteTypes = , Remote.S3.remote , Remote.Bup.remote , Remote.Directory.remote + , Remote.Rsync.remote ] {- Builds a list of all available Remotes. diff --git a/Remote/Git.hs b/Remote/Git.hs index bab452a331..e6df6be46e 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -158,7 +158,7 @@ rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool) rsynchelper r sending key file = do showProgress -- make way for progress bar p <- rsyncParams r sending key file - res <- liftIO $ boolSystem "rsync" p + res <- liftIO $ rsync p if res then return res else do diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs new file mode 100644 index 0000000000..0a62ff92fc --- /dev/null +++ b/Remote/Rsync.hs @@ -0,0 +1,193 @@ +{- A remote that is only accessible by rsync. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Rsync (remote) where + +import qualified Data.ByteString.Lazy.Char8 as L +import Control.Exception.Extensible (IOException) +import qualified Data.Map as M +import Control.Monad.State (liftIO, when) +import System.FilePath +import System.Directory +import System.Posix.Files +import System.Posix.Process + +import RemoteClass +import Types +import qualified GitRepo as Git +import qualified Annex +import UUID +import Locations +import Config +import Content +import Utility +import Remote.Special +import Remote.Encryptable +import Crypto +import Messages +import RsyncFile + +type RsyncUrl = String + +data RsyncOpts = RsyncOpts { + rsyncUrl :: RsyncUrl, + rsyncOptions :: [CommandParam] +} + +remote :: RemoteType Annex +remote = RemoteType { + typename = "rsync", + enumerate = findSpecialRemotes "rsyncurl", + generate = gen, + setup = rsyncSetup +} + +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) +gen r u c = do + url <- getConfig r "rsyncurl" (error "missing rsyncurl") + opts <- getConfig r "rsync-options" "" + let o = RsyncOpts url $ map Param $ words opts + cst <- remoteCost r expensiveRemoteCost + return $ encryptableRemote c + (storeEncrypted o) + (retrieveEncrypted o) + Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store o, + retrieveKeyFile = retrieve o, + removeKey = remove o, + hasKey = checkPresent r o, + hasKeyCheap = True, + config = Nothing + } + +rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +rsyncSetup u c = do + -- verify configuration is sane + let url = case M.lookup "rsyncurl" c of + Nothing -> error "Specify rsyncurl=" + Just d -> d + c' <- encryptionSetup c + + -- The rsyncurl is stored in git config, not only in this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' "rsyncurl" url + return c' + +rsyncKey :: RsyncOpts -> Key -> String +rsyncKey o k = rsyncUrl o hashDirMixed k f f + where + f = keyFile k + +store :: RsyncOpts -> Key -> Annex Bool +store o k = do + g <- Annex.gitRepo + rsyncSend o k (gitAnnexLocation g k) + +storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do + g <- Annex.gitRepo + let f = gitAnnexLocation g k + liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s + rsyncSend o enck tmp + +retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool +retrieve o k f = rsyncRemote o + -- use inplace when retrieving to support resuming + [ Param "--inplace" + , Param $ rsyncKey o k + , Param f + ] + +retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do + res <- retrieve o enck tmp + if res + then liftIO $ catchBool $ do + withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f + return True + else return res + +remove :: RsyncOpts -> Key -> Annex Bool +remove o k = withRsyncScratchDir $ \tmp -> do + {- Send an empty directory to rysnc as the parent directory + - of the file to remove. -} + let dummy = tmp keyFile k + liftIO $ createDirectoryIfMissing True dummy + liftIO $ rsync $ rsyncOptions o ++ + [ Params "--delete --recursive" + , partialParams + , Param $ addTrailingPathSeparator dummy + , Param $ parentDir $ rsyncKey o k + ] + +checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool) +checkPresent r o k = do + showNote ("checking " ++ Git.repoDescribe r ++ "...") + -- note: Does not currently differnetiate between rsync failing + -- to connect, and the file not being present. + res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd] + return $ Right res + where + cmd = "rsync --quiet " ++ testfile ++ " 2>/dev/null" + testfile = shellEscape $ rsyncKey o k + +{- Rsync params to enable resumes of sending files safely, + - ensure that files are only moved into place once complete + -} +partialParams :: CommandParam +partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial" + +{- Runs an action in an empty scratch directory that can be used to build + - up trees for rsync. -} +withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool +withRsyncScratchDir a = do + g <- Annex.gitRepo + pid <- liftIO $ getProcessID + let tmp = gitAnnexTmpDir g "rsynctmp" show pid + nuke tmp + liftIO $ createDirectoryIfMissing True $ tmp + res <- a tmp + nuke tmp + return res + where + nuke d = liftIO $ do + e <- doesDirectoryExist d + when e $ liftIO $ removeDirectoryRecursive d + +rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool +rsyncRemote o params = do + showProgress -- make way for progress bar + res <- liftIO $ rsync $ rsyncOptions o ++ defaultParams ++ params + if res + then return res + else do + showLongNote "rsync failed -- run git annex again to resume file transfer" + return res + where + defaultParams = [Params "--progress"] + +{- To send a single key is slightly tricky; need to build up a temporary + directory structure to pass to rsync so it can create the hash + directories. -} +rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool +rsyncSend o k src = withRsyncScratchDir $ \tmp -> do + let dest = tmp hashDirMixed k f f + liftIO $ createDirectoryIfMissing True $ parentDir $ dest + liftIO $ createLink src dest + res <- rsyncRemote o + [ Param "--recursive" + , partialParams + -- tmp/ to send contents of tmp dir + , Param $ addTrailingPathSeparator tmp + , Param $ rsyncUrl o + ] + return res + where + f = keyFile k diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 07e33368ea..2e198f79d1 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -19,7 +19,6 @@ import Control.Monad (when) import Control.Monad.State (liftIO) import System.Environment import System.Posix.Files -import System.Directory import RemoteClass import Types @@ -33,7 +32,7 @@ import Remote.Special import Remote.Encryptable import Crypto import Key -import Utility +import Content remote :: RemoteType Annex remote = RemoteType { @@ -108,18 +107,15 @@ store r k = s3Action r False $ \(conn, bucket) -> do s3Bool res storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool -storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do - g <- Annex.gitRepo - let f = gitAnnexLocation g k +storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> -- To get file size of the encrypted content, have to use a temp file. -- (An alternative would be chunking to to a constant size.) - let tmp = gitAnnexTmpLocation g enck - liftIO $ createDirectoryIfMissing True (parentDir tmp) - liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s - res <- liftIO $ storeHelper (conn, bucket) r enck tmp - tmp_exists <- liftIO $ doesFileExist tmp - when tmp_exists $ liftIO $ removeFile tmp - s3Bool res + withTmp enck $ \tmp -> do + g <- Annex.gitRepo + let f = gitAnnexLocation g k + liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s + res <- liftIO $ storeHelper (conn, bucket) r enck tmp + s3Bool res storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> FilePath -> IO (AWSResult ()) storeHelper (conn, bucket) r k file = do diff --git a/Utility.hs b/Utility.hs index 13ebbfccba..51bbc17a32 100644 --- a/Utility.hs +++ b/Utility.hs @@ -95,8 +95,9 @@ boolSystem command params = do restoresignals oldint oldset executeFile command True (toCommand params) Nothing -{- Escapes a filename to be safely able to be exposed to the shell. -} -shellEscape :: FilePath -> String +{- Escapes a filename or other parameter to be safely able to be exposed to + - the shell. -} +shellEscape :: String -> String shellEscape f = "'" ++ escaped ++ "'" where -- replace ' with '"'"' diff --git a/debian/changelog b/debian/changelog index ce5e651b06..78f65e8b97 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,8 @@ git-annex (0.20110426) UNRELEASED; urgency=low * Switch back to haskell SHA library, so git-annex remains buildable on Debian stable. + * Added rsync special remotes. This could be used, for example, to + store annexed content on rsync.net, encrypted naturally. Or anywhere else. -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn index becfe23c79..9807bf91ed 100644 --- a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn @@ -1,5 +1,21 @@ -i think it would be useful to have a fourth kind of [[special remote]]s that connects to a dumb storage using sftp or rsync. this can be emulated by using sshfs, but that means lots of round-trips through the system and is limited to platforms where sshfs is available. +i think it would be useful to have a fourth kind of [[special remote]]s +that connects to a dumb storage using sftp or rsync. this can be emulated +by using sshfs, but that means lots of round-trips through the system and +is limited to platforms where sshfs is available. -typical use cases are backups to storate shared between a group of people where each user only has limited access (sftp or rsync), when using [[bup]] is not an option. +typical use cases are backups to storate shared between a group of people +where each user only has limited access (sftp or rsync), when using [[bup]] +is not an option. -an alternative to implementing yet another special remote would be to have some kind of plugin system by which external programs can provide an interface to key-value stores (i'd implement the sftp backend myself, but haven't learned haskell yet). +an alternative to implementing yet another special remote would be to have +some kind of plugin system by which external programs can provide an +interface to key-value stores (i'd implement the sftp backend myself, but +haven't learned haskell yet). + +> Ask and ye [[shall receive|special_remotes/rsync]]. +> +> Sometimes I almost think that a generic configurable special remote that +> just uses configured shell commands would be useful.. But there's really +> no comparison with sitting down and writing code tuned to work with +> a given transport like rsync, when it comes to reliability and taking +> advantage of its abilities (like resuming). --[[Joey]] diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index a33d3f6127..210f995d2f 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -9,6 +9,7 @@ They cannot be used by other git commands though. * [[Amazon_S3]] * [[bup]] * [[directory]] +* [[rsync]] ## Unused content on special remotes diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn index 8006c44fc9..9e4bfa33bd 100644 --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -7,4 +7,4 @@ the drive's mountpoint as a directory remote. Setup example: - # git annex initremote usbdrive directory=/media/usbdrive/ encryption=none + # git annex initremote usbdrive type=directory directory=/media/usbdrive/ encryption=none diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn new file mode 100644 index 0000000000..5936442916 --- /dev/null +++ b/doc/special_remotes/rsync.mdwn @@ -0,0 +1,23 @@ +This special remote type rsyncs file contents to somewhere else. + +Setup example: + + # git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync encryption=joey@kitenet.net + +## configuration + +These parameters can be passed to `git annex initremote` to configure rsync: + +* `encryption` - Required. Either "none" to disable encryption of content + stored in rsync, + or a value that can be looked up (using gpg -k) to find a gpg encryption + key that will be given access to the remote. Note that additional gpg + keys can be given access to a remote by rerunning initremote with + the new key id. See [[encryption]]. + +* `rsyncurl` - Required. This is the url or `hostname:/directory` to + pass to rsync to tell it where to store content. + +The `annex-rsync-options` git configuration setting can be used to pass +parameters to rsync. Note that it is **not safe** to put "--delete" +in `annex-rsync-options` when using rsync special remotes. From 39966ba4eeb6046c511d3f3b630a3ee2ced5019a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Apr 2011 20:30:43 -0400 Subject: [PATCH 1621/2835] filter out --delete rsync option rsync does not have a --no-delete, so do it this way instead --- Remote/Rsync.hs | 18 +++++++++++++++--- debian/changelog | 2 +- doc/special_remotes/rsync.mdwn | 3 +-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 0a62ff92fc..21c570a87c 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -48,9 +48,7 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do - url <- getConfig r "rsyncurl" (error "missing rsyncurl") - opts <- getConfig r "rsync-options" "" - let o = RsyncOpts url $ map Param $ words opts + o <- genRsyncOpts r cst <- remoteCost r expensiveRemoteCost return $ encryptableRemote c (storeEncrypted o) @@ -67,6 +65,20 @@ gen r u c = do config = Nothing } +genRsyncOpts :: Git.Repo -> Annex RsyncOpts +genRsyncOpts r = do + url <- getConfig r "rsyncurl" (error "missing rsyncurl") + opts <- getConfig r "rsync-options" "" + return $ RsyncOpts url $ map Param $ filter safe $ words opts + where + safe o + -- Don't allow user to pass --delete to rsync; + -- that could cause it to delete other keys + -- in the same hash bucket as a key it sends. + | o == "--delete" = False + | o == "--delete-excluded" = False + | otherwise = True + rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig rsyncSetup u c = do -- verify configuration is sane diff --git a/debian/changelog b/debian/changelog index 78f65e8b97..991c9e0bb9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,7 @@ git-annex (0.20110426) UNRELEASED; urgency=low * Switch back to haskell SHA library, so git-annex remains buildable on Debian stable. * Added rsync special remotes. This could be used, for example, to - store annexed content on rsync.net, encrypted naturally. Or anywhere else. + store annexed content on rsync.net (encrypted naturally). Or anywhere else. -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn index 5936442916..b13f9ace76 100644 --- a/doc/special_remotes/rsync.mdwn +++ b/doc/special_remotes/rsync.mdwn @@ -19,5 +19,4 @@ These parameters can be passed to `git annex initremote` to configure rsync: pass to rsync to tell it where to store content. The `annex-rsync-options` git configuration setting can be used to pass -parameters to rsync. Note that it is **not safe** to put "--delete" -in `annex-rsync-options` when using rsync special remotes. +parameters to rsync. From 7a338031933cbba7b021468ee83bb63fb3d6d42a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Apr 2011 22:10:57 -0400 Subject: [PATCH 1622/2835] Avoid pipeline stall when running git annex drop or fsck on a lot of files. When it's stalled, there are 3 processes: git annex git ls-files git check-attr git-annex stalls trying to write to git check-attr, which stalls trying to write to stdout (read by git-annex). git ls-files does not seem to be involved directly; I've seen the stall when it was still streaming out the file list, and after it had exited and zombified. The read and write are supposed to be handled by two different threads, which pipeBoth forks off, thus avoiding deadlock. But it does deadlock. (Certian signals unblock the deadlock for a while, then it stalls again.) So, this is another case of WTF is the ghc IO manager doing today? I avoid the issue by converting the writer to a separate process. Possibly this was caused by some change in ghc 7 -- I'm offline and cannot verify now, but I'm sure I used to be able to run git annex drop w/o it hanging! And the code does not seem to have changed, except for commit c1dc4079419cff94cca72441d5e67a866110ec7e, which I tried reverting without success. In fact, I reverted all the way back to 0.20110316 and still saw the stall. Update: Minimal test case: import System.Cmd.Utils main = do as <- checkAttr "blah" $ map show [1..100000] sequence $ map (putStrLn . show) as checkAttr attr files = do (_, s) <- pipeBoth "git" params $ unlines files return $ lines s where params = ["check-attr", attr, "--stdin"] Bug filed on ghc in debian, #624389 --- GitRepo.hs | 10 +++++++++- debian/changelog | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 2bf320eda2..9ecaa8ffcb 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -78,6 +78,7 @@ import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf import Data.List (isInfixOf, isPrefixOf) +import System.Exit import Utility @@ -482,7 +483,14 @@ checkAttr repo attr files = do -- in its output back to relative. cwd <- getCurrentDirectory let absfiles = map (absPathFrom cwd) files - (_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles + (_, fromh, toh) <- hPipeBoth "git" (toCommand params) + _ <- forkProcess $ do + hClose fromh + hPutStr toh $ join "\0" absfiles + hClose toh + exitSuccess + hClose toh + s <- hGetContents fromh return $ map (topair $ cwd++"/") $ lines s where params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"] diff --git a/debian/changelog b/debian/changelog index 991c9e0bb9..a69f03f5ef 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,8 @@ git-annex (0.20110426) UNRELEASED; urgency=low Debian stable. * Added rsync special remotes. This could be used, for example, to store annexed content on rsync.net (encrypted naturally). Or anywhere else. + * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a + lot of files. Possibly only occured with ghc 7. -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 From 4e6dd7f31988834e513a285eb338e4428e892e3e Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 28 Apr 2011 07:47:39 +0000 Subject: [PATCH 1623/2835] Added a comment --- ...comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment new file mode 100644 index 0000000000..c513ed4008 --- /dev/null +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2011-04-28T07:47:38Z" + content=""" ++1 for a generic user configurable backend that a user can put shell commands in, which has a disclaimer such that if a user hangs themselves with misconfiguration then its their own fault :P + +I would love to be able to quickly plugin an irods/sector set of put/get/delete/stat(get info) commands into git-annex to access my private clouds which aren't s3 compatible. +"""]] From d980a55b47136eb313ceecc6f18f82fb2e3cdbf5 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Thu, 28 Apr 2011 11:27:51 +0000 Subject: [PATCH 1624/2835] thanks & notes on migration --- doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn index 9807bf91ed..47e8b85622 100644 --- a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn @@ -19,3 +19,8 @@ haven't learned haskell yet). > no comparison with sitting down and writing code tuned to work with > a given transport like rsync, when it comes to reliability and taking > advantage of its abilities (like resuming). --[[Joey]] + +>> big thanks, and bonus points for identical formats, so converting from +>> directory to rsync is just a matter of changing ``type`` from ``directory`` +>> to ``rsync`` in ``.git-annex/remote.log`` and replacing the directory info +>> with ``annex-rsyncurl = :`` in ``.git/config``. --[[chrysn]] From 5201c43787e2c712a3917c418228c2fd8feca9ae Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Thu, 28 Apr 2011 13:45:32 +0000 Subject: [PATCH 1625/2835] added example fo rusing rsync over ssh --- doc/special_remotes/rsync.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn index b13f9ace76..7a7f3ab149 100644 --- a/doc/special_remotes/rsync.mdwn +++ b/doc/special_remotes/rsync.mdwn @@ -4,6 +4,10 @@ Setup example: # git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync encryption=joey@kitenet.net +Or for using rsync over SSH + + # git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:/myrsync encryption=joey@kitenet.net + ## configuration These parameters can be passed to `git annex initremote` to configure rsync: From 84e1ebfb0eab5490ace84c1e8a66688bd52a1a54 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 14:38:01 -0400 Subject: [PATCH 1626/2835] erm, thought I committed this release? --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index a69f03f5ef..cb3913470e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110426) UNRELEASED; urgency=low +git-annex (0.20110427) unstable; urgency=low * Switch back to haskell SHA library, so git-annex remains buildable on Debian stable. @@ -7,7 +7,7 @@ git-annex (0.20110426) UNRELEASED; urgency=low * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a lot of files. Possibly only occured with ghc 7. - -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 + -- Joey Hess Wed, 27 Apr 2011 22:50:26 -0400 git-annex (0.20110425) unstable; urgency=low From d7b330b33bcfa57164dad05b0fc9990ad4a35275 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 14:39:51 -0400 Subject: [PATCH 1627/2835] Fix hasKeyCheap setting for bup and rsync special remotes. --- Remote/Bup.hs | 2 +- Remote/Rsync.hs | 2 +- debian/changelog | 6 ++++++ doc/news/version_0.20110401.mdwn | 11 ----------- doc/news/version_0.20110427.mdwn | 8 ++++++++ 5 files changed, 16 insertions(+), 13 deletions(-) delete mode 100644 doc/news/version_0.20110401.mdwn create mode 100644 doc/news/version_0.20110427.mdwn diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 6ae002c3b9..0aaff06b25 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -61,7 +61,7 @@ gen r u c = do retrieveKeyFile = retrieve buprepo, removeKey = remove, hasKey = checkPresent r bupr', - hasKeyCheap = True, + hasKeyCheap = bupLocal buprepo, config = c } diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 21c570a87c..682c961748 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -61,7 +61,7 @@ gen r u c = do retrieveKeyFile = retrieve o, removeKey = remove o, hasKey = checkPresent r o, - hasKeyCheap = True, + hasKeyCheap = False, config = Nothing } diff --git a/debian/changelog b/debian/changelog index cb3913470e..870a45820e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.20110428) UNRELEASED; urgency=low + + * Fix hasKeyCheap setting for bup and rsync special remotes. + + -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 + git-annex (0.20110427) unstable; urgency=low * Switch back to haskell SHA library, so git-annex remains buildable on diff --git a/doc/news/version_0.20110401.mdwn b/doc/news/version_0.20110401.mdwn deleted file mode 100644 index 7c9ca6f5ca..0000000000 --- a/doc/news/version_0.20110401.mdwn +++ /dev/null @@ -1,11 +0,0 @@ -git-annex 0.20110401 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Amazon S3 is now supported as a special type of remote. - Warning: Encrypting data before sending it to S3 is not yet supported. - * Note that Amazon S3 support is not built in by default on Debian yet, - as hS3 is not packaged. - * fsck: Ensure that files and directories in .git/annex/objects - have proper permissions. - * Added a special type of remote called a directory remote, which - simply stores files in an arbitrary local directory. - * Bugfix: copy --to --fast never really copied, fixed."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110427.mdwn b/doc/news/version_0.20110427.mdwn new file mode 100644 index 0000000000..2764c6de56 --- /dev/null +++ b/doc/news/version_0.20110427.mdwn @@ -0,0 +1,8 @@ +git-annex 0.20110427 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Switch back to haskell SHA library, so git-annex remains buildable on + Debian stable. + * Added rsync special remotes. This could be used, for example, to + store annexed content on rsync.net (encrypted naturally). Or anywhere else. + * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a + lot of files. Possibly only occured with ghc 7."""]] \ No newline at end of file From 07576f2a2c84a54dc3b0e2aa6050e747f29c3a43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 15:26:21 -0400 Subject: [PATCH 1628/2835] documentation for hook special remotes Releasing before I have quite finished the code. Got a little caught up in Anathem references. Time for a walk and then a tiny bit more coding and possibly testing. --- debian/changelog | 1 + doc/special_remotes.mdwn | 1 + doc/special_remotes/hook.mdwn | 65 +++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 doc/special_remotes/hook.mdwn diff --git a/debian/changelog b/debian/changelog index 870a45820e..b7f33a7041 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.20110428) UNRELEASED; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. + * Add hook special remotes. -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index 210f995d2f..7d55a88612 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -10,6 +10,7 @@ They cannot be used by other git commands though. * [[bup]] * [[directory]] * [[rsync]] +* [[hook]] ## Unused content on special remotes diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn new file mode 100644 index 0000000000..521b440277 --- /dev/null +++ b/doc/special_remotes/hook.mdwn @@ -0,0 +1,65 @@ +This special remote type runs hooks that you configure to store content. + +It's not recommended to use this remote type when another like [[rsync]] +or [[directory]] will do. If your hooks are not carefully written, data +could be lost. + +## example + +Here's a simple example that stores content on clay tablets. If you +implement this example in the real world, I'd appreciate a tour +next Apert! :) --[[Joey]] + + # git config annex.cuneiform-store-hook 'tocuneiform < $ANNEX_FILE | tablet-writer --implement=stylus --title=$ANNEX_KEY | tablet-proofreader | librarian --shelve --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2' + # git config annex.cuneiform-retrieve-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY | tablet-reader --implement=coffee --implement=glasses --force-monastic-dedication | fromcuneiform > $ANNEX_FILE' + # git config annex.cuneiform-remove-hook 'goon --hit-with-hammer --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY' + # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY --shout-title' + # git annex initremote special type=hook hooktype=cuneiform encryption=none + +Can you spot the potential data loss bugs in the above simple example? +(Hint: What happens when the `tablet-proofreader` exits nonzero?) + +## configuration + +These parameters can be passed to `git annex initremote`: + +* `encryption` - Required. Either "none" to disable encryption of content, + or a value that can be looked up (using gpg -k) to find a gpg encryption + key that will be given access to the remote. Note that additional gpg + keys can be given access to a remote by rerunning initremote with + the new key id. See [[encryption]]. + +* `hooktype` - Required. This specifies a collection of hooks to use for + this remote. + +## hooks + +Each type of hook remote is specified by a collection of hook commands. +Each hook command is run as a shell command line, and should return nonzero +on failure, and zero on success. + +These environment variables are used to communicate with the hook commands: + +* `ANNEX_KEY` - name of a key to store, retrieve, remove, or check. +* `ANNEX_FILE` - a file containing the key's content +* `ANNEX_HASH_1` - short stable value, based on the key, can be used for hashing +* `ANNEX_HASH_2` - another hash value, can be used for a second level of hashing + +The setting to use in git config for the hook commands are as follows: + +* `annex.$hooktype-store-hook` - Command run to store a key in the special remote. + `ANNEX_FILE` contains the content to be stored. + +* `annex.$hooktype-retrieve-hook` - Command run to retrieve a key from the special remote. + `ANNEX_FILE` is a file that the retrieved content should be written to. + The file may already exist with a partial + copy of the content (or possibly just garbage), to allow for resuming + of partial transfers. + +* `annex.$hooktype-remove-hook` - Command to remove a key from the special remote. + +* `annex.$hooktype-checkpresent-hook` - Command to check if a key is present + in the special remote. Should output the key name to stdout, on its own line, + if and only if the key has been actively verified to be present in the + special remote (caching presence information is a very bad idea); + all other output to stdout will be ignored. From 56eaf4470aee904f3dd6fe245fc59d35618d2b2e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 16:08:10 -0400 Subject: [PATCH 1629/2835] bugfix --- doc/special_remotes/hook.mdwn | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 521b440277..7bdd317b98 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -10,11 +10,11 @@ Here's a simple example that stores content on clay tablets. If you implement this example in the real world, I'd appreciate a tour next Apert! :) --[[Joey]] - # git config annex.cuneiform-store-hook 'tocuneiform < $ANNEX_FILE | tablet-writer --implement=stylus --title=$ANNEX_KEY | tablet-proofreader | librarian --shelve --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2' - # git config annex.cuneiform-retrieve-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY | tablet-reader --implement=coffee --implement=glasses --force-monastic-dedication | fromcuneiform > $ANNEX_FILE' - # git config annex.cuneiform-remove-hook 'goon --hit-with-hammer --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY' - # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY --shout-title' - # git annex initremote special type=hook hooktype=cuneiform encryption=none + # git config annex.cuneiform-store-hook 'tocuneiform < "$ANNEX_FILE" | tablet-writer --implement=stylus --title="$ANNEX_KEY" | tablet-proofreader | librarian --shelve --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2' + # git config annex.cuneiform-retrieve-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" | tablet-reader --implement=coffee --implement=glasses --force-monastic-dedication | fromcuneiform > "$ANNEX_FILE"' + # git config annex.cuneiform-remove-hook 'goon --hit-with-hammer --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY"' + # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" --shout-title' + # git annex initremote library type=hook hooktype=cuneiform encryption=none Can you spot the potential data loss bugs in the above simple example? (Hint: What happens when the `tablet-proofreader` exits nonzero?) From b5072b7b4cab21118f60c55a58497f363f749244 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 16:08:18 -0400 Subject: [PATCH 1630/2835] add boolSystemEnv --- Utility.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Utility.hs b/Utility.hs index 51bbc17a32..0dab371045 100644 --- a/Utility.hs +++ b/Utility.hs @@ -16,6 +16,7 @@ module Utility ( relPathCwdToFile, relPathDirToFile, boolSystem, + boolSystemEnv, shellEscape, shellUnEscape, unsetFileMode, @@ -73,7 +74,10 @@ toCommand l = concat $ map unwrap l - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -} boolSystem :: FilePath -> [CommandParam] -> IO Bool -boolSystem command params = do +boolSystem command params = boolSystemEnv command params Nothing + +boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool +boolSystemEnv command params env = do -- Going low-level because all the high-level system functions -- block SIGINT etc. We need to block SIGCHLD, but allow -- SIGINT to do its default program termination. @@ -93,7 +97,7 @@ boolSystem command params = do setSignalMask oldset childaction oldint oldset = do restoresignals oldint oldset - executeFile command True (toCommand params) Nothing + executeFile command True (toCommand params) env {- Escapes a filename or other parameter to be safely able to be exposed to - the shell. -} From 3ab3f41aea78f6816493d094d2daca7cc0067a91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 17:21:45 -0400 Subject: [PATCH 1631/2835] hook special remote implemented, and tested --- Remote.hs | 2 + Remote/Hook.hs | 157 ++++++++++++++++++++++++++++++++++ doc/special_remotes/hook.mdwn | 3 +- 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 Remote/Hook.hs diff --git a/Remote.hs b/Remote.hs index f47bea560b..bbecdb9995 100644 --- a/Remote.hs +++ b/Remote.hs @@ -49,6 +49,7 @@ import qualified Remote.S3 import qualified Remote.Bup import qualified Remote.Directory import qualified Remote.Rsync +import qualified Remote.Hook remoteTypes :: [RemoteType Annex] remoteTypes = @@ -57,6 +58,7 @@ remoteTypes = , Remote.Bup.remote , Remote.Directory.remote , Remote.Rsync.remote + , Remote.Hook.remote ] {- Builds a list of all available Remotes. diff --git a/Remote/Hook.hs b/Remote/Hook.hs new file mode 100644 index 0000000000..2613fda7a6 --- /dev/null +++ b/Remote/Hook.hs @@ -0,0 +1,157 @@ +{- A remote that provides hooks to run shell commands. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Hook (remote) where + +import qualified Data.ByteString.Lazy.Char8 as L +import Control.Exception.Extensible (IOException) +import qualified Data.Map as M +import Control.Monad.State (liftIO) +import System.FilePath +import System.Posix.Process +import System.Posix.IO +import System.IO +import System.IO.Error (try) +import System.Exit + +import RemoteClass +import Types +import qualified GitRepo as Git +import qualified Annex +import UUID +import Locations +import Config +import Content +import Utility +import Remote.Special +import Remote.Encryptable +import Crypto +import Messages + +remote :: RemoteType Annex +remote = RemoteType { + typename = "hook", + enumerate = findSpecialRemotes "hooktype", + generate = gen, + setup = hookSetup +} + +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) +gen r u c = do + hooktype <- getConfig r "hooktype" (error "missing hooktype") + cst <- remoteCost r expensiveRemoteCost + return $ encryptableRemote c + (storeEncrypted hooktype) + (retrieveEncrypted hooktype) + Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store hooktype, + retrieveKeyFile = retrieve hooktype, + removeKey = remove hooktype, + hasKey = checkPresent r hooktype, + hasKeyCheap = False, + config = Nothing + } + +hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +hookSetup u c = do + let hooktype = case M.lookup "hooktype" c of + Nothing -> error "Specify hooktype=" + Just r -> r + c' <- encryptionSetup c + gitConfigSpecialRemote u c' "hooktype" hooktype + return c' + +hookEnv :: Key -> Maybe FilePath -> Maybe [(String, String)] +hookEnv k f = Just $ keyenv : fileenv f + where + env s v = ("ANNEX_" ++ s, v) + keyenv = env "KEY" (show k) + fileenv Nothing = [] + fileenv (Just file) = + [ env "FILE" file + , env "HASH_1" (hashbits !! 0) + , env "HASH_2" (hashbits !! 1) + ] + hashbits = map takeDirectory $ splitPath $ hashDirMixed k + +lookupHook :: String -> String -> Annex (Maybe String) +lookupHook hooktype hook =do + g <- Annex.gitRepo + command <- getConfig g hookname "" + if null command + then do + warning $ "missing configuration for " ++ hookname + return Nothing + else return $ Just command + where + hookname = hooktype ++ "-" ++ hook ++ "-hook" + +runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool +runHook hooktype hook k f a = do + command <- lookupHook hooktype hook + case command of + Nothing -> return False + Just c -> do + showProgress -- make way for hook output + res <- liftIO $ boolSystemEnv + "sh" [Param "-c", Param c] $ hookEnv k f + if res + then a + else do + warning $ hook ++ " hook exited nonzero!" + return res + +store :: String -> Key -> Annex Bool +store h k = do + g <- Annex.gitRepo + runHook h "store" k (Just $ gitAnnexLocation g k) $ return True + +storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do + g <- Annex.gitRepo + let f = gitAnnexLocation g k + liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s + runHook h "store" enck (Just tmp) $ return True + +retrieve :: String -> Key -> FilePath -> Annex Bool +retrieve h k f = runHook h "retrieve" k (Just f) $ return True + +retrieveEncrypted :: String -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp -> + runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBool $ do + withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f + return True + +remove :: String -> Key -> Annex Bool +remove h k = runHook h "remove" k Nothing $ do return True + +checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool) +checkPresent r h k = do + showNote ("checking " ++ Git.repoDescribe r ++ "...") + v <- lookupHook h "checkpresent" + liftIO (try (check v) ::IO (Either IOException Bool)) + where + findkey s = (show k) `elem` (lines s) + env = hookEnv k Nothing + check Nothing = error "checkpresent hook misconfigured" + check (Just hook) = do + (frompipe, topipe) <- createPipe + pid <- forkProcess $ do + _ <- dupTo topipe stdOutput + closeFd frompipe + executeFile "sh" True ["-c", hook] env + closeFd topipe + fromh <- fdToHandle frompipe + reply <- hGetContentsStrict fromh + hClose fromh + s <- getProcessStatus True False pid + case s of + Just (Exited (ExitSuccess)) -> return $ findkey reply + _ -> error "checkpresent hook failed" diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 7bdd317b98..74c4029cdd 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -1,4 +1,5 @@ -This special remote type runs hooks that you configure to store content. +This special remote lets you store content in a remote of your own +devising. It's not recommended to use this remote type when another like [[rsync]] or [[directory]] will do. If your hooks are not carefully written, data From 847bbe30b1a00ec5ef419895f96ab5d7cb767a83 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 28 Apr 2011 21:22:04 +0000 Subject: [PATCH 1632/2835] Added a comment --- .../comment_2_84e4414c88ae91c048564a2cdc2d3250._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment new file mode 100644 index 0000000000..6243708f94 --- /dev/null +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-28T21:22:03Z" + content=""" +Ask and ye shalle receive with an Abbot on top: [[special_remotes/hook]] +"""]] From b0efff86b688e6ea68921221ac56e02a838d2c51 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 17:28:38 -0400 Subject: [PATCH 1633/2835] wording --- doc/special_remotes/hook.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 74c4029cdd..502375fb94 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -1,4 +1,4 @@ -This special remote lets you store content in a remote of your own +This special remote type lets you store content in a remote of your own devising. It's not recommended to use this remote type when another like [[rsync]] From 8b950c5fe8a89bcdefe65ac709e2d3640eb03b1a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Thu, 28 Apr 2011 23:38:36 +0000 Subject: [PATCH 1634/2835] This demonstrates a git-annex failure when hard links are involved --- ...ateSymbolicLink:_already_exists__34__.mdwn | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn diff --git a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn new file mode 100644 index 0000000000..70881fcc84 --- /dev/null +++ b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -0,0 +1,35 @@ +I'm importing a directory where some files are hard links of each other. + +This is confusing git-annex. Here's a small test of that: + +
+paulproteus@pathi:/tmp$ mkdir annex-test
+paulproteus@pathi:/tmp$ cd annex-test
+paulproteus@pathi:/tmp/annex-test$ git init
+Initialized empty Git repository in /tmp/annex-test/.git/
+paulproteus@pathi:/tmp/annex-test$ git annex init testing
+init testing ok
+paulproteus@pathi:/tmp/annex-test$ echo '* annex.backend=SHA1' >> .gitattributes 
+paulproteus@pathi:/tmp/annex-test$ git commit .gitattributes -m 'Default to sha1'
+[master dd54b41] Default to sha1
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+paulproteus@pathi:/tmp/annex-test$ echo "Look at me" > file1
+paulproteus@pathi:/tmp/annex-test$ cp -l file1 file2
+paulproteus@pathi:/tmp/annex-test$ git annex add file1
+add file1 (checksum...) ok
+(Recording state in git...)
+paulproteus@pathi:/tmp/annex-test$ git commit -m 'So far, so good'
+[master eb43084] So far, so good
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 100644 .git-annex/9a3/f1f/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c.log
+ create mode 120000 file1
+paulproteus@pathi:/tmp/annex-test$ git annex add file2
+add file2 (checksum...) 
+  git-annex: .git/annex/objects/PM/7p/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c: createSymbolicLink: already exists (File exists)
+git-annex: 1 failed
+paulproteus@pathi:/tmp/annex-test$ 
+
+ +I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. + +-- Asheesh. From b8e114bce153875db8afd20182eae03e35662737 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Thu, 28 Apr 2011 23:39:28 +0000 Subject: [PATCH 1635/2835] --- ..._a_file___34__createSymbolicLink:_already_exists__34__.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn index 70881fcc84..6bcd2fc4c7 100644 --- a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn +++ b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -30,6 +30,9 @@ git-annex: 1 failed paulproteus@pathi:/tmp/annex-test$ +When trying to make a small test case for this bug, I noticed that if file1 and file2 have the same contents but are not hard links of each other, they both get annexed just fine. + I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. + -- Asheesh. From eef3f634e9f92e7af486e5ee4afdac9a79b034cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 20:41:40 -0400 Subject: [PATCH 1636/2835] Avoid crashing when an existing key is readded to the annex. --- Content.hs | 37 ++++++++++++--- debian/changelog | 1 + ...ateSymbolicLink:_already_exists__34__.mdwn | 46 +++++++++++++++++++ ...ateSymbolicLink:_already_exists__34__.mdwn | 39 +--------------- 4 files changed, 78 insertions(+), 45 deletions(-) create mode 100644 doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn diff --git a/Content.hs b/Content.hs index 99770f553d..ade936da37 100644 --- a/Content.hs +++ b/Content.hs @@ -182,18 +182,41 @@ allowWrite f = do s <- getFileStatus f setFileMode f $ fileMode s `unionFileModes` ownerWriteMode -{- Moves a file into .git/annex/objects/ -} +{- Moves a file into .git/annex/objects/ + - + - What if the key there already has content? This could happen for + - various reasons; perhaps the same content is being annexed again. + - Perhaps there has been a hash collision generating the keys. + - + - The current strategy is to assume that in this case it's safe to delete + - one of the two copies of the content; and the one already in the annex + - is left there, assuming it's the original, canonical copy. + - + - I considered being more paranoid, and checking that both files had + - the same content. Decided against it because A) users explicitly choose + - a backend based on its hashing properties and so if they're dealing + - with colliding files it's their own fault and B) adding such a check + - would not catch all cases of colliding keys. For example, perhaps + - a remote has a key; if it's then added again with different content then + - the overall system now has two different peices of content for that + - key, and one of them will probably get deleted later. So, adding the + - check here would only raise expectations that git-annex cannot truely + - meet. + -} moveAnnex :: Key -> FilePath -> Annex () moveAnnex key src = do g <- Annex.gitRepo let dest = gitAnnexLocation g key let dir = parentDir dest - liftIO $ do - createDirectoryIfMissing True dir - allowWrite dir -- in case the directory already exists - renameFile src dest - preventWrite dest - preventWrite dir + e <- liftIO $ doesFileExist dest + if e + then liftIO $ removeFile src + else liftIO $ do + createDirectoryIfMissing True dir + allowWrite dir -- in case the directory already exists + renameFile src dest + preventWrite dest + preventWrite dir {- Removes a key's file from .git/annex/objects/ -} removeAnnex :: Key -> Annex () diff --git a/debian/changelog b/debian/changelog index b7f33a7041..92c05a5a69 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.20110428) UNRELEASED; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. * Add hook special remotes. + * Avoid crashing when an existing key is readded to the annex. -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 diff --git a/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn new file mode 100644 index 0000000000..e943127740 --- /dev/null +++ b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -0,0 +1,46 @@ +I'm importing a directory where some files are hard links of each other. + +This is confusing git-annex. Here's a small test of that: + +
+paulproteus@pathi:/tmp$ mkdir annex-test
+paulproteus@pathi:/tmp$ cd annex-test
+paulproteus@pathi:/tmp/annex-test$ git init
+Initialized empty Git repository in /tmp/annex-test/.git/
+paulproteus@pathi:/tmp/annex-test$ git annex init testing
+init testing ok
+paulproteus@pathi:/tmp/annex-test$ echo '* annex.backend=SHA1' >> .gitattributes 
+paulproteus@pathi:/tmp/annex-test$ git commit .gitattributes -m 'Default to sha1'
+[master dd54b41] Default to sha1
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+paulproteus@pathi:/tmp/annex-test$ echo "Look at me" > file1
+paulproteus@pathi:/tmp/annex-test$ cp -l file1 file2
+paulproteus@pathi:/tmp/annex-test$ git annex add file1
+add file1 (checksum...) ok
+(Recording state in git...)
+paulproteus@pathi:/tmp/annex-test$ git commit -m 'So far, so good'
+[master eb43084] So far, so good
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 100644 .git-annex/9a3/f1f/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c.log
+ create mode 120000 file1
+paulproteus@pathi:/tmp/annex-test$ git annex add file2
+add file2 (checksum...) 
+  git-annex: .git/annex/objects/PM/7p/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c: createSymbolicLink: already exists (File exists)
+git-annex: 1 failed
+paulproteus@pathi:/tmp/annex-test$ 
+
+ +When trying to make a small test case for this bug, I noticed that if file1 and file2 have the same contents but are not hard links of each other, they both get annexed just fine. + +I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. + + +-- Asheesh. + +> The same thing happens anytime the key for a file collides with a key +> already in the annex, AFAICS. Including when the files have the same +> content but are not hard links. +> +> I've fixed this bug. The first file in wins. See commit for some +> interesting discussion about why it should not check for hash collisions +> in this situation. [[done]] --[[Joey]] diff --git a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn index 6bcd2fc4c7..38865f49aa 100644 --- a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn +++ b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -1,38 +1 @@ -I'm importing a directory where some files are hard links of each other. - -This is confusing git-annex. Here's a small test of that: - -
-paulproteus@pathi:/tmp$ mkdir annex-test
-paulproteus@pathi:/tmp$ cd annex-test
-paulproteus@pathi:/tmp/annex-test$ git init
-Initialized empty Git repository in /tmp/annex-test/.git/
-paulproteus@pathi:/tmp/annex-test$ git annex init testing
-init testing ok
-paulproteus@pathi:/tmp/annex-test$ echo '* annex.backend=SHA1' >> .gitattributes 
-paulproteus@pathi:/tmp/annex-test$ git commit .gitattributes -m 'Default to sha1'
-[master dd54b41] Default to sha1
- 1 files changed, 1 insertions(+), 0 deletions(-)
-paulproteus@pathi:/tmp/annex-test$ echo "Look at me" > file1
-paulproteus@pathi:/tmp/annex-test$ cp -l file1 file2
-paulproteus@pathi:/tmp/annex-test$ git annex add file1
-add file1 (checksum...) ok
-(Recording state in git...)
-paulproteus@pathi:/tmp/annex-test$ git commit -m 'So far, so good'
-[master eb43084] So far, so good
- 2 files changed, 2 insertions(+), 0 deletions(-)
- create mode 100644 .git-annex/9a3/f1f/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c.log
- create mode 120000 file1
-paulproteus@pathi:/tmp/annex-test$ git annex add file2
-add file2 (checksum...) 
-  git-annex: .git/annex/objects/PM/7p/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c: createSymbolicLink: already exists (File exists)
-git-annex: 1 failed
-paulproteus@pathi:/tmp/annex-test$ 
-
- -When trying to make a small test case for this bug, I noticed that if file1 and file2 have the same contents but are not hard links of each other, they both get annexed just fine. - -I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. - - --- Asheesh. +Moved to [[bugs|bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__]] --[[Joey]] From 8fcac59852e28c36c4512d5ee0a32ed76774aa2d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 20:44:08 -0400 Subject: [PATCH 1637/2835] closing bug my explanation seems to have sufficed --- .../git_annex_migrate_leaves_old_backend_versions_around.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn index e37ee06bb3..7f586b5ff0 100644 --- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -16,3 +16,4 @@ would be great if these were purged automatically somehow. > > This way a lot of migrations can be done, and only when you're done you > can do the more expensive cleanup pass if you want to. --[[Joey]] +> [[done]] From 6e95521b85263d9b734fa3f77030a96f850b1e1e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 20:47:36 -0400 Subject: [PATCH 1638/2835] tweak --- ...a_file___34__createSymbolicLink:_already_exists__34__.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn index e943127740..21293af547 100644 --- a/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn +++ b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -38,8 +38,8 @@ I think the right behavior here is to annex file2 just fine, as if they weren't -- Asheesh. > The same thing happens anytime the key for a file collides with a key -> already in the annex, AFAICS. Including when the files have the same -> content but are not hard links. +> already in the annex, AFAICS. (Including when the files have the same +> content but are not hard links... unless you're using WORM backend.) > > I've fixed this bug. The first file in wins. See commit for some > interesting discussion about why it should not check for hash collisions From f581481a75d6f098b622f2079aa183e06c1273fc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:33:27 +0000 Subject: [PATCH 1639/2835] --- ...95__remotes__47__hook_with_tahoe-lafs.mdwn | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn new file mode 100644 index 0000000000..2d04a46946 --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn @@ -0,0 +1,20 @@ +This is work in progress, since there is now a "hook" for users to plug in whatever they want on the back end, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file + + tahoe-store-hook = tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY + tahoe-retrieve-hook = tahoe get tahoe:$ANNEX_KEY $ANNEX_FILE + tahoe-remove-hook = tahoe rm tahoe:$ANNEX_KEY + tahoe-checkpresent-hook = tahoe ls tahoe:$ANNEX_KEY 2>&1 || echo FAIL + +The only quirk I've noticed is this... + +
+$ git annex whereis .
+whereis frink.jar (2 copies) 
+  	084603a8-7243-11e0-b1f5-83102bcd7953  -- testtest <-- here
+   	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a
+ok
+
+ +1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a is my [[!google tahoe-lafs]] remote, but there is no label/description on it. The checkpresent-hook was a little confusing when I was setting up, I'm currently unsure if I am doing the right thing or not with my hook. My get and put commands are a little verbose for now, i might redirect it to /dev/null once I am happier with the overall performance/behaviour my setup. + +Other than the quirks above, I am able to put and get files from my tahoe-lafs remote. The only thing that I have not figured out is how to "remove a file" on the remote to free up space on the remote. From 66dbf7799f38e0191f317b51fedca6a0f7ee7c3e Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:34:02 +0000 Subject: [PATCH 1640/2835] --- .../tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn index 2d04a46946..a1bf942546 100644 --- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn @@ -1,4 +1,4 @@ -This is work in progress, since there is now a "hook" for users to plug in whatever they want on the back end, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file +This is work in progress, since there is now a "hook" for users to plug in whatever they want as a remote, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file tahoe-store-hook = tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY tahoe-retrieve-hook = tahoe get tahoe:$ANNEX_KEY $ANNEX_FILE From 05fe77c110b0b8f4571380c01bbbd09201a099c0 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:36:37 +0000 Subject: [PATCH 1641/2835] --- .../tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn index a1bf942546..50b3078b2d 100644 --- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn @@ -5,6 +5,8 @@ This is work in progress, since there is now a "hook" for users to plug in whate tahoe-remove-hook = tahoe rm tahoe:$ANNEX_KEY tahoe-checkpresent-hook = tahoe ls tahoe:$ANNEX_KEY 2>&1 || echo FAIL +Where `tahoe:` is a tahoe-lafs alias, ideally you should create a new alias (DIR-CAP or whatever the terminolgy is) to store your files, I just used the default `tahoe:` alias for testing. + The only quirk I've noticed is this... 
@@ -15,6 +17,6 @@ whereis frink.jar (2 copies)
 ok
-1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a is my [[!google tahoe-lafs]] remote, but there is no label/description on it. The checkpresent-hook was a little confusing when I was setting up, I'm currently unsure if I am doing the right thing or not with my hook. My get and put commands are a little verbose for now, i might redirect it to /dev/null once I am happier with the overall performance/behaviour my setup. +1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a is my [[!google tahoe-lafs]] remote, but there is no label/description on it. The checkpresent-hook was a little confusing when I was setting it up, I'm currently unsure if I am doing the right thing or not with my hook. My get and put commands are a little verbose for now, i might redirect it to /dev/null once I am happier with the overall performance/behaviour my setup. Other than the quirks above, I am able to put and get files from my tahoe-lafs remote. The only thing that I have not figured out is how to "remove a file" on the remote to free up space on the remote. From 75f902844671f363a9f934e760226ff8815a9e12 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:37:17 +0000 Subject: [PATCH 1642/2835] --- .../tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn index 50b3078b2d..4f5f089a89 100644 --- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn @@ -1,4 +1,4 @@ -This is work in progress, since there is now a "hook" for users to plug in whatever they want as a remote, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file +This is work in progress, since there is now a [[special_remotes/hook]] for users to plug in whatever they want as a remote, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file tahoe-store-hook = tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY tahoe-retrieve-hook = tahoe get tahoe:$ANNEX_KEY $ANNEX_FILE From db2786bf0ec357ebb4d2b5b694a023ac6f495632 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:43:31 +0000 Subject: [PATCH 1643/2835] Added a comment --- .../comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment new file mode 100644 index 0000000000..dc21ec4885 --- /dev/null +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2011-04-29T10:43:31Z" + content=""" +Cool!, I just tried adding tahoe-lafs as a remote, and it wasn't too hard. +"""]] From 9b133a81221cdca4f880d200906e75dd1319370d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Fri, 29 Apr 2011 11:48:23 +0000 Subject: [PATCH 1644/2835] Added a comment: Duplication of the filenames is what I am concerned about --- ...3_076cb22057583957d5179d8ba9004605._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment new file mode 100644 index 0000000000..d11119bc3d --- /dev/null +++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" + nickname="Asheesh" + subject="Duplication of the filenames is what I am concerned about" + date="2011-04-29T11:48:22Z" + content=""" +For what it's worth, yes, I want to actually forget I ever had the same file in the filesystem with a duplicated name. I'm not just aiming to clean up the disk's space usage; I'm also aiming to clean things up so that navigating the filesystem is easier. + +I can write my own script to do that based on the symlinks' target (and I wrote something along those lines), but I still think it'd be nicer if git-annex supported this use case. + +Perhaps: + +
git annex drop --by-contents
+ +could let me remove a file from git-annex if the contents are available through a different name. (Right now, \"git annex drop\" requires the name *and* contents match.) + +-- Asheesh. +"""]] From 446585351b0ca0590715a4cab1711425c9767344 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Fri, 29 Apr 2011 13:08:36 +0000 Subject: [PATCH 1645/2835] Added a comment: whereis labels --- ...comment_1_76bb33ce45ce6a91b86454147463193b._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment new file mode 100644 index 0000000000..388641f69e --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="whereis labels" + date="2011-04-29T13:08:35Z" + content=""" +You should be able to fix the missing label by editing .git-annex/uuid.log and adding + + 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a tahoe +"""]] From eca02546286d881bb6ff71d9246a21c79e8679a3 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Fri, 29 Apr 2011 14:48:08 +0000 Subject: [PATCH 1646/2835] no way the goon would be admitted in the library; he doesn't understand the floor/shelf system anyway --- doc/special_remotes/hook.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 502375fb94..e3b70d2026 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -13,7 +13,7 @@ next Apert! :) --[[Joey]] # git config annex.cuneiform-store-hook 'tocuneiform < "$ANNEX_FILE" | tablet-writer --implement=stylus --title="$ANNEX_KEY" | tablet-proofreader | librarian --shelve --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2' # git config annex.cuneiform-retrieve-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" | tablet-reader --implement=coffee --implement=glasses --force-monastic-dedication | fromcuneiform > "$ANNEX_FILE"' - # git config annex.cuneiform-remove-hook 'goon --hit-with-hammer --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY"' + # git config annex.cuneiform-remove-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" | goon --hit-with-hammer' # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" --shout-title' # git annex initremote library type=hook hooktype=cuneiform encryption=none From 0ea7f966dc61948d107372fac6f2ec90a1dc28f5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 11:16:54 -0400 Subject: [PATCH 1647/2835] add git annex describe to special remote setup examples --- doc/special_remotes/directory.mdwn | 1 + doc/special_remotes/hook.mdwn | 1 + doc/special_remotes/rsync.mdwn | 2 ++ 3 files changed, 4 insertions(+) diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn index 9e4bfa33bd..0a38c763cc 100644 --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -8,3 +8,4 @@ the drive's mountpoint as a directory remote. Setup example: # git annex initremote usbdrive type=directory directory=/media/usbdrive/ encryption=none + # git annex describe usbdrive "usb drive on /media/usbdrive/" diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index e3b70d2026..5b636613c9 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -16,6 +16,7 @@ next Apert! :) --[[Joey]] # git config annex.cuneiform-remove-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" | goon --hit-with-hammer' # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" --shout-title' # git annex initremote library type=hook hooktype=cuneiform encryption=none + # git annex describe library "the reborn Library of Alexandria (upgrade to bronze plates pending)" Can you spot the potential data loss bugs in the above simple example? (Hint: What happens when the `tablet-proofreader` exits nonzero?) diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn index 7a7f3ab149..90d544a1e1 100644 --- a/doc/special_remotes/rsync.mdwn +++ b/doc/special_remotes/rsync.mdwn @@ -3,10 +3,12 @@ This special remote type rsyncs file contents to somewhere else. Setup example: # git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync encryption=joey@kitenet.net + # git annex describe myrsync "rsync server" Or for using rsync over SSH # git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:/myrsync encryption=joey@kitenet.net + # git annex describe myrsync "rsync server" ## configuration From a83adb0acddba7accb3c7f30aaa4f084a28b632b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 11:22:55 -0400 Subject: [PATCH 1648/2835] note number of hash buckets --- doc/special_remotes/hook.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 5b636613c9..9a7dbf7a19 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -45,6 +45,7 @@ These environment variables are used to communicate with the hook commands: * `ANNEX_KEY` - name of a key to store, retrieve, remove, or check. * `ANNEX_FILE` - a file containing the key's content * `ANNEX_HASH_1` - short stable value, based on the key, can be used for hashing + into 1024 buckets. * `ANNEX_HASH_2` - another hash value, can be used for a second level of hashing The setting to use in git config for the hook commands are as follows: From 15711872f7e9e9b0b3267f1a58592b48bd94f93b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 29 Apr 2011 15:24:56 +0000 Subject: [PATCH 1649/2835] Added a comment --- ...comment_2_4d9b9d47d01d606a475678f630797bf9._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment new file mode 100644 index 0000000000..e7c3d619dd --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-29T15:24:56Z" + content=""" +If `tahoe ls` outputs only the key, on its own line, and exits nonzero if it's not present, then I think you did the right thing. + +To remove a file, use `git annex move file --from tahoe` and then you can drop it locally. +"""]] From 4a13bdc3df8f0a66885157813d1090d6008a0b6f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 15:33:54 +0000 Subject: [PATCH 1650/2835] Added a comment --- ...mment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment new file mode 100644 index 0000000000..16ad9e9886 --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2011-04-29T15:33:24Z" + content=""" +@justin, I discovered that \"git annex describe\" did what I wanted + +@joey, yep that is the behaviour of \"tahoe ls\", thanks for the tip on removing the file from the remote. + +It seems to be working okay for now, the only concern is that on the remote everything is dumped into the same directory, but I can live with that, since I want to track biggish blobs and not lots of small little files. +"""]] From 976ed575d5dcfc08ce8a43f43732b0bc9dd0d8f8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 16:17:23 +0000 Subject: [PATCH 1651/2835] Added a comment --- ..._fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment new file mode 100644 index 0000000000..5d271c6f3c --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2011-04-29T16:17:11Z" + content=""" +I've just tried to use the ANNEX_HASH_ variables, example of my configuration + +
+    git config annex.tahoe-store-hook 'tahoe mkdir $ANNEX_HASH_1 && tahoe put $ANNEX_FILE tahoe:$ANNEX_HASH_1/$ANNEX_KEY'
+    git config annex.tahoe-retrieve-hook 'tahoe get tahoe:$ANNEX_HASH_1/$ANNEX_KEY $ANNEX_FILE'
+    git config annex.tahoe-remove-hook 'tahoe rm tahoe:$ANNEX_HASH_1/$ANNEX_KEY'
+    git config annex.tahoe-checkpresent-hook 'tahoe ls tahoe:$ANNEX_HASH_1/$ANNEX_KEY 2>&1 || echo FAIL'
+    git annex initremote library type=hook hooktype=tahoe encryption=none
+    git annex describe 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a library
+
+ +It's seems to work quite well for me now, I did run across this when I tried to drop a file locally, leaving the file on my remote + +
+jtang@x00:/tmp/annex3 $ git annex drop .
+drop frink.sh (checking library...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+  Try making some of these repositories available:
+  	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a  -- library
+  (Use --force to override this check, or adjust annex.numcopies.)
+failed
+drop t/frink.jar (checking library...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+  Try making some of these repositories available:
+  	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a  -- library
+  (Use --force to override this check, or adjust annex.numcopies.)
+failed
+git-annex: 2 failed
+1|jtang@x00:/tmp/annex3 $ 
+
+ +I do know that the files exist in my library as I have just inserted them, it seemed to work when I didnt have the hashing, it appears that the checkpresent doesn't seem to pass the ANNEX_HASH_* variables (from the limited debugging I did) +"""]] From 5d8e0d5a1c09b95043d02d7517f1e00604b37244 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 12:20:59 -0400 Subject: [PATCH 1652/2835] remove unused file --- Backend/SHA256.hs | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 Backend/SHA256.hs diff --git a/Backend/SHA256.hs b/Backend/SHA256.hs deleted file mode 100644 index 42d3d48c7d..0000000000 --- a/Backend/SHA256.hs +++ /dev/null @@ -1,14 +0,0 @@ -{- git-annex "SHA256" backend - - - - Copyright 2011 Joey Hess - - - - Licensed under the GNU GPL version 3 or higher. - -} - -module Backend.SHA256 (backend) where - -import Types -import Backend.SHA - -backend :: Backend Annex -backend = genBackend 256 From 49efc6c39928baec03d7dd0d5cb37f346432f1d3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 13:12:26 -0400 Subject: [PATCH 1653/2835] add -fspec-constr-count workaround ghc 7 produces these warnings http://hackage.haskell.org/trac/ghc/ticket/4288 The specialization is enabled by -O2, and the default limit of 3 is there to avoid specialization blowing up binary size. Perhaps that default is a little low? I needed 4 to avoid a warning on Unused.hs, and 5 to avoid warnings on test.hs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 69fb518074..01a1a6a54c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr IGNORE=-ignore-package monads-fd -GHCFLAGS=-O2 -Wall $(IGNORE) +GHCFLAGS=-O2 -Wall $(IGNORE) -fspec-constr-count=5 ifdef PROFILE GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE) endif From 43f0a666f0f6cc152a2b778921831d6d7daedcaf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 13:59:00 -0400 Subject: [PATCH 1654/2835] unused: Now also lists files fsck places in .git/annex/bad/ --- Command/DropUnused.hs | 78 ++++++++++++++++++-------------- Command/Unused.hs | 100 +++++++++++++++++++++++++----------------- Locations.hs | 11 +++-- debian/changelog | 1 + doc/git-annex.mdwn | 2 +- 5 files changed, 114 insertions(+), 78 deletions(-) diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 1eec688202..b129235e1d 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -21,54 +21,66 @@ import qualified Annex import qualified Command.Drop import qualified Command.Move import qualified Remote +import qualified GitRepo as Git import Backend import Key +type UnusedMap = M.Map String Key + command :: [Command] command = [repoCommand "dropunused" (paramRepeating paramNumber) seek "drop unused file content"] seek :: [CommandSeek] -seek = [withUnusedMap] +seek = [withUnusedMaps] -{- Read unusedlog once, and pass the map to each start action. -} -withUnusedMap :: CommandSeek -withUnusedMap params = do - m <- readUnusedLog - return $ map (start m) params +{- Read unused logs once, and pass the maps to each start action. -} +withUnusedMaps :: CommandSeek +withUnusedMaps params = do + unused <- readUnusedLog "" + unusedbad <- readUnusedLog "bad" + unusedtmp <- readUnusedLog "tmp" + return $ map (start (unused, unusedbad, unusedtmp)) params -start :: M.Map String Key -> CommandStartString -start m s = notBareRepo $ do - case M.lookup s m of - Nothing -> return Nothing - Just key -> do - showStart "dropunused" s - from <- Annex.getState Annex.fromremote - case from of - Just name -> do - r <- Remote.byName name - return $ Just $ performRemote r key - _ -> return $ Just $ perform key +start :: (UnusedMap, UnusedMap, UnusedMap) -> CommandStartString +start (unused, unusedbad, unusedtmp) s = notBareRepo $ search + [ (unused, perform) + , (unusedbad, performOther gitAnnexBadLocation) + , (unusedtmp, performOther gitAnnexTmpLocation) + ] + where + search [] = return Nothing + search ((m, a):rest) = do + case M.lookup s m of + Nothing -> search rest + Just key -> do + showStart "dropunused" s + return $ Just $ a key -{- drop both content in the backend and any tmp file for the key -} perform :: Key -> CommandPerform perform key = do - g <- Annex.gitRepo - let tmp = gitAnnexTmpLocation g key - tmp_exists <- liftIO $ doesFileExist tmp - when tmp_exists $ liftIO $ removeFile tmp - backend <- keyBackend key - Command.Drop.perform key backend (Just 0) -- force drop + from <- Annex.getState Annex.fromremote + case from of + Just name -> do + r <- Remote.byName name + showNote $ "from " ++ Remote.name r ++ "..." + return $ Just $ Command.Move.fromCleanup r True key + _ -> do + backend <- keyBackend key + Command.Drop.perform key backend (Just 0) -- force drop -performRemote :: Remote.Remote Annex -> Key -> CommandPerform -performRemote r key = do - showNote $ "from " ++ Remote.name r ++ "..." - return $ Just $ Command.Move.fromCleanup r True key - -readUnusedLog :: Annex (M.Map String Key) -readUnusedLog = do +performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform +performOther filespec key = do g <- Annex.gitRepo - let f = gitAnnexUnusedLog g + let f = filespec g key + e <- liftIO $ doesFileExist f + when e $ liftIO $ removeFile f + return $ Just $ return True + +readUnusedLog :: FilePath -> Annex UnusedMap +readUnusedLog prefix = do + g <- Annex.gitRepo + let f = gitAnnexUnusedLog prefix g e <- liftIO $ doesFileExist f if e then do diff --git a/Command/Unused.hs b/Command/Unused.hs index a3fb6fe232..67f10581d6 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -7,7 +7,7 @@ module Command.Unused where -import Control.Monad (filterM, unless, forM_) +import Control.Monad (filterM, unless, forM_, when) import Control.Monad.State (liftIO) import qualified Data.Set as S import Data.Maybe @@ -51,14 +51,17 @@ perform = do checkUnused :: Annex () checkUnused = do - (unused, staletmp) <- unusedKeys - let unusedlist = number 0 unused - let staletmplist = number (length unused) staletmp - let list = unusedlist ++ staletmplist - writeUnusedFile list - unless (null unused) $ showLongNote $ unusedMsg unusedlist - unless (null staletmp) $ showLongNote $ staleTmpMsg staletmplist - unless (null list) $ showLongNote $ "\n" + (unused, stalebad, staletmp) <- unusedKeys + n <- list "" unusedMsg unused 0 + n' <- list "bad" staleBadMsg stalebad n + _ <- list "tmp" staleTmpMsg staletmp n' + return () + where + list file msg l c = do + let unusedlist = number c l + when (not $ null l) $ showLongNote $ msg unusedlist + writeUnusedFile file unusedlist + return $ length l checkRemoteUnused :: Remote.Remote Annex -> Annex () checkRemoteUnused r = do @@ -69,7 +72,7 @@ checkRemoteUnused r = do remotehas <- filterM isthere logged let remoteunused = remotehas `exclude` referenced let list = number 0 remoteunused - writeUnusedFile list + writeUnusedFile "" list unless (null remoteunused) $ do showLongNote $ remoteUnusedMsg r list showLongNote $ "\n" @@ -80,10 +83,10 @@ checkRemoteUnused r = do return $ uuid `elem` us uuid = Remote.uuid r -writeUnusedFile :: [(Int, Key)] -> Annex () -writeUnusedFile l = do +writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex () +writeUnusedFile prefix l = do g <- Annex.gitRepo - liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ + liftIO $ safeWriteFile (gitAnnexUnusedLog prefix g) $ unlines $ map (\(n, k) -> show n ++ " " ++ show k) l table :: [(Int, Key)] -> [String] @@ -100,7 +103,12 @@ staleTmpMsg :: [(Int, Key)] -> String staleTmpMsg t = unlines $ ["Some partially transferred data exists in temporary files:"] ++ table t ++ [dropMsg Nothing] - + +staleBadMsg :: [(Int, Key)] -> String +staleBadMsg t = unlines $ + ["Some corrupted files have been preserved by fsck, just in case:"] + ++ table t ++ [dropMsg Nothing] + unusedMsg :: [(Int, Key)] -> String unusedMsg u = unusedMsg' u ["Some annexed data is no longer used by any files in the repository:"] @@ -127,36 +135,28 @@ dropMsg :: Maybe (Remote.Remote Annex) -> String dropMsg Nothing = dropMsg' "" dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r dropMsg' :: String -> String -dropMsg' s = "(To remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER)" +dropMsg' s = "(To remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER)\n" {- Finds keys whose content is present, but that do not seem to be used - - by any files in the git repo, or that are only present as tmp files. -} -unusedKeys :: Annex ([Key], [Key]) + - by any files in the git repo, or that are only present as bad or tmp + - files. -} +unusedKeys :: Annex ([Key], [Key], [Key]) unusedKeys = do - g <- Annex.gitRepo - fast <- Annex.getState Annex.fast if fast then do - showNote "fast mode enabled; only finding temporary files" - tmps <- tmpKeys - return ([], tmps) + showNote "fast mode enabled; only finding stale files" + tmp <- staleKeys' gitAnnexTmpDir + bad <- staleKeys' gitAnnexBadDir + return ([], bad, tmp) else do showNote "checking for unused data..." present <- getKeysPresent referenced <- getKeysReferenced - tmps <- tmpKeys - let unused = present `exclude` referenced - let staletmp = tmps `exclude` present - let duptmp = tmps `exclude` staletmp - - -- Tmp files that are dups of content already present - -- can simply be removed. - liftIO $ forM_ duptmp $ \t -> removeFile $ - gitAnnexTmpLocation g t - - return (unused, staletmp) + staletmp <- staleKeys gitAnnexTmpDir present + stalebad <- staleKeys gitAnnexBadDir present + return (unused, stalebad, staletmp) {- Finds items in the first, smaller list, that are not - present in the second, larger list. @@ -178,16 +178,34 @@ getKeysReferenced = do keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs -{- List of keys that have temp files in the git repo. -} -tmpKeys :: Annex [Key] -tmpKeys = do +{- Looks in the specified directory for bad/tmp keys, and returns a list + - of those that might still have value, or might be stale and removable. + - + - When a list of presently available keys is provided, stale keys + - that no longer have value are deleted. + -} +staleKeys :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key] +staleKeys dirspec present = do + contents <- staleKeys' dirspec + + let stale = contents `exclude` present + let dup = contents `exclude` stale + g <- Annex.gitRepo - let tmp = gitAnnexTmpDir g - exists <- liftIO $ doesDirectoryExist tmp - if (not exists) + let dir = dirspec g + liftIO $ forM_ dup $ \t -> removeFile $ dir keyFile t + + return stale + +staleKeys' :: (Git.Repo -> FilePath) -> Annex [Key] +staleKeys' dirspec = do + g <- Annex.gitRepo + let dir = dirspec g + exists <- liftIO $ doesDirectoryExist dir + if not exists then return [] else do - contents <- liftIO $ getDirectoryContents tmp + contents <- liftIO $ getDirectoryContents dir files <- liftIO $ filterM doesFileExist $ - map (tmp ) contents + map (dir ) contents return $ catMaybes $ map (fileKey . takeFileName) files diff --git a/Locations.hs b/Locations.hs index f263ea526b..1c4f8296e8 100644 --- a/Locations.hs +++ b/Locations.hs @@ -17,6 +17,7 @@ module Locations ( gitAnnexTmpDir, gitAnnexTmpLocation, gitAnnexBadDir, + gitAnnexBadLocation, gitAnnexUnusedLog, isLinkToAnnex, logFile, @@ -105,9 +106,13 @@ gitAnnexTmpLocation r key = gitAnnexTmpDir r keyFile key gitAnnexBadDir :: Git.Repo -> FilePath gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r "bad" -{- .git/annex/unused is used to number possibly unused keys -} -gitAnnexUnusedLog :: Git.Repo -> FilePath -gitAnnexUnusedLog r = gitAnnexDir r "unused" +{- The bad file to use for a given key. -} +gitAnnexBadLocation :: Git.Repo -> Key -> FilePath +gitAnnexBadLocation r key = gitAnnexBadDir r keyFile key + +{- .git/annex/*unused is used to number possibly unused keys -} +gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath +gitAnnexUnusedLog prefix r = gitAnnexDir r (prefix ++ "unused") {- Checks a symlink target to see if it appears to point to annexed content. -} isLinkToAnnex :: FilePath -> Bool diff --git a/debian/changelog b/debian/changelog index 92c05a5a69..813816079a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.20110428) UNRELEASED; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. * Add hook special remotes. * Avoid crashing when an existing key is readded to the annex. + * unused: Now also lists files fsck places in .git/annex/bad/ -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3e91e7ad92..450b95a0dd 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -158,7 +158,7 @@ Many git-annex commands will stage changes for later `git commit` by you. Checks the annex for data that does not correspond to any files currently in the respository, and prints a numbered list of the data. - To only show unused temp files, specify --fast + To only show unused temp and bad files, specify --fast To check data on a remote that does not correspond to any files currently in the local repository, specify --from. From cf501d3b9be89931bfede402da85cb3bdc455041 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 14:04:20 -0400 Subject: [PATCH 1655/2835] set ANNEX_HASH_* always --- Remote/Hook.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Remote/Hook.hs b/Remote/Hook.hs index 2613fda7a6..ba38355caf 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -69,16 +69,16 @@ hookSetup u c = do return c' hookEnv :: Key -> Maybe FilePath -> Maybe [(String, String)] -hookEnv k f = Just $ keyenv : fileenv f +hookEnv k f = Just $ fileenv f ++ keyenv where env s v = ("ANNEX_" ++ s, v) - keyenv = env "KEY" (show k) - fileenv Nothing = [] - fileenv (Just file) = - [ env "FILE" file + keyenv = + [ env "KEY" (show k) , env "HASH_1" (hashbits !! 0) , env "HASH_2" (hashbits !! 1) ] + fileenv Nothing = [] + fileenv (Just file) = [env "FILE" file] hashbits = map takeDirectory $ splitPath $ hashDirMixed k lookupHook :: String -> String -> Annex (Maybe String) From a14368aa8746bd12c599b7ae02a11ec9ef419c5a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 29 Apr 2011 18:01:04 +0000 Subject: [PATCH 1656/2835] Added a comment --- .../comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment new file mode 100644 index 0000000000..9127cdeeaa --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 5" + date="2011-04-29T18:01:04Z" + content=""" +I've corrected the missing `ANNEX_HASH_*` oversight. (It also affected removal, btw.) +"""]] From 110b1e2b0a4d8355b3de5ebde1710b6b7cd61911 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 20:11:09 +0000 Subject: [PATCH 1657/2835] Added a comment --- ..._2e9da5a919bbbc27b32de3b243867d4f._comment | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment new file mode 100644 index 0000000000..748b4f29d6 --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 6" + date="2011-04-29T20:11:08Z" + content=""" +Cool, that seems to make things work as expected, here's an updated recipe + + +
+git config annex.tahoe-store-hook 'tahoe mkdir tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2 && tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY'
+git config annex.tahoe-retrieve-hook 'tahoe get tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY $ANNEX_FILE'
+git config annex.tahoe-remove-hook 'tahoe rm tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY'
+git config annex.tahoe-checkpresent-hook 'tahoe ls tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY 2>&1 || echo FAIL'
+git annex initremote library type=hook hooktype=tahoe encryption=none
+git annex describe 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a library
+
+ + +I just needs some of the output redirected to /dev/null. +"""]] From 1f84c7a9642378e26d2b076def52255361591a04 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 May 2011 14:05:10 -0400 Subject: [PATCH 1658/2835] S3: When encryption is enabled, the Amazon S3 login credentials are stored, encrypted, in .git-annex/remotes.log, so environment variables need not be set after the remote is initialized. --- Crypto.hs | 3 + Remote/Encryptable.hs | 30 ++++++---- Remote/S3real.hs | 88 ++++++++++++++++++++-------- debian/changelog | 3 + doc/special_remotes/Amazon_S3.mdwn | 6 ++ doc/walkthrough/using_Amazon_S3.mdwn | 4 +- 6 files changed, 96 insertions(+), 38 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 20b999f65a..2f544f21fd 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -22,6 +22,8 @@ module Crypto ( withDecryptedHandle, withEncryptedContent, withDecryptedContent, + toB64, + fromB64, prop_hmacWithCipher_sane ) where @@ -252,6 +254,7 @@ fromB64 s = case B64.decode s of Nothing -> error "bad base64 encoded data" Just ws -> w82s ws + hmacWithCipher :: Cipher -> String -> String hmacWithCipher c = hmacWithCipher' (cipherHmac c) hmacWithCipher' :: String -> String -> String diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index 493ff12143..27c4e7f46c 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -70,23 +70,27 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = Nothing -> a k Just (_, k') -> a k' -{- Gets encryption Cipher, and encrypted version of Key. - - - - The decrypted Cipher is cached in the Annex state. -} -cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key)) -cipherKey Nothing _ = return Nothing -cipherKey (Just c) k = do +{- Gets encryption Cipher. The decrypted Cipher is cached in the Annex + - state. -} +remoteCipher :: RemoteConfig -> Annex (Maybe Cipher) +remoteCipher c = do cache <- Annex.getState Annex.cipher case cache of - Just cipher -> ret cipher + Just cipher -> return $ Just cipher Nothing -> case extractCipher c of Nothing -> return Nothing Just encipher -> do - showNote "gpg" cipher <- liftIO $ decryptCipher c encipher Annex.changeState (\s -> s { Annex.cipher = Just cipher }) - ret cipher - where - ret cipher = do - k' <- liftIO $ encryptKey cipher k - return $ Just (cipher, k') + return $ Just cipher + +{- Gets encryption Cipher, and encrypted version of Key. -} +cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key)) +cipherKey Nothing _ = return Nothing +cipherKey (Just c) k = do + cipher <- remoteCipher c + case cipher of + Just ciphertext -> do + k' <- liftIO $ encryptKey ciphertext k + return $ Just (ciphertext, k') + Nothing -> return Nothing diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 2e198f79d1..c1ae0fbcc6 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -15,7 +15,7 @@ import Network.AWS.AWSResult import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import Data.Maybe -import Control.Monad (when) +import Control.Monad (when, liftM) import Control.Monad.State (liftIO) import System.Environment import System.Posix.Files @@ -45,9 +45,10 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do cst <- remoteCost r expensiveRemoteCost - return $ gen' r u c cst + c' <- s3GetCreds c + return $ gen' r u c' cst gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex -gen' r u c cst = +gen' r u c cst = do encryptableRemote c (storeEncrypted this) (retrieveEncrypted this) @@ -69,12 +70,12 @@ s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane c' <- encryptionSetup c - let fullconfig = M.union c' defaults + c'' <- liftM fromJust (s3GetCreds $ Just c') + let fullconfig = M.union c'' defaults -- check bucket location to see if the bucket exists, and create it let datacenter = fromJust $ M.lookup "datacenter" fullconfig conn <- s3ConnectionRequired fullconfig - showNote "checking bucket" loc <- liftIO $ getBucketLocation conn bucket case loc of @@ -88,7 +89,7 @@ s3Setup u c = do Left err -> s3Error err gitConfigSpecialRemote u fullconfig "s3" "true" - return fullconfig + s3SetCreds fullconfig where remotename = fromJust (M.lookup "name" c) bucket = remotename ++ "-" ++ u @@ -186,6 +187,19 @@ s3Bool res = do Right _ -> return True Left e -> s3Warning e +s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a +s3Action r noconn action = do + when (config r == Nothing) $ + error $ "Missing configuration for special remote " ++ name r + let bucket = M.lookup "bucket" $ fromJust $ config r + conn <- s3Connection $ fromJust $ config r + case (bucket, conn) of + (Just b, Just c) -> action (c, b) + _ -> return noconn + +bucketKey :: String -> Key -> S3Object +bucketKey bucket k = S3Object bucket (show k) "" [] L.empty + s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection s3ConnectionRequired c = do conn <- s3Connection c @@ -195,30 +209,58 @@ s3ConnectionRequired c = do s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) s3Connection c = do - ak <- getEnvKey "AWS_ACCESS_KEY_ID" - sk <- getEnvKey "AWS_SECRET_ACCESS_KEY" - if (null ak || null sk) - then do - warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3" + case (M.lookup s3AccessKey c, M.lookup s3SecretKey c) of + (Just ak, Just sk) -> return $ Just $ AWSConnection host port ak sk + _ -> do + warning $ "Set both " ++ s3AccessKey ++ " and " ++ s3SecretKey ++ " to use S3" return Nothing - else return $ Just $ AWSConnection host port ak sk where host = fromJust $ (M.lookup "host" c) port = let s = fromJust $ (M.lookup "port" c) in case reads s of [(p, _)] -> p _ -> error $ "bad S3 port value: " ++ s + +{- S3 creds come from the environment if set. + - Otherwise, might be stored encrypted in the remote's config. -} +s3GetCreds :: Maybe RemoteConfig -> Annex (Maybe RemoteConfig) +s3GetCreds Nothing = return Nothing +s3GetCreds (Just c) = do + ak <- getEnvKey s3AccessKey + sk <- getEnvKey s3SecretKey + if (null ak || null sk) + then do + mcipher <- remoteCipher c + case (M.lookup "s3creds" c, mcipher) of + (Just encrypted, Just cipher) -> do + s <- liftIO $ withDecryptedContent cipher + (return $ L.pack $ fromB64 encrypted) + (return . L.unpack) + let line = lines s + creds (line !! 0) (line !! 1) + _ -> return $ Just c + else creds ak sk + where getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") + creds ak sk = return $ Just $ M.insert s3AccessKey ak $ M.insert s3SecretKey sk c -s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a -s3Action r noconn action = do - when (config r == Nothing) $ - error $ "Missing configuration for special remote " ++ name r - let bucket = M.lookup "bucket" $ fromJust $ config r - conn <- s3Connection (fromJust $ config r) - case (bucket, conn) of - (Just b, Just c) -> action (c, b) - _ -> return noconn +{- Stores S3 creds encrypted in the remote's config if possible. -} +s3SetCreds :: RemoteConfig -> Annex RemoteConfig +s3SetCreds c = do + let cleanconfig = M.delete s3AccessKey $ M.delete s3SecretKey c + case (M.lookup s3AccessKey c, M.lookup s3SecretKey c) of + (Just ak, Just sk) -> do + mcipher <- remoteCipher c + case mcipher of + Just cipher -> do + s <- liftIO $ withEncryptedContent cipher + (return $ L.pack $ unlines [ak, sk]) + (return . L.unpack) + return $ M.insert "s3creds" (toB64 s) cleanconfig + Nothing -> return cleanconfig + _ -> return cleanconfig -bucketKey :: String -> Key -> S3Object -bucketKey bucket k = S3Object bucket (show k) "" [] L.empty +s3AccessKey :: String +s3AccessKey = "AWS_ACCESS_KEY_ID" +s3SecretKey :: String +s3SecretKey = "AWS_SECRET_ACCESS_KEY" diff --git a/debian/changelog b/debian/changelog index 813816079a..b28728b0ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ git-annex (0.20110428) UNRELEASED; urgency=low * Add hook special remotes. * Avoid crashing when an existing key is readded to the annex. * unused: Now also lists files fsck places in .git/annex/bad/ + * S3: When encryption is enabled, the Amazon S3 login credentials + are stored, encrypted, in .git-annex/remotes.log, so environment + variables need not be set after the remote is initialized. -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn index 87cde32993..35397dc2a9 100644 --- a/doc/special_remotes/Amazon_S3.mdwn +++ b/doc/special_remotes/Amazon_S3.mdwn @@ -29,3 +29,9 @@ the S3 remote. * `bucket` - S3 requires that buckets have a globally unique name, so by default, a bucket name is chosen based on the remote name and UUID. This can be specified to pick a bucket name. + +The standard environment variables `ANNEX_S3_ACCESS_KEY_ID` and +`ANNEX_S3_SECRET_ACCESS_KEY` can be used to supply login credentials +for Amazon. When encryption is enabled, they are stored in encrypted form +by `git annex initremote`, so you do not need to keep the environment +variables set after the initial initalization of the remote. diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index c842583546..63bed5d63a 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -30,8 +30,8 @@ repository use the same S3 remote is easy: Now the remote can be used like any other remote. # git annex copy my_cool_big_file --to cloud - copy my_cool_big_file (gpg) (checking cloud...) (to cloud...) ok + copy my_cool_big_file (checking cloud...) (to cloud...) ok # git annex move video/hackity_hack_and_kaxxt.mov --to cloud - move video/hackity_hack_and_kaxxt.mov (gpg) (checking cloud...) (to cloud...) ok + move video/hackity_hack_and_kaxxt.mov (checking cloud...) (to cloud...) ok See [[special_remotes/Amazon_S3]] for details. From 3095e1631180d87cba112c210dfdfeee9b57ef54 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 May 2011 14:09:07 -0400 Subject: [PATCH 1659/2835] mention that the cipher can also be used to crypt access keys --- doc/design/encryption.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index b30e01cdda..bcd6a11bc6 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -85,6 +85,15 @@ really have content. If it's later determined to be safe to not encrypt the HMAC cipher, the current design allows changing that, even for existing remotes. +## other use of the symmetric cipher + +The symmetric cipher can be used to encrypt other content than the content +sent to the remote. In particular, it may make sense to encrypt whatever +access keys are used by the special remote with the cipher, and store that +in remotes.log. This way anyone whose gpg key has been given access to +the cipher can get access to whatever other credentials are needed to +use the special remote. + ## risks A risk of this scheme is that, once the symmetric cipher has been obtained, it From 2ddade8132169ea751628f72ae5b03c5921abafc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 May 2011 14:27:40 -0400 Subject: [PATCH 1660/2835] factor out base64 code --- Base64.hs | 20 ++++++++++++++++++++ Crypto.hs | 14 +------------- Remote/S3real.hs | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 Base64.hs diff --git a/Base64.hs b/Base64.hs new file mode 100644 index 0000000000..cc6346b415 --- /dev/null +++ b/Base64.hs @@ -0,0 +1,20 @@ +{- Simple Base64 access + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Base64 (toB64, fromB64) where + +import Codec.Binary.Base64 +import Data.Bits.Utils + +toB64 :: String -> String +toB64 = encode . s2w8 + +fromB64 :: String -> String +fromB64 s = + case decode s of + Nothing -> error "bad base64 encoded data" + Just ws -> w82s ws diff --git a/Crypto.hs b/Crypto.hs index 2f544f21fd..53cd48dd59 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -22,21 +22,17 @@ module Crypto ( withDecryptedHandle, withEncryptedContent, withDecryptedContent, - toB64, - fromB64, prop_hmacWithCipher_sane ) where import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M -import qualified Codec.Binary.Base64 as B64 import Data.ByteString.Lazy.UTF8 (fromString) import Data.Digest.Pure.SHA import System.Cmd.Utils import Data.String.Utils import Data.List -import Data.Bits.Utils import System.IO import System.Posix.IO import System.Posix.Types @@ -50,6 +46,7 @@ import Types import Key import RemoteClass import Utility +import Base64 import CryptoTypes {- The first half of a Cipher is used for HMAC; the remainder @@ -246,15 +243,6 @@ configGet c key = Just v -> v Nothing -> error $ "missing " ++ key ++ " in remote config" -toB64 :: String -> String -toB64 = B64.encode . s2w8 - -fromB64 :: String -> String -fromB64 s = - case B64.decode s of - Nothing -> error "bad base64 encoded data" - Just ws -> w82s ws - hmacWithCipher :: Cipher -> String -> String hmacWithCipher c = hmacWithCipher' (cipherHmac c) hmacWithCipher' :: String -> String -> String diff --git a/Remote/S3real.hs b/Remote/S3real.hs index c1ae0fbcc6..d6bfe54874 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -33,6 +33,7 @@ import Remote.Encryptable import Crypto import Key import Content +import Base64 remote :: RemoteType Annex remote = RemoteType { From 3c319cd844f23edeab800d37ed0256d92a88a818 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 May 2011 15:13:54 -0400 Subject: [PATCH 1661/2835] avoid always decrypting cipher Last change moved cipher decryption to remote setup time. Fixed this with a bit of a hack. --- Remote/Encryptable.hs | 1 + Remote/S3real.hs | 43 +++++++++++++++------------- doc/walkthrough/using_Amazon_S3.mdwn | 6 ++-- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index 27c4e7f46c..31ef1f37a7 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -80,6 +80,7 @@ remoteCipher c = do Nothing -> case extractCipher c of Nothing -> return Nothing Just encipher -> do + showNote "gpg" cipher <- liftIO $ decryptCipher c encipher Annex.changeState (\s -> s { Annex.cipher = Just cipher }) return $ Just cipher diff --git a/Remote/S3real.hs b/Remote/S3real.hs index d6bfe54874..b0371eb5ea 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -15,10 +15,11 @@ import Network.AWS.AWSResult import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import Data.Maybe -import Control.Monad (when, liftM) +import Control.Monad (when) import Control.Monad.State (liftIO) import System.Environment import System.Posix.Files +import System.Posix.Env (setEnv) import RemoteClass import Types @@ -46,8 +47,7 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do cst <- remoteCost r expensiveRemoteCost - c' <- s3GetCreds c - return $ gen' r u c' cst + return $ gen' r u c cst gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex gen' r u c cst = do encryptableRemote c @@ -71,8 +71,7 @@ s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane c' <- encryptionSetup c - c'' <- liftM fromJust (s3GetCreds $ Just c') - let fullconfig = M.union c'' defaults + let fullconfig = M.union c' defaults -- check bucket location to see if the bucket exists, and create it let datacenter = fromJust $ M.lookup "datacenter" fullconfig @@ -210,8 +209,9 @@ s3ConnectionRequired c = do s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) s3Connection c = do - case (M.lookup s3AccessKey c, M.lookup s3SecretKey c) of - (Just ak, Just sk) -> return $ Just $ AWSConnection host port ak sk + creds <- s3GetCreds c + case creds of + Just (ak, sk) -> return $ Just $ AWSConnection host port ak sk _ -> do warning $ "Set both " ++ s3AccessKey ++ " and " ++ s3SecretKey ++ " to use S3" return Nothing @@ -224,9 +224,8 @@ s3Connection c = do {- S3 creds come from the environment if set. - Otherwise, might be stored encrypted in the remote's config. -} -s3GetCreds :: Maybe RemoteConfig -> Annex (Maybe RemoteConfig) -s3GetCreds Nothing = return Nothing -s3GetCreds (Just c) = do +s3GetCreds :: RemoteConfig -> Annex (Maybe (String, String)) +s3GetCreds c = do ak <- getEnvKey s3AccessKey sk <- getEnvKey s3SecretKey if (null ak || null sk) @@ -238,28 +237,32 @@ s3GetCreds (Just c) = do (return $ L.pack $ fromB64 encrypted) (return . L.unpack) let line = lines s - creds (line !! 0) (line !! 1) - _ -> return $ Just c - else creds ak sk + let ak' = line !! 0 + let sk' = line !! 1 + liftIO $ do + setEnv s3AccessKey ak True + setEnv s3SecretKey sk True + return $ Just (ak', sk') + _ -> return Nothing + else return $ Just (ak, sk) where getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") - creds ak sk = return $ Just $ M.insert s3AccessKey ak $ M.insert s3SecretKey sk c {- Stores S3 creds encrypted in the remote's config if possible. -} s3SetCreds :: RemoteConfig -> Annex RemoteConfig s3SetCreds c = do - let cleanconfig = M.delete s3AccessKey $ M.delete s3SecretKey c - case (M.lookup s3AccessKey c, M.lookup s3SecretKey c) of - (Just ak, Just sk) -> do + creds <- s3GetCreds c + case creds of + Just (ak, sk) -> do mcipher <- remoteCipher c case mcipher of Just cipher -> do s <- liftIO $ withEncryptedContent cipher (return $ L.pack $ unlines [ak, sk]) (return . L.unpack) - return $ M.insert "s3creds" (toB64 s) cleanconfig - Nothing -> return cleanconfig - _ -> return cleanconfig + return $ M.insert "s3creds" (toB64 s) c + Nothing -> return c + _ -> return c s3AccessKey :: String s3AccessKey = "AWS_ACCESS_KEY_ID" diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index 63bed5d63a..512ef961f9 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -15,7 +15,7 @@ like "2512E3C7" Next, create the S3 remote, and describe it. # git annex initremote cloud type=S3 encryption=2512E3C7 - initremote cloud (encryption setup with gpg key C910D9222512E3C7) (checking bucket) (creating bucket in US) ok + initremote cloud (encryption setup with gpg key C910D9222512E3C7) (checking bucket) (creating bucket in US) (gpg) ok # git annex describe cloud "at Amazon's US datacenter" describe cloud ok @@ -25,12 +25,12 @@ repository use the same S3 remote is easy: # cd /media/usb/annex # git pull laptop master # git annex initremote cloud - initremote cloud (checking bucket) ok + initremote cloud (gpg) (checking bucket) ok Now the remote can be used like any other remote. # git annex copy my_cool_big_file --to cloud - copy my_cool_big_file (checking cloud...) (to cloud...) ok + copy my_cool_big_file (gpg) (checking cloud...) (to cloud...) ok # git annex move video/hackity_hack_and_kaxxt.mov --to cloud move video/hackity_hack_and_kaxxt.mov (checking cloud...) (to cloud...) ok From 86d3205061dd6f10b126e1578eec135376ae6e99 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 3 May 2011 21:49:20 -0400 Subject: [PATCH 1662/2835] releasing version 0.20110503 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index b28728b0ff..73f5924a95 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110428) UNRELEASED; urgency=low +git-annex (0.20110503) unstable; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. * Add hook special remotes. @@ -8,7 +8,7 @@ git-annex (0.20110428) UNRELEASED; urgency=low are stored, encrypted, in .git-annex/remotes.log, so environment variables need not be set after the remote is initialized. - -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 + -- Joey Hess Tue, 03 May 2011 20:56:01 -0400 git-annex (0.20110427) unstable; urgency=low From e70fc1340a9cf4e33356e85b4ec4793fe04fa2f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 3 May 2011 21:50:01 -0400 Subject: [PATCH 1663/2835] add news item for git-annex 0.20110503 --- doc/news/version_0.20110417.mdwn | 21 --------------------- doc/news/version_0.20110503.mdwn | 9 +++++++++ 2 files changed, 9 insertions(+), 21 deletions(-) delete mode 100644 doc/news/version_0.20110417.mdwn create mode 100644 doc/news/version_0.20110503.mdwn diff --git a/doc/news/version_0.20110417.mdwn b/doc/news/version_0.20110417.mdwn deleted file mode 100644 index 7e28ea2138..0000000000 --- a/doc/news/version_0.20110417.mdwn +++ /dev/null @@ -1,21 +0,0 @@ -git-annex 0.20110417 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * bup is now supported as a special type of remote. - * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted - using GPG for privacy. - * Use lowercase hash directories for locationlog files, to avoid - some issues with git on OSX with the mixed-case directories. - No migration is needed; the old mixed case hash directories are still - read; new information is written to the new directories. - * Unused files on remotes, particulary special remotes, can now be - identified and dropped, by using "--from remote" with git annex unused - and git annex dropunused. - * Clear up short option confusion between --from and --force (-f is now - --from, and there is no short option for --force). - * Add build depend on perlmagick so docs are consistently built. - Closes: #[621410](http://bugs.debian.org/621410) - * Add doc-base file. Closes: #[621408](http://bugs.debian.org/621408) - * Periodically flush git command queue, to avoid boating memory usage - too much. - * Support "sha1" and "sha512" commands on FreeBSD, and allow building - if any/all SHA commands are not available. Thanks, Fraser Tweedale"""]] \ No newline at end of file diff --git a/doc/news/version_0.20110503.mdwn b/doc/news/version_0.20110503.mdwn new file mode 100644 index 0000000000..f24f0c48f6 --- /dev/null +++ b/doc/news/version_0.20110503.mdwn @@ -0,0 +1,9 @@ +git-annex 0.20110503 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Fix hasKeyCheap setting for bup and rsync special remotes. + * Add hook special remotes. + * Avoid crashing when an existing key is readded to the annex. + * unused: Now also lists files fsck places in .git/annex/bad/ + * S3: When encryption is enabled, the Amazon S3 login credentials + are stored, encrypted, in .git-annex/remotes.log, so environment + variables need not be set after the remote is initialized."""]] \ No newline at end of file From 405745acdd1300d8a1e1db1ca653d867c556adf5 Mon Sep 17 00:00:00 2001 From: "https://me.yahoo.com/a/wpdhh7Et0MiET3shW4BlKe60GFs_mXI-#16fd9" Date: Thu, 5 May 2011 22:35:44 +0000 Subject: [PATCH 1664/2835] --- ...hlist:_Prevent_repeated_password_prompts_for_one_command.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn new file mode 100644 index 0000000000..6c12b1d155 --- /dev/null +++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn @@ -0,0 +1 @@ +Simple, when performing various git annex command over ssh, in particular a multi-file get, and using password authentication, git annex will prompt more than once for a user password. This makes batch updates very inconvenient. From d3dfdb9f9d48cd914a6ce2ac54645502a8932a2f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Thu, 5 May 2011 23:57:40 +0000 Subject: [PATCH 1665/2835] Initial commit --- ...ncorrectly_parses_bare_IPv6_addresses.mdwn | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn diff --git a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn new file mode 100644 index 0000000000..e5a2c03377 --- /dev/null +++ b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn @@ -0,0 +1,41 @@ +I have a git remote in a git-annex-enabled repository. Here's what it looks like in .git/config: + +
+[remote "renaissance"]
+        url = ssh://[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]/home/paulproteus/Music/annex/
+        fetch = +refs/heads/*:refs/remotes/renaissance/*
+        annex-uuid = 2992752e-1a13-11e0-ba68-57d3c800da64
+
+ +I wanted to "git annex get" some data. git-annex appears to pass incorrectly-formatted IPv6 addresses to rsync: + +
+get primary/emusiq/Arab Strap/Monday At The Hug And Pint/01-The Shy Retirer.mp3 (copying from renaissance...) 
+ssh: Could not resolve hostname [2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]: Name or service not known
+rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]
+rsync error: unexplained error (code 255) at io.c(601) [Receiver=3.0.7]
+
+  rsync failed -- run git annex again to resume file transfer
+  Unable to access these remotes: renaissance
+  Try making some of these repositories available:
+  	2992752e-1a13-11e0-ba68-57d3c800da64
+failed
+
+ +In this case, the square brackets should not be there. + +I tried changing the .git/config syntax slightly, and got a different, also-incorrect behavior: + +
+[remote "renaissance"]
+        url = [2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulproteus/Music/annex/
+        fetch = +refs/heads/*:refs/remotes/renaissance/*
+        annex-uuid = 2992752e-1a13-11e0-ba68-57d3c800da64
+
+ +
+paulproteus@pathi:~/Music/annex$ git annex get
+git-annex: bad url ssh://[2001/~/0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulproteus/Music/annex/
+
+ +(Note that both these .git/config entries work fine with "git fetch".) From c7ff9d3dab19c2d504e74d5d92e8447b4719e872 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Fri, 6 May 2011 00:00:17 +0000 Subject: [PATCH 1666/2835] Add sign-off --- doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn index e5a2c03377..fdaf5d5d1c 100644 --- a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn +++ b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn @@ -39,3 +39,5 @@ git-annex: bad url ssh://[2001/~/0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulprote (Note that both these .git/config entries work fine with "git fetch".) + +-- Asheesh. From d59666aed614f85e6d4fbdc0e19d7870ebe1961d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 6 May 2011 18:28:43 +0000 Subject: [PATCH 1667/2835] Added a comment --- ...ment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment new file mode 100644 index 0000000000..3cc0b08344 --- /dev/null +++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-05-06T18:28:41Z" + content=""" +Unless you are forced to use a password, you should really be using a ssh key. + +ssh-keygen +put local .ssh/id_?sa.pub into remote .ssh/authorized_keys (which needs to be chmod 600) +ssh-add +git annex whatever +"""]] From 489156f255c0bbd1195db00fe0158e6f5c0be549 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 6 May 2011 18:29:23 +0000 Subject: [PATCH 1668/2835] removed --- ...ment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment deleted file mode 100644 index 3cc0b08344..0000000000 --- a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment +++ /dev/null @@ -1,13 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" - nickname="Richard" - subject="comment 1" - date="2011-05-06T18:28:41Z" - content=""" -Unless you are forced to use a password, you should really be using a ssh key. - -ssh-keygen -put local .ssh/id_?sa.pub into remote .ssh/authorized_keys (which needs to be chmod 600) -ssh-add -git annex whatever -"""]] From 33041e4afb66984a56e646ff4c8240adc8c5e01d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 6 May 2011 18:30:03 +0000 Subject: [PATCH 1669/2835] Added a comment --- ...ent_1_3f9c0d08932c2ede61c802a91261a1f7._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment new file mode 100644 index 0000000000..2801d8e68f --- /dev/null +++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-05-06T18:30:02Z" + content=""" +Unless you are forced to use a password, you should really be using a ssh key. + + ssh-keygen + #put local .ssh/id_?sa.pub into remote .ssh/authorized_keys (which needs to be chmod 600) + ssh-add + git annex whatever + +"""]] From f1fbe33cfeba70123a702b3cdd1139361b75b875 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 May 2011 14:43:40 -0400 Subject: [PATCH 1670/2835] response --- ..._Prevent_repeated_password_prompts_for_one_command.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn index 6c12b1d155..6d1552fe4e 100644 --- a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn +++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn @@ -1 +1,8 @@ Simple, when performing various git annex command over ssh, in particular a multi-file get, and using password authentication, git annex will prompt more than once for a user password. This makes batch updates very inconvenient. + +> I'd suggest using ssh-agent, or a passwordless ssh key. Possibly in +> combination with [[git-annex-shell]] if you want to lock down a +> particular ssh key to only being able to use git-annex and git-daemon. +> +> Combining multiple operations into a single ssh is on the todo list, but +> very far down it. --[[Joey]] From 078a6fbd76190c48cfa5c588bb9d2174baef5852 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 May 2011 15:21:30 -0400 Subject: [PATCH 1671/2835] Work around a bug in Network.URI's handling of bracketed ipv6 addresses. --- GitRepo.hs | 18 +++++++++++++++--- debian/changelog | 6 ++++++ ...incorrectly_parses_bare_IPv6_addresses.mdwn | 16 ++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 9ecaa8ffcb..49024abe0e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -280,9 +280,21 @@ urlScheme :: Repo -> String urlScheme Repo { location = Url u } = uriScheme u urlScheme repo = assertUrl repo $ error "internal" +{- Work around a bug in the real uriRegName + - -} +uriRegName' :: URIAuth -> String +uriRegName' a = fixup $ uriRegName a + where + fixup x@('[':rest) + | rest !! len == ']' = take len rest + | otherwise = x + where + len = (length rest) - 1 + fixup x = x + {- Hostname of an URL repo. -} urlHost :: Repo -> String -urlHost = urlAuthPart uriRegName +urlHost = urlAuthPart uriRegName' {- Port of an URL repo, if it has a nonstandard one. -} urlPort :: Repo -> Maybe Integer @@ -294,11 +306,11 @@ urlPort r = {- Hostname of an URL repo, including any username (ie, "user@host") -} urlHostUser :: Repo -> String -urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName r +urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r {- The full authority portion an URL repo. (ie, "user@host:port") -} urlAuthority :: Repo -> String -urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a +urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName' a ++ uriPort a where a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) urlAuthority repo = assertUrl repo $ error "internal" diff --git a/debian/changelog b/debian/changelog index 73f5924a95..09dde346c0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.20110504) UNRELEASED; urgency=low + + * Work around a bug in Network.URI's handling of bracketed ipv6 addresses. + + -- Joey Hess Fri, 06 May 2011 15:20:38 -0400 + git-annex (0.20110503) unstable; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. diff --git a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn index fdaf5d5d1c..c94952b490 100644 --- a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn +++ b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn @@ -41,3 +41,19 @@ git-annex: bad url ssh://[2001/~/0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulprote (Note that both these .git/config entries work fine with "git fetch".) -- Asheesh. + +> Technically, this seems to be a bug in the haskell URI library; it honors +> the `[]` in parsing, but does not remove them when the URI is queried for +> the host part. + +
+Prelude Network.URI> let (Just u) = parseURI "http://foo@[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]/bar"
+Prelude Network.URI> let (Just a) = uriAuthority u
+Prelude Network.URI> uriRegName a
+"[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]"
+Prelude Network.URI> isIPv6address $ uriRegName a
+False
+
+ +> I have filed a [bug upstream](http://trac.haskell.org/network/ticket/40), and put a workaround in git-annex. [[done]] +> --[[Joey]] From 7020d5bc5bada2533cda27ffa696c0a924158a8c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 6 May 2011 22:35:28 +0000 Subject: [PATCH 1672/2835] --- doc/todo/wishlist:_swift_backend.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/todo/wishlist:_swift_backend.mdwn diff --git a/doc/todo/wishlist:_swift_backend.mdwn b/doc/todo/wishlist:_swift_backend.mdwn new file mode 100644 index 0000000000..498f9588a7 --- /dev/null +++ b/doc/todo/wishlist:_swift_backend.mdwn @@ -0,0 +1,5 @@ +[swift](http://swift.openstack.org/) is the object storage of Openstack. Think S3, but fully open source. As it's backed by rackspace.com, NASA, Dell and several other major players, adoption rates will explode. + +I can provide a test account soonish if need be, else rackspace.com might already be offering their object storage via swift. + +Richard From 579ade130aa10b5907536f9c6f698689d246171f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 May 2011 18:44:49 -0400 Subject: [PATCH 1673/2835] update --- doc/install/OSX.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn index c8c381486b..46a285e131 100644 --- a/doc/install/OSX.mdwn +++ b/doc/install/OSX.mdwn @@ -6,6 +6,7 @@ sudo cabal install utf8-string sudo port install pcre sudo cabal install pcre-light sudo cabal install quickcheck +sudo cabal install SHA # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland) export PATH=$PATH:/opt/local/libexec/gnubin From 50fecbcb410ebb69a0013ab54630fcca59550291 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 9 May 2011 12:23:50 +0000 Subject: [PATCH 1674/2835] --- doc/todo/wishlist:_swift_backend.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/todo/wishlist:_swift_backend.mdwn b/doc/todo/wishlist:_swift_backend.mdwn index 498f9588a7..5e2991871e 100644 --- a/doc/todo/wishlist:_swift_backend.mdwn +++ b/doc/todo/wishlist:_swift_backend.mdwn @@ -1,5 +1,5 @@ [swift](http://swift.openstack.org/) is the object storage of Openstack. Think S3, but fully open source. As it's backed by rackspace.com, NASA, Dell and several other major players, adoption rates will explode. -I can provide a test account soonish if need be, else rackspace.com might already be offering their object storage via swift. +I can provide a test account soonish if need be, else https://auth.api.rackspacecloud.com/v1.0 Richard From 59e960a51cf88ffa8bc664f41a026ce1ca08e4d4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 9 May 2011 12:24:28 +0000 Subject: [PATCH 1675/2835] --- doc/todo/wishlist:_swift_backend.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/todo/wishlist:_swift_backend.mdwn b/doc/todo/wishlist:_swift_backend.mdwn index 5e2991871e..28bd265faf 100644 --- a/doc/todo/wishlist:_swift_backend.mdwn +++ b/doc/todo/wishlist:_swift_backend.mdwn @@ -1,5 +1,5 @@ [swift](http://swift.openstack.org/) is the object storage of Openstack. Think S3, but fully open source. As it's backed by rackspace.com, NASA, Dell and several other major players, adoption rates will explode. -I can provide a test account soonish if need be, else https://auth.api.rackspacecloud.com/v1.0 +I can provide a test account soonish if need be, else rackspace.com if offering swift storage. Their API gateway lives at https://auth.api.rackspacecloud.com/v1.0 Richard From 62cd26c64a5c66c86e6c834aba51d314457cca9f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 12 May 2011 00:07:31 +0000 Subject: [PATCH 1676/2835] Added a comment --- ...1_6a41bf7e2db83db3a01722b516fb6886._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment b/doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment new file mode 100644 index 0000000000..59c30de534 --- /dev/null +++ b/doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-05-12T00:07:29Z" + content=""" +I followed this to re-inject files which git annex fsck listed as missing. + +For everyone of those files, I get + + git-annex-shell: key is already present in annex + rsync: connection unexpectedly closed (0 bytes received so far) [sender] + rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.8] + +when trying to copy the files to the remote. + +-- Richard +"""]] From f7174d09b60992ba0bf106956f83b35bac038b16 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 12 May 2011 01:01:34 +0000 Subject: [PATCH 1677/2835] Added a comment --- .../comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment b/doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment new file mode 100644 index 0000000000..44aab3baa0 --- /dev/null +++ b/doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-05-12T01:01:34Z" + content=""" +Sounds like you probably didn't commit after the fsck, or didn't push so the other repository did not know the first had the content again -- but I'm not 100% sure. +"""]] From 1fdc6b3aadc336a3a4ed2e35db9ad6797fd3b4d5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 14:55:27 -0400 Subject: [PATCH 1678/2835] update --- doc/design/encryption.mdwn | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index bcd6a11bc6..2b9769aa7a 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -1,5 +1,6 @@ -This was the design doc for [[encryption]] and is preserved for -the curious. +This was the design doc for [[/encryption]] and is preserved for +the curious. For an example of using git-annex with an encrypted S3 remote, +see [[using_Amazon_S3]]. [[!toc]] @@ -32,7 +33,7 @@ Allowing multiple ciphers to be used within a single remote would add a lot of complexity, so is not planned to be supported. Instead, if you want a new cipher, create a new S3 bucket, or whatever. There does not seem to be much benefit to using the same cipher for -two different enrypted remotes. +two different encrypted remotes. So, the encrypted cipher could just be stored with the rest of a remote's configuration in `.git-annex/remotes.log` (see [[internals]]). When `git From e9815a2eb64232da7f7f594bdfaeeaf716e5818d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 14:56:25 -0400 Subject: [PATCH 1679/2835] update --- doc/design/encryption.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 2b9769aa7a..11056478bd 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -1,6 +1,6 @@ This was the design doc for [[/encryption]] and is preserved for the curious. For an example of using git-annex with an encrypted S3 remote, -see [[using_Amazon_S3]]. +see [[walkthrough/using_Amazon_S3]]. [[!toc]] From 552aee694c75e331eb796c6ef93220d1b5feabdb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 14:58:48 -0400 Subject: [PATCH 1680/2835] layout --- doc/walkthrough/using_Amazon_S3.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index 512ef961f9..6b0f496392 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -23,7 +23,7 @@ The configuration for the S3 remote is stored in git. So to make another repository use the same S3 remote is easy: # cd /media/usb/annex - # git pull laptop master + # git pull laptop master # git annex initremote cloud initremote cloud (gpg) (checking bucket) ok From 6cf797e8c3f779f1cc3b17c82fd1685613a8b4d3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 19:15:15 -0400 Subject: [PATCH 1681/2835] add github repo mirror should be populated by Branchable when it sees this commit --- doc/download.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/download.mdwn b/doc/download.mdwn index 748ac0ca34..ff780bd4ca 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,5 +1,6 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] +(The repo is mirrored to [github](https://github.com/joeyh/git-annex).) Some operating systems include git-annex in easily prepackaged form and others need some manual work. See [[install]] for details. From d0ea28e9c7731a629e6722e69490da81518f0f0a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 19:17:54 -0400 Subject: [PATCH 1682/2835] wording --- doc/download.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/download.mdwn b/doc/download.mdwn index ff780bd4ca..405bc2fc4b 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,6 +1,6 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] -(The repo is mirrored to [github](https://github.com/joeyh/git-annex).) +(The repo is mirrored [at github](https://github.com/joeyh/git-annex).) Some operating systems include git-annex in easily prepackaged form and others need some manual work. See [[install]] for details. From 536036cfa2c1b28932f72408f1244f8e345eba82 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 19:20:54 -0400 Subject: [PATCH 1683/2835] add README --- README | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 README diff --git a/README b/README new file mode 100644 index 0000000000..ce67d68166 --- /dev/null +++ b/README @@ -0,0 +1,6 @@ +git-annex allows managing files with git, without checking the file +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. + +For documentation, see doc/ or From e72c1c33ae60f7fe544acd0a1f1331138b880e22 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 23:05:14 -0400 Subject: [PATCH 1684/2835] would be nice.. --- doc/todo/support_S3_multipart_uploads.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/todo/support_S3_multipart_uploads.mdwn diff --git a/doc/todo/support_S3_multipart_uploads.mdwn b/doc/todo/support_S3_multipart_uploads.mdwn new file mode 100644 index 0000000000..711ac41b2a --- /dev/null +++ b/doc/todo/support_S3_multipart_uploads.mdwn @@ -0,0 +1,14 @@ +Did not know of this when I wrote S3 support. Ability to resume large +uploads would be good. + + + +Also allows supporting files > 5 gb, a S3 limit I was not aware of. + +NB: It would work just as well to split the object and upload the N parts +to S3, but not bother with S3's paperwork to rejoin them into one object. +Only reasons not to do that are a) backwards compatability with +the existing S3 remote and b) this would not allow accessing the content +in S3 w/o using git-annex, which could be useful in some scenarios. + +--[[Joey]] From fcdbfbdc9132d3c023da0d567807e5bb29fb695b Mon Sep 17 00:00:00 2001 From: zooko Date: Sat, 14 May 2011 05:07:18 +0000 Subject: [PATCH 1685/2835] Added a comment: request for information, plus some ideas --- ..._d636c868524b2055ee85832527437f90._comment | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment new file mode 100644 index 0000000000..1d75fb9631 --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="zooko" + ip="97.118.97.117" + subject="request for information, plus some ideas" + date="2011-05-14T05:07:17Z" + content=""" +Hey Jimmy: how's this working for you now? I would expect it to go slower and slower since Tahoe-LAFS has an O(N) algorithm for reading or updating directories. + +Of course, if it is still fast enough for your uses then that's okay. :-) + +(We're working on optimizations of this for future releases of Tahoe-LAFS.) + +I'd like to understand the desired behavior of store-hook and retrieve-hook better, in order to see if there is a more efficient way to use Tahoe-LAFS for this. + +Off to look for docs. + +Regards, + +Zooko +"""]] From 037fa707aa4f8ab43b5d18055e90c3bd42d818e0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 14 May 2011 02:55:12 -0400 Subject: [PATCH 1686/2835] fix bug: put file in hashed directory structure, not tahoe root --- .../comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment index 748b4f29d6..80874db31d 100644 --- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment @@ -8,7 +8,7 @@ Cool, that seems to make things work as expected, here's an updated recipe 
-git config annex.tahoe-store-hook 'tahoe mkdir tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2 && tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY'
+git config annex.tahoe-store-hook 'tahoe mkdir tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2 && tahoe put $ANNEX_FILE tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY'
 git config annex.tahoe-retrieve-hook 'tahoe get tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY $ANNEX_FILE'
 git config annex.tahoe-remove-hook 'tahoe rm tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY'
 git config annex.tahoe-checkpresent-hook 'tahoe ls tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY 2>&1 || echo FAIL'
@@ -18,4 +18,6 @@ git annex describe 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a library
 
 
 I just needs some of the output redirected to /dev/null.
+
+(I updated this comment to fix a bug. --[[Joey]])
 """]]

From 99c6dd556f940a51b862bacd00ce5f67eb1937a2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 14 May 2011 09:06:54 +0000
Subject: [PATCH 1687/2835] Added a comment

---
 ...ent_3_b596b5cfd3377e58dbbb5d509d026b90._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment b/doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
new file mode 100644
index 0000000000..4744db995c
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-05-14T09:06:54Z"
+ content="""
+As my comment from work is stuck in moderation:
+
+I ran this twice:
+
+    git pull && git annex add . && git annex copy . --to  --fast --quiet && git commit -a -m \"$HOST $(date +%F--%H-%M-%S-%Z)\" && git push
+
+but nothing changed
+"""]]

From 70d97467491f613bac30790462920c37b221c2d6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 14 May 2011 10:02:30 +0000
Subject: [PATCH 1688/2835] Added a comment

---
 ...comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment

diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment
new file mode 100644
index 0000000000..dc97128bd1
--- /dev/null
+++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 8"
+ date="2011-05-14T10:02:26Z"
+ content="""
+@joey thanks for the update in the previous comment, I had forgotten about updating it.
+
+@zooko it's working okay for me right now, since I'm only putting fairly big blogs on stuff on to it and only things that I *really* care about. On the performance side, if it ran faster then it would be nicer :)
+"""]]

From 12ecf6c0c624ecb883140b456e9933fd3cd4a5b4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 14 May 2011 10:04:37 +0000
Subject: [PATCH 1689/2835] Added a comment

---
 .../comment_1_e6efbb35f61ee521b473a92674036788._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment

diff --git a/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment b/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment
new file mode 100644
index 0000000000..98a998c1cf
--- /dev/null
+++ b/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-05-14T10:04:36Z"
+ content="""
+I don't suppose this SWIFT api is compatible with the eucalytpus walrus api ?
+"""]]

From d96749588b65ef3f28775d140bb9e8b67cde90cc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 14 May 2011 10:23:44 +0000
Subject: [PATCH 1690/2835] mention that tahoe-lafs has limited testing as a
 special remote (me being the only user isn't very good testing)

---
 doc/special_remotes.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 7d55a88612..7e4b9d377b 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -11,6 +11,7 @@ They cannot be used by other git commands though.
 * [[directory]]
 * [[rsync]]
 * [[hook]]
+  * [tahoe-lafs][] - limited testing
 
 ## Unused content on special remotes
 
@@ -34,3 +35,5 @@ on special remotes, instead use `git annex unused --from`. Example:
 Do be cautious when using this; it cannot detect if content in a remote
 is used by that remote, or is the last copy of data that is used by
 some *other* remote.
+
+[tahoe-lafs]: http://git-annex.branchable.com/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/

From 8c152836ab2acc6142c544330823b79a214bb32c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 14 May 2011 15:00:51 +0000
Subject: [PATCH 1691/2835] Added a comment

---
 .../comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment

diff --git a/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment b/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment
new file mode 100644
index 0000000000..97863b095f
--- /dev/null
+++ b/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-05-14T15:00:51Z"
+ content="""
+It does offer a S3 compability layer, but that is de facto non-functioning as of right now.
+"""]]

From 3623c17a7ef3cc15bfd5ab7f0e13c10887bd4e9f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 May 2011 11:57:40 -0400
Subject: [PATCH 1692/2835] use wikilink

---
 doc/special_remotes.mdwn | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 7e4b9d377b..13c18122e7 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -11,7 +11,7 @@ They cannot be used by other git commands though.
 * [[directory]]
 * [[rsync]]
 * [[hook]]
-  * [tahoe-lafs][] - limited testing
+* [[tahoe-lafs|forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] - limited testing
 
 ## Unused content on special remotes
 
@@ -35,5 +35,3 @@ on special remotes, instead use `git annex unused --from`. Example:
 Do be cautious when using this; it cannot detect if content in a remote
 is used by that remote, or is the last copy of data that is used by
 some *other* remote.
-
-[tahoe-lafs]: http://git-annex.branchable.com/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/

From 56182e2609aee14c2e743e8b83310899efbbecb0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 15:58:57 +0000
Subject: [PATCH 1693/2835] Comment moderation

---
 ...ment_2_c2b0ce025805b774dc77ce264a222824._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment

diff --git a/doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment b/doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment
new file mode 100644
index 0000000000..21f9d713cf
--- /dev/null
+++ b/doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="format, respect working directory"
+ date="2011-04-26T12:31:02Z"
+ content="""
+we could include the information about the current directory as well, if the command is not issued in the local git root directory. to avoid large numbers of similar lines, that could look like this:
+
+    Estimated annex size: B MiB (of C MiB; [B/C]%)
+    Estimated annex size in $PWD: B' MiB (of C' MiB; [B'/C']%)
+
+with the percentages being replaced with \"complete\" if really all files are present (and not just many enough for the value to be rounded to 100%).
+"""]]

From 81bca602dacfb038a5bb1f66146dbc85ba184223 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 16:13:58 +0000
Subject: [PATCH 1694/2835] Added a comment

---
 ...mment_4_d7112c315fb016a8a399e24e9b6461d8._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment b/doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
new file mode 100644
index 0000000000..1fb19ab192
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-05-14T16:13:58Z"
+ content="""
+Hmm. Old versions may have forgotten to git add a .git-annex location log file when recovering content with fsck. That could be another reason things are out of sync.
+
+But I'm not clear on which repo is trying to copy files to which.
+
+(NB: If the files were recovered on a bare git repo, fsck cannot update the location log there, which could also explain this.)
+"""]]

From 60a1aa418ce68b573ed2aebb808c4186add4768a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 May 2011 12:23:26 -0400
Subject: [PATCH 1695/2835] typo

---
 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn
index c447f75f0c..118f6fbb7c 100644
--- a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn
@@ -16,4 +16,4 @@ make: *** [Touch.hs] Error 1
 
 I dug around the OSX documentation and fcntl.h header file and it seems that UTIME_OMIT, UTIME_NOW, AT_FDCWD and AT_SYMLINK_NOFOLLOW aren't defined (at least on OSX). I suspect the BSD's in general will have problems compiling git-annex.
 
-[[!meta title="annexed symlink mtime matching code is disabled on non-linux systems; needs testing"]
+[[!meta title="annexed symlink mtime matching code is disabled on non-linux systems; needs testing"]]

From 4ff8b30d2c8390bfca8b345b3dd7c300477e95f6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 May 2011 12:26:06 -0400
Subject: [PATCH 1696/2835] layout

---
 doc/git-annex-shell.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 060d23ca18..1fc9647c88 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -42,7 +42,7 @@ Any command not listed below is passed through to git-shell.
 
   This runs rsync in server mode to transfer out the content of a key.
 
-## OPTIONS
+# OPTIONS
 
 Same as git-annex or git-shell, depending on the command being run.
 

From fdc5e5f56c6fca3d4a1f72bd5338b5d2ec9ab10c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 16:28:36 +0000
Subject: [PATCH 1697/2835] Added a comment

---
 ...ent_3_be62be5fe819acc0cb8b878802decd46._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment

diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment
new file mode 100644
index 0000000000..9c56452e53
--- /dev/null
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-05-14T16:28:36Z"
+ content="""
+To re-inject new content for a file, you really want to get a new key for the file. Otherwise, other repos that have the old file will never get the new content. So:
+
+
+git rm file
+mv ~/newcontent file
+git annex add file
+

+"""]]

From 020e9b013dea9e7429cee49cb19a5b7bc958a19f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 May 2011 12:28:42 -0400
Subject: [PATCH 1698/2835] close

---
 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn
index ced4fc5a0f..8df3bde481 100644
--- a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn
@@ -8,3 +8,5 @@ Then, I copied the annex object to the correct place in .git/annex/objects/...,
 
 
 Long story short, I think there should be a `git annex reinject $file` or similar which will take a file, either one replacing the symlink or with an arbitrary path, and put it into the correct place in the object store. Called normally, it should reject all reinjects where the checksum does not match. With --force, this should be overridden. For reasons of safety, WORM should always require --force.
+
+> [[closing|done]], seems addressed --[[Joey]] 

From a43a2731764e59e743fce4a75a5a416f24697754 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 16:29:35 +0000
Subject: [PATCH 1699/2835] Added a comment

---
 .../comment_4_480a4f72445a636eab1b1c0f816d365c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment

diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
new file mode 100644
index 0000000000..4a22d414b2
--- /dev/null
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-05-14T16:29:35Z"
+ content="""
+Although, if you really do want to shoot yourself in the foot, or know you have the old content, you can use `git-annex setkey`.
+"""]]

From 9a42ab53bef804f7e13ecfbbe3eba6ca3ec5eac1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 14 May 2011 19:03:44 +0000
Subject: [PATCH 1700/2835] Added a comment

---
 ...ent_5_4ea29a6f8152eddf806c536de33ef162._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment b/doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
new file mode 100644
index 0000000000..0a546bd88c
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 5"
+ date="2011-05-14T19:03:43Z"
+ content="""
+Version: 0.20110503
+
+My local non-bare repo is copying to a remote bare repo.
+
+I have been recovering in a non-bare repo.
+
+If there is anything I can send you to help... If I removed said files and went through http://git-annex.branchable.com/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/ -- would that help?
+"""]]

From 385268da2ea0fb66d6cda5be6d0b643db9e4f706 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 19:23:45 +0000
Subject: [PATCH 1701/2835] Added a comment

---
 .../comment_6_0d85f114a103bd6532a3b3b24466012e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment b/doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
new file mode 100644
index 0000000000..1e3f325319
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-05-14T19:23:45Z"
+ content="""
+Well, focus on a specific file that exhibits the problem. What does `git annex whereis` say about it? Is the content actually present in annex/objects/ on the bare repository? Does that contradict whereis?
+"""]]

From a9ba88dfbd0cef84fd5bdc438d1c89f16f23b89c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 14 May 2011 23:13:15 +0000
Subject: [PATCH 1702/2835] Added a comment

---
 ...mment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment b/doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
new file mode 100644
index 0000000000..f7dfad68ca
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 7"
+ date="2011-05-14T23:13:15Z"
+ content="""
+It exists locally, whereis tells me it exists locally and locally, only.
+
+The object is _not_ in the bare repo.
+
+The file _might_ have gone missing before I upgraded my annex backend version to 2. Could this be a factor?
+"""]]

From 1b2e0c183df47456e7225198cc7ebed1b74d5b87 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 15 May 2011 00:09:34 +0000
Subject: [PATCH 1703/2835] Added a comment

---
 ...comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment b/doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
new file mode 100644
index 0000000000..01248914c3
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-05-15T00:09:34Z"
+ content="""
+What you're describing should be impossible; the error message shown can only occur if the object is present in the annex where `git-annex-shell recvkey` is run. So something strange is going on.
+
+Try reproducing it by running on the remote system, `git-annex-shell recvkey /remote/repo.git $key` .. if you can reproduce it, I guess the next thing to do will be to strace the command and see why it's thinking the object is there.
+"""]]

From b400984ddf9aeb24c3d67e87cfeb29470618636c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 01:33:25 -0400
Subject: [PATCH 1704/2835] fixup

---
 test.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index 4e0eb3fb2a..73eff662b9 100644
--- a/test.hs
+++ b/test.hs
@@ -466,7 +466,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	where
 		checkunused expectedkeys = do
 			git_annex "unused" ["-q"] @? "unused failed"
-			unusedmap <- annexeval $ Command.DropUnused.readUnusedLog
+			unusedmap <- annexeval $ Command.DropUnused.readUnusedLog ""
 			let unusedkeys = M.elems unusedmap
 			assertEqual "unused keys differ"
 				(sort expectedkeys) (sort unusedkeys)

From 56bc3e95cabb85e5f23e30b453f90438c33efbb8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 02:02:46 -0400
Subject: [PATCH 1705/2835] refactor some boilerplate

---
 Command.hs            | 10 ++++++++++
 Command/Add.hs        |  8 ++++----
 Command/ConfigList.hs |  2 +-
 Command/Describe.hs   |  4 ++--
 Command/Drop.hs       | 12 ++++++------
 Command/DropKey.hs    |  7 +++----
 Command/DropUnused.hs |  8 ++++----
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  6 +++---
 Command/FromKey.hs    |  4 ++--
 Command/Fsck.hs       |  6 +++---
 Command/Get.hs        |  9 ++++-----
 Command/InAnnex.hs    |  2 +-
 Command/Init.hs       |  6 +++---
 Command/InitRemote.hs |  4 ++--
 Command/Lock.hs       |  4 ++--
 Command/Map.hs        |  2 +-
 Command/Migrate.hs    | 11 +++++------
 Command/Move.hs       | 22 +++++++++++-----------
 Command/PreCommit.hs  |  4 ++--
 Command/Semitrust.hs  |  4 ++--
 Command/SetKey.hs     |  4 ++--
 Command/Trust.hs      |  4 ++--
 Command/Unannex.hs    |  8 ++++----
 Command/Uninit.hs     |  4 ++--
 Command/Unlock.hs     |  4 ++--
 Command/Untrust.hs    |  4 ++--
 Command/Unused.hs     |  4 ++--
 Command/Upgrade.hs    |  2 +-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  6 +++---
 31 files changed, 93 insertions(+), 86 deletions(-)

diff --git a/Command.hs b/Command.hs
index 9c908b8004..d9e94a2f36 100644
--- a/Command.hs
+++ b/Command.hs
@@ -65,12 +65,22 @@ data Command = Command {
 	cmdusesrepo :: Bool
 }
 
+{- Most commands operate on files in a git repo. -}
 repoCommand :: String -> String -> [CommandSeek] -> String -> Command
 repoCommand n p s d = Command n p s d True
 
+{- Others can run anywhere. -}
 standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command
 standaloneCommand n p s d = Command n p s d False
 
+{- For start and perform stages to indicate what step to run next. -}
+next :: a -> Annex (Maybe a)
+next a = return $ Just a
+
+{- Or to indicate nothing needs to be done. -}
+stop :: Annex (Maybe a)
+stop = return Nothing
+
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
 prepCommand :: Command -> [String] -> Annex [Annex Bool]
diff --git a/Command/Add.hs b/Command/Add.hs
index b532ab045d..29a1518e84 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -34,19 +34,19 @@ start :: CommandStartBackendFile
 start pair@(file, _) = notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
 	if (isSymbolicLink s) || (not $ isRegularFile s)
-		then return Nothing
+		then stop
 		else do
 			showStart "add" file
-			return $ Just $ perform pair
+			next $ perform pair
 
 perform :: BackendFile -> CommandPerform
 perform (file, backend) = do
 	stored <- Backend.storeFileKey file backend
 	case stored of
-		Nothing -> return Nothing
+		Nothing -> stop
 		Just (key, _) -> do
 			moveAnnex key file
-			return $ Just $ cleanup file key
+			next $ cleanup file key
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 476d73cfbb..d8dbff03af 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -25,4 +25,4 @@ start = do
 	g <- Annex.gitRepo
 	u <- getUUID g
 	liftIO $ putStrLn $ "annex.uuid=" ++ u
-	return Nothing
+	stop
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 9e98a81437..dcabef7fbf 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -29,9 +29,9 @@ start params = notBareRepo $ do
 	
 	showStart "describe" name
 	u <- Remote.nameToUUID name
-	return $ Just $ perform u description
+	next $ perform u description
 
 perform :: UUID -> String -> CommandPerform
 perform u description = do
 	describeUUID u description
-	return $ Just $ Command.Init.cleanup
+	next $ Command.Init.cleanup
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 52b724e62b..05c956fddf 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -29,11 +29,11 @@ seek = [withAttrFilesInGit "annex.numcopies" start]
 start :: CommandStartAttrFile
 start (file, attr) = isAnnexed file $ \(key, backend) -> do
 	inbackend <- Backend.hasKey key
-	if not inbackend
-		then return Nothing
-		else do
+	if inbackend
+		then do
 			showStart "drop" file
-			return $ Just $ perform key backend numcopies
+			next $ perform key backend numcopies
+		else stop
 	where
 		numcopies = readMaybe attr :: Maybe Int
 
@@ -41,8 +41,8 @@ perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform
 perform key backend numcopies = do
 	success <- Backend.removeKey backend key numcopies
 	if success
-		then return $ Just $ cleanup key
-		else return Nothing
+		then next $ cleanup key
+		else stop
 
 cleanup :: Key -> CommandCleanup
 cleanup key = do
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 4c6f1ab2e1..780fe0adf0 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -26,20 +26,19 @@ start key = do
 	present <- inAnnex key
 	force <- Annex.getState Annex.force
 	if not present
-		then return Nothing
+		then stop
 		else if not force
 			then error "dropkey is can cause data loss; use --force if you're sure you want to do this"
 			else do
 				showStart "dropkey" (show key)
-				return $ Just $ perform key
+				next $ perform key
 
 perform :: Key -> CommandPerform
 perform key = do
 	removeAnnex key
-	return $ Just $ cleanup key
+	next $ cleanup key
 
 cleanup :: Key -> CommandCleanup
 cleanup key = do
 	logStatus key ValueMissing
 	return True
-
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index b129235e1d..861c78c905 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -49,13 +49,13 @@ start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
 	, (unusedtmp, performOther gitAnnexTmpLocation)
 	]
 	where
-		search [] = return Nothing
+		search [] = stop
 		search ((m, a):rest) = do
 			case M.lookup s m of
 				Nothing -> search rest
 				Just key -> do
 					showStart "dropunused" s
-					return $ Just $ a key
+					next $ a key
 
 perform :: Key -> CommandPerform
 perform key = do
@@ -64,7 +64,7 @@ perform key = do
 		Just name -> do
 			r <- Remote.byName name
 			showNote $ "from " ++ Remote.name r ++ "..."
-			return $ Just $ Command.Move.fromCleanup r True key
+			next $ Command.Move.fromCleanup r True key
 		_ -> do
 			backend <- keyBackend key
 			Command.Drop.perform key backend (Just 0) -- force drop
@@ -75,7 +75,7 @@ performOther filespec key = do
 	let f = filespec g key
 	e <- liftIO $ doesFileExist f
 	when e $ liftIO $ removeFile f
-	return $ Just $ return True
+	next $ return True
 
 readUnusedLog :: FilePath -> Annex UnusedMap
 readUnusedLog prefix = do
diff --git a/Command/Find.hs b/Command/Find.hs
index 6a6ae29787..eecf3cd7da 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -25,4 +25,4 @@ start :: CommandStartString
 start file = isAnnexed file $ \(key, _) -> do
 	exists <- inAnnex key
 	when exists $ liftIO $ putStrLn file
-	return Nothing
+	stop
diff --git a/Command/Fix.hs b/Command/Fix.hs
index d898ce517d..60627e9df0 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -30,17 +30,17 @@ start file = isAnnexed file $ \(key, _) -> do
 	link <- calcGitLink file key
 	l <- liftIO $ readSymbolicLink file
 	if link == l
-		then return Nothing
+		then stop
 		else do
 			showStart "fix" file
-			return $ Just $ perform file link
+			next $ perform file link
 
 perform :: FilePath -> FilePath -> CommandPerform
 perform file link = do
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 	liftIO $ removeFile file
 	liftIO $ createSymbolicLink link file
-	return $ Just $ cleanup file
+	next $ cleanup file
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index eadaa13e1f..ca61094eb4 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -34,7 +34,7 @@ start file = notBareRepo $ do
 	unless inbackend $ error $
 		"key ("++keyName key++") is not present in backend"
 	showStart "fromkey" file
-	return $ Just $ perform file
+	next $ perform file
 
 perform :: FilePath -> CommandPerform
 perform file = do
@@ -42,7 +42,7 @@ perform file = do
 	link <- calcGitLink file key
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 	liftIO $ createSymbolicLink link file
-	return $ Just $ cleanup file
+	next $ cleanup file
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 20ef2c8083..adfd702de7 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -31,7 +31,7 @@ seek = [withAttrFilesInGit "annex.numcopies" start]
 start :: CommandStartAttrFile
 start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
-	return $ Just $ perform key file backend numcopies
+	next $ perform key file backend numcopies
 	where
 		numcopies = readMaybe attr :: Maybe Int
 
@@ -42,8 +42,8 @@ perform key file backend numcopies = do
 	locationlogok <- verifyLocationLog key file
 	backendok <- Backend.fsckKey backend key (Just file) numcopies
 	if locationlogok && backendok
-		then return $ Just $ return True
-		else return Nothing
+		then next $ return True
+		else stop
 
 {- Checks that the location log reflects the current status of the key,
    in this repository only. -}
diff --git a/Command/Get.hs b/Command/Get.hs
index 0463dccb05..90c0540960 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -25,15 +25,14 @@ start :: CommandStartString
 start file = isAnnexed file $ \(key, backend) -> do
 	inannex <- inAnnex key
 	if inannex
-		then return Nothing
+		then stop
 		else do
 			showStart "get" file
-			return $ Just $ perform key backend
+			next $ perform key backend
 
 perform :: Key -> Backend Annex -> CommandPerform
 perform key backend = do
 	ok <- getViaTmp key (Backend.retrieveKeyFile backend key)
 	if ok
-		then return $ Just $ return True -- no cleanup needed
-		else return Nothing
-
+		then next $ return True -- no cleanup needed
+		else stop
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index a7e2ecff60..b5b59ccf7d 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -24,5 +24,5 @@ start :: CommandStartKey
 start key = do
 	present <- inAnnex key
 	if present
-		then return Nothing
+		then stop
 		else liftIO $ exitFailure
diff --git a/Command/Init.hs b/Command/Init.hs
index cca2e8faef..668b5c87d6 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -35,7 +35,7 @@ start description = do
 	when (null description) $
 		error "please specify a description of this repository\n"
 	showStart "init" description
-	return $ Just $ perform description
+	next $ perform description
 
 perform :: String -> CommandPerform
 perform description = do
@@ -48,12 +48,12 @@ perform description = do
 				"This is a bare repository, so its description cannot be committed.\n" ++
 				"To record the description, run this command in a clone of this repository:\n" ++
 				"   git annex describe " ++ show u ++ " " ++ show description ++ "\n\n"
-			return $ Just $ return True
+			next $ return True
 		else do
 			describeUUID u description
 			liftIO $ gitAttributesWrite g
 			gitPreCommitHookWrite g
-			return $ Just cleanup
+			next cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 4c2fc3a078..eda50ee5de 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -39,7 +39,7 @@ start params = notBareRepo $ do
 	t <- findType fullconfig
 	
 	showStart "initremote" name
-	return $ Just $ perform t u $ M.union config c
+	next $ perform t u $ M.union config c
 
 	where
 		ws = words params
@@ -49,7 +49,7 @@ start params = notBareRepo $ do
 perform :: RemoteClass.RemoteType Annex -> UUID -> RemoteClass.RemoteConfig -> CommandPerform
 perform t u c = do
 	c' <- RemoteClass.setup t u c
-	return $ Just $ cleanup u c'
+	next $ cleanup u c'
 
 cleanup :: UUID -> RemoteClass.RemoteConfig -> CommandCleanup
 cleanup u c = do
diff --git a/Command/Lock.hs b/Command/Lock.hs
index cdbc560194..1ae4882272 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -26,7 +26,7 @@ seek = [withFilesUnlocked start]
 start :: CommandStartBackendFile
 start (file, _) = do
 	showStart "lock" file
-	return $ Just $ perform file
+	next $ perform file
 
 perform :: FilePath -> CommandPerform
 perform file = do
@@ -36,4 +36,4 @@ perform file = do
 	liftIO $ Git.run g "reset" [Params "-q --", File file]
 	-- checkout the symlink
 	liftIO $ Git.run g "checkout" [Param "--", File file]
-	return $ Just $ return True -- no cleanup needed
+	next $ return True -- no cleanup needed
diff --git a/Command/Map.hs b/Command/Map.hs
index 2325c87e14..3c94fc75b5 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -45,7 +45,7 @@ start = do
 	showLongNote $ "running: dot -Tx11 " ++ file
 	showProgress
 	r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file]
-	return $ Just $ return $ Just $ return r
+	next $ next $ return r
 	where
 		file = "map.dot"
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 0d21fcbdf9..35855d5270 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -35,9 +35,8 @@ start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	if (newbackend /= oldbackend || upgradable) && exists
 		then do
 			showStart "migrate" file
-			return $ Just $ perform file key newbackend
-		else
-			return Nothing
+			next $ perform file key newbackend
+		else stop
 	where
 		choosebackend Nothing = do
 			backends <- Backend.list
@@ -55,7 +54,7 @@ perform file oldkey newbackend = do
 	let src = gitAnnexLocation g oldkey
 	stored <- Backend.storeFileKey src $ Just newbackend
 	case stored of
-		Nothing -> return Nothing
+		Nothing -> stop
 		Just (newkey, _) -> do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
@@ -68,5 +67,5 @@ perform file oldkey newbackend = do
 				then do
 					-- Update symlink to use the new key.
 					liftIO $ removeFile file
-					return $ Just $ Command.Add.cleanup file newkey
-				else return Nothing
+					next $ Command.Add.cleanup file newkey
+				else stop
diff --git a/Command/Move.hs b/Command/Move.hs
index 476bf866a0..623003e47a 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -73,10 +73,10 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 	u <- getUUID g
 	ishere <- inAnnex key
 	if not ishere || u == Remote.uuid dest
-		then return Nothing -- not here, so nothing to do
+		then stop -- not here, so nothing to do
 		else do
 			showAction move file
-			return $ Just $ toPerform dest move key
+			next $ toPerform dest move key
 toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
 	-- Checking the remote is expensive, so not done in the start step.
@@ -92,14 +92,14 @@ toPerform dest move key = do
 	case isthere of
 		Left err -> do
 			showNote $ show err
-			return Nothing
+			stop
 		Right False -> do
 			showNote $ "to " ++ Remote.name dest ++ "..."
 			ok <- Remote.storeKey dest key
 			if ok
-				then return $ Just $ toCleanup dest move key
-				else return Nothing -- failed
-		Right True -> return $ Just $ toCleanup dest move key
+				then next $ toCleanup dest move key
+				else stop -- failed
+		Right True -> next $ toCleanup dest move key
 toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 toCleanup dest move key = do
 	remoteHasKey dest key True
@@ -119,21 +119,21 @@ fromStart src move file = isAnnexed file $ \(key, _) -> do
 	u <- getUUID g
 	(remotes, _) <- Remote.keyPossibilities key
 	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
-		then return Nothing
+		then stop
 		else do
 			showAction move file
-			return $ Just $ fromPerform src move key
+			next $ fromPerform src move key
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = do
 	ishere <- inAnnex key
 	if ishere
-		then return $ Just $ fromCleanup src move key
+		then next $ fromCleanup src move key
 		else do
 			showNote $ "from " ++ Remote.name src ++ "..."
 			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
 			if ok
-				then return $ Just $ fromCleanup src move key
-				else return Nothing -- fail
+				then next $ fromCleanup src move key
+				else stop -- fail
 fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 fromCleanup src True key = do
 	ok <- Remote.removeKey src key
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 1db40f75fa..d7f2487137 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -27,13 +27,13 @@ seek = [withFilesToBeCommitted Command.Fix.start,
 	withFilesUnlockedToBeCommitted start]
 
 start :: CommandStartBackendFile
-start pair = return $ Just $ perform pair
+start pair = next $ perform pair
 
 perform :: BackendFile -> CommandPerform
 perform pair@(file, _) = do
 	ok <- doCommand $ Command.Add.start pair
 	if ok
-		then return $ Just $ cleanup file
+		then next $ cleanup file
 		else error $ "failed to add " ++ file ++ "; canceling commit"
 
 cleanup :: FilePath -> CommandCleanup
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index e64d418f83..fc1bcbbcdc 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -24,9 +24,9 @@ start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "semitrust" name
 	u <- Remote.nameToUUID name
-	return $ Just $ perform u
+	next $ perform u
 
 perform :: UUID -> CommandPerform
 perform uuid = do
 	trustSet uuid SemiTrusted
-	return $ Just $ return True
+	next $ return True
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 6f6078e4ba..dbad148b25 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -26,7 +26,7 @@ seek = [withTempFile start]
 start :: CommandStartString
 start file = do
 	showStart "setkey" file
-	return $ Just $ perform file
+	next $ perform file
 
 perform :: FilePath -> CommandPerform
 perform file = do
@@ -40,7 +40,7 @@ perform file = do
 				boolSystem "mv" [File file, File dest]
 			else return True
 	if ok
-		then return $ Just $ cleanup
+		then next cleanup
 		else error "mv failed!"
 
 cleanup :: CommandCleanup
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 05505cd045..ef03828c23 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -24,9 +24,9 @@ start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "trust" name
 	u <- Remote.nameToUUID name
-	return $ Just $ perform u
+	next $ perform u
 
 perform :: UUID -> CommandPerform
 perform uuid = do
 	trustSet uuid Trusted
-	return $ Just $ return True
+	next $ return True
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 94db500c68..0a5381d562 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -43,16 +43,16 @@ start file = isAnnexed file $ \(key, backend) -> do
 				Annex.changeState $ \s -> s { Annex.force = True }
 
 			showStart "unannex" file
-			return $ Just $ perform file key backend
-		else return Nothing
+			next $ perform file key backend
+		else stop
 
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file key backend = do
 	-- force backend to always remove
 	ok <- Backend.removeKey backend key (Just 0)
 	if ok
-		then return $ Just $ cleanup file key
-		else return Nothing
+		then next $ cleanup file key
+		else stop
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index ee0cbde6b3..d3d7ac3398 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -30,7 +30,7 @@ seek = [withFilesInGit Command.Unannex.start, withNothing start]
 start :: CommandStartNothing
 start = do
 	showStart "uninit" ""
-	return $ Just $ perform
+	next perform
 
 perform :: CommandPerform
 perform = do
@@ -39,7 +39,7 @@ perform = do
 	gitPreCommitHookUnWrite g
 	liftIO $ gitAttributesUnWrite g
 
-	return $ Just $ return True
+	next $ return True
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index bf593e1e99..d65579ec73 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -34,7 +34,7 @@ seek = [withFilesInGit start]
 start :: CommandStartString
 start file = isAnnexed file $ \(key, _) -> do
 	showStart "unlock" file
-	return $ Just $ perform file key
+	next $ perform file key
 
 perform :: FilePath -> Key -> CommandPerform
 perform dest key = do
@@ -52,5 +52,5 @@ perform dest key = do
         if ok
                 then do
 			liftIO $ allowWrite dest
-			return $ Just $ return True
+			next $ return True
                 else error "copy failed!"
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 311ec6eeb7..ebe9c31b3b 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -24,9 +24,9 @@ start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "untrust" name
 	u <- Remote.nameToUUID name
-	return $ Just $ perform u
+	next $ perform u
 
 perform :: UUID -> CommandPerform
 perform uuid = do
 	trustSet uuid UnTrusted
-	return $ Just $ return True
+	next $ return True
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 67f10581d6..7570dfe90a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -37,7 +37,7 @@ seek = [withNothing start]
 start :: CommandStartNothing
 start = notBareRepo $ do
 	showStart "unused" ""
-	return $ Just perform
+	next perform
 
 perform :: CommandPerform
 perform = do
@@ -47,7 +47,7 @@ perform = do
 			r <- Remote.byName name
 			checkRemoteUnused r
 		_ -> checkUnused
-	return $ Just $ return True
+	next $ return True
 
 checkUnused :: Annex ()
 checkUnused = do
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 880a5324f7..b3c0468039 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -24,4 +24,4 @@ start = do
 	showStart "upgrade" ""
 	r <- upgrade
 	checkVersion
-	return $ Just $ return $ Just $ return r
+	next $ next $ return r
diff --git a/Command/Version.hs b/Command/Version.hs
index 2b294c80be..755b95acca 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -28,6 +28,6 @@ start = do
 	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion
 	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions
 	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
-	return Nothing
+	stop
 	where
 		vs l = join " " l
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 599df44676..2e0fa15f6f 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -26,7 +26,7 @@ seek = [withFilesInGit start]
 start :: CommandStartString
 start file = isAnnexed file $ \(key, _) -> do
 	showStart "whereis" file
-	return $ Just $ perform key
+	next $ perform key
 
 perform :: Key -> CommandPerform
 perform key = do
@@ -35,12 +35,12 @@ perform key = do
 	let num = length uuids
 	showNote $ show num ++ " " ++ copiesplural num
 	if null $ uuids
-		then return Nothing
+		then stop
 		else do
 			pp <- prettyPrintUUIDs uuids
 			showLongNote $ pp
 			showProgress	
-			return $ Just $ return True
+			next $ return True
 	where
 		copiesplural 1 = "copy"
 		copiesplural _ = "copies"

From efa7f544050c0d5be6bc1b0fc0125278e475c213 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 02:12:17 -0400
Subject: [PATCH 1706/2835] cleanup

---
 Command.hs | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/Command.hs b/Command.hs
index d9e94a2f36..0e3958c180 100644
--- a/Command.hs
+++ b/Command.hs
@@ -58,20 +58,20 @@ type CommandSeekNothing = CommandStart -> CommandSeek
 type CommandStartNothing = CommandStart
 
 data Command = Command {
+	cmdusesrepo :: Bool,
 	cmdname :: String,
 	cmdparams :: String,
 	cmdseek :: [CommandSeek],
-	cmddesc :: String,
-	cmdusesrepo :: Bool
+	cmddesc :: String
 }
 
 {- Most commands operate on files in a git repo. -}
 repoCommand :: String -> String -> [CommandSeek] -> String -> Command
-repoCommand n p s d = Command n p s d True
+repoCommand = Command True
 
 {- Others can run anywhere. -}
 standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command
-standaloneCommand n p s d = Command n p s d False
+standaloneCommand = Command False
 
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
@@ -102,13 +102,8 @@ doCommand start = do
 					return False
 				Just cleanup -> do
 					c <- cleanup
-					if c
-						then do
-							showEndOk
-							return True
-						else do
-							showEndFail
-							return False
+					if c then showEndOk else showEndFail
+					return c
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = do

From cad0e1c8b7eb21f8dceca8dd9fa3bc1d1aa7eabd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 02:49:43 -0400
Subject: [PATCH 1707/2835] simplified a bunch of Maybe handling

---
 Backend.hs            | 29 ++++++++++-------------------
 Base64.hs             |  6 ++----
 Command.hs            | 26 +++++++-------------------
 Command/DropUnused.hs |  9 ++++-----
 Command/InitRemote.hs | 17 +++++++++--------
 Command/Map.hs        | 16 +++++-----------
 Command/Unused.hs     | 14 ++++++--------
 Content.hs            | 10 ++++------
 Crypto.hs             |  6 ++----
 Dot.hs                |  5 +----
 GitRepo.hs            | 13 ++++---------
 LocationLog.hs        |  4 +---
 Remote/Bup.hs         |  5 ++---
 Remote/Directory.hs   |  5 ++---
 Remote/Encryptable.hs |  9 ++++-----
 Remote/Hook.hs        | 15 ++++++---------
 Remote/Rsync.hs       |  5 ++---
 Remote/S3real.hs      | 16 ++++++----------
 Utility.hs            | 11 ++++-------
 19 files changed, 81 insertions(+), 140 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index aec87ce665..6140664cee 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -76,10 +76,9 @@ list = do
 
 {- Looks up a backend in a list. May fail if unknown. -}
 lookupBackendName :: [Backend Annex] -> String -> Backend Annex
-lookupBackendName bs s =
-	case maybeLookupBackendName bs s of
-		Just b -> b
-		Nothing -> error $ "unknown backend " ++ s
+lookupBackendName bs s = maybe unknown id $ maybeLookupBackendName bs s
+	where
+		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: [Backend Annex] -> String -> Maybe (Backend Annex)
 maybeLookupBackendName bs s =
 	if 1 /= length matches
@@ -91,23 +90,18 @@ maybeLookupBackendName bs s =
 storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
 storeFileKey file trybackend = do
 	bs <- list
-	let bs' = case trybackend of
-		Nothing -> bs
-		Just backend -> backend:bs
+	let bs' = maybe bs (:bs) trybackend
 	storeFileKey' bs' file
 storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
 storeFileKey' [] _ = return Nothing
-storeFileKey' (b:bs) file = do
-	result <- (B.getKey b) file
-	case result of
-		Nothing -> nextbackend
-		Just key -> do
+storeFileKey' (b:bs) file = maybe nextbackend store =<< (B.getKey b) file
+	where
+		nextbackend = storeFileKey' bs file
+		store key = do
 			stored <- (B.storeFileKey b) file key
 			if (not stored)
 				then nextbackend
 				else return $ Just (key, b)
-	where
-		nextbackend = storeFileKey' bs file
 
 {- Attempts to retrieve an key from one of the backends, saving it to
  - a specified location. -}
@@ -148,11 +142,8 @@ lookupFile file = do
 		getsymlink = do
 			l <- readSymbolicLink file
 			return $ takeFileName l
-		makekey bs l =
-			case fileKey l of
-				Just k -> makeret k l bs
-				Nothing -> return Nothing
-		makeret k l bs =
+		makekey bs l = maybe (return Nothing) (makeret bs l) (fileKey l)
+		makeret bs l k =
 			case maybeLookupBackendName bs bname of
 					Just backend -> return $ Just (k, backend)
 					Nothing -> do
diff --git a/Base64.hs b/Base64.hs
index cc6346b415..153049751d 100644
--- a/Base64.hs
+++ b/Base64.hs
@@ -14,7 +14,5 @@ toB64 :: String -> String
 toB64 = encode . s2w8
 
 fromB64 :: String -> String
-fromB64 s =
-	case decode s of
-		Nothing -> error "bad base64 encoded data"
-		Just ws -> w82s ws
+fromB64 s = maybe bad w82s $ decode s
+	where bad = error "bad base64 encoded data"
diff --git a/Command.hs b/Command.hs
index 0e3958c180..c6c1fe5c55 100644
--- a/Command.hs
+++ b/Command.hs
@@ -14,6 +14,7 @@ import Control.Monad (filterM, liftM, when)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
+import Data.Maybe
 
 import Types
 import qualified Backend
@@ -106,18 +107,10 @@ doCommand start = do
 					return c
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
-notAnnexed file a = do
-	r <- Backend.lookupFile file
-	case r of
-		Just _ -> return Nothing
-		Nothing -> a
+notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
 
 isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
-isAnnexed file a = do
-	r <- Backend.lookupFile file
-	case r of
-		Just v -> a v
-		Nothing -> return Nothing
+isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
@@ -183,9 +176,7 @@ withFilesUnlocked' typechanged a params = do
 withKeys :: CommandSeekKeys
 withKeys a params = return $ map a $ map parse params
 	where
-		parse p = case readKey p of
-			Just k -> k
-			Nothing -> error "bad key"
+		parse p = maybe (error "bad key") id $ readKey p
 withTempFile :: CommandSeekStrings
 withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
@@ -206,9 +197,7 @@ filterFiles l = do
 		else return $ filter (notExcluded $ wildsRegex exclude) l'
 	where
 		notState f = not $ stateDir `isPrefixOf` f
-		notExcluded r f = case match r f [] of
-			Nothing -> True
-			Just _ -> False
+		notExcluded r f = isJust $ match r f []
 
 wildsRegex :: [String] -> Regex
 wildsRegex ws = compile regex []
@@ -257,11 +246,10 @@ cmdlineKey  = do
 	case k of
 		Nothing -> nokey
 		Just "" -> nokey
-		Just kstring -> case readKey kstring of
-			Nothing -> error "bad key"
-			Just key -> return key
+		Just kstring -> maybe badkey return $ readKey kstring
 	where
 		nokey = error "please specify the key with --key"
+		badkey = error "bad key"
 
 {- Given an original list of files, and an expanded list derived from it,
  - ensures that the original list's ordering is preserved. 
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 861c78c905..965a99ed56 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -58,14 +58,13 @@ start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
 					next $ a key
 
 perform :: Key -> CommandPerform
-perform key = do
-	from <- Annex.getState Annex.fromremote
-	case from of
-		Just name -> do
+perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
+	where
+		dropremote name = do
 			r <- Remote.byName name
 			showNote $ "from " ++ Remote.name r ++ "..."
 			next $ Command.Move.fromCleanup r True key
-		_ -> do
+		droplocal = do
 			backend <- keyBackend key
 			Command.Drop.perform key backend (Just 0) -- force drop
 
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index eda50ee5de..261ccdc8b7 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -68,11 +68,11 @@ cleanup u c = do
 findByName :: String -> Annex (UUID, RemoteClass.RemoteConfig)
 findByName name = do
 	m <- Remote.readRemoteLog
-	case findByName' name m of
-		Just i -> return i
-		Nothing -> do
+	maybe generate return $ findByName' name m
+	where
+		generate = do
 			uuid <- liftIO $ genUUID
-			return $ (uuid, M.insert nameKey name M.empty)
+			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID RemoteClass.RemoteConfig -> Maybe (UUID, RemoteClass.RemoteConfig)
 findByName' n m = if null matches then Nothing else Just $ head matches
@@ -86,12 +86,13 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 
 {- find the specified remote type -}
 findType :: RemoteClass.RemoteConfig -> Annex (RemoteClass.RemoteType Annex)
-findType config = 
-	case M.lookup typeKey config of
-		Nothing -> error "Specify the type of remote with type="
-		Just s -> case filter (\i -> RemoteClass.typename i == s) Remote.remoteTypes of
+findType config = maybe unspecified specified $ M.lookup typeKey config
+	where
+		unspecified = error "Specify the type of remote with type="
+		specified s = case filter (findtype s) Remote.remoteTypes of
 			[] -> error $ "Unknown remote type " ++ s
 			(t:_) -> return t
+		findtype s i = RemoteClass.typename i == s
 
 {- The name of a configured remote is stored in its config using this key. -}
 nameKey :: String
diff --git a/Command/Map.hs b/Command/Map.hs
index 3c94fc75b5..7a9121b69b 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -84,10 +84,7 @@ repoName umap r
 	| otherwise = M.findWithDefault fallback repouuid umap
 	where
 		repouuid = getUncachedUUID r
-		fallback =
-			case (Git.repoRemoteName r) of
-				Just n -> n
-				Nothing -> "unknown"
+		fallback = maybe "unknown" id $ Git.repoRemoteName r
 
 {- A unique id for the node for a repo. Uses the annex.uuid if available. -}
 nodeId :: Git.Repo -> String
@@ -121,13 +118,10 @@ edge umap fullinfo from to =
 		{- Only name an edge if the name is different than the name
 		 - that will be used for the destination node, and is
 		 - different from its hostname. (This reduces visual clutter.) -}
-		edgename =
-			case (Git.repoRemoteName to) of
-				Nothing -> Nothing
-				Just n ->
-					if (n == repoName umap fullto || n == hostname fullto)
-						then Nothing
-						else Just n
+		edgename = maybe Nothing calcname $ Git.repoRemoteName to
+		calcname n
+			| n == repoName umap fullto || n == hostname fullto = Nothing
+			| otherwise = Just n
 
 unreachable :: String -> String
 unreachable = Dot.fillColor "red"
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 7570dfe90a..a2e1c86de1 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -41,12 +41,7 @@ start = notBareRepo $ do
 
 perform :: CommandPerform
 perform = do
-	from <- Annex.getState Annex.fromremote
-	case from of
-		Just name -> do
-			r <- Remote.byName name
-			checkRemoteUnused r
-		_ -> checkUnused
+	maybe checkUnused checkRemoteUnused =<< Annex.getState Annex.fromremote
 	next $ return True
 
 checkUnused :: Annex ()
@@ -63,8 +58,11 @@ checkUnused = do
 			writeUnusedFile file unusedlist
 			return $ length l
 
-checkRemoteUnused :: Remote.Remote Annex -> Annex ()
-checkRemoteUnused r = do
+checkRemoteUnused :: String -> Annex ()
+checkRemoteUnused name = checkRemoteUnused' =<< Remote.byName name
+
+checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
+checkRemoteUnused' r = do
 	g <- Annex.gitRepo
 	showNote $ "checking for unused data on " ++ Remote.name r ++ "..."
 	referenced <- getKeysReferenced
diff --git a/Content.hs b/Content.hs
index ade936da37..9040383be9 100644
--- a/Content.hs
+++ b/Content.hs
@@ -57,11 +57,11 @@ calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
 	g <- Annex.gitRepo
 	cwd <- liftIO $ getCurrentDirectory
-	let absfile = case absNormPath cwd file of
-		Just f -> f
-		Nothing -> error $ "unable to normalize " ++ file
+	let absfile = maybe whoops id $ absNormPath cwd file
 	return $ relPathDirToFile (parentDir absfile) 
 			(Git.workTree g)  ".git"  annexLocation key
+	where
+		whoops = error $ "unable to normalize " ++ file
 
 {- Updates the LocationLog when a key's presence changes in the current
  - repository.
@@ -148,9 +148,7 @@ checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- Annex.gitRepo
 	r <- getConfig g "diskreserve" ""
-	let reserve = case readSize dataUnits r of
-		Nothing -> megabyte
-		Just v -> v
+	let reserve = maybe megabyte id $ readSize dataUnits r
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
diff --git a/Crypto.hs b/Crypto.hs
index 53cd48dd59..42f1389507 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -238,10 +238,8 @@ configKeyIds c = do
 		keyIdField s = (split ":" s) !! 4
 
 configGet :: RemoteConfig -> String -> String
-configGet c key =
-	case M.lookup key c of
-		Just v -> v
-		Nothing -> error $ "missing " ++ key ++ " in remote config"
+configGet c key = maybe missing id $ M.lookup key c
+	where missing = error $ "missing " ++ key ++ " in remote config"
 
 hmacWithCipher :: Cipher -> String -> String
 hmacWithCipher c = hmacWithCipher' (cipherHmac c) 
diff --git a/Dot.hs b/Dot.hs
index 592b21f691..deba10201e 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -20,10 +20,7 @@ graphNode nodeid desc = label desc $ quote nodeid
 
 {- an edge between two nodes -}
 graphEdge :: String -> String -> Maybe String -> String
-graphEdge fromid toid desc = indent $
-	case desc of
-		Nothing -> edge
-		Just d -> label d edge
+graphEdge fromid toid desc = indent $ maybe edge (\d -> label d edge) desc
 	where
 		edge = quote fromid ++ " -> " ++ quote toid
 
diff --git a/GitRepo.hs b/GitRepo.hs
index 49024abe0e..b20ff7db3a 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -122,9 +122,8 @@ repoFromUrl url
 	| startswith "file://" url = repoFromAbsPath $ uriPath u
 	| otherwise = return $ newFrom $ Url u
 		where
-			u = case (parseURI url) of
-				Just v -> v
-				Nothing -> error $ "bad url " ++ url
+			u = maybe bad id $ parseURI url
+			bad = error $ "bad url " ++ url
 
 {- Creates a repo that has an unknown location. -}
 repoFromUnknown :: Repo
@@ -264,9 +263,7 @@ workTreeFile repo@(Repo { location = Dir d }) file = do
 		absrepo = case (absNormPath "/" d) of
 			Just f -> addTrailingPathSeparator f
 			Nothing -> error $ "bad repo" ++ repoDescribe repo
-		absfile c = case (secureAbsNormPath c file) of
-			Just f -> f
-			Nothing -> file
+		absfile c = maybe file id $ secureAbsNormPath c file
 		inrepo f = absrepo `isPrefixOf` f
 workTreeFile repo _ = assertLocal repo $ error "internal"
 
@@ -352,9 +349,7 @@ reap :: IO ()
 reap = do
 	-- throws an exception when there are no child processes
 	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
-	case r of
-		Nothing -> return ()
-		Just _ -> reap
+	maybe (return ()) (const reap) r
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/LocationLog.hs b/LocationLog.hs
index e0ccb642b3..6759b47fec 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -71,9 +71,7 @@ instance Read LogLine where
 	-- Such lines have a status of Undefined.
 	readsPrec _ string = 
 		if length w == 3
-			then case pdate of
-				Just v -> good v
-				Nothing -> bad
+			then maybe bad good pdate
 			else bad
 		where
 			w = words string
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 0aaff06b25..d2b771bf77 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -68,9 +68,8 @@ gen r u c = do
 bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 bupSetup u c = do
 	-- verify configuration is sane
-	let buprepo = case M.lookup "buprepo" c of
-		Nothing -> error "Specify buprepo="
-		Just r -> r
+	let buprepo = maybe (error "Specify buprepo=") id $
+		M.lookup "buprepo" c
 	c' <- encryptionSetup c
 
 	-- bup init will create the repository.
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index c680d61212..0cd3760d63 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -60,9 +60,8 @@ gen r u c = do
 directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 directorySetup u c = do
 	-- verify configuration is sane
-	let dir = case M.lookup "directory" c of
-		Nothing -> error "Specify directory="
-		Just d -> d
+	let dir = maybe (error "Specify directory=") id $
+		M.lookup "directory" c
 	e <- liftIO $ doesDirectoryExist dir
 	when (not e) $ error $ "Directory does not exist: " ++ dir
 	c' <- encryptionSetup c
diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs
index 31ef1f37a7..f9b388c8ae 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Encryptable.hs
@@ -73,11 +73,10 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 {- Gets encryption Cipher. The decrypted Cipher is cached in the Annex
  - state. -}
 remoteCipher :: RemoteConfig -> Annex (Maybe Cipher)
-remoteCipher c = do
-	cache <- Annex.getState Annex.cipher
-	case cache of
-		Just cipher -> return $ Just cipher
-		Nothing -> case extractCipher c of
+remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher
+	where
+		cached cipher = return $ Just cipher
+		expensive = case extractCipher c of
 			Nothing -> return Nothing
 			Just encipher -> do
 				showNote "gpg"
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index ba38355caf..7f2d5dbee2 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -61,9 +61,8 @@ gen r u c = do
 
 hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 hookSetup u c = do
-	let hooktype = case M.lookup "hooktype" c of
-		Nothing -> error "Specify hooktype="
-		Just r -> r
+	let hooktype = maybe (error "Specify hooktype=") id $
+		M.lookup "hooktype" c
 	c' <- encryptionSetup c
 	gitConfigSpecialRemote u c' "hooktype" hooktype
 	return c'
@@ -94,14 +93,12 @@ lookupHook hooktype hook =do
 		hookname =  hooktype ++ "-" ++ hook ++ "-hook"
 
 runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool
-runHook hooktype hook k f a = do
-	command <- lookupHook hooktype hook
-	case command of
-		Nothing -> return False
-		Just c -> do
+runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype hook
+	where
+		run command = do
 			showProgress -- make way for hook output
 			res <- liftIO $ boolSystemEnv
-				"sh" [Param "-c", Param c] $ hookEnv k f
+				"sh" [Param "-c", Param command] $ hookEnv k f
 			if res
 				then a
 				else do
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 682c961748..c15ab37a75 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -82,9 +82,8 @@ genRsyncOpts r = do
 rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 rsyncSetup u c = do
 	-- verify configuration is sane
-	let url = case M.lookup "rsyncurl" c of
-		Nothing -> error "Specify rsyncurl="
-		Just d -> d
+	let url = maybe (error "Specify rsyncurl=") id $
+		M.lookup "rsyncurl" c
 	c' <- encryptionSetup c
 
 	-- The rsyncurl is stored in git config, not only in this remote's
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index b0371eb5ea..eaa6590b19 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -123,11 +123,7 @@ storeHelper (conn, bucket) r k file = do
 	content <- liftIO $ L.readFile file
 	-- size is provided to S3 so the whole content does not need to be
 	-- buffered to calculate it
-	size <- case keySize k of
-		Just s -> return $ fromIntegral s
-		Nothing -> do
-			s <- liftIO $ getFileStatus file
-			return $ fileSize s
+	size <- maybe getsize (return . fromIntegral) $ keySize k
 	let object = setStorageClass storageclass $ 
 		S3Object bucket (show k) ""
 			[("Content-Length",(show size))] content
@@ -137,6 +133,9 @@ storeHelper (conn, bucket) r k file = do
 			case fromJust $ M.lookup "storageclass" $ fromJust $ config r of
 				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
 				_ -> STANDARD
+		getsize = do
+			s <- liftIO $ getFileStatus file
+			return $ fileSize s
 
 retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
 retrieve r k f = s3Action r False $ \(conn, bucket) -> do
@@ -201,11 +200,8 @@ bucketKey :: String -> Key -> S3Object
 bucketKey bucket k = S3Object bucket (show k) "" [] L.empty
 
 s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection
-s3ConnectionRequired c = do
-	conn <- s3Connection c
-	case conn of
-		Nothing -> error "Cannot connect to S3"
-		Just conn' -> return conn'
+s3ConnectionRequired c =
+	maybe (error "Cannot connect to S3") return =<< s3Connection c
 
 s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection)
 s3Connection c = do
diff --git a/Utility.hs b/Utility.hs
index 0dab371045..44c8cdd650 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -165,9 +165,7 @@ prop_parentDir_basics dir
 dirContains :: FilePath -> FilePath -> Bool
 dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
 	where
-		norm p = case (absNormPath p ".") of
-			Just r -> r
-			Nothing -> ""
+		norm p = maybe "" id $ absNormPath p "."
 		a' = norm a
 		b' = norm b
 
@@ -180,10 +178,9 @@ absPath file = do
 {- Converts a filename into a normalized, absolute path
  - from the specified cwd. -}
 absPathFrom :: FilePath -> FilePath -> FilePath
-absPathFrom cwd file =
-	case absNormPath cwd file of
-		Just f -> f
-		Nothing -> error $ "unable to normalize " ++ file
+absPathFrom cwd file = maybe bad id $ absNormPath cwd file
+	where
+		bad = error $ "unable to normalize " ++ file
 
 {- Constructs a relative path from the CWD to a file.
  -

From dfdfbe5f22116b73da8ae439ac4a7ab8b662f8fe Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 09:16:53 +0000
Subject: [PATCH 1708/2835] Added a comment

---
 .../comment_9_2cee4f6bd6db7518fd61453c595162c6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment b/doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment
new file mode 100644
index 0000000000..2755cf3317
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 9"
+ date="2011-05-15T09:16:49Z"
+ content="""
+Just to make sure: How do I get $key? What I did was look at the path in the object store of the local repo and see if that exact same path & file existed in the remote.
+"""]]

From 3e15a8a791d15c166557fa18f240639891a8754f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 12:25:58 -0400
Subject: [PATCH 1709/2835] Maybe reduction pass 2

---
 Command.hs            | 20 ++++++--------------
 GitRepo.hs            | 29 ++++++++++++-----------------
 Messages.hs           |  4 ++++
 Remote/Encryptable.hs | 31 +++++++++++--------------------
 4 files changed, 33 insertions(+), 51 deletions(-)

diff --git a/Command.hs b/Command.hs
index c6c1fe5c55..4f835a3adc 100644
--- a/Command.hs
+++ b/Command.hs
@@ -91,20 +91,12 @@ prepCommand Command { cmdseek = seek } params = do
 
 {- Runs a command through the start, perform and cleanup stages -}
 doCommand :: CommandStart -> CommandCleanup
-doCommand start = do
-	s <- start
-	case s of
-		Nothing -> return True
-		Just perform -> do
-			p <- perform
-			case p of
-				Nothing -> do
-					showEndFail
-					return False
-				Just cleanup -> do
-					c <- cleanup
-					if c then showEndOk else showEndFail
-					return c
+doCommand = start
+	where
+		start   = stage $ maybe (return True) perform
+		perform = stage $ maybe (showEndFail >> return False) cleanup
+		cleanup = stage $ \r -> showEndResult r >> return r
+		stage a b = b >>= a
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
diff --git a/GitRepo.hs b/GitRepo.hs
index b20ff7db3a..3c5a1e129e 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -210,9 +210,9 @@ assertUrl repo action =
 				" not supported"
 
 configBare :: Repo -> Bool
-configBare repo = case Map.lookup "core.bare" $ config repo of
-	Just v -> configTrue v
-	Nothing -> error $ "it is not known if git repo " ++
+configBare repo = maybe unknown configTrue $ Map.lookup "core.bare" $ config repo
+	where
+		unknown = error $ "it is not known if git repo " ++
 			repoDescribe repo ++
 			" is a bare repository; config not read"
 
@@ -260,11 +260,10 @@ workTreeFile repo@(Repo { location = Dir d }) file = do
 	where
 		-- normalize both repo and file, so that repo
 		-- will be substring of file
-		absrepo = case (absNormPath "/" d) of
-			Just f -> addTrailingPathSeparator f
-			Nothing -> error $ "bad repo" ++ repoDescribe repo
+		absrepo = maybe bad addTrailingPathSeparator $ absNormPath "/" d
 		absfile c = maybe file id $ secureAbsNormPath c file
 		inrepo f = absrepo `isPrefixOf` f
+		bad = error $ "bad repo" ++ repoDescribe repo
 workTreeFile repo _ = assertLocal repo $ error "internal"
 
 {- Path of an URL repo. -}
@@ -627,23 +626,19 @@ expandTilde = expandt True
 
 {- Finds the current git repository, which may be in a parent directory. -}
 repoFromCwd :: IO Repo
-repoFromCwd = do
-	cwd <- getCurrentDirectory
-	top <- seekUp cwd isRepoTop
-	case top of
-		-- repoFromAbsPath is not used to avoid looking for
-		-- "dir.git" directories.
-		(Just dir) -> return $ newFrom $ Dir dir
-		Nothing -> error "Not in a git repository."
+repoFromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
+	where
+		makerepo = return . newFrom . Dir
+		norepo = error "Not in a git repository."
 
-seekUp :: FilePath -> (FilePath -> IO Bool) -> IO (Maybe FilePath)
-seekUp dir want = do
+seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
+seekUp want dir = do
 	ok <- want dir
 	if ok
 		then return (Just dir)
 		else case (parentDir dir) of
 			"" -> return Nothing
-			d -> seekUp d want
+			d -> seekUp want d
 
 isRepoTop :: FilePath -> IO Bool
 isRepoTop dir = do
diff --git a/Messages.hs b/Messages.hs
index 733638ce12..c44e44eea1 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -45,6 +45,10 @@ showEndOk = verbose $ liftIO $ putStrLn "ok"
 showEndFail :: Annex ()
 showEndFail = verbose $ liftIO $ putStrLn "\nfailed"
 
+showEndResult :: Bool -> Annex ()
+showEndResult True = showEndOk
+showEndResult False = showEndFail
+
 showErr :: (Show a) => a -> Annex ()
 showErr e = warning $ "git-annex: " ++ show e
 
diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs
index f9b388c8ae..68ecfd01e6 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Encryptable.hs
@@ -54,21 +54,14 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 		cost = cost r + encryptedRemoteCostAdj
 	}
 	where
-		store k = do
-			v <- cipherKey c k
-			case v of
-				Nothing -> (storeKey r) k
-				Just x -> storeKeyEncrypted x k
-		retrieve k f = do
-			v <- cipherKey c k
-			case v of
-				Nothing -> (retrieveKeyFile r) k f
-				Just x -> retrieveKeyFileEncrypted x f
-		withkey a k = do
-			v <- cipherKey c k
-			case v of
-				Nothing -> a k
-				Just (_, k') -> a k'
+		store k = cip k >>= maybe
+			(storeKey r k)
+			(\x -> storeKeyEncrypted x k)
+		retrieve k f = cip k >>= maybe
+			(retrieveKeyFile r k f)
+			(\x -> retrieveKeyFileEncrypted x f)
+		withkey a k = cip k >>= maybe (a k) (a . snd)
+		cip = cipherKey c
 
 {- Gets encryption Cipher. The decrypted Cipher is cached in the Annex
  - state. -}
@@ -87,10 +80,8 @@ remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher
 {- Gets encryption Cipher, and encrypted version of Key. -}
 cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))
 cipherKey Nothing _ = return Nothing
-cipherKey (Just c) k = do
-	cipher <- remoteCipher c
-	case cipher of	
-		Just ciphertext -> do
+cipherKey (Just c) k = remoteCipher c >>= maybe (return Nothing) encrypt
+	where
+		encrypt ciphertext = do
 			k' <- liftIO $ encryptKey ciphertext k
 			return $ Just (ciphertext, k')
-		Nothing -> return Nothing

From db540b746b0eb93fa76f8bec0bdb63dd9ea5a5e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 12:32:18 -0400
Subject: [PATCH 1710/2835] don't optimise test suite

This avoids needing to adjust -fspec-constr-count, which
continues to need high values (> 8 now) when building the test suite.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 01a1a6a54c..286c3a6e54 100644
--- a/Makefile
+++ b/Makefile
@@ -45,7 +45,7 @@ install: all
 	fi
 
 test: $(bins)
-	if ! $(GHCMAKE) test; then \
+	if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
 		./test; \

From 7010b9e5c5c1519ba5d87d4c14bc36a01e298e1f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 15 May 2011 16:47:53 +0000
Subject: [PATCH 1711/2835] Added a comment

---
 .../comment_10_435f87d54052f264096a8f23e99eae06._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment b/doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment
new file mode 100644
index 0000000000..ec24c478d8
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 10"
+ date="2011-05-15T16:47:53Z"
+ content="""
+The key is the basename of the symlink target.
+"""]]

From 9708940cedcbc4e9f0ce49826bdfb710cfb25e7f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 18:53:27 +0000
Subject: [PATCH 1712/2835] Added a comment

---
 ..._9be0aef403a002c1706d17deee45763c._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment b/doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment
new file mode 100644
index 0000000000..7bc54573ed
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 11"
+ date="2011-05-15T18:53:26Z"
+ content="""
+It seems the objects are in the remote after all, but the remote is unaware of this fact. No idea where/why the remote lost that info, but.. Anyway, with the SHA backends, wouldn't it make sense to simply return \"OK\" and update the annex logs accordingly, no?
+
+Local:
+
+    % ls -l foo
+    lrwxrwxrwx 1 richih richih 312 Apr  3 01:18 foo -> .git/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    % 
+
+Remote:
+
+    % git-annex-shell recvkey  SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    git-annex-shell: key is already present in annex
+    % strace git-annex-shell recvkey /base/git-annex/fun SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 2>&1 | grep SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    stat64(\"/base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491\", {st_mode=S_IFREG|0444, st_size=80781, ...}) = 0
+    % ls -l /base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    -r--r--r-- 1 richih richih 80781 2011-04-01 12:44 /base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    % 
+"""]]

From 6aab88fa251a14fbf31c7a8d80296c78db0ed048 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 15:27:49 -0400
Subject: [PATCH 1713/2835] more monadic operator use

---
 Annex.hs       |  4 +---
 Content.hs     |  3 +--
 LocationLog.hs |  6 ++----
 Remote.hs      | 12 +++++-------
 UUID.hs        |  6 ++----
 5 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 9915112a59..b2257281fd 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -86,9 +86,7 @@ getState c = liftM c get
  - Example: changeState (\s -> s { quiet = True })
  -}
 changeState :: (AnnexState -> AnnexState) -> Annex ()
-changeState a = do
-	state <- get
-	put (a state)
+changeState a = put . a =<< get
 
 {- Returns the git repository being acted on -}
 gitRepo :: Annex Git.Repo
diff --git a/Content.hs b/Content.hs
index 9040383be9..0758fcdb13 100644
--- a/Content.hs
+++ b/Content.hs
@@ -72,8 +72,7 @@ calcGitLink file key = do
  - updated instead. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
-	g <- Annex.gitRepo
-	u <- getUUID g
+	u <- getUUID =<< Annex.gitRepo
 	logStatusFor u key status
 
 {- Updates the LocationLog when a key's presence changes in a repository
diff --git a/LocationLog.hs b/LocationLog.hs
index 6759b47fec..b2d423cf99 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -150,16 +150,13 @@ mapLog m l =
 		then Map.insert u l m
 		else m
 	where
-		better = case Map.lookup u m of
-			Just l' -> (date l' <= date l)
-			Nothing -> True
+		better = maybe True (\l' -> date l' <= date l) $ Map.lookup u m
 		u = uuid l
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Git.Repo -> IO [Key]
 loggedKeys repo = do
-	let dir = gitStateDir repo
 	exists <- doesDirectoryExist dir
 	if exists
 		then do
@@ -172,3 +169,4 @@ loggedKeys repo = do
 		else return []
 	where
 		tryDirContents d = catch (dirContents d) (return . const [])
+		dir = gitStateDir repo
diff --git a/Remote.hs b/Remote.hs
index bbecdb9995..211168b153 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -75,10 +75,10 @@ genList = do
 			return rs'
 		else return rs
 	where
-		process m t = do
-			l <- enumerate t
-			l' <- filterM remoteNotIgnored l
-			mapM (gen m t) l'
+		process m t = 
+			enumerate t >>=
+			filterM remoteNotIgnored >>=
+			mapM (gen m t)
 		gen m t r = do
 			u <- getUUID r
 			generate t r u (M.lookup u m)
@@ -97,9 +97,7 @@ byName n = do
 
 {- Looks up a remote by name (or by UUID), and returns its UUID. -}
 nameToUUID :: String -> Annex UUID
-nameToUUID "." = do -- special case for current repo
-	g <- Annex.gitRepo
-	getUUID g
+nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = liftM uuid (byName n)
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
diff --git a/UUID.hs b/UUID.hs
index eb1fb319c6..0d7aee1414 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -79,8 +79,7 @@ getUncachedUUID r = Git.configGet r configkey ""
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
 prepUUID = do
-	g <- Annex.gitRepo
-	u <- getUUID g
+	u <- getUUID =<< Annex.gitRepo
 	when ("" == u) $ do
 		uuid <- liftIO $ genUUID
 		setConfig configkey uuid
@@ -88,8 +87,7 @@ prepUUID = do
 {- Pretty-prints a list of UUIDs -}
 prettyPrintUUIDs :: [UUID] -> Annex String
 prettyPrintUUIDs uuids = do
-	g <- Annex.gitRepo
-	here <- getUUID g
+	here <- getUUID =<< Annex.gitRepo
 	m <- uuidMap
 	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
 	where

From 59ae1df7a0180263c5434c13b01eb4e7e4ed3267 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 15 May 2011 19:40:47 +0000
Subject: [PATCH 1714/2835] Added a comment

---
 ...mment_12_26d60661196f63fd01ee4fbb6e2340e7._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment b/doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
new file mode 100644
index 0000000000..b458a37b69
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 12"
+ date="2011-05-15T19:40:47Z"
+ content="""
+So, it appears that you're using git annex copy --fast. As documented that assumes the location log is correct. So it avoids directly checking if the bare repo contains the file, and tries to upload it, and the bare repo is all like \"but I've already got this file!\". The only way to improve that behavior might be to let rsync go ahead and retransfer the file, which, with recovery, should require sending little data etc. But I can't say I like the idea much, as the repo already has the content, so unlocking it and letting rsync mess with it is an unnecessary risk. I think it's ok for --force to blow up
+if its assumptions turn out to be wrong.
+
+If you use git annex copy without --fast in this situation, it will do the right thing.
+"""]]

From 4d140e3901b7e3c8d66a95817d845b9f35ac6b05 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 20:25:25 +0000
Subject: [PATCH 1715/2835] Added a comment

---
 ..._ead55b915d3b92a62549b2957ad211c8._comment | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment b/doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
new file mode 100644
index 0000000000..d92ecbba03
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 13"
+ date="2011-05-15T20:25:25Z"
+ content="""
+Yes, makes sense. I am so used to using --fast, I forgot a non-fast mode existed. I still think it would be a good idea to fall back to non-fast mode if --fast runs into an error from the remote, but as that is well without my abilities how about this patch?
+
+
+    From 4855510c7a84eb5d28fdada429580a8a42b7112a Mon Sep 17 00:00:00 2001
+    From: Richard Hartmann 
+    Date: Sun, 15 May 2011 22:20:42 +0200
+    Subject: [PATCH] Make error in RecvKey.hs suggest possible solution
+    
+    ---
+     Command/RecvKey.hs |    2 +-
+     1 files changed, 1 insertions(+), 1 deletions(-)
+    
+    diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
+    index 126608f..b917a1c 100644
+    --- a/Command/RecvKey.hs
+    +++ b/Command/RecvKey.hs
+    @@ -27,7 +27,7 @@ start :: CommandStartKey
+     start key = do
+        present <- inAnnex key
+        when present $
+    -       error \"key is already present in annex\"
+    +       error \"key is already present in annex. If you are running copy, try without '--fast'\"
+        
+        ok <- getViaTmp key (liftIO . rsyncServerReceive)
+        if ok
+    -- 
+    1.7.4.4
+
+"""]]

From 120bc4208d45f848e4ab0890b54a255bef6a90bb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 20:50:27 +0000
Subject: [PATCH 1716/2835] Added a comment

---
 ...omment_14_191de89d3988083d9cf001799818ff4a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment b/doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment
new file mode 100644
index 0000000000..f45bd70468
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 14"
+ date="2011-05-15T20:50:26Z"
+ content="""
+Or, even better, wouldn't it make sense to have SHA backends always default to --fast and only use non-fast when any snags are hit, use non-fast mode for that file.
+
+Though if we continue here, we should probably move this to its own page.
+"""]]

From 4c312d8d8fba8840edd86fe8b951b6b20349985f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 21:38:47 +0000
Subject: [PATCH 1717/2835] Added a comment

---
 .../comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment b/doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment
new file mode 100644
index 0000000000..b4a00bd7e1
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 15"
+ date="2011-05-15T21:38:47Z"
+ content="""
+PS: Just to make this clear, I am using a custom alias for all my copying needs and thus didn't even see that I used --fast. :p
+"""]]

From af54d40d5b61df9c2172d6b533b063d19c29438e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 20:33:39 -0400
Subject: [PATCH 1718/2835] move conversation to a real bug report

---
 doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn       | 2 ++
 .../comment_1_6a41bf7e2db83db3a01722b516fb6886._comment         | 0
 .../comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment         | 0
 .../comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment         | 0
 .../comment_4_d7112c315fb016a8a399e24e9b6461d8._comment         | 0
 .../comment_5_4ea29a6f8152eddf806c536de33ef162._comment         | 0
 .../comment_6_0d85f114a103bd6532a3b3b24466012e._comment         | 0
 .../comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment         | 0
 .../comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment         | 0
 .../comment_9_2cee4f6bd6db7518fd61453c595162c6._comment         | 0
 10 files changed, 2 insertions(+)
 create mode 100644 doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_5_4ea29a6f8152eddf806c536de33ef162._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_6_0d85f114a103bd6532a3b3b24466012e._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment (100%)

diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
new file mode 100644
index 0000000000..4ed72f76d6
--- /dev/null
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
@@ -0,0 +1,2 @@
+Conversation moved from [[walkthrough/recover_data_from_lost+found]]
+to a proper bug. --[[Joey]]
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment

From c91764b257b93c523b7884d96a1f027472ba652c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 20:35:39 -0400
Subject: [PATCH 1719/2835] move other comments

---
 .../comment_10_435f87d54052f264096a8f23e99eae06._comment          | 0
 .../comment_11_9be0aef403a002c1706d17deee45763c._comment          | 0
 .../comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment          | 0
 .../comment_13_ead55b915d3b92a62549b2957ad211c8._comment          | 0
 .../comment_14_191de89d3988083d9cf001799818ff4a._comment          | 0
 .../comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment          | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_10_435f87d54052f264096a8f23e99eae06._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_11_9be0aef403a002c1706d17deee45763c._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_13_ead55b915d3b92a62549b2957ad211c8._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_14_191de89d3988083d9cf001799818ff4a._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment (100%)

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment

From d67998b3d37acc8c31df8d6200385805d24921ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 20:40:03 -0400
Subject: [PATCH 1720/2835] meh

---
 doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
index 4ed72f76d6..2767b5aa9c 100644
--- a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
@@ -1,2 +1,4 @@
 Conversation moved from [[walkthrough/recover_data_from_lost+found]]
 to a proper bug. --[[Joey]]
+
+(Unfortunatly that scrambled the comment creation times and thus order.)

From 647f7cf47cd659ae34d27a18d3aa068c1a0755eb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 02:07:59 -0400
Subject: [PATCH 1721/2835] added documentation for using the Internet Archive
 as a remote via S3

Renamed Amazon_S3 page to just S3.
---
 doc/cheatsheet.mdwn                           |  1 +
 doc/index.mdwn                                |  2 +-
 doc/special_remotes.mdwn                      |  2 +-
 .../{Amazon_S3.mdwn => S3.mdwn}               |  0
 doc/walkthrough/Internet_Archive_via_S3.mdwn  | 48 +++++++++++++++++++
 doc/walkthrough/using_Amazon_S3.mdwn          |  2 +-
 6 files changed, 52 insertions(+), 3 deletions(-)
 rename doc/special_remotes/{Amazon_S3.mdwn => S3.mdwn} (100%)
 create mode 100644 doc/walkthrough/Internet_Archive_via_S3.mdwn

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 4287756a6c..3f5960972d 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -11,4 +11,5 @@ A suppliment to the [[walkthrough]].
 	walkthrough/untrusted_repositories
 	walkthrough/what_to_do_when_you_lose_a_repository
 	walkthrough/recover_data_from_lost+found
+	walkthrough/Internet_Archive_via_S3
 """]]
diff --git a/doc/index.mdwn b/doc/index.mdwn
index eb6307f327..8975c82de7 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -43,7 +43,7 @@ files with git.
 
 * [[git-annex man page|git-annex]]
 * [[key-value backends|backends]] for data storage
-* [[special_remotes]] (including [[special_remotes/Amazon_S3]] and [[special_remotes/bup]])
+* [[special_remotes]] (including [[special_remotes/S3]] and [[special_remotes/bup]])
 * [[encryption]]
 * [[bare_repositories]]
 * [[internals]]
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 13c18122e7..dcb6b5063e 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -6,7 +6,7 @@ But, git-annex also extends git's concept of remotes, with these special
 types of remotes. These can be used just like any normal remote by git-annex.
 They cannot be used by other git commands though.
 
-* [[Amazon_S3]]
+* [[S3]] (Amazon S3, and other compatible services)
 * [[bup]]
 * [[directory]]
 * [[rsync]]
diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/S3.mdwn
similarity index 100%
rename from doc/special_remotes/Amazon_S3.mdwn
rename to doc/special_remotes/S3.mdwn
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
new file mode 100644
index 0000000000..089102d142
--- /dev/null
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -0,0 +1,48 @@
+[The Internet Archive](http://www.archive.org/) allows members to upload
+collections using an Amazon S3 
+[compatible API](http://www.archive.org/help/abouts3.txt), and this can
+be used with git-annex's [[special_remotes/S3]] support. 
+
+So, if you're an archivist, you can locally archive things with git-annex,
+and define remotes that correspond to "items" at the Internet Archive,
+and use git-annex to upload your files to there.
+Of course, your use of the Internet Archive must comply with their
+[terms of service](http://www.archive.org/about/terms.php).
+
+## step 0
+
+Sign up for an account, and get your access keys here:
+
+	
+	# export AWS_ACCESS_KEY_ID=blahblah
+	# export AWS_SECRET_ACCESS_KEY=xxxxxxx
+
+Now go to  and create the item.
+This allows you to fill in metadata which git-annex cannot provide to the
+Internet Archive. (It also works around a bug with bucket creation.)
+
+(Note that there seems to be a bug in either hS3 or the archive that
+breaks authentication when the item name contains spaces or upper-case
+letters.. use all lowercase and no spaces.)
+
+Specify `host=s3.us.archive.org` when doing initremote to set up
+a remote at the Archive. It does not make sense to use encryption.
+For the bucket name, specify the item name created in step 1.
+
+	# git annex initremote panama type=S3 encryption=none host=s3.us.archive.org bucket=panama-canal-lock-blueprints
+	initremote archive-panama (checking bucket) (creating bucket in US) ok
+	# git annex describe archive-panama "Internet Archive item for my grandfather's Panama Canal lock design blueprints"
+	describe archive-panama ok
+
+Then you can annex files and copy them to the remote as usual:
+
+	# git annex add photo1.jpeg
+	add photo1.jpeg ok
+	# git annex copy photo1.jpeg --to archive-panama
+	copy (checking archive-panama...) (to archive-panama...) ok
+
+Note that it probably makes the most sense to use the WORM backend
+for files, since that exposes the original filename in the key stored
+in the Archive, which allows its special processing for sound files,
+movies, etc to be done. Also, the Internet Archive has restrictions
+on what is allowed in a filename; particularly no spaces are allowed.
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 6b0f496392..7f972afe11 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -34,4 +34,4 @@ Now the remote can be used like any other remote.
 	# git annex move video/hackity_hack_and_kaxxt.mov --to cloud
 	move video/hackity_hack_and_kaxxt.mov (checking cloud...) (to cloud...) ok
 
-See [[special_remotes/Amazon_S3]] for details.
+See [[special_remotes/S3]] for details.

From 8564498b5d87a424effec695825562b56f1391d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 02:15:00 -0400
Subject: [PATCH 1722/2835] cleanup

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 089102d142..554f4b03b1 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -9,8 +9,6 @@ and use git-annex to upload your files to there.
 Of course, your use of the Internet Archive must comply with their
 [terms of service](http://www.archive.org/about/terms.php).
 
-## step 0
-
 Sign up for an account, and get your access keys here:
 
 	

From 275c72850943d78677929b0bac5899d1e7e5e5b1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 02:18:28 -0400
Subject: [PATCH 1723/2835] cleanup

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 554f4b03b1..3cd83a2e7c 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -25,7 +25,7 @@ letters.. use all lowercase and no spaces.)
 
 Specify `host=s3.us.archive.org` when doing initremote to set up
 a remote at the Archive. It does not make sense to use encryption.
-For the bucket name, specify the item name created in step 1.
+For the bucket name, specify the item name you created earlier.
 
 	# git annex initremote panama type=S3 encryption=none host=s3.us.archive.org bucket=panama-canal-lock-blueprints
 	initremote archive-panama (checking bucket) (creating bucket in US) ok

From e259c86975a6ec1ab604811684dec7166f57b7bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 02:21:40 -0400
Subject: [PATCH 1724/2835] cleanup

---
 doc/special_remotes/S3.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index 35397dc2a9..abd61ac797 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -1,6 +1,5 @@
 This special remote type stores file contents in a bucket in Amazon S3
-or a similar service, such as 
-[Archive.org's S3 API](http://www.archive.org/help/abouts3.txt).
+or a similar service.
 
 See [[walkthrough/using_Amazon_S3]] for usage examples.
 

From 79c74bf27dfb9795ad35bc4e4c2061004212621d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 09:42:54 -0400
Subject: [PATCH 1725/2835] refactor

---
 Remote/S3real.hs | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index eaa6590b19..e8c700e2c7 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -73,21 +73,7 @@ s3Setup u c = do
 	c' <- encryptionSetup c
 	let fullconfig = M.union c' defaults
 
-	-- check bucket location to see if the bucket exists, and create it
-	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
-	conn <- s3ConnectionRequired fullconfig
-	showNote "checking bucket"
-	loc <- liftIO $ getBucketLocation conn bucket 
-	case loc of
-		Right _ -> return ()
-		Left err@(NetworkError _) -> s3Error err
-		Left (AWSError _ _) -> do
-			showNote $ "creating bucket in " ++ datacenter
-			res <- liftIO $ createBucketIn conn bucket datacenter
-			case res of
-				Right _ -> return ()
-				Left err -> s3Error err
-
+	genBucket fullconfig
 	gitConfigSpecialRemote u fullconfig "s3" "true"
 	s3SetCreds fullconfig
 	where
@@ -126,7 +112,7 @@ storeHelper (conn, bucket) r k file = do
 	size <- maybe getsize (return . fromIntegral) $ keySize k
 	let object = setStorageClass storageclass $ 
 		S3Object bucket (show k) ""
-			[("Content-Length",(show size))] content
+			[("Content-Length",(show size)), ("x-amz-auto-make-bucket","1")] content
 	sendObject conn object
 	where
 		storageclass =
@@ -199,6 +185,24 @@ s3Action r noconn action = do
 bucketKey :: String -> Key -> S3Object
 bucketKey bucket k = S3Object bucket (show k) "" [] L.empty
 
+genBucket :: RemoteConfig -> Annex ()
+genBucket c = do
+	conn <- s3ConnectionRequired c
+	showNote "checking bucket"
+	loc <- liftIO $ getBucketLocation conn bucket 
+	case loc of
+		Right _ -> return ()
+		Left err@(NetworkError _) -> s3Error err
+		Left (AWSError _ _) -> do
+			showNote $ "creating bucket in " ++ datacenter
+			res <- liftIO $ createBucketIn conn bucket datacenter
+			case res of
+				Right _ -> return ()
+				Left err -> s3Error err
+	where
+		bucket = fromJust $ M.lookup "bucket" c
+		datacenter = fromJust $ M.lookup "datacenter" c
+
 s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection
 s3ConnectionRequired c =
 	maybe (error "Cannot connect to S3") return =<< s3Connection c

From 1d2984441c654f01e88e427f3289f8066cd2e6b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:20:30 -0400
Subject: [PATCH 1726/2835] add a few tweaks to make it easy to use the
 Internet Archive's variant of S3

In particular, munge key filenames to comply with the IA's filename limits,
disable encryption, support their nonstandard way of creating buckets, and
allow x-amz-* headers to be specified in initremote to set item metadata.

Still TODO: initremote does not handle multiword metadata headers right.
---
 Remote/S3real.hs                             | 90 ++++++++++++++++----
 debian/changelog                             |  5 ++
 doc/special_remotes/S3.mdwn                  | 13 +--
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 40 +++++----
 4 files changed, 108 insertions(+), 40 deletions(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index e8c700e2c7..7d6b5d5bab 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -15,6 +15,9 @@ import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.Maybe
+import Data.List
+import Data.Char
+import Data.String.Utils
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Environment
@@ -68,24 +71,53 @@ gen' r u c cst = do
 		}
 
 s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
-s3Setup u c = do
-	-- verify configuration is sane
-	c' <- encryptionSetup c
-	let fullconfig = M.union c' defaults
-
-	genBucket fullconfig
-	gitConfigSpecialRemote u fullconfig "s3" "true"
-	s3SetCreds fullconfig
+s3Setup u c = handlehost $ M.lookup "host" c
 	where
 		remotename = fromJust (M.lookup "name" c)
-		bucket = remotename ++ "-" ++ u
+		defbucket = remotename ++ "-" ++ u
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
 			, ("host", defaultAmazonS3Host)
 			, ("port", show defaultAmazonS3Port)
-			, ("bucket", bucket)
+			, ("bucket", defbucket)
 			]
+		
+		handlehost Nothing = defaulthost
+		handlehost (Just h)
+			| ".archive.org" `isSuffixOf` (map toLower h) = archiveorg
+			| otherwise = defaulthost
+
+		use fullconfig = do
+			genBucket fullconfig
+			gitConfigSpecialRemote u fullconfig "s3" "true"
+			s3SetCreds fullconfig	
+
+		defaulthost = do
+			c' <- encryptionSetup c
+			use $ M.union c' defaults
+
+		archiveorg = do
+			showNote $ "Internet Archive mode"
+			maybe (error "specify bucket=") (const $ return ()) $
+				M.lookup "bucket" archiveconfig
+			use archiveconfig
+			where
+				archiveconfig =
+					-- hS3 does not pass through
+					-- x-archive-* headers
+					M.mapKeys (replace "x-archive-" "x-amz-") $
+					-- encryption does not make sense here
+					M.insert "encryption" "none" $
+					M.union c $
+					-- special constraints on key names
+					M.insert "mungekeys" "ia" $
+					-- buckets created only as files
+					-- are uploaded
+					M.insert "x-amz-auto-make-bucket" "1" $
+					-- no default bucket name; should
+					-- be human-readable
+					M.delete "bucket" defaults
 
 store :: Remote Annex -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
@@ -111,8 +143,8 @@ storeHelper (conn, bucket) r k file = do
 	-- buffered to calculate it
 	size <- maybe getsize (return . fromIntegral) $ keySize k
 	let object = setStorageClass storageclass $ 
-		S3Object bucket (show k) ""
-			[("Content-Length",(show size)), ("x-amz-auto-make-bucket","1")] content
+		S3Object bucket (bucketFile r k) ""
+			(("Content-Length", show size) : xheaders) content
 	sendObject conn object
 	where
 		storageclass =
@@ -122,10 +154,13 @@ storeHelper (conn, bucket) r k file = do
 		getsize = do
 			s <- liftIO $ getFileStatus file
 			return $ fileSize s
+		
+		xheaders = filter isxheader $ M.assocs $ fromJust $ config r
+		isxheader (h, _) = "x-amz-" `isPrefixOf` h
 
 retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
 retrieve r k f = s3Action r False $ \(conn, bucket) -> do
-	res <- liftIO $ getObject conn $ bucketKey bucket k
+	res <- liftIO $ getObject conn $ bucketKey r bucket k
 	case res of
 		Right o -> do
 			liftIO $ L.writeFile f $ obj_data o
@@ -134,7 +169,7 @@ retrieve r k f = s3Action r False $ \(conn, bucket) -> do
 
 retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do
-	res <- liftIO $ getObject conn $ bucketKey bucket enck
+	res <- liftIO $ getObject conn $ bucketKey r bucket enck
 	case res of
 		Right o -> liftIO $ 
 			withDecryptedContent cipher (return $ obj_data o) $ \content -> do
@@ -144,13 +179,13 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do
 
 remove :: Remote Annex -> Key -> Annex Bool
 remove r k = s3Action r False $ \(conn, bucket) -> do
-	res <- liftIO $ deleteObject conn $ bucketKey bucket k
+	res <- liftIO $ deleteObject conn $ bucketKey r bucket k
 	s3Bool res
 
 checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
 checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
 	showNote ("checking " ++ name r ++ "...")
-	res <- liftIO $ getObjectInfo conn $ bucketKey bucket k
+	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k
 	case res of
 		Right _ -> return $ Right True
 		Left (AWSError _ _) -> return $ Right False
@@ -182,8 +217,27 @@ s3Action r noconn action = do
 		(Just b, Just c) -> action (c, b)
 		_ -> return noconn
 
-bucketKey :: String -> Key -> S3Object
-bucketKey bucket k = S3Object bucket (show k) "" [] L.empty
+bucketFile :: Remote Annex -> Key -> FilePath
+bucketFile r k = (munge $ show k)
+	where
+		munge s = case M.lookup "mungekeys" $ fromJust $ config r of
+			Just "ia" -> iaMunge s
+			_ -> s
+
+bucketKey :: Remote Annex -> String -> Key -> S3Object
+bucketKey r bucket k = S3Object bucket (bucketFile r k) "" [] L.empty
+
+{- Internet Archive limits filenames to a subset of ascii,
+ - with no whitespace. Other characters are xml entity
+ - encoded. -}
+iaMunge :: String -> String
+iaMunge = concat . (map munge)
+	where
+		munge c
+			| isAsciiUpper c || isAsciiLower c || isNumber c = [c]
+			| c `elem` "_-.\"" = [c]
+			| isSpace c = []
+			| otherwise = "&" ++ show (ord c) ++ ";"
 
 genBucket :: RemoteConfig -> Annex ()
 genBucket c = do
diff --git a/debian/changelog b/debian/changelog
index 09dde346c0..59ebb1deb7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,11 @@
 git-annex (0.20110504) UNRELEASED; urgency=low
 
   * Work around a bug in Network.URI's handling of bracketed ipv6 addresses.
+  * Add a few tweaks to make it easy to use the Internet Archive's variant
+    of S3. In particular, munge key filenames to comply with the IA's filename
+    limits, disable encryption, support their nonstandard way of creating
+    buckets, and allow x-amz-* headers to be specified in initremote to set
+    item metadata.
 
  -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400
 
diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index abd61ac797..d6a7229e33 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -5,6 +5,12 @@ See [[walkthrough/using_Amazon_S3]] for usage examples.
 
 ## configuration
 
+The standard environment variables `ANNEX_S3_ACCESS_KEY_ID` and 
+`ANNEX_S3_SECRET_ACCESS_KEY` are used to supply login credentials
+for Amazon. When encryption is enabled, they are stored in encrypted form
+by `git annex initremote`, so you do not need to keep the environment
+variables set after the initial initalization of the remote.
+
 A number of parameters can be passed to `git annex initremote` to configure
 the S3 remote.
 
@@ -29,8 +35,5 @@ the S3 remote.
   so by default, a bucket name is chosen based on the remote name
   and UUID. This can be specified to pick a bucket name.
 
-The standard environment variables `ANNEX_S3_ACCESS_KEY_ID` and 
-`ANNEX_S3_SECRET_ACCESS_KEY` can be used to supply login credentials
-for Amazon. When encryption is enabled, they are stored in encrypted form
-by `git annex initremote`, so you do not need to keep the environment
-variables set after the initial initalization of the remote.
+* `x-amz-*` are passed through as http headers when storing keys
+  in S3.
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 3cd83a2e7c..e0f8fafb44 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -15,20 +15,18 @@ Sign up for an account, and get your access keys here:
 	# export AWS_ACCESS_KEY_ID=blahblah
 	# export AWS_SECRET_ACCESS_KEY=xxxxxxx
 
-Now go to  and create the item.
-This allows you to fill in metadata which git-annex cannot provide to the
-Internet Archive. (It also works around a bug with bucket creation.)
+Specify `host=s3.us.archive.org` when doing `initremote` to set up
+a remote at the Archive. This will enable a special Internet Archive mode:
+Encryption is not allowed; you are required to specify a bucket name
+rather than letting git-annex pick a random one; and you can optionally
+specify `x-archive-meta*` headers to add metadata as explained in their
+[documentation](http://www.archive.org/help/abouts3.txt).
 
-(Note that there seems to be a bug in either hS3 or the archive that
-breaks authentication when the item name contains spaces or upper-case
-letters.. use all lowercase and no spaces.)
-
-Specify `host=s3.us.archive.org` when doing initremote to set up
-a remote at the Archive. It does not make sense to use encryption.
-For the bucket name, specify the item name you created earlier.
-
-	# git annex initremote panama type=S3 encryption=none host=s3.us.archive.org bucket=panama-canal-lock-blueprints
-	initremote archive-panama (checking bucket) (creating bucket in US) ok
+	# git annex initremote archive-panama type=S3
+	# host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
+		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
+		x-archive-meta-title="original Panama Canal lock design blueprints"
+	initremote archive-panama (Internet Archive mode) (checking bucket) (creating bucket in US) ok
 	# git annex describe archive-panama "Internet Archive item for my grandfather's Panama Canal lock design blueprints"
 	describe archive-panama ok
 
@@ -36,11 +34,19 @@ Then you can annex files and copy them to the remote as usual:
 
 	# git annex add photo1.jpeg
 	add photo1.jpeg ok
-	# git annex copy photo1.jpeg --to archive-panama
-	copy (checking archive-panama...) (to archive-panama...) ok
+	# git annex copy photo1.jpeg --fast --to archive-panama
+	copy (to archive-panama...) ok
+
+-----
 
 Note that it probably makes the most sense to use the WORM backend
 for files, since that exposes the original filename in the key stored
 in the Archive, which allows its special processing for sound files,
-movies, etc to be done. Also, the Internet Archive has restrictions
-on what is allowed in a filename; particularly no spaces are allowed.
+movies, etc to be done. 
+
+Also, the Internet Archive has restrictions on what is allowed in a
+filename; particularly no spaces are allowed. 
+
+There seems to be a bug in either hS3 or the archive that breaks
+authentication when the bucket name contains spaces or upper-case letters..
+use all lowercase and no spaces when making the bucket with `initremote`.

From 2a8efc7af19aa149dbf0ebc158954bb376f9c3a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:46:34 -0400
Subject: [PATCH 1727/2835] Added filename extension preserving variant
 backends SHA1E, SHA256E, etc.

---
 Backend/SHA.hs                               | 44 ++++++++++++++++----
 debian/changelog                             |  1 +
 doc/backends.mdwn                            |  3 ++
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 18 ++++----
 4 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index d9aeb72aa4..6d721038c3 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -14,6 +14,7 @@ import System.IO
 import System.Directory
 import Data.Maybe
 import System.Posix.Files
+import System.FilePath
 
 import qualified Backend.File
 import BackendClass
@@ -27,11 +28,14 @@ import qualified SysConfig
 import Key
 
 type SHASize = Int
- 
+
+sizes :: [Int]
+sizes = [1, 256, 512, 224, 384]
+
 backends :: [Backend Annex]
 -- order is slightly significant; want sha1 first ,and more general
 -- sizes earlier
-backends = catMaybes $ map genBackend [1, 256, 512, 224, 384]
+backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
 genBackend :: SHASize -> Maybe (Backend Annex)
 genBackend size
@@ -44,6 +48,15 @@ genBackend size
 			, fsckKey = Backend.File.checkKey $ checkKeyChecksum size
 			}
 
+genBackendE :: SHASize -> Maybe (Backend Annex)
+genBackendE size =
+	case genBackend size of
+		Nothing -> Nothing
+		Just b -> Just $ b
+			{ name = shaNameE size
+			, getKey = keyValueE size
+			}
+
 shaCommand :: SHASize -> Maybe String
 shaCommand 1 = SysConfig.sha1
 shaCommand 256 = SysConfig.sha256
@@ -55,6 +68,9 @@ shaCommand _ = Nothing
 shaName :: SHASize -> String
 shaName size = "SHA" ++ show size
 
+shaNameE :: SHASize -> String
+shaNameE size = shaName size ++ "E"
+
 shaN :: SHASize -> FilePath -> Annex String
 shaN size file = do
 	showNote "checksum..."
@@ -72,11 +88,25 @@ keyValue :: SHASize -> FilePath -> Annex (Maybe Key)
 keyValue size file = do
 	s <- shaN size file	
 	stat <- liftIO $ getFileStatus file
-	return $ Just $ stubKey {
-		keyName = s,
-		keyBackendName = shaName size,
-		keySize = Just $ fromIntegral $ fileSize stat
-	}
+	return $ Just $ stubKey
+		{ keyName = s
+		, keyBackendName = shaName size
+		, keySize = Just $ fromIntegral $ fileSize stat
+		}
+
+{- Extension preserving keys. -}
+keyValueE :: SHASize -> FilePath -> Annex (Maybe Key)
+keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
+	where
+		addE k = return $ Just $ k
+			{ keyName = keyName k ++ extension
+			, keyBackendName = shaNameE size
+			}
+		naiveextension = takeExtension file
+		extension = 
+			if length naiveextension > 6
+				then "" -- probably not really an extension
+				else naiveextension
 
 -- A key's checksum is checked during fsck.
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
diff --git a/debian/changelog b/debian/changelog
index 59ebb1deb7..a7f6039810 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (0.20110504) UNRELEASED; urgency=low
     limits, disable encryption, support their nonstandard way of creating
     buckets, and allow x-amz-* headers to be specified in initremote to set
     item metadata.
+  * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
 
  -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400
 
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index b0a2c882aa..4290da33ba 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -23,6 +23,9 @@ these backends.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
   researching checksum collisions and wants to annex their colliding data. ;)
+* `SHA1E`, `SHA512E`, etc -- Variants that preserve filename extension as
+  part of the key. Useful for archival tasks where the filename extension
+  contains metadata that should be preserved.
 
 These backends store file contents in other key/value stores.
 
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index e0f8fafb44..f92e0ee9d7 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -32,20 +32,18 @@ specify `x-archive-meta*` headers to add metadata as explained in their
 
 Then you can annex files and copy them to the remote as usual:
 
-	# git annex add photo1.jpeg
-	add photo1.jpeg ok
+	# git annex add photo1.jpeg --backend=SHA1E
+	add photo1.jpeg (checksum...) ok
 	# git annex copy photo1.jpeg --fast --to archive-panama
 	copy (to archive-panama...) ok
 
------
+Note the use of the SHA1E [[backend|backends]]. It makes most sense 
+to use the WORM or SHA1E backend for files that will be stored in
+the Internet Archive, since the key name will be exposed as the filename
+there, and since the Archive does special processing of files based on
+their extension.
 
-Note that it probably makes the most sense to use the WORM backend
-for files, since that exposes the original filename in the key stored
-in the Archive, which allows its special processing for sound files,
-movies, etc to be done. 
-
-Also, the Internet Archive has restrictions on what is allowed in a
-filename; particularly no spaces are allowed. 
+----
 
 There seems to be a bug in either hS3 or the archive that breaks
 authentication when the bucket name contains spaces or upper-case letters..

From e7b309ce02902474b0073d4bb3577bdca1686b67 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:49:52 -0400
Subject: [PATCH 1728/2835] clarify

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index a7f6039810..666abc94e7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,8 +4,8 @@ git-annex (0.20110504) UNRELEASED; urgency=low
   * Add a few tweaks to make it easy to use the Internet Archive's variant
     of S3. In particular, munge key filenames to comply with the IA's filename
     limits, disable encryption, support their nonstandard way of creating
-    buckets, and allow x-amz-* headers to be specified in initremote to set
-    item metadata.
+    buckets, and allow x-archive-* headers to be specified in initremote to
+    set item metadata.
   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
 
  -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400

From b4301c208f96414899f3ba40c2fa5d3bb43089a3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:52:33 -0400
Subject: [PATCH 1729/2835] cleanup

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 25 ++++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index f92e0ee9d7..89e8564705 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -3,11 +3,10 @@ collections using an Amazon S3
 [compatible API](http://www.archive.org/help/abouts3.txt), and this can
 be used with git-annex's [[special_remotes/S3]] support. 
 
-So, if you're an archivist, you can locally archive things with git-annex,
-and define remotes that correspond to "items" at the Internet Archive,
-and use git-annex to upload your files to there.
-Of course, your use of the Internet Archive must comply with their
-[terms of service](http://www.archive.org/about/terms.php).
+So, you can locally archive things with git-annex, define remotes that
+correspond to "items" at the Internet Archive, and use git-annex to upload
+your files to there. Of course, your use of the Internet Archive must
+comply with their [terms of service](http://www.archive.org/about/terms.php).
 
 Sign up for an account, and get your access keys here:
 
@@ -22,8 +21,14 @@ rather than letting git-annex pick a random one; and you can optionally
 specify `x-archive-meta*` headers to add metadata as explained in their
 [documentation](http://www.archive.org/help/abouts3.txt).
 
-	# git annex initremote archive-panama type=S3
-	# host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
+[[!template id=note text="""
+There seems to be a bug in either hS3 or the archive that breaks
+authentication when the bucket name contains spaces or upper-case letters..
+use all lowercase and no spaces when making the bucket with `initremote`.
+"""]]
+
+	# git annex initremote archive-panama type=S3 \
+		host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
 		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
 		x-archive-meta-title="original Panama Canal lock design blueprints"
 	initremote archive-panama (Internet Archive mode) (checking bucket) (creating bucket in US) ok
@@ -42,9 +47,3 @@ to use the WORM or SHA1E backend for files that will be stored in
 the Internet Archive, since the key name will be exposed as the filename
 there, and since the Archive does special processing of files based on
 their extension.
-
-----
-
-There seems to be a bug in either hS3 or the archive that breaks
-authentication when the bucket name contains spaces or upper-case letters..
-use all lowercase and no spaces when making the bucket with `initremote`.

From 27285adc202afee35f6198d0e3aadcff2c662ef8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:55:07 -0400
Subject: [PATCH 1730/2835] link

---
 doc/special_remotes/S3.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index d6a7229e33..047798e238 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -1,7 +1,8 @@
 This special remote type stores file contents in a bucket in Amazon S3
 or a similar service.
 
-See [[walkthrough/using_Amazon_S3]] for usage examples.
+See [[walkthrough/using_Amazon_S3]] and
+[[walkthrough/Internet_Archive_via_S3]] for usage examples.
 
 ## configuration
 

From 267eeb995b8bfb779c017086df75b4700a103485 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:55:33 -0400
Subject: [PATCH 1731/2835] tweak

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 89e8564705..5ed71a123e 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -22,7 +22,7 @@ specify `x-archive-meta*` headers to add metadata as explained in their
 [documentation](http://www.archive.org/help/abouts3.txt).
 
 [[!template id=note text="""
-There seems to be a bug in either hS3 or the archive that breaks
+/!\ There seems to be a bug in either hS3 or the archive that breaks
 authentication when the bucket name contains spaces or upper-case letters..
 use all lowercase and no spaces when making the bucket with `initremote`.
 """]]

From 5256a6b011ba7c8ddd4e07232a5a25a5562c1b88 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 12:10:08 -0400
Subject: [PATCH 1732/2835] migrate: Use current filename when generating new
 key, for backends where the filename affects the key name.

---
 Command/Migrate.hs | 12 ++++++++++--
 debian/changelog   |  2 ++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 35855d5270..790d5d365b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,9 +8,10 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (unless)
+import Control.Monad (unless, when)
 import System.Posix.Files
 import System.Directory
+import System.FilePath
 
 import Command
 import qualified Annex
@@ -52,7 +53,10 @@ perform file oldkey newbackend = do
 	-- The old backend's key is not dropped from it, because there may
 	-- be other files still pointing at that key.
 	let src = gitAnnexLocation g oldkey
-	stored <- Backend.storeFileKey src $ Just newbackend
+	let tmpfile = gitAnnexTmpDir g  takeFileName file
+	liftIO $ createLink src tmpfile
+	stored <- Backend.storeFileKey tmpfile $ Just newbackend
+	liftIO $ cleantmp tmpfile
 	case stored of
 		Nothing -> stop
 		Just (newkey, _) -> do
@@ -69,3 +73,7 @@ perform file oldkey newbackend = do
 					liftIO $ removeFile file
 					next $ Command.Add.cleanup file newkey
 				else stop
+	where
+		cleantmp t = do
+			exists <- doesFileExist t
+			when exists $ removeFile t
diff --git a/debian/changelog b/debian/changelog
index 666abc94e7..3827a4632b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,8 @@ git-annex (0.20110504) UNRELEASED; urgency=low
     buckets, and allow x-archive-* headers to be specified in initremote to
     set item metadata.
   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
+  * migrate: Use current filename when generating new key, for backends
+    where the filename affects the key name.
 
  -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400
 

From 8fa17eaba08b99250d6290915379f048844d83d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 12:12:03 -0400
Subject: [PATCH 1733/2835] tweak

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 5ed71a123e..1d5fc88f53 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -17,7 +17,7 @@ Sign up for an account, and get your access keys here:
 Specify `host=s3.us.archive.org` when doing `initremote` to set up
 a remote at the Archive. This will enable a special Internet Archive mode:
 Encryption is not allowed; you are required to specify a bucket name
-rather than letting git-annex pick a random one; and you can optionally
+rather than having git-annex pick a random one; and you can optionally
 specify `x-archive-meta*` headers to add metadata as explained in their
 [documentation](http://www.archive.org/help/abouts3.txt).
 

From ceff04ff3e7fff4b0ea6e8ad4334cca80d291880 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 12:25:54 -0400
Subject: [PATCH 1734/2835] better multiword parameter handling

This way, individual words as entered on the command line are available
to commands.
---
 Command.hs            | 6 ++++--
 Command/Describe.hs   | 8 ++++----
 Command/Init.hs       | 8 +++++---
 Command/InitRemote.hs | 9 +++++----
 Command/Semitrust.hs  | 7 ++++---
 Command/Trust.hs      | 7 ++++---
 Command/Untrust.hs    | 7 ++++---
 7 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/Command.hs b/Command.hs
index 4f835a3adc..abbe897b5b 100644
--- a/Command.hs
+++ b/Command.hs
@@ -47,6 +47,8 @@ type CommandCleanup = Annex Bool
  - functions. -}
 type CommandSeekStrings = CommandStartString -> CommandSeek
 type CommandStartString = String -> CommandStart
+type CommandSeekWords = CommandStartWords -> CommandSeek
+type CommandStartWords = [String] -> CommandStart
 type CommandSeekKeys = CommandStartKey -> CommandSeek
 type CommandStartKey = Key -> CommandStart
 type BackendFile = (FilePath, Maybe (Backend Annex))
@@ -143,8 +145,8 @@ withFilesNotInGit a params = do
 	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
-withString :: CommandSeekStrings
-withString a params = return [a $ unwords params]
+withWords :: CommandSeekWords
+withWords a params = return [a params]
 withStrings :: CommandSeekStrings
 withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
diff --git a/Command/Describe.hs b/Command/Describe.hs
index dcabef7fbf..57f884e037 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -18,12 +18,12 @@ command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start params = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
 	let (name, description) =
-		case (words params) of
+		case ws of
 			(n:d) -> (n,unwords d)
 			_ -> error "Specify a repository and a description."
 	
diff --git a/Command/Init.hs b/Command/Init.hs
index 668b5c87d6..b7a0787991 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -27,15 +27,17 @@ command = [repoCommand "init" paramDesc seek
 		"initialize git-annex with repository description"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
 {- Stores description for the repository etc. -}
-start :: CommandStartString
-start description = do
+start :: CommandStartWords
+start ws = do
 	when (null description) $
 		error "please specify a description of this repository\n"
 	showStart "init" description
 	next $ perform description
+	where
+		description = unwords ws
 
 perform :: String -> CommandPerform
 perform description = do
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 261ccdc8b7..ae22e3564e 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -28,21 +28,22 @@ command = [repoCommand "initremote"
 	"sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start params = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
 	when (null ws) $ error "Specify a name for the remote"
 
 	(u, c) <- findByName name
 	let fullconfig = M.union config c	
 	t <- findType fullconfig
 	
+	liftIO $ putStrLn $ show fullconfig
+
 	showStart "initremote" name
 	next $ perform t u $ M.union config c
 
 	where
-		ws = words params
 		name = head ws
 		config = Remote.keyValToConfig $ tail ws
 
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index fc1bcbbcdc..11742137f1 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -18,10 +18,11 @@ command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start name = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
+	let name = unwords ws
 	showStart "semitrust" name
 	u <- Remote.nameToUUID name
 	next $ perform u
diff --git a/Command/Trust.hs b/Command/Trust.hs
index ef03828c23..d5444affe6 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -18,10 +18,11 @@ command = [repoCommand "trust" (paramRepeating paramRemote) seek
 	"trust a repository"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start name = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
+	let name = unwords ws
 	showStart "trust" name
 	u <- Remote.nameToUUID name
 	next $ perform u
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index ebe9c31b3b..174c395066 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -18,10 +18,11 @@ command = [repoCommand "untrust" (paramRepeating paramRemote) seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start name = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
+	let name = unwords ws
 	showStart "untrust" name
 	u <- Remote.nameToUUID name
 	next $ perform u

From 93c5fb5da7f085cc772e28d8ded08f4ea0b0bf15 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 13:07:56 -0400
Subject: [PATCH 1735/2835] support remote config values with spaces and other
 characters

---
 Command/InitRemote.hs |  2 --
 Remote.hs             | 38 +++++++++++++++++++++++++++++++++++---
 test.hs               |  1 +
 3 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index ae22e3564e..ad0718e38e 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -37,8 +37,6 @@ start ws = notBareRepo $ do
 	(u, c) <- findByName name
 	let fullconfig = M.union config c	
 	t <- findType fullconfig
-	
-	liftIO $ putStrLn $ show fullconfig
 
 	showStart "initremote" name
 	next $ perform t u $ M.union config c
diff --git a/Remote.hs b/Remote.hs
index 211168b153..7bbe910000 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -25,7 +25,10 @@ module Remote (
 	remoteLog,
 	readRemoteLog,
 	configSet,
-	keyValToConfig
+	keyValToConfig,
+	configToKeyVal,
+	
+	prop_idempotent_configEscape
 ) where
 
 import Control.Monad.State (liftIO)
@@ -33,6 +36,7 @@ import Control.Monad (when, liftM, filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
+import Data.Char
 
 import RemoteClass
 import Types
@@ -176,9 +180,37 @@ keyValToConfig ws = M.fromList $ map (/=/) ws
 		(/=/) s = (k, v)
 			where
 				k = takeWhile (/= '=') s
-				v = drop (1 + length k) s
+				v = configUnEscape $ drop (1 + length k) s
 
 configToKeyVal :: M.Map String String -> [String]
 configToKeyVal m = map toword $ sort $ M.toList m
 	where
-		toword (k, v) = k ++ "=" ++ v
+		toword (k, v) = k ++ "=" ++ configEscape v
+
+configEscape :: String -> String
+configEscape = concat . (map escape)
+	where
+		escape c
+			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"
+			| otherwise = [c]
+
+configUnEscape :: String -> String
+configUnEscape = unescape
+	where
+		unescape [] = []
+		unescape (c:rest)
+			| c == '&' = entity rest
+			| otherwise = c : unescape rest
+		entity s = if ok
+				then chr (read num) : unescape rest
+				else '&' : unescape s
+			where
+				num = takeWhile isNumber s
+				r = drop (length num) s
+				rest = drop 1 r
+				ok = not (null num) && 
+					not (null r) && r !! 0 == ';'
+
+{- for quickcheck -}
+prop_idempotent_configEscape :: String -> Bool
+prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
diff --git a/test.hs b/test.hs
index 73eff662b9..456c09060c 100644
--- a/test.hs
+++ b/test.hs
@@ -75,6 +75,7 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_idempotent_key_read_show" Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
 	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
+	, qctest "prop_idempotent_configEscape" Remote.prop_idempotent_configEscape
 	, qctest "prop_parentDir_basics" Utility.prop_parentDir_basics
 	, qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane

From 0a7bcd47aeec9a2d1c9a42ef8d9ea539a6aef0d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 13:10:26 -0400
Subject: [PATCH 1736/2835] IA: do not create bucket at initremote time

This way, the metadata sent when uploading a file is applied to the bucket
then.
---
 Remote/S3real.hs                             | 7 ++++---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 7d6b5d5bab..1359669036 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -89,13 +89,14 @@ s3Setup u c = handlehost $ M.lookup "host" c
 			| otherwise = defaulthost
 
 		use fullconfig = do
-			genBucket fullconfig
 			gitConfigSpecialRemote u fullconfig "s3" "true"
 			s3SetCreds fullconfig	
 
 		defaulthost = do
 			c' <- encryptionSetup c
-			use $ M.union c' defaults
+			let fullconfig = M.union c' defaults
+			genBucket fullconfig
+			use fullconfig
 
 		archiveorg = do
 			showNote $ "Internet Archive mode"
@@ -112,7 +113,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 					M.union c $
 					-- special constraints on key names
 					M.insert "mungekeys" "ia" $
-					-- buckets created only as files
+					-- bucket created only when files
 					-- are uploaded
 					M.insert "x-amz-auto-make-bucket" "1" $
 					-- no default bucket name; should
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 1d5fc88f53..b80f0a4b7c 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -31,7 +31,7 @@ use all lowercase and no spaces when making the bucket with `initremote`.
 		host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
 		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
 		x-archive-meta-title="original Panama Canal lock design blueprints"
-	initremote archive-panama (Internet Archive mode) (checking bucket) (creating bucket in US) ok
+	initremote archive-panama (Internet Archive mode) ok
 	# git annex describe archive-panama "Internet Archive item for my grandfather's Panama Canal lock design blueprints"
 	describe archive-panama ok
 

From d006586cd0b706c9cc92b2747b2ba3487f52c04a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 13:27:19 -0400
Subject: [PATCH 1737/2835] add a message in potenatially confusing copy --fast
 failure situation

---
 Command/Move.hs                                        | 10 ++++++++--
 .../copy_fast_confusing_with_broken_locationlog.mdwn   |  2 ++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 623003e47a..f49fe20e00 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -7,6 +7,8 @@
 
 module Command.Move where
 
+import Control.Monad (when)
+
 import Command
 import qualified Command.Drop
 import qualified Annex
@@ -84,7 +86,8 @@ toPerform dest move key = do
 	-- and an explicit check is not done, when copying. When moving,
 	-- it has to be done, to avoid inaverdent data loss.
 	fast <- Annex.getState Annex.fast
-	isthere <- if fast && not move && not (Remote.hasKeyCheap dest)
+	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)
+	isthere <- if fastcheck
 		then do
 			(remotes, _) <- Remote.keyPossibilities key
 			return $ Right $ dest `elem` remotes
@@ -98,7 +101,10 @@ toPerform dest move key = do
 			ok <- Remote.storeKey dest key
 			if ok
 				then next $ toCleanup dest move key
-				else stop -- failed
+				else do
+					when fastcheck $
+						warning "This could have failed because --fast is enabled."
+					stop
 		Right True -> next $ toCleanup dest move key
 toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 toCleanup dest move key = do
diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
index 2767b5aa9c..47fc77fa4c 100644
--- a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
@@ -2,3 +2,5 @@ Conversation moved from [[walkthrough/recover_data_from_lost+found]]
 to a proper bug. --[[Joey]]
 
 (Unfortunatly that scrambled the comment creation times and thus order.)
+
+> Added a message [[done]] --[[Joey]] 

From 57428c356ea81ea13193ac5dfc32d9a824ed4d65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 13:33:33 -0400
Subject: [PATCH 1738/2835] heh

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index b80f0a4b7c..8c0f2dde74 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -32,7 +32,7 @@ use all lowercase and no spaces when making the bucket with `initremote`.
 		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
 		x-archive-meta-title="original Panama Canal lock design blueprints"
 	initremote archive-panama (Internet Archive mode) ok
-	# git annex describe archive-panama "Internet Archive item for my grandfather's Panama Canal lock design blueprints"
+	# git annex describe archive-panama "a man, a plan, a canal: panama"
 	describe archive-panama ok
 
 Then you can annex files and copy them to the remote as usual:

From 760cde28b67c14e0ad68e8649c0abe0544c44947 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 14:49:28 -0400
Subject: [PATCH 1739/2835] more pointless monadic golfing

---
 GitRepo.hs       | 3 +--
 Locations.hs     | 2 +-
 Remote/S3real.hs | 2 +-
 Utility.hs       | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 3c5a1e129e..87cceece41 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -577,8 +577,7 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""
 				e_num c = showoctal $ ord c
 				-- unicode character is decomposed to
 				-- Word8s and each is shown in octal
-				e_utf c = concat $ map showoctal $
-						(encode [c] :: [Word8])
+				e_utf c = showoctal =<< (encode [c] :: [Word8])
 
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
diff --git a/Locations.hs b/Locations.hs
index 1c4f8296e8..38a320a2b2 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -175,7 +175,7 @@ prop_idempotent_fileKey s = Just k == fileKey (keyFile k)
 hashDirMixed :: Key -> FilePath
 hashDirMixed k = addTrailingPathSeparator $ take 2 dir  drop 2 dir
 	where
-		dir = take 4 $ concat $ map display_32bits_as_dir [a,b,c,d]
+		dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d]
 		ABCD (a,b,c,d) = md5 $ Str $ show k
 
 {- Generates a hash directory that is all lower case. -}
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 1359669036..baf570593e 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -232,7 +232,7 @@ bucketKey r bucket k = S3Object bucket (bucketFile r k) "" [] L.empty
  - with no whitespace. Other characters are xml entity
  - encoded. -}
 iaMunge :: String -> String
-iaMunge = concat . (map munge)
+iaMunge = (>>= munge)
 	where
 		munge c
 			| isAsciiUpper c || isAsciiLower c || isNumber c = [c]
diff --git a/Utility.hs b/Utility.hs
index 44c8cdd650..6dd7d329c8 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -59,7 +59,7 @@ data CommandParam = Params String | Param String | File FilePath
 {- Used to pass a list of CommandParams to a function that runs
  - a command and expects Strings. -}
 toCommand :: [CommandParam] -> [String]
-toCommand l = concat $ map unwrap l
+toCommand = (>>= unwrap)
 	where
 		unwrap (Param s) = [s]
 		unwrap (Params s) = filter (not . null) (split " " s)

From 3ab15b9f4f22aaa77a09e198d03dd4296a83bb44 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 15:01:05 -0400
Subject: [PATCH 1740/2835] releasing version 0.20110516

---
 Remote.hs        | 2 +-
 debian/changelog | 6 +++---
 debian/control   | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 7bbe910000..7df84a5da6 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -188,7 +188,7 @@ configToKeyVal m = map toword $ sort $ M.toList m
 		toword (k, v) = k ++ "=" ++ configEscape v
 
 configEscape :: String -> String
-configEscape = concat . (map escape)
+configEscape = (>>= escape)
 	where
 		escape c
 			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"
diff --git a/debian/changelog b/debian/changelog
index 3827a4632b..5cae0a8b51 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,5 @@
-git-annex (0.20110504) UNRELEASED; urgency=low
+git-annex (0.20110516) unstable; urgency=low
 
-  * Work around a bug in Network.URI's handling of bracketed ipv6 addresses.
   * Add a few tweaks to make it easy to use the Internet Archive's variant
     of S3. In particular, munge key filenames to comply with the IA's filename
     limits, disable encryption, support their nonstandard way of creating
@@ -9,8 +8,9 @@ git-annex (0.20110504) UNRELEASED; urgency=low
   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
   * migrate: Use current filename when generating new key, for backends
     where the filename affects the key name.
+  * Work around a bug in Network.URI's handling of bracketed ipv6 addresses.
 
- -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400
+ -- Joey Hess   Mon, 16 May 2011 14:16:52 -0400
 
 git-annex (0.20110503) unstable; urgency=low
 
diff --git a/debian/control b/debian/control
index b75b35afbc..f2f9cecd39 100644
--- a/debian/control
+++ b/debian/control
@@ -17,7 +17,7 @@ Build-Depends:
 	uuid,
 	rsync,
 Maintainer: Joey Hess 
-Standards-Version: 3.9.1
+Standards-Version: 3.9.2
 Vcs-Git: git://git.kitenet.net/git-annex
 Homepage: http://git-annex.branchable.com/
 

From 8d4d84b80f8d652a28baa12a51bf5e24681aada4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 15:01:42 -0400
Subject: [PATCH 1741/2835] add news item for git-annex 0.20110516

---
 doc/news/version_0.20110419.mdwn |  7 -------
 doc/news/version_0.20110516.mdwn | 11 +++++++++++
 2 files changed, 11 insertions(+), 7 deletions(-)
 delete mode 100644 doc/news/version_0.20110419.mdwn
 create mode 100644 doc/news/version_0.20110516.mdwn

diff --git a/doc/news/version_0.20110419.mdwn b/doc/news/version_0.20110419.mdwn
deleted file mode 100644
index 2506ad0a87..0000000000
--- a/doc/news/version_0.20110419.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 0.20110419 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Don't run gpg in batch mode, so it can prompt for passphrase when
-     there is no agent.
-   * Add missing build dep on dataenc.
-   * S3: Fix stalls when transferring encrypted data.
-   * bup: Avoid memory leak when transferring encrypted data."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110516.mdwn b/doc/news/version_0.20110516.mdwn
new file mode 100644
index 0000000000..aa350a4848
--- /dev/null
+++ b/doc/news/version_0.20110516.mdwn
@@ -0,0 +1,11 @@
+git-annex 0.20110516 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Add a few tweaks to make it easy to use the Internet Archive's variant
+     of S3. In particular, munge key filenames to comply with the IA's filename
+     limits, disable encryption, support their nonstandard way of creating
+     buckets, and allow x-archive-* headers to be specified in initremote to
+     set item metadata.
+   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
+   * migrate: Use current filename when generating new key, for backends
+     where the filename affects the key name.
+   * Work around a bug in Network.URI's handling of bracketed ipv6 addresses."""]]
\ No newline at end of file

From 786b990d5127ce21e2924336f6f679fb7569358d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 16 May 2011 20:01:29 +0000
Subject: [PATCH 1742/2835] Added a comment

---
 .../comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment

diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment
new file mode 100644
index 0000000000..6d3dabb92b
--- /dev/null
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 16"
+ date="2011-05-16T20:01:28Z"
+ content="""
+Thanks.
+"""]]

From a5beb61b90d318d6ab367fd761d182b658a577f4 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 17 May 2011 01:15:12 +0000
Subject: [PATCH 1743/2835] Added a comment

---
 ..._d1fd70c67243971c96d59e1ffb7ef6e7._comment | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment

diff --git a/doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment b/doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment
new file mode 100644
index 0000000000..39986144be
--- /dev/null
+++ b/doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-05-17T01:15:10Z"
+ content="""
+What a good idea!
+
+150 lines of haskell later, I have this:
+
+
+# git annex status
+supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+supported remote types: git S3 bup directory rsync hook
+local annex keys: 32
+local annex size: 58 megabytes
+total annex keys: 38158
+total annex size: 6 terabytes (but 1632 keys have unknown size)
+backend usage: 
+	SHA1: 1789
+	WORM: 36369
+

+"""]]

From a8816efc140108cc62713cc6227db69ef96cd913 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 21:18:34 -0400
Subject: [PATCH 1744/2835] status: New subcommand to show info about an annex,
 including its size.

---
 Command/Status.hs  | 151 +++++++++++++++++++++++++++++++++++++++++++++
 Command/Unused.hs  |  18 +++---
 GitAnnex.hs        |   2 +
 debian/changelog   |   6 ++
 doc/git-annex.mdwn |  10 +++
 5 files changed, 178 insertions(+), 9 deletions(-)
 create mode 100644 Command/Status.hs

diff --git a/Command/Status.hs b/Command/Status.hs
new file mode 100644
index 0000000000..e8fce3bca1
--- /dev/null
+++ b/Command/Status.hs
@@ -0,0 +1,151 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Status where
+
+import Control.Monad.State
+import Data.Maybe
+import System.IO
+import Data.List
+import qualified Data.Map as M
+
+import qualified Annex
+import qualified BackendClass
+import qualified RemoteClass
+import qualified Remote
+import qualified Command.Unused
+import Command
+import Types
+import DataUnits
+import Content
+import Key
+
+-- a named computation that produces a statistic
+type Stat = (String, StatState String)
+
+-- cached info that multiple Stats may need
+type SizeList a = ([a], Int)
+data StatInfo = StatInfo
+	{ keysPresentCache :: (Maybe (SizeList Key))
+	, keysReferencedCache :: (Maybe (SizeList Key))
+	}
+
+-- a state monad for running Stats in
+type StatState = StateT StatInfo Annex
+
+command :: [Command]
+command = [repoCommand "status" (paramNothing) seek
+	"shows status information about the annex"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+{- Order is significant. Less expensive operations, and operations
+ - that share data go together.
+ -}
+faststats :: [Stat]
+faststats = 
+	[ supported_backends
+	, supported_remote_types
+	, local_annex_keys
+	, local_annex_size
+	]
+slowstats :: [Stat]
+slowstats =
+	[ total_annex_keys
+	, total_annex_size
+	, backend_usage
+	]
+
+start :: CommandStartNothing
+start = do
+	fast <- Annex.getState Annex.fast
+	let todo = if fast then faststats else faststats ++ slowstats
+	evalStateT (mapM_ showStat todo) (StatInfo Nothing Nothing)
+	stop
+
+stat :: String -> StatState String -> Stat
+stat desc a = (desc, a)
+
+showStat :: Stat -> StatState ()
+showStat (desc, a) = do
+	liftIO $ putStr $ desc ++ ": "
+	liftIO $ hFlush stdout
+	liftIO . putStrLn =<< a
+
+
+supported_backends :: Stat
+supported_backends = stat "supported backends" $
+	lift (Annex.getState Annex.supportedBackends) >>=
+		return . unwords . (map BackendClass.name)
+
+supported_remote_types :: Stat
+supported_remote_types = stat "supported remote types" $
+	return $ unwords $ map RemoteClass.typename Remote.remoteTypes
+
+local_annex_size :: Stat
+local_annex_size = stat "local annex size" $
+	cachedKeysPresent >>= keySizeSum
+
+total_annex_size :: Stat
+total_annex_size = stat "total annex size" $
+	cachedKeysReferenced >>= keySizeSum
+
+local_annex_keys :: Stat
+local_annex_keys = stat "local annex keys" $ 
+	return . show . snd =<< cachedKeysPresent
+
+total_annex_keys :: Stat
+total_annex_keys = stat "total annex keys" $
+	return . show . snd =<< cachedKeysReferenced
+
+backend_usage :: Stat
+backend_usage = stat "backend usage" $
+	return . usage =<< cachedKeysReferenced
+	where
+		usage (ks, _) = pp "" $ sort $ map tflip $ splits ks
+		splits :: [Key] -> [(String, Integer)]
+		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
+		tcount k = (keyBackendName k, 1)
+		tflip (a, b) = (b, a)
+		pp c [] = c
+		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
+
+cachedKeysPresent :: StatState (SizeList Key)
+cachedKeysPresent = do
+	s <- get
+	case keysPresentCache s of
+		Just v -> return v
+		Nothing -> do
+			keys <- lift $ getKeysPresent
+			let v = (keys, length keys)
+			put s { keysPresentCache = Just v }
+			return v
+
+cachedKeysReferenced :: StatState (SizeList Key)
+cachedKeysReferenced = do
+	s <- get
+	case keysReferencedCache s of
+		Just v -> return v
+		Nothing -> do
+			keys <- lift $ Command.Unused.getKeysReferenced
+			-- A given key may be referenced repeatedly.
+			-- nub does not seem too slow (yet)..
+			let uniques = nub keys
+			let v = (uniques, length uniques)
+			put s { keysReferencedCache = Just v }
+			return v
+
+keySizeSum :: SizeList Key -> StatState String
+keySizeSum (keys, len) = do
+	let knownsize = catMaybes $ map keySize keys
+	let total = roughSize storageUnits False $ foldl (+) 0 knownsize
+	let missing = len - length knownsize
+	return $ total ++
+		if missing > 0
+			then " (but " ++ show missing ++ " keys have unknown size)"
+			else ""
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a2e1c86de1..1482f057e8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -144,16 +144,16 @@ unusedKeys = do
 	if fast
 		then do
 			showNote "fast mode enabled; only finding stale files"
-			tmp <- staleKeys' gitAnnexTmpDir
-			bad <- staleKeys' gitAnnexBadDir
+			tmp <- staleKeys gitAnnexTmpDir
+			bad <- staleKeys gitAnnexBadDir
 			return ([], bad, tmp)
 		else do
 			showNote "checking for unused data..."
 			present <- getKeysPresent
 			referenced <- getKeysReferenced
 			let unused = present `exclude` referenced
-			staletmp <- staleKeys gitAnnexTmpDir present
-			stalebad <- staleKeys gitAnnexBadDir present
+			staletmp <- staleKeysPrune gitAnnexTmpDir present
+			stalebad <- staleKeysPrune gitAnnexBadDir present
 			return (unused, stalebad, staletmp)
 
 {- Finds items in the first, smaller list, that are not
@@ -182,9 +182,9 @@ getKeysReferenced = do
  - When a list of presently available keys is provided, stale keys
  - that no longer have value are deleted.
  -}
-staleKeys :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key]
-staleKeys dirspec present = do
-	contents <- staleKeys' dirspec
+staleKeysPrune :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key]
+staleKeysPrune dirspec present = do
+	contents <- staleKeys dirspec
 		
 	let stale = contents `exclude` present
 	let dup = contents `exclude` stale
@@ -195,8 +195,8 @@ staleKeys dirspec present = do
 
 	return stale
 
-staleKeys' :: (Git.Repo -> FilePath) -> Annex [Key]
-staleKeys' dirspec = do
+staleKeys :: (Git.Repo -> FilePath) -> Annex [Key]
+staleKeys dirspec = do
 	g <- Annex.gitRepo
 	let dir = dirspec g
 	exists <- liftIO $ doesDirectoryExist dir
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 736b430e60..99aec187a9 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -36,6 +36,7 @@ import qualified Command.Lock
 import qualified Command.PreCommit
 import qualified Command.Find
 import qualified Command.Whereis
+import qualified Command.Status
 import qualified Command.Migrate
 import qualified Command.Uninit
 import qualified Command.Trust
@@ -72,6 +73,7 @@ cmds = concat
 	, Command.DropUnused.command
 	, Command.Find.command
 	, Command.Whereis.command
+	, Command.Status.command
 	, Command.Migrate.command
 	, Command.Map.command
 	, Command.Upgrade.command
diff --git a/debian/changelog b/debian/changelog
index 5cae0a8b51..d759d3672a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.20110517) UNRELEASED; urgency=low
+
+  * status: New subcommand to show info about an annex, including its size.
+
+ -- Joey Hess   Mon, 16 May 2011 20:27:46 -0400
+
 git-annex (0.20110516) unstable; urgency=low
 
   * Add a few tweaks to make it easy to use the Internet Archive's variant
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 450b95a0dd..e2a04d27b9 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -182,6 +182,16 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Displays a list of repositories known to contain the content of the
   specified file or files.
 
+* status
+
+  Displays some statistics and other information, including how much data
+  is in the annex.
+
+  Some of the statistics can take a while to generate, and those
+  come last. You can ctrl-c this command once it's displayed the
+  information you wanted to see. Or, use --fast to only display
+  the first, fast(ish) statistics.
+
 * migrate [path ...]
 
   Changes the specified annexed files to store their content in the

From 1e3da8efb0f303f8e03c39c0c0e521c19708c88d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:01:50 -0400
Subject: [PATCH 1745/2835] add info about any temp files and bad content files

---
 Command/Status.hs | 50 ++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index e8fce3bca1..a82fc9e1c1 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -18,17 +18,18 @@ import qualified BackendClass
 import qualified RemoteClass
 import qualified Remote
 import qualified Command.Unused
+import qualified GitRepo as Git
 import Command
 import Types
 import DataUnits
 import Content
 import Key
+import Locations
 
 -- a named computation that produces a statistic
-type Stat = (String, StatState String)
+type Stat = StatState (Maybe (String, StatState String))
 
 -- cached info that multiple Stats may need
-type SizeList a = ([a], Int)
 data StatInfo = StatInfo
 	{ keysPresentCache :: (Maybe (SizeList Key))
 	, keysReferencedCache :: (Maybe (SizeList Key))
@@ -37,6 +38,11 @@ data StatInfo = StatInfo
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
 
+type SizeList a = ([a], Int)
+
+sizeList :: [a] -> SizeList a
+sizeList l = (l, length l)
+
 command :: [Command]
 command = [repoCommand "status" (paramNothing) seek
 	"shows status information about the annex"]
@@ -53,6 +59,8 @@ faststats =
 	, supported_remote_types
 	, local_annex_keys
 	, local_annex_size
+	, tmp_size
+	, bad_data_size
 	]
 slowstats :: [Stat]
 slowstats =
@@ -69,14 +77,19 @@ start = do
 	stop
 
 stat :: String -> StatState String -> Stat
-stat desc a = (desc, a)
+stat desc a = return $ Just (desc, a)
+
+nostat :: Stat
+nostat = return $ Nothing
 
 showStat :: Stat -> StatState ()
-showStat (desc, a) = do
-	liftIO $ putStr $ desc ++ ": "
-	liftIO $ hFlush stdout
-	liftIO . putStrLn =<< a
-
+showStat s = calc =<< s
+	where
+		calc (Just (desc, a)) = do
+			liftIO $ putStr $ desc ++ ": "
+			liftIO $ hFlush stdout
+			liftIO . putStrLn =<< a
+		calc Nothing = return ()
 
 supported_backends :: Stat
 supported_backends = stat "supported backends" $
@@ -103,6 +116,12 @@ total_annex_keys :: Stat
 total_annex_keys = stat "total annex keys" $
 	return . show . snd =<< cachedKeysReferenced
 
+tmp_size :: Stat
+tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
+
+bad_data_size :: Stat
+bad_data_size = staleSize "bad keys size" gitAnnexBadDir
+
 backend_usage :: Stat
 backend_usage = stat "backend usage" $
 	return . usage =<< cachedKeysReferenced
@@ -115,6 +134,7 @@ backend_usage = stat "backend usage" $
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 
+
 cachedKeysPresent :: StatState (SizeList Key)
 cachedKeysPresent = do
 	s <- get
@@ -122,7 +142,7 @@ cachedKeysPresent = do
 		Just v -> return v
 		Nothing -> do
 			keys <- lift $ getKeysPresent
-			let v = (keys, length keys)
+			let v = sizeList keys
 			put s { keysPresentCache = Just v }
 			return v
 
@@ -135,8 +155,7 @@ cachedKeysReferenced = do
 			keys <- lift $ Command.Unused.getKeysReferenced
 			-- A given key may be referenced repeatedly.
 			-- nub does not seem too slow (yet)..
-			let uniques = nub keys
-			let v = (uniques, length uniques)
+			let v = sizeList $ nub keys
 			put s { keysReferencedCache = Just v }
 			return v
 
@@ -149,3 +168,12 @@ keySizeSum (keys, len) = do
 		if missing > 0
 			then " (but " ++ show missing ++ " keys have unknown size)"
 			else ""
+
+staleSize :: String -> (Git.Repo -> FilePath) -> Stat
+staleSize label dirspec = do
+	keys <- lift (Command.Unused.staleKeys dirspec)
+	if null keys
+		then nostat
+		else stat label $ do
+			s <- keySizeSum $ sizeList keys
+			return $ s ++ " (clean up with git-annex unused)"

From 5068985020a4fbd699721cd506250316c55f129f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:19:15 -0400
Subject: [PATCH 1746/2835] rejigger what's --fast

---
 Command/Status.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index a82fc9e1c1..85a6a5a4be 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -57,14 +57,14 @@ faststats :: [Stat]
 faststats = 
 	[ supported_backends
 	, supported_remote_types
-	, local_annex_keys
-	, local_annex_size
 	, tmp_size
 	, bad_data_size
 	]
 slowstats :: [Stat]
 slowstats =
-	[ total_annex_keys
+	[ local_annex_keys
+	, local_annex_size
+	, total_annex_keys
 	, total_annex_size
 	, backend_usage
 	]

From 21953a802a0f55399288b52834cbfa970fa40d0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:22:37 -0400
Subject: [PATCH 1747/2835] am I silly to worry about length overflowing int
 max?

---
 Command/Status.hs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 85a6a5a4be..c2f7692c5f 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -38,10 +38,13 @@ data StatInfo = StatInfo
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
 
-type SizeList a = ([a], Int)
+-- a list with a known length
+-- (Integer is used for the length to avoid
+-- blowing up if someone annexed billions of files..)
+type SizeList a = ([a], Integer)
 
 sizeList :: [a] -> SizeList a
-sizeList l = (l, length l)
+sizeList l = (l, genericLength l)
 
 command :: [Command]
 command = [repoCommand "status" (paramNothing) seek
@@ -163,7 +166,7 @@ keySizeSum :: SizeList Key -> StatState String
 keySizeSum (keys, len) = do
 	let knownsize = catMaybes $ map keySize keys
 	let total = roughSize storageUnits False $ foldl (+) 0 knownsize
-	let missing = len - length knownsize
+	let missing = len - genericLength knownsize
 	return $ total ++
 		if missing > 0
 			then " (but " ++ show missing ++ " keys have unknown size)"

From 51cc71fac176878de2ccb960f62db419bb63d00f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:37:31 -0400
Subject: [PATCH 1748/2835] longterm todo item

---
 doc/todo/cache_key_info.mdwn | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 doc/todo/cache_key_info.mdwn

diff --git a/doc/todo/cache_key_info.mdwn b/doc/todo/cache_key_info.mdwn
new file mode 100644
index 0000000000..d26d055129
--- /dev/null
+++ b/doc/todo/cache_key_info.mdwn
@@ -0,0 +1,36 @@
+Most of git-annex is designed to be fast no matter how many other files are
+in the annex. Things like add/get/drop/move/fsck have good locality;
+they will only operate on as many files as you need them to. 
+
+(git commit can get a little slow with a great deal of files,
+but that's out of scope -- and recent git-annex versions use queuing
+to save git add from piling up too much in the index.)
+
+But currently two git-annex commands are quite slow when annexes become large
+in quantity of files. These are unused and stats
+(Both have --fast versions that don't do as much).
+
+Both are slow because both need two peices of information that are not
+quick to look up, and require examining the whole repo, very seekily:
+
+1. The keys present in the annex. Found by looking thru .git/annex/objects.
+2. The keys referenced by files in git. Found by finding every file
+   in git, and looking at its symlink.
+
+Of these, the first is less expensive (typically, an annex does not have every
+key in it). It could be optimized fairly simply, by adding a database
+of keys present in the annex that is optimised to list them all. The
+database would be updated by the few functions that move content in and
+out.
+
+The second is harder to optimise, because the user can delete, revert,
+copy, add, etc files in git at will, and git-annex does not have a good way
+to watch that and maintain a database of what keys are being referenced.
+
+It could use a post-commit hook and examine files changed by commits, etc.
+But then staged files would be left out. It might be sufficient to 
+make --fast trust the database... except unused will suggest *deleting*
+data if nothing references it. Or maybe it could be required to have a
+clean tree with nothing staged before running git-annex unused.
+
+Anyway, this is a semi-longterm item for me. --[[Joey]] 

From 33e6425d102af17f32593e87286aaf8b11a7e1d4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:49:41 -0400
Subject: [PATCH 1749/2835] tweak

---
 Command/Status.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index c2f7692c5f..39e2300219 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -169,7 +169,7 @@ keySizeSum (keys, len) = do
 	let missing = len - genericLength knownsize
 	return $ total ++
 		if missing > 0
-			then " (but " ++ show missing ++ " keys have unknown size)"
+			then aside $ "but " ++ show missing ++ " keys have unknown size"
 			else ""
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
@@ -179,4 +179,7 @@ staleSize label dirspec = do
 		then nostat
 		else stat label $ do
 			s <- keySizeSum $ sizeList keys
-			return $ s ++ " (clean up with git-annex unused)"
+			return $ s ++ aside "clean up with git-annex unused"
+
+aside :: String -> String
+aside s = "\t(" ++ s ++ ")"

From 75a3f5027f74565d909fb940893636d081d9872a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 01:59:44 -0400
Subject: [PATCH 1750/2835] tweak

---
 Command/Status.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 39e2300219..43a9484433 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -11,6 +11,7 @@ import Control.Monad.State
 import Data.Maybe
 import System.IO
 import Data.List
+import Data.Tuple
 import qualified Data.Map as M
 
 import qualified Annex
@@ -129,11 +130,10 @@ backend_usage :: Stat
 backend_usage = stat "backend usage" $
 	return . usage =<< cachedKeysReferenced
 	where
-		usage (ks, _) = pp "" $ sort $ map tflip $ splits ks
+		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
-		tflip (a, b) = (b, a)
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 

From c91929f6934fc4e94603d0fa004e824d5e2cfb65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 03:10:13 -0400
Subject: [PATCH 1751/2835] add whenM and unlessM

Just more golfing.. I am pretty sure something in a library somewhere can
do this, but I have been unable to find it.
---
 Command/Drop.hs       |  5 +----
 Command/DropUnused.hs |  5 ++---
 Command/Find.hs       |  5 ++---
 Command/Migrate.hs    | 10 +++-------
 Command/RecvKey.hs    |  6 ++----
 Command/SendKey.hs    |  7 +++----
 Command/Uninit.hs     |  7 ++-----
 Command/Unlock.hs     |  6 ++----
 Content.hs            |  6 ++----
 CopyFile.hs           |  4 +---
 GitRepo.hs            |  6 +++---
 Remote/Bup.hs         |  9 ++++-----
 Remote/Directory.hs   |  4 ++--
 Remote/Rsync.hs       |  7 +++----
 Utility.hs            | 32 +++++++++++++++++++++++++++-----
 git-annex-shell.hs    |  4 +---
 16 files changed, 60 insertions(+), 63 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index 05c956fddf..07cec1a677 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -7,8 +7,6 @@
 
 module Command.Drop where
 
-import Control.Monad (when)
-
 import Command
 import qualified Backend
 import LocationLog
@@ -46,7 +44,6 @@ perform key backend numcopies = do
 
 cleanup :: Key -> CommandCleanup
 cleanup key = do
-	inannex <- inAnnex key
-	when inannex $ removeAnnex key
+	whenM (inAnnex key) $ removeAnnex key
 	logStatus key ValueMissing
 	return True
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 965a99ed56..1bb3b7f970 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -7,7 +7,6 @@
 
 module Command.DropUnused where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Directory
@@ -24,6 +23,7 @@ import qualified Remote
 import qualified GitRepo as Git
 import Backend
 import Key
+import Utility
 
 type UnusedMap = M.Map String Key
 
@@ -72,8 +72,7 @@ performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
 	g <- Annex.gitRepo
 	let f = filespec g key
-	e <- liftIO $ doesFileExist f
-	when e $ liftIO $ removeFile f
+	liftIO $ whenM (doesFileExist f) $ removeFile f
 	next $ return True
 
 readUnusedLog :: FilePath -> Annex UnusedMap
diff --git a/Command/Find.hs b/Command/Find.hs
index eecf3cd7da..9d760ff5a8 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,11 +7,11 @@
 
 module Command.Find where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 
 import Command
 import Content
+import Utility
 
 command :: [Command]
 command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
@@ -23,6 +23,5 @@ seek = [withFilesInGit start]
 {- Output a list of files. -}
 start :: CommandStartString
 start file = isAnnexed file $ \(key, _) -> do
-	exists <- inAnnex key
-	when exists $ liftIO $ putStrLn file
+	whenM (inAnnex key) $ liftIO $ putStrLn file
 	stop
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 790d5d365b..09ff6df7da 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,7 +8,6 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (unless, when)
 import System.Posix.Files
 import System.Directory
 import System.FilePath
@@ -20,6 +19,7 @@ import Locations
 import Types
 import Content
 import Messages
+import Utility
 import qualified Command.Add
 
 command :: [Command]
@@ -63,9 +63,7 @@ perform file oldkey newbackend = do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
 				-- cached key, to avoid wasting disk space.
-				liftIO $ do
-					exists <- doesFileExist t
-					unless exists $ createLink src t
+				liftIO $ unlessM (doesFileExist t) $ createLink src t
 				return True
 			if ok
 				then do
@@ -74,6 +72,4 @@ perform file oldkey newbackend = do
 					next $ Command.Add.cleanup file newkey
 				else stop
 	where
-		cleantmp t = do
-			exists <- doesFileExist t
-			when exists $ removeFile t
+		cleantmp t = whenM (doesFileExist t) $ removeFile t
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 126608f614..b49116de45 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -7,13 +7,13 @@
 
 module Command.RecvKey where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Exit
 
 import Command
 import CmdLine
 import Content
+import Utility
 import RsyncFile
 
 command :: [Command]
@@ -25,9 +25,7 @@ seek = [withKeys start]
 
 start :: CommandStartKey
 start key = do
-	present <- inAnnex key
-	when present $
-		error "key is already present in annex"
+	whenM (inAnnex key) $ error "key is already present in annex"
 	
 	ok <- getViaTmp key (liftIO . rsyncServerReceive)
 	if ok
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 871a530af7..7497ce3bfe 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -7,7 +7,6 @@
 
 module Command.SendKey where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Exit
 
@@ -15,6 +14,7 @@ import Locations
 import qualified Annex
 import Command
 import Content
+import Utility
 import RsyncFile
 
 command :: [Command]
@@ -26,9 +26,8 @@ seek = [withKeys start]
 
 start :: CommandStartKey
 start key = do
-	present <- inAnnex key
 	g <- Annex.gitRepo
 	let file = gitAnnexLocation g key
-	when present $
-		liftIO $ rsyncServerSend file
+	whenM (inAnnex key) $
+		liftIO $ rsyncServerSend file -- does not return
 	liftIO exitFailure
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index d3d7ac3398..1e96e1e6f7 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -8,7 +8,6 @@
 module Command.Uninit where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when)
 import System.Directory
 
 import Command
@@ -44,8 +43,7 @@ perform = do
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
 	let hook = Command.Init.preCommitHook repo
-	hookexists <- liftIO $ doesFileExist hook
-	when hookexists $ do
+	whenM (liftIO $ doesFileExist hook) $ do
 		c <- liftIO $ readFile hook
 		if c == Command.Init.preCommitScript
 			then liftIO $ removeFile hook
@@ -56,8 +54,7 @@ gitPreCommitHookUnWrite repo = do
 gitAttributesUnWrite :: Git.Repo -> IO ()
 gitAttributesUnWrite repo = do
 	let attributes = Git.attributes repo
-	attrexists <- doesFileExist attributes
-	when attrexists $ do
+	whenM (doesFileExist attributes) $ do
 		c <- readFileStrict attributes
 		safeWriteFile attributes $ unlines $
 			filter (\l -> not $ l `elem` Command.Init.attrLines) $ lines c
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index d65579ec73..161df2ddf9 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -7,7 +7,6 @@
 
 module Command.Unlock where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory hiding (copyFile)
 
@@ -19,6 +18,7 @@ import Messages
 import Locations
 import Content
 import CopyFile
+import Utility
 
 command :: [Command]
 command =
@@ -38,9 +38,7 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: FilePath -> Key -> CommandPerform
 perform dest key = do
-	inbackend <- Backend.hasKey key
-	when (not inbackend) $
-		error "content not present"
+	unlessM (Backend.hasKey key) $ error "content not present"
 	
 	checkDiskSpace key
 
diff --git a/Content.hs b/Content.hs
index 0758fcdb13..ec7a3776bf 100644
--- a/Content.hs
+++ b/Content.hs
@@ -134,8 +134,7 @@ withTmp key action = do
 	let tmp = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	res <- action tmp
-	tmp_exists <- liftIO $ doesFileExist tmp
-	when tmp_exists $ liftIO $ removeFile tmp
+	liftIO $ whenM (doesFileExist tmp) $ liftIO $ removeFile tmp
 	return res
 
 {- Checks that there is disk space available to store a given key,
@@ -160,8 +159,7 @@ checkDiskSpace' adjustment key = do
 		megabyte :: Integer
 		megabyte = 1000000
 		needmorespace n = do
-			force <- Annex.getState Annex.force
-			unless force $ 
+			unlessM (Annex.getState Annex.force) $
 				error $ "not enough free space, need " ++ 
 					roughSize storageUnits True n ++
 					" more (use --force to override this check or adjust annex.diskreserve)"
diff --git a/CopyFile.hs b/CopyFile.hs
index 4575fb08ad..b08ede3c88 100644
--- a/CopyFile.hs
+++ b/CopyFile.hs
@@ -7,7 +7,6 @@
 
 module CopyFile (copyFile) where
 
-import Control.Monad (when)
 import System.Directory (doesFileExist, removeFile)
 
 import Utility
@@ -17,8 +16,7 @@ import qualified SysConfig
  - and because this allows easy access to features like cp --reflink. -}
 copyFile :: FilePath -> FilePath -> IO Bool
 copyFile src dest = do
-	e <- doesFileExist dest
-	when e $
+	whenM (doesFileExist dest) $
 		removeFile dest
 	boolSystem "cp" [params, File src, File dest]
 	where
diff --git a/GitRepo.hs b/GitRepo.hs
index 87cceece41..d070bc89ef 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -329,9 +329,9 @@ gitCommandLine repo _ = assertLocal repo $ error "internal"
 
 {- Runs git in the specified repo, throwing an error if it fails. -}
 run :: Repo -> String -> [CommandParam] -> IO ()
-run repo subcommand params = assertLocal repo $ do
-	ok <- boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
-	unless ok $ error $ "git " ++ show params ++ " failed"
+run repo subcommand params = assertLocal repo $
+	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
+		<|> error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns it output, lazily. 
  -
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index d2b771bf77..51a5d05d17 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -11,7 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import IO
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad (unless, when)
+import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Process
 import System.Exit
@@ -75,8 +75,7 @@ bupSetup u c = do
 	-- bup init will create the repository.
 	-- (If the repository already exists, bup init again appears safe.)
 	showNote "bup init"
-	ok <- bup "init" buprepo []
-	unless ok $ error "bup init failed"
+	bup "init" buprepo [] <|> error "bup init failed"
 
 	storeBupUUID u buprepo
 
@@ -172,9 +171,9 @@ storeBupUUID u buprepo = do
 	if Git.repoIsUrl r
 		then do
 			showNote "storing uuid"
-			ok <- onBupRemote r boolSystem "git"
+			onBupRemote r boolSystem "git"
 				[Params $ "config annex.uuid " ++ u]
-			unless ok $ do error "ssh failed"
+				<|> error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
 			let olduuid = Git.configGet r' "annex.uuid" ""
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 0cd3760d63..f69aa1256b 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -62,8 +62,8 @@ directorySetup u c = do
 	-- verify configuration is sane
 	let dir = maybe (error "Specify directory=") id $
 		M.lookup "directory" c
-	e <- liftIO $ doesDirectoryExist dir
-	when (not e) $ error $ "Directory does not exist: " ++ dir
+	liftIO $ doesDirectoryExist dir
+		<|> error $ "Directory does not exist: " ++ dir
 	c' <- encryptionSetup c
 
 	-- The directory is stored in git config, not in this remote's
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index c15ab37a75..53418a9ef8 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -10,7 +10,7 @@ module Remote.Rsync (remote) where
 import qualified Data.ByteString.Lazy.Char8 as L
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad.State (liftIO, when)
+import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
 import System.Posix.Files
@@ -168,9 +168,8 @@ withRsyncScratchDir a = do
 	nuke tmp
 	return res
 	where
-		nuke d = liftIO $ do
-			e <- doesDirectoryExist d
-			when e $ liftIO $ removeDirectoryRecursive d
+		nuke d = liftIO $ 
+			doesDirectoryExist d <&> removeDirectoryRecursive d
 
 rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool
 rsyncRemote o params = do
diff --git a/Utility.hs b/Utility.hs
index 6dd7d329c8..5aa0afea7d 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -1,4 +1,4 @@
-{- git-annex utility functions
+{- general purpose utility functions
  -
  - Copyright 2010-2011 Joey Hess 
  -
@@ -26,6 +26,10 @@ module Utility (
 	dirContents,
 	myHomeDir,
 	catchBool,
+	whenM,
+	(<&>),
+	unlessM,
+	(<|>),
 	
 	prop_idempotent_shellEscape,
 	prop_idempotent_shellEscape_multiword,
@@ -46,7 +50,8 @@ import System.FilePath
 import System.Directory
 import Foreign (complement)
 import Data.List
-import Control.Monad (liftM2)
+import Data.Maybe
+import Control.Monad (liftM2, when, unless)
 
 {- A type for parameters passed to a shell command. A command can
  - be passed either some Params (multiple parameters can be included,
@@ -110,7 +115,7 @@ shellEscape f = "'" ++ escaped ++ "'"
 {- Unescapes a set of shellEscaped words or filenames. -}
 shellUnEscape :: String -> [String]
 shellUnEscape [] = []
-shellUnEscape s = word:(shellUnEscape rest)
+shellUnEscape s = word : shellUnEscape rest
 	where
 		(word, rest) = findword "" s
 		findword w [] = (w, "")
@@ -165,7 +170,7 @@ prop_parentDir_basics dir
 dirContains :: FilePath -> FilePath -> Bool
 dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
 	where
-		norm p = maybe "" id $ absNormPath p "."
+		norm p = fromMaybe "" $ absNormPath p "."
 		a' = norm a
 		b' = norm b
 
@@ -178,7 +183,7 @@ absPath file = do
 {- Converts a filename into a normalized, absolute path
  - from the specified cwd. -}
 absPathFrom :: FilePath -> FilePath -> FilePath
-absPathFrom cwd file = maybe bad id $ absNormPath cwd file
+absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
 	where
 		bad = error $ "unable to normalize " ++ file
 
@@ -258,3 +263,20 @@ myHomeDir = do
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
+
+{- when with a monadic conditional -}
+whenM :: Monad m => m Bool -> m () -> m ()
+whenM c a = c >>= flip when a
+
+unlessM :: Monad m => m Bool -> m () -> m ()
+unlessM c a = c >>= flip unless a
+
+(<&>) :: Monad m => m Bool -> m () -> m ()
+(<&>) = whenM
+
+(<|>) :: Monad m => m Bool -> m () -> m ()
+(<|>) = unlessM
+
+-- low fixity allows eg, foo bar <|> error $ "failed " ++ meep
+infixr 0 <&>
+infixr 0 <|>
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index e8a744748b..1487a61616 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -6,7 +6,6 @@
  -}
 
 import System.Environment
-import Control.Monad (when)
 import Data.List
 
 import qualified GitRepo as Git
@@ -66,8 +65,7 @@ builtin cmd dir params = do
 
 external :: [String] -> IO ()
 external params = do
-	ret <- boolSystem "git-shell" $ map Param $ ("-c":filterparams params)
-	when (not ret) $
+	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $
 		error "git-shell failed"
 
 -- Drop all args after "--".

From e6c95a6a5e514fbe35b051c6bbec6c22793dbc05 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 17 May 2011 07:25:33 +0000
Subject: [PATCH 1752/2835]

---
 doc/todo/cache_key_info.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/todo/cache_key_info.mdwn b/doc/todo/cache_key_info.mdwn
index d26d055129..791584836d 100644
--- a/doc/todo/cache_key_info.mdwn
+++ b/doc/todo/cache_key_info.mdwn
@@ -10,7 +10,7 @@ But currently two git-annex commands are quite slow when annexes become large
 in quantity of files. These are unused and stats
 (Both have --fast versions that don't do as much).
 
-Both are slow because both need two peices of information that are not
+Both are slow because both need two pieces of information that are not
 quick to look up, and require examining the whole repo, very seekily:
 
 1. The keys present in the annex. Found by looking thru .git/annex/objects.

From 3154dc036abf36c59df28e78da3a8748a8240b67 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 17 May 2011 07:27:03 +0000
Subject: [PATCH 1753/2835] Added a comment

---
 ...omment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment

diff --git a/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment b/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment
new file mode 100644
index 0000000000..086e7f3e84
--- /dev/null
+++ b/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-05-17T07:27:02Z"
+ content="""
+Sounds like a good idea.
+
+* git annex fsck (or similar) should check/rebuild the caches
+* I would simply require a clean tree with a verbose error. 80/20 rule and defaulting to save actions.
+"""]]

From 21d9c84e7292a8984ea5d46c0134ddc6ff19babc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 11:44:13 -0400
Subject: [PATCH 1754/2835] more standard names for whenM and unlessM operators

These are defined in ifelse, but it's not currently available and I don't
want to pull in a library for 6 lines of code anyhow.

Also, ifelse sets the fixity to 1, which does not allow >>? error $ ...
---
 GitRepo.hs          |  2 +-
 Remote/Bup.hs       |  4 ++--
 Remote/Directory.hs |  2 +-
 Remote/Rsync.hs     |  2 +-
 Utility.hs          | 16 ++++++++--------
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index d070bc89ef..f489dfe35d 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -331,7 +331,7 @@ gitCommandLine repo _ = assertLocal repo $ error "internal"
 run :: Repo -> String -> [CommandParam] -> IO ()
 run repo subcommand params = assertLocal repo $
 	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
-		<|> error $ "git " ++ show params ++ " failed"
+		>>! error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns it output, lazily. 
  -
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 51a5d05d17..c40826e5eb 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -75,7 +75,7 @@ bupSetup u c = do
 	-- bup init will create the repository.
 	-- (If the repository already exists, bup init again appears safe.)
 	showNote "bup init"
-	bup "init" buprepo [] <|> error "bup init failed"
+	bup "init" buprepo [] >>! error "bup init failed"
 
 	storeBupUUID u buprepo
 
@@ -173,7 +173,7 @@ storeBupUUID u buprepo = do
 			showNote "storing uuid"
 			onBupRemote r boolSystem "git"
 				[Params $ "config annex.uuid " ++ u]
-				<|> error "ssh failed"
+					>>! error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
 			let olduuid = Git.configGet r' "annex.uuid" ""
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index f69aa1256b..dedab473f3 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -63,7 +63,7 @@ directorySetup u c = do
 	let dir = maybe (error "Specify directory=") id $
 		M.lookup "directory" c
 	liftIO $ doesDirectoryExist dir
-		<|> error $ "Directory does not exist: " ++ dir
+		>>! error $ "Directory does not exist: " ++ dir
 	c' <- encryptionSetup c
 
 	-- The directory is stored in git config, not in this remote's
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 53418a9ef8..9d32ad19b9 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -169,7 +169,7 @@ withRsyncScratchDir a = do
 	return res
 	where
 		nuke d = liftIO $ 
-			doesDirectoryExist d <&> removeDirectoryRecursive d
+			doesDirectoryExist d >>? removeDirectoryRecursive d
 
 rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool
 rsyncRemote o params = do
diff --git a/Utility.hs b/Utility.hs
index 5aa0afea7d..816464373b 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -27,9 +27,9 @@ module Utility (
 	myHomeDir,
 	catchBool,
 	whenM,
-	(<&>),
+	(>>?),
 	unlessM,
-	(<|>),
+	(>>!),
 	
 	prop_idempotent_shellEscape,
 	prop_idempotent_shellEscape_multiword,
@@ -271,12 +271,12 @@ whenM c a = c >>= flip when a
 unlessM :: Monad m => m Bool -> m () -> m ()
 unlessM c a = c >>= flip unless a
 
-(<&>) :: Monad m => m Bool -> m () -> m ()
-(<&>) = whenM
+(>>?) :: Monad m => m Bool -> m () -> m ()
+(>>?) = whenM
 
-(<|>) :: Monad m => m Bool -> m () -> m ()
-(<|>) = unlessM
+(>>!) :: Monad m => m Bool -> m () -> m ()
+(>>!) = unlessM
 
 -- low fixity allows eg, foo bar <|> error $ "failed " ++ meep
-infixr 0 <&>
-infixr 0 <|>
+infixr 0 >>?
+infixr 0 >>!

From ebfa50b72959de4c4e917771c57b0efcea98e55b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 12:14:46 -0400
Subject: [PATCH 1755/2835] add

---
 doc/todo/tahoe_lfs_for_reals.mdwn | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/todo/tahoe_lfs_for_reals.mdwn

diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn
new file mode 100644
index 0000000000..053a763d7e
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals.mdwn
@@ -0,0 +1,28 @@
+[[forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] is a good
+start, but Zooko points out that using Tahoe's directory translation layer
+incurs O(N^2) overhead as the number of objects grows. Also, making
+hash subdirectories in Tahoe is expensive. Instead it would be good to use
+it as a key/value store directly. The catch is that doing so involves
+sending the content to Tahoe, and getting back a key identifier.
+
+This would be fairly easy to do as a [[backend]], which can assign its
+own key names (although typically done before data is stored in it),
+but a tahoe-lafs special remote would be more flexible.
+
+To support a special remote, a mapping is needed from git-annex keys to
+Tahoe keys.
+
+The best place to store this mapping is perhaps as a new field in the
+location log:
+
+	date present repo-uuid newfields
+
+This way, each remote can store its own key-specfic data in the same place
+as other key-specific data, with minimal overhead.
+
+Unfortunatly, the current location log parser throws out lines that it 
+cannot parse, so making this change would involve something of a flag day
+upgrade. Also unfortunatly, the location log and other .git-annex/ data
+does not have its own version that can be checked to force an upgrade
+across all clones. It might be best to deal with this at the same time
+the ideas in [[branching]] are done. --[[Joey]]

From f664a2538a848d204922de908fac9add7c1db0fe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 12:18:50 -0400
Subject: [PATCH 1756/2835] fix

---
 doc/todo/tahoe_lfs_for_reals.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn
index 053a763d7e..4e0bf2fb16 100644
--- a/doc/todo/tahoe_lfs_for_reals.mdwn
+++ b/doc/todo/tahoe_lfs_for_reals.mdwn
@@ -5,7 +5,7 @@ hash subdirectories in Tahoe is expensive. Instead it would be good to use
 it as a key/value store directly. The catch is that doing so involves
 sending the content to Tahoe, and getting back a key identifier.
 
-This would be fairly easy to do as a [[backend]], which can assign its
+This would be fairly easy to do as a [[backend|backends]], which can assign its
 own key names (although typically done before data is stored in it),
 but a tahoe-lafs special remote would be more flexible.
 

From f63412a42c3cc3df35e0e1c945d415b81f811caa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 12:19:52 -0400
Subject: [PATCH 1757/2835] close

---
 doc/todo/S3.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index dd0174cc55..7e417336f0 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -20,3 +20,5 @@ and you forget to drop it from S3, cruft can build up there.
 This could be fixed by adding a hook to list all keys present in a remote.
 Then unused could scan remotes for keys, and if they were not used locally,
 offer the possibility to drop them from the remote.
+
+[[done]]

From 44499bbdf8317664e5a2e83aa63c904cefbf64c1 Mon Sep 17 00:00:00 2001
From: zooko 
Date: Tue, 17 May 2011 19:20:39 +0000
Subject: [PATCH 1758/2835] Added a comment: performance

---
 ...comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment

diff --git a/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment b/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment
new file mode 100644
index 0000000000..16ef882a42
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="zooko"
+ ip="97.118.97.117"
+ subject="performance"
+ date="2011-05-17T19:20:39Z"
+ content="""
+Hm... O(N^2)? I think it just takes O(N). To read an entry out of a directory you have to download the entire directory (and store it in RAM and parse it). The constants are basically \"too big to be good but not big enough to be prohibitive\", I think. jctang has reported that his special remote hook performs well enough to use, but it would be nice if it were faster.
+
+The Tahoe-LAFS folks are working on speeding up mutable files, by the way, after which we would be able to speed up directories.
+"""]]

From 93b0f21c95c32b15e8fc611f2da3dd910d8045f1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 17 May 2011 19:57:33 +0000
Subject: [PATCH 1759/2835] Added a comment

---
 ...comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment

diff --git a/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
new file mode 100644
index 0000000000..a32461615f
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-05-17T19:57:33Z"
+ content="""
+Whoops! You'd only told me O(N) twice before..
+
+So this is not too high priority. I think I would like to get the per-remote storage sorted out anyway, since probably it will be the thing needed to convert the URL backend into a special remote, which would then allow ripping out the otherwise unused pluggable backend infrastructure.
+"""]]

From dd44e53c0ca3e5eec78f03a3a222b4922d6d097c Mon Sep 17 00:00:00 2001
From: zooko 
Date: Wed, 18 May 2011 04:32:17 +0000
Subject: [PATCH 1760/2835] Added a comment: Tahoe-LAFS comes with encryption

---
 ...comment_1_1afca8d7182075d46db41f6ad3dd5911._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment

diff --git a/doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment b/doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment
new file mode 100644
index 0000000000..db93bf63f8
--- /dev/null
+++ b/doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="zooko"
+ ip="75.220.153.232"
+ subject="Tahoe-LAFS comes with encryption"
+ date="2011-05-18T04:32:14Z"
+ content="""
+The Tahoe-LAFS special remote automatically encrypts and adds cryptography integrity checks/digital signatures. For that special remote you should not use the git-annex encryption scheme.
+
+Tahoe-LAFS encryption generates a new independent key for each file. This means that you can share access to one of the files without thereby sharing access to all of them, and it means that individual files can be deduplicated among multiple users.
+"""]]

From cd8354187273ee07c8953bea72df2d60b8b50c5d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 18 May 2011 19:34:46 -0400
Subject: [PATCH 1761/2835] --backend now overrides any backend configured in
 .gitattributes files.

---
 Annex.hs         |  4 ++--
 Backend.hs       | 15 +++++++++++----
 Options.hs       |  6 +++---
 debian/changelog |  1 +
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index b2257281fd..2148dd6252 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -38,7 +38,7 @@ data AnnexState = AnnexState
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
-	, defaultbackend :: Maybe String
+	, forcebackend :: Maybe String
 	, defaultkey :: Maybe String
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
@@ -56,7 +56,7 @@ newState gitrepo allbackends = AnnexState
 	, quiet = False
 	, force = False
 	, fast = False
-	, defaultbackend = Nothing
+	, forcebackend = Nothing
 	, defaultkey = Nothing
 	, toremote = Nothing
 	, fromremote = Nothing
diff --git a/Backend.hs b/Backend.hs
index 6140664cee..ab15974c8a 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -33,6 +33,7 @@ module Backend (
 ) where
 
 import Control.Monad.State (liftIO, when)
+import Control.Monad (liftM)
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
@@ -56,7 +57,7 @@ list = do
 		then return l
 		else do
 			s <- getstandard
-			d <- Annex.getState Annex.defaultbackend
+			d <- Annex.getState Annex.forcebackend
 			handle d s
 	where
 		parseBackendList l [] = l
@@ -161,9 +162,15 @@ lookupFile file = do
 chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))]
 chooseBackends fs = do
 	g <- Annex.gitRepo
-	bs <- Annex.getState Annex.supportedBackends
-	pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
-	return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs
+	forced <- Annex.getState Annex.forcebackend
+	if forced /= Nothing
+		then do
+			l <- list
+			return $ map (\f -> (f, Just $ head l)) fs
+		else do
+			bs <- Annex.getState Annex.supportedBackends
+			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
+			return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs
 
 {- Returns the backend to use for a key. -}
 keyBackend :: Key -> Annex (Backend Annex)
diff --git a/Options.hs b/Options.hs
index 31b73e25a8..ae5707e4a0 100644
--- a/Options.hs
+++ b/Options.hs
@@ -28,11 +28,11 @@ commonOptions =
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setquiet False))
 		"allow verbose output"
-	, Option ['b'] ["backend"] (ReqArg setdefaultbackend paramName)
-		"specify default key-value backend to use"
+	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
+		"specify key-value backend to use"
 	]
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
-		setdefaultbackend v = Annex.changeState $ \s -> s { Annex.defaultbackend = Just v }
+		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
diff --git a/debian/changelog b/debian/changelog
index d759d3672a..f8b5f02cf2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (0.20110517) UNRELEASED; urgency=low
 
   * status: New subcommand to show info about an annex, including its size.
+  * --backend now overrides any backend configured in .gitattributes files.
 
  -- Joey Hess   Mon, 16 May 2011 20:27:46 -0400
 

From 22bf25b102722b35af79d85a9014801ae18d5f64 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
 
Date: Thu, 19 May 2011 08:19:09 +0000
Subject: [PATCH 1762/2835] Added a comment: gitweb supplies --no-name as of
 1.7.5.1

---
 .../comment_4_10fc013865c7542c2ed9d6c0963bb391._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment

diff --git a/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment b/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment
new file mode 100644
index 0000000000..9bb9aa8ae3
--- /dev/null
+++ b/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
+ nickname="Fraser"
+ subject="gitweb supplies --no-name as of 1.7.5.1"
+ date="2011-05-19T08:19:02Z"
+ content="""
+git v1.7.5.1 fixes the gitweb gzip issue.  If the git instance is updated we
+can have stable distributions (and I can finally write a FreeBSD port ^_^)
+"""]]

From fa8ae628f28486ad5114767f1ee3cd3d4a3c5af0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 19 May 2011 16:10:36 +0000
Subject: [PATCH 1763/2835] Added a comment

---
 .../comment_5_c6b1bc40226fc2c8ba3e558150856992._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment

diff --git a/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment b/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment
new file mode 100644
index 0000000000..76ba75edc4
--- /dev/null
+++ b/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-05-19T16:10:35Z"
+ content="""
+Hmm, I've upgraded to that version, but I see nothing in its changelog, commit log, code, or runtime behavior to indicate that it's producing stable gzip output.
+"""]]

From 463fc3ea58a1273704d1f500d494a9e85e2c4ea6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 19 May 2011 19:21:52 +0000
Subject: [PATCH 1764/2835] Added a comment: anonymous git push

---
 ...comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment

diff --git a/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment b/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment
new file mode 100644
index 0000000000..830d678ca1
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="anonymous git push"
+ date="2011-05-19T19:21:51Z"
+ content="""
+@Jimmy mentioned anonymous git push -- that is now enabled for this wiki. Enjoy!
+
+I may try to spend more time on #vcs-home -- or I can be summoned there from my other lurking places on irc, I guess.
+"""]]

From dbf02220d57948132cd0a7a76714f2b562806a08 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 May 2011 15:44:22 -0400
Subject: [PATCH 1765/2835] update with irc channel

---
 doc/contact.mdwn | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/contact.mdwn b/doc/contact.mdwn
index 1f122fdad9..22f072cb2b 100644
--- a/doc/contact.mdwn
+++ b/doc/contact.mdwn
@@ -1,4 +1,10 @@
-Joey Hess  is the author of git-annex.
+Joey Hess  is the author of git-annex. If you need to
+talk about something privatly, email me.
+
+The [[forum]] is the best place to discuss git-annex.
 
 The [VCS-home mailing list](http://lists.madduck.net/listinfo/vcs-home)
-is a good place to discuss it, or use the [[forum]].
+is a good mailing list for users who want to use git-annex in the context
+of managing their large personal files.
+
+For realtime chat, use the `#vcs-home` channel on irc.oftc.net.

From fbdaf212f0346a3543187c586f20ac2bef73ac20 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 May 2011 15:56:19 -0400
Subject: [PATCH 1766/2835] main git repo switching to branchable one

It allows anonymous git push, and it's simpler to just have it there.
The kite repo will remain, as a mirror.
---
 doc/download.mdwn | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 405bc2fc4b..19070a70aa 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -1,6 +1,11 @@
-The main git repository for git-annex is `git://git.kitenet.net/git-annex`
-[[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)]
-(The repo is mirrored [at github](https://github.com/joeyh/git-annex).)
+The main git repository for git-annex is `git://git-annex.branchable.com/`
+
+(You can push changes to this wiki from that anonymous git checkout.)
+
+Other mirrors of the git repository:
+
+* `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)]
+* [at github](https://github.com/joeyh/git-annex)
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.

From 4c8ce2d86681fbc7bb7b56c556fce4d58c612d3c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 19 May 2011 20:09:34 +0000
Subject: [PATCH 1767/2835] test commit at 4:09

---
 doc/sandbox.mdwn | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/sandbox.mdwn

diff --git a/doc/sandbox.mdwn b/doc/sandbox.mdwn
new file mode 100644
index 0000000000..ed220722ee
--- /dev/null
+++ b/doc/sandbox.mdwn
@@ -0,0 +1,34 @@
+This is the SandBox, a page anyone can edit to learn how to use the wiki.
+
+Hello!
+
+----
+
+Here's a paragraph.
+
+Here's another one with *emphasised* text.
+
+# Header
+
+## Subheader
+
+> This is a blockquote.
+>
+> This is the first level of quoting.
+>
+> > This is nested blockquote.
+>
+> Back to the first level.
+
+Numbered list 
+
+1. First item.
+1. Another.
+1. And another..
+
+Bulleted list
+
+* *item*
+* item
+
+[[ikiwiki/WikiLink]]

From 813ff7f4b0cef294e020e50d4c6f973d1e7f35ae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 May 2011 16:23:05 -0400
Subject: [PATCH 1768/2835] fixed url

---
 doc/download.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 19070a70aa..3607e403f0 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -4,7 +4,7 @@ The main git repository for git-annex is `git://git-annex.branchable.com/`
 
 Other mirrors of the git repository:
 
-* `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)]
+* `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
 Some operating systems include git-annex in easily prepackaged form and

From c7b7595f50ed276d97f18931eb7b2092f22104f4 Mon Sep 17 00:00:00 2001
From: "http://ertai.myopenid.com/" 
Date: Fri, 20 May 2011 20:17:15 +0000
Subject: [PATCH 1769/2835] Added a comment

---
 ...mment_2_adb76f06a7997abe4559d3169a3181c3._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment

diff --git a/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment b/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment
new file mode 100644
index 0000000000..6ecce52c42
--- /dev/null
+++ b/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="comment 2"
+ date="2011-05-20T20:14:15Z"
+ content="""
+I agree with Christian.
+
+One should first make a better use of connections to remotes before exploring parallel possibilities. One should pipeline the requests and answers.
+
+Of course this could be implemented using parallel&concurrency features of Haskell to do this.
+"""]]

From 1876db50f258a1a6a67d874049e93a84d34cac32 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 11:07:08 -0400
Subject: [PATCH 1770/2835] found a few places I can use newtype

for presumably some speedups
---
 Backend.hs     | 1 -
 CryptoTypes.hs | 5 +++--
 Touch.hsc      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index ab15974c8a..645bfdfc3f 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -33,7 +33,6 @@ module Backend (
 ) where
 
 import Control.Monad.State (liftIO, when)
-import Control.Monad (liftM)
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
diff --git a/CryptoTypes.hs b/CryptoTypes.hs
index 944a9d34e0..ba22c4cbe8 100644
--- a/CryptoTypes.hs
+++ b/CryptoTypes.hs
@@ -9,11 +9,12 @@ module CryptoTypes where
 
 import Data.String.Utils
 
-data Cipher = Cipher String -- XXX ideally, this would be a locked memory region
+-- XXX ideally, this would be a locked memory region
+newtype Cipher = Cipher String
 
 data EncryptedCipher = EncryptedCipher String KeyIds
 
-data KeyIds = KeyIds [String]
+newtype KeyIds = KeyIds [String]
 
 instance Show KeyIds where
 	show (KeyIds ks) = join "," ks
diff --git a/Touch.hsc b/Touch.hsc
index fd3500f869..4f26855d21 100644
--- a/Touch.hsc
+++ b/Touch.hsc
@@ -16,7 +16,7 @@ module Touch (
 import Foreign
 import Foreign.C
 
-data TimeSpec = TimeSpec CTime
+newtype TimeSpec = TimeSpec CTime
 
 {- Changes the access and modification times of an existing file.
    Can follow symlinks, or not. Throws IO error on failure. -}

From 93a4f3d4e6970b05116fc25b8d57f0dd9d9ec675 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 11:52:13 -0400
Subject: [PATCH 1771/2835] Add --debug option. Closes: #627499

This takes advantage of the debug logging done by missingh, and I added
my own debug messages for executeFile calls. There are still some other
low-level ways git-annex runs stuff that are not shown by debugging,
but this gets most of it easily.
---
 Options.hs         |  7 ++++++-
 Remote/Hook.hs     |  2 +-
 RsyncFile.hs       |  1 -
 Utility.hs         | 12 +++++++++++-
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  4 ++++
 6 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/Options.hs b/Options.hs
index ae5707e4a0..f8dbfb6bc0 100644
--- a/Options.hs
+++ b/Options.hs
@@ -8,6 +8,8 @@
 module Options where
 
 import System.Console.GetOpt
+import System.Log.Logger
+import Control.Monad.State (liftIO)
 
 import qualified Annex
 import Types
@@ -27,7 +29,9 @@ commonOptions =
 	, Option ['q'] ["quiet"] (NoArg (setquiet True))
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setquiet False))
-		"allow verbose output"
+		"allow verbose output (default)"
+	, Option ['d'] ["debug"] (NoArg (setdebug))
+		"show debug messages"
 	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
 		"specify key-value backend to use"
 	]
@@ -36,3 +40,4 @@ commonOptions =
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
+		setdebug = liftIO $ updateGlobalLogger "" $ setLevel DEBUG
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 7f2d5dbee2..dc4d392741 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -12,7 +12,7 @@ import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Control.Monad.State (liftIO)
 import System.FilePath
-import System.Posix.Process
+import System.Posix.Process hiding (executeFile)
 import System.Posix.IO
 import System.IO
 import System.IO.Error (try)
diff --git a/RsyncFile.hs b/RsyncFile.hs
index afff46c0ce..48d927fcf2 100644
--- a/RsyncFile.hs
+++ b/RsyncFile.hs
@@ -7,7 +7,6 @@
 
 module RsyncFile where
 
-import System.Posix.Process
 import Data.String.Utils
 
 import Utility
diff --git a/Utility.hs b/Utility.hs
index 816464373b..47d10ed759 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -17,6 +17,7 @@ module Utility (
 	relPathDirToFile,
 	boolSystem,
 	boolSystemEnv,
+	executeFile,
 	shellEscape,
 	shellUnEscape,
 	unsetFileMode,
@@ -39,7 +40,8 @@ module Utility (
 
 import System.IO
 import System.Exit
-import System.Posix.Process
+import qualified System.Posix.Process
+import System.Posix.Process hiding (executeFile)
 import System.Posix.Signals
 import System.Posix.Files
 import System.Posix.Types
@@ -52,6 +54,7 @@ import Foreign (complement)
 import Data.List
 import Data.Maybe
 import Control.Monad (liftM2, when, unless)
+import System.Log.Logger
 
 {- A type for parameters passed to a shell command. A command can
  - be passed either some Params (multiple parameters can be included,
@@ -104,6 +107,13 @@ boolSystemEnv command params env = do
 			restoresignals oldint oldset
 			executeFile command True (toCommand params) env
 
+{- executeFile with debug logging -}
+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO a
+executeFile c path p e = do
+	debugM "Utility.executeFile" $
+		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
+	System.Posix.Process.executeFile c path p e
+
 {- Escapes a filename or other parameter to be safely able to be exposed to
  - the shell. -}
 shellEscape :: String -> String
diff --git a/debian/changelog b/debian/changelog
index f8b5f02cf2..16821c848d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (0.20110517) UNRELEASED; urgency=low
 
   * status: New subcommand to show info about an annex, including its size.
   * --backend now overrides any backend configured in .gitattributes files.
+  * Add --debug option. Closes: #627499
 
  -- Joey Hess   Mon, 16 May 2011 20:27:46 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e2a04d27b9..7f2fce9d23 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -330,6 +330,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Enable verbose logging.
 
+* --debug
+
+  Show debug messages.
+
 * --from=repository
 
   Specifies a repository that content will be retrieved from, or that

From 944b1207dcedac199729842e27d47ce89a1dd90c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 11:58:35 -0400
Subject: [PATCH 1772/2835] releasing version 0.20110521

---
 debian/changelog |  4 ++--
 doc/sandbox.mdwn | 34 ----------------------------------
 2 files changed, 2 insertions(+), 36 deletions(-)
 delete mode 100644 doc/sandbox.mdwn

diff --git a/debian/changelog b/debian/changelog
index 16821c848d..cf1c9baf95 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,10 +1,10 @@
-git-annex (0.20110517) UNRELEASED; urgency=low
+git-annex (0.20110521) unstable; urgency=low
 
   * status: New subcommand to show info about an annex, including its size.
   * --backend now overrides any backend configured in .gitattributes files.
   * Add --debug option. Closes: #627499
 
- -- Joey Hess   Mon, 16 May 2011 20:27:46 -0400
+ -- Joey Hess   Sat, 21 May 2011 11:52:53 -0400
 
 git-annex (0.20110516) unstable; urgency=low
 
diff --git a/doc/sandbox.mdwn b/doc/sandbox.mdwn
deleted file mode 100644
index ed220722ee..0000000000
--- a/doc/sandbox.mdwn
+++ /dev/null
@@ -1,34 +0,0 @@
-This is the SandBox, a page anyone can edit to learn how to use the wiki.
-
-Hello!
-
-----
-
-Here's a paragraph.
-
-Here's another one with *emphasised* text.
-
-# Header
-
-## Subheader
-
-> This is a blockquote.
->
-> This is the first level of quoting.
->
-> > This is nested blockquote.
->
-> Back to the first level.
-
-Numbered list 
-
-1. First item.
-1. Another.
-1. And another..
-
-Bulleted list
-
-* *item*
-* item
-
-[[ikiwiki/WikiLink]]

From abfd8dd6c7126f012008f422725ed5b7505281d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 12:02:41 -0400
Subject: [PATCH 1773/2835] add news item for git-annex 0.20110521

---
 doc/news/version_0.20110420.mdwn | 8 --------
 doc/news/version_0.20110521.mdwn | 5 +++++
 2 files changed, 5 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_0.20110420.mdwn
 create mode 100644 doc/news/version_0.20110521.mdwn

diff --git a/doc/news/version_0.20110420.mdwn b/doc/news/version_0.20110420.mdwn
deleted file mode 100644
index bb7fee219b..0000000000
--- a/doc/news/version_0.20110420.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 0.20110420 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Update Debian build dependencies for ghc 7.
-   * Debian package is now built with S3 support.
-     Thanks Joachim Breitner for making this possible.
-   * Somewhat improved memory usage of S3, still work to do.
-     Thanks Greg Heartsfield for ongoing work to improve the hS3 library
-     for git-annex."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110521.mdwn b/doc/news/version_0.20110521.mdwn
new file mode 100644
index 0000000000..f64392a760
--- /dev/null
+++ b/doc/news/version_0.20110521.mdwn
@@ -0,0 +1,5 @@
+git-annex 0.20110521 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * status: New subcommand to show info about an annex, including its size.
+   * --backend now overrides any backend configured in .gitattributes files.
+   * Add --debug option. Closes: #[627499](http://bugs.debian.org/627499)"""]]
\ No newline at end of file

From 8ed27db18fa38c0ba975805e5d3a32b40825387d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 13:03:13 -0400
Subject: [PATCH 1774/2835] add explict build dep on hslogger

pulled in by missingh, but now used directly by git-annex
---
 debian/control | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/control b/debian/control
index f2f9cecd39..4294a64820 100644
--- a/debian/control
+++ b/debian/control
@@ -5,6 +5,7 @@ Build-Depends:
 	debhelper (>= 7.0.50),
 	ghc,
 	libghc-missingh-dev,
+	ligghc-hslogger-dev,
 	libghc-pcre-light-dev,
 	libghc-sha-dev,
 	libghc-dataenc-dev,

From 078d1dd0cbb22dbdd8d97a96c77a391d7df829c0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 20:18:35 -0400
Subject: [PATCH 1775/2835] clarify synopsis re options

---
 git-annex-shell.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 1487a61616..940db71c34 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -32,7 +32,7 @@ cmds = map adddirparam $ concat
 		adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c }
 
 header :: String
-header = "Usage: git-annex-shell [-c] command [option ..]"
+header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
 
 main :: IO ()
 main = do

From f81c1f10e6ce452636eb06209c3702d2da05c49f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 May 2011 14:03:06 -0400
Subject: [PATCH 1776/2835] show a warning message when failing to find
 requested key

Otherwise, the user sees only a rsync protocol error message and then
git-annex's less specific failure message.
---
 Command/SendKey.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 7497ce3bfe..c2f793f8fe 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -16,6 +16,7 @@ import Command
 import Content
 import Utility
 import RsyncFile
+import Messages
 
 command :: [Command]
 command = [repoCommand "sendkey" paramKey seek
@@ -30,4 +31,5 @@ start key = do
 	let file = gitAnnexLocation g key
 	whenM (inAnnex key) $
 		liftIO $ rsyncServerSend file -- does not return
+	warning "requested key is not present"
 	liftIO exitFailure

From 5b941980aa47ced29f702e2cc84346d517b78391 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 May 2011 14:12:16 -0400
Subject: [PATCH 1777/2835] Closer emulation of git's behavior when told to use
 "foo/.git" as a git repository instead of just "foo". Closes: #627563

---
 GitRepo.hs                               | 20 +++++++++++++++++---
 debian/changelog                         |  7 +++++++
 doc/bugs/weird_local_clone_confuses.mdwn |  2 ++
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index f489dfe35d..24bc9b5c2b 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -77,7 +77,7 @@ import Data.Char
 import Data.Word (Word8)
 import Codec.Binary.UTF8.String (encode)
 import Text.Printf
-import Data.List (isInfixOf, isPrefixOf)
+import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
 
 import Utility
@@ -111,10 +111,24 @@ repoFromAbsPath dir
 	| "/" `isPrefixOf` dir = do
  		-- Git always looks for "dir.git" in preference to
 		-- to "dir", even if dir ends in a "/".
-		let dir' = (dropTrailingPathSeparator dir) ++ ".git"
+		let canondir = dropTrailingPathSeparator dir
+		let dir' = canondir ++ ".git"
 		e <- doesDirectoryExist dir'
-		return $ newFrom $ Dir $ if e then dir' else dir
+		if e
+			then ret dir'
+			else if "/.git" `isSuffixOf` canondir
+				then do
+					-- When dir == "foo/.git", git looks
+					-- for "foo/.git/.git", and failing
+					-- that, uses "foo" as the repository.
+					e' <- doesDirectoryExist $ dir  ".git"
+					if e'
+						then ret dir
+						else ret $ takeDirectory canondir
+				else ret dir
 	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
+	where
+		ret = return . newFrom . Dir
 
 {- Remote Repo constructor. Throws exception on invalid url. -}
 repoFromUrl :: String -> IO Repo
diff --git a/debian/changelog b/debian/changelog
index cf1c9baf95..76495c9017 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (0.20110522) UNRELEASED; urgency=low
+
+  * Closer emulation of git's behavior when told to use "foo/.git" as a
+    git repository instead of just "foo". Closes: #627563
+
+ -- Joey Hess   Sun, 22 May 2011 14:03:42 -0400
+
 git-annex (0.20110521) unstable; urgency=low
 
   * status: New subcommand to show info about an annex, including its size.
diff --git a/doc/bugs/weird_local_clone_confuses.mdwn b/doc/bugs/weird_local_clone_confuses.mdwn
index 371f6d9a5e..aa838f1670 100644
--- a/doc/bugs/weird_local_clone_confuses.mdwn
+++ b/doc/bugs/weird_local_clone_confuses.mdwn
@@ -16,3 +16,5 @@ Just tested, and the new support for bare repositories didn't solve this.
 I think this is not something git-annex should go out of its way to
 support. [[done]]
 --[[Joey]] 
+
+Later.. Fixed this after all. --[[Joey]] 

From 1ab9743b5cd0282942c28086b840aaea459acfe9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
 
Date: Sun, 22 May 2011 23:02:40 +0000
Subject: [PATCH 1778/2835] Added a comment

---
 ...omment_6_3a52993d3553deb9a413debec9a5f92d._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment

diff --git a/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment b/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment
new file mode 100644
index 0000000000..0dbd88b1e5
--- /dev/null
+++ b/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
+ nickname="Fraser"
+ subject="comment 6"
+ date="2011-05-22T23:02:39Z"
+ content="""
+Whups, the fix landed in git's `maint' branch just after 1.7.5 but 1.7.5.1 was
+tagged on a different branch.
+
+Will look closer in future, and let you know when it's really released.
+"""]]

From 32a946af7d87d4c58c6f1d9dcaf201d906e4298f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 May 2011 19:07:20 -0400
Subject: [PATCH 1779/2835] clarify

---
 Options.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Options.hs b/Options.hs
index f8dbfb6bc0..7f78f44f62 100644
--- a/Options.hs
+++ b/Options.hs
@@ -40,4 +40,5 @@ commonOptions =
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
-		setdebug = liftIO $ updateGlobalLogger "" $ setLevel DEBUG
+		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
+			setLevel DEBUG

From 7a22f9d5161a24f1a98ddcead3c2ea3fd56e159c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 23 May 2011 00:38:51 -0400
Subject: [PATCH 1780/2835] update

---
 doc/install/OSX.mdwn | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 46a285e131..a6afd408bd 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -1,12 +1,12 @@
 
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils
+sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils pcre
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string
-sudo port install pcre
 sudo cabal install pcre-light
 sudo cabal install quickcheck  
-sudo cabal install SHA  
+sudo cabal install SHA
+sudo cabal install dataenc
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin

From 224977900d1b9961149b753df33b54d8045448ef Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
 
Date: Fri, 27 May 2011 01:27:38 +0000
Subject: [PATCH 1781/2835] Added a comment

---
 .../comment_7_a5eebd214b135f34b18274a682211943._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment

diff --git a/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment b/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment
new file mode 100644
index 0000000000..9960e0ea85
--- /dev/null
+++ b/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
+ nickname="Fraser"
+ subject="comment 7"
+ date="2011-05-27T01:27:37Z"
+ content="""
+v1.7.5.3 has it.
+"""]]

From 001edb008aee205a4f1885fe5f3335b2e2a47c33 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 May 2011 20:20:20 -0400
Subject: [PATCH 1782/2835] Fix bug in --exclude introduced in 0.20110516.

---
 Command.hs       | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command.hs b/Command.hs
index abbe897b5b..fd49531f44 100644
--- a/Command.hs
+++ b/Command.hs
@@ -191,7 +191,7 @@ filterFiles l = do
 		else return $ filter (notExcluded $ wildsRegex exclude) l'
 	where
 		notState f = not $ stateDir `isPrefixOf` f
-		notExcluded r f = isJust $ match r f []
+		notExcluded r f = isNothing $ match r f []
 
 wildsRegex :: [String] -> Regex
 wildsRegex ws = compile regex []
diff --git a/debian/changelog b/debian/changelog
index 76495c9017..1834bea15e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (0.20110522) UNRELEASED; urgency=low
 
   * Closer emulation of git's behavior when told to use "foo/.git" as a
     git repository instead of just "foo". Closes: #627563
+  * Fix bug in --exclude introduced in 0.20110516.
 
  -- Joey Hess   Sun, 22 May 2011 14:03:42 -0400
 

From 82b88d0676a26e5ce62da91560f6bd179d663601 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 May 2011 20:21:13 -0400
Subject: [PATCH 1783/2835] typo

---
 debian/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index 4294a64820..8c533af114 100644
--- a/debian/control
+++ b/debian/control
@@ -5,7 +5,7 @@ Build-Depends:
 	debhelper (>= 7.0.50),
 	ghc,
 	libghc-missingh-dev,
-	ligghc-hslogger-dev,
+	libghc-hslogger-dev,
 	libghc-pcre-light-dev,
 	libghc-sha-dev,
 	libghc-dataenc-dev,

From 7ea54e1c6ec04b644f47deb997076f804c4b558a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 May 2011 20:28:01 -0400
Subject: [PATCH 1784/2835] releasing version 0.20110522

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 1834bea15e..991244a885 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,10 +1,10 @@
-git-annex (0.20110522) UNRELEASED; urgency=low
+git-annex (0.20110522) unstable; urgency=low
 
   * Closer emulation of git's behavior when told to use "foo/.git" as a
     git repository instead of just "foo". Closes: #627563
   * Fix bug in --exclude introduced in 0.20110516.
 
- -- Joey Hess   Sun, 22 May 2011 14:03:42 -0400
+ -- Joey Hess   Fri, 27 May 2011 20:20:41 -0400
 
 git-annex (0.20110521) unstable; urgency=low
 

From de817bafa0ed871c26d59a572a2e6495cdaf55f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 May 2011 20:30:58 -0400
Subject: [PATCH 1785/2835] add news item for git-annex 0.20110522

---
 doc/news/version_0.20110425.mdwn | 7 -------
 doc/news/version_0.20110522.mdwn | 5 +++++
 2 files changed, 5 insertions(+), 7 deletions(-)
 delete mode 100644 doc/news/version_0.20110425.mdwn
 create mode 100644 doc/news/version_0.20110522.mdwn

diff --git a/doc/news/version_0.20110425.mdwn b/doc/news/version_0.20110425.mdwn
deleted file mode 100644
index 8f5c6515a0..0000000000
--- a/doc/news/version_0.20110425.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 0.20110425 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Use haskell Crypto library instead of haskell SHA library.
-   * Remove testpack from build depends for non x86 architectures where it
-     is not available. The test suite will not be run if it cannot be compiled.
-   * Avoid using absolute paths when staging location log, as that can
-     confuse git when a remote's path contains a symlink. Closes: #[621386](http://bugs.debian.org/621386)"""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110522.mdwn b/doc/news/version_0.20110522.mdwn
new file mode 100644
index 0000000000..5dccb993e2
--- /dev/null
+++ b/doc/news/version_0.20110522.mdwn
@@ -0,0 +1,5 @@
+git-annex 0.20110522 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Closer emulation of git's behavior when told to use "foo/.git" as a
+     git repository instead of just "foo". Closes: #[627563](http://bugs.debian.org/627563)
+   * Fix bug in --exclude introduced in 0.20110516."""]]
\ No newline at end of file

From 27847585cbe4792e92f8b718f003605e5a329421 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 28 May 2011 12:18:07 +0000
Subject: [PATCH 1786/2835]

---
 ..._issue_with_latest_release_0.20110522-1-gde817ba.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn

diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
new file mode 100644
index 0000000000..662e35b4b5
--- /dev/null
+++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
@@ -0,0 +1,9 @@
+
+[70 of 81] Compiling Command.DropUnused ( Command/DropUnused.hs, Command/DropUnused.o )
+[71 of 81] Compiling Command.Status   ( Command/Status.hs, Command/Status.o )
+
+Command/Status.hs:133:37: Not in scope: `swap'
+make: *** [git-annex] Error 1
+

+
+it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install.

From f13863281c01f7ce8aaeea3b90c8b6b5180b35a0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 28 May 2011 12:22:14 +0000
Subject: [PATCH 1787/2835]

---
 .../build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
index 662e35b4b5..5743a219b5 100644
--- a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
+++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
@@ -1,3 +1,5 @@
+A recent checkout of git-annex fails to build for me (I've installed the new dependancies as well)
+
 
 [70 of 81] Compiling Command.DropUnused ( Command/DropUnused.hs, Command/DropUnused.o )
 [71 of 81] Compiling Command.Status   ( Command/Status.hs, Command/Status.o )

From 7db690bdc2d434f9bbe36a2d3bdac7202eb355f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 28 May 2011 12:26:08 +0000
Subject: [PATCH 1788/2835]

---
 .../build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
index 5743a219b5..c5fddd63c6 100644
--- a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
+++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
@@ -8,4 +8,4 @@ Command/Status.hs:133:37: Not in scope: `swap'
 make: *** [git-annex] Error 1
 

 
-it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install.
+it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install. I ran a bisect and found that commit 75a3f5027f74565d909fb940893636d081d9872a seems to have broken git-annex for me, reverting the commit allows me to build git-annex, I have not run the tests to verify everything is working correctly though.

From c38f8264355ebf99c2e1594279120d4fbdb0efc1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 11:56:48 -0400
Subject: [PATCH 1789/2835] Data.Tuple.swap not available with ghc 6.12.3

---
 Command/Status.hs                                              | 2 +-
 .../build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 43a9484433..dd518416cf 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -11,7 +11,6 @@ import Control.Monad.State
 import Data.Maybe
 import System.IO
 import Data.List
-import Data.Tuple
 import qualified Data.Map as M
 
 import qualified Annex
@@ -134,6 +133,7 @@ backend_usage = stat "backend usage" $
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
+		swap (a, b) = (b, a)
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 
diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
index c5fddd63c6..a7bae50b8b 100644
--- a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
+++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
@@ -9,3 +9,6 @@ make: *** [git-annex] Error 1
 

 
 it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install. I ran a bisect and found that commit 75a3f5027f74565d909fb940893636d081d9872a seems to have broken git-annex for me, reverting the commit allows me to build git-annex, I have not run the tests to verify everything is working correctly though.
+
+> Probably `swap` appeared only in a newer GHC. I've reverted to avoid a
+> versioned build dependency. [[done]] --[[Joey]]

From d147b822bdfe46c1bd38a9490344e108944963b0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 28 May 2011 16:04:51 +0000
Subject: [PATCH 1790/2835] Added a comment

---
 .../comment_8_59a976de6c7d333709b92f7cd5830850._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment

diff --git a/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment b/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment
new file mode 100644
index 0000000000..5aa4f8c94a
--- /dev/null
+++ b/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-05-28T16:04:51Z"
+ content="""
+And that is now installed on kitenet.net and verified to work.
+"""]]

From 0359b9b63802fcf4eca2b72fb454f7e7f5c18e28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:07:10 -0400
Subject: [PATCH 1791/2835] add link to stable tarball download from new gitweb
 version

---
 doc/download.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 3607e403f0..c0d2d261d3 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,5 +7,8 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
+To download a tarball of a particular release, use an url like
+
+
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.

From 1c85e878ac3b6a181b28f72c7ab613776ee1fd88 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:24:59 -0400
Subject: [PATCH 1792/2835] add a redirect to the current version's tarball

---
 doc/download.mdwn  | 3 +--
 doc/latest-version | 1 +
 doc/tarball.mdwn   | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 doc/latest-version
 create mode 100644 doc/tarball.mdwn

diff --git a/doc/download.mdwn b/doc/download.mdwn
index c0d2d261d3..78ab9537cf 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,8 +7,7 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
-To download a tarball of a particular release, use an url like
-
+Or download a [[tarball]] of the latest release.
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
diff --git a/doc/latest-version b/doc/latest-version
new file mode 100644
index 0000000000..1c4efa2b79
--- /dev/null
+++ b/doc/latest-version
@@ -0,0 +1 @@
+0.20110522
diff --git a/doc/tarball.mdwn b/doc/tarball.mdwn
new file mode 100644
index 0000000000..d544268ad4
--- /dev/null
+++ b/doc/tarball.mdwn
@@ -0,0 +1 @@
+[[!meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes files=latest-version]]"]]

From 8581ba59234a4fd1b60100e929f05ccd166624aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:27:02 -0400
Subject: [PATCH 1793/2835] typo

---
 doc/tarball.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tarball.mdwn b/doc/tarball.mdwn
index d544268ad4..84df11d3c0 100644
--- a/doc/tarball.mdwn
+++ b/doc/tarball.mdwn
@@ -1 +1 @@
-[[!meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes files=latest-version]]"]]
+[[!meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]]"]]

From d2b199219db7d3b19f8b5e34bffa8ee4b685635a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:30:08 -0400
Subject: [PATCH 1794/2835] debugging my inline

---
 doc/tarball.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tarball.mdwn b/doc/tarball.mdwn
index 84df11d3c0..5512dbd0ec 100644
--- a/doc/tarball.mdwn
+++ b/doc/tarball.mdwn
@@ -1 +1 @@
-[[!meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]]"]]
+meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]]"

From f336878765baabf55cba7519fbce5d4ee05a3349 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:33:08 -0400
Subject: [PATCH 1795/2835] lose redir page

meta does not support nested inlines inside redir urls
---
 doc/download.mdwn | 3 ++-
 doc/tarball.mdwn  | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
 delete mode 100644 doc/tarball.mdwn

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 78ab9537cf..1e65624837 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,7 +7,8 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
-Or download a [[tarball]] of the latest release.
+Or download a [tarball](http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]])
+of the latest release.
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
diff --git a/doc/tarball.mdwn b/doc/tarball.mdwn
deleted file mode 100644
index 5512dbd0ec..0000000000
--- a/doc/tarball.mdwn
+++ /dev/null
@@ -1 +0,0 @@
-meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]]"

From e8ab4f7292f059ba028275131aa4f12e7e494f79 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:34:52 -0400
Subject: [PATCH 1796/2835] remove newline

---
 doc/latest-version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/latest-version b/doc/latest-version
index 1c4efa2b79..3015879ce5 100644
--- a/doc/latest-version
+++ b/doc/latest-version
@@ -1 +1 @@
-0.20110522
+0.20110522
\ No newline at end of file

From ed2b1eda909d14b604b487b9ddeb7a39bd2b8ce4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:37:12 -0400
Subject: [PATCH 1797/2835] bleagh

---
 doc/download.mdwn  | 4 ++--
 doc/latest-version | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)
 delete mode 100644 doc/latest-version

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 1e65624837..c0d2d261d3 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,8 +7,8 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
-Or download a [tarball](http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]])
-of the latest release.
+To download a tarball of a particular release, use an url like
+
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
diff --git a/doc/latest-version b/doc/latest-version
deleted file mode 100644
index 3015879ce5..0000000000
--- a/doc/latest-version
+++ /dev/null
@@ -1 +0,0 @@
-0.20110522
\ No newline at end of file

From 8a4a3be9f6573c090d8d919ea1453d18264941ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 16:09:11 -0400
Subject: [PATCH 1798/2835] simplify

---
 Command.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index fd49531f44..a07ae6e097 100644
--- a/Command.hs
+++ b/Command.hs
@@ -15,6 +15,7 @@ import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
 import Data.Maybe
+import Data.String.Utils
 
 import Types
 import qualified Backend
@@ -195,11 +196,9 @@ filterFiles l = do
 
 wildsRegex :: [String] -> Regex
 wildsRegex ws = compile regex []
-	where regex = "^(" ++ wildsRegex' ws "" ++ ")"
-wildsRegex' :: [String] -> String -> String
-wildsRegex' [] c = c
-wildsRegex' (w:ws) "" = wildsRegex' ws (wildToRegex w)
-wildsRegex' (w:ws) c = wildsRegex' ws (c ++ "|" ++ wildToRegex w)
+	where
+		regex = "^(" ++ alternatives ++ ")"
+		alternatives = join "|" $ map wildToRegex ws
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool

From a19d81a42e4af0e8fc2ad0c09f1ebd72fb98cdd4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 20:01:45 -0400
Subject: [PATCH 1799/2835] show error message on unexpected parameters to
 commands that take none

Before it would exit nonzero w/o doing anything, which was confusing.
---
 Command.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command.hs b/Command.hs
index a07ae6e097..0da847d24d 100644
--- a/Command.hs
+++ b/Command.hs
@@ -176,7 +176,7 @@ withTempFile :: CommandSeekStrings
 withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
 withNothing a [] = return [a]
-withNothing _ _ = return []
+withNothing _ _ = error "This command takes no parameters."
 
 backendPairs :: CommandSeekBackendFiles
 backendPairs a files = liftM (map a) $ Backend.chooseBackends files

From fdead6b9bb0883dd982e72e955fc58dee844f2ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 21:58:48 -0400
Subject: [PATCH 1800/2835] improve error message when no remote name is
 specified

list available remotes in case user wanted to use or change one of them
---
 Command/InitRemote.hs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index ad0718e38e..460f14de2c 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -10,6 +10,8 @@ module Command.InitRemote where
 import qualified Data.Map as M
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
+import Data.Maybe
+import Data.String.Utils
 
 import Command
 import qualified Annex
@@ -32,7 +34,7 @@ seek = [withWords start]
 
 start :: CommandStartWords
 start ws = notBareRepo $ do
-	when (null ws) $ error "Specify a name for the remote"
+	when (null ws) $ needname
 
 	(u, c) <- findByName name
 	let fullconfig = M.union config c	
@@ -44,6 +46,13 @@ start ws = notBareRepo $ do
 	where
 		name = head ws
 		config = Remote.keyValToConfig $ tail ws
+		needname = do
+			let err s = error $ "Specify a name for the remote. " ++ s
+			names <- remoteNames
+			if null names
+				then err ""
+				else err $ "Either a new name, or one of these existing special remotes: " ++ join " " names
+			
 
 perform :: RemoteClass.RemoteType Annex -> UUID -> RemoteClass.RemoteConfig -> CommandPerform
 perform t u c = do
@@ -83,6 +92,11 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 				| n' == n -> True
 				| otherwise -> False
 
+remoteNames :: Annex [String]
+remoteNames = do
+	m <- Remote.readRemoteLog
+	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m
+
 {- find the specified remote type -}
 findType :: RemoteClass.RemoteConfig -> Annex (RemoteClass.RemoteType Annex)
 findType config = maybe unspecified specified $ M.lookup typeKey config

From 25b13673f0574ad2cc7baa055d08c13da134821a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 22:20:22 -0400
Subject: [PATCH 1801/2835] improve unused command's output

Display the name of the remote being checked, with "." for the current
remote, echoing the way describe takes that to change its description.
---
 Command/Unused.hs                             | 27 ++++++++++---------
 doc/special_remotes.mdwn                      |  2 +-
 .../fsck:_verifying_your_data.mdwn            |  2 +-
 doc/walkthrough/unused_data.mdwn              |  2 +-
 4 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 1482f057e8..7e5549c093 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -36,21 +36,22 @@ seek = [withNothing start]
 {- Finds unused content in the annex. -} 
 start :: CommandStartNothing
 start = notBareRepo $ do
-	showStart "unused" ""
-	next perform
+	from <- Annex.getState Annex.fromremote
+	case from of
+		Nothing -> pass "." checkUnused
+		Just n -> pass n $ checkRemoteUnused n
+	where
+		pass n a = do
+			showStart "unused" n
+			next a
 
-perform :: CommandPerform
-perform = do
-	maybe checkUnused checkRemoteUnused =<< Annex.getState Annex.fromremote
-	next $ return True
-
-checkUnused :: Annex ()
+checkUnused :: CommandPerform
 checkUnused = do
 	(unused, stalebad, staletmp) <- unusedKeys
 	n  <- list "" unusedMsg unused 0
 	n' <- list "bad" staleBadMsg stalebad n
 	_  <- list "tmp" staleTmpMsg staletmp n'
-	return ()
+	next $ return True
 	where
 		list file msg l c = do
 			let unusedlist = number c l
@@ -58,13 +59,15 @@ checkUnused = do
 			writeUnusedFile file unusedlist
 			return $ length l
 
-checkRemoteUnused :: String -> Annex ()
-checkRemoteUnused name = checkRemoteUnused' =<< Remote.byName name
+checkRemoteUnused :: String -> CommandPerform
+checkRemoteUnused name = do
+	checkRemoteUnused' =<< Remote.byName name
+	next $ return True
 
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
+	showNote $ "checking for unused data..."
 	g <- Annex.gitRepo
-	showNote $ "checking for unused data on " ++ Remote.name r ++ "..."
 	referenced <- getKeysReferenced
 	logged <- liftIO $ loggedKeys g
 	remotehas <- filterM isthere logged
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index dcb6b5063e..12e0aedb1b 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -21,7 +21,7 @@ repository is found by running `git annex unused`. To detect unused content
 on special remotes, instead use `git annex unused --from`. Example:
 
 	$ git annex unused --from mys3
-	unused  (checking for unused data on mys3...) 
+	unused mys3 (checking for unused data...) 
 	  Some annexed data on mys3 is not used by any files in this repository.
 	    NUMBER  KEY
 	    1       WORM-s3-m1301674316--foo
diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn
index cd3a47a8a9..7e05469a12 100644
--- a/doc/walkthrough/fsck:_verifying_your_data.mdwn
+++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn
@@ -5,7 +5,7 @@ the checksums of your files are good. Fsck also checks that the annex.numcopies
 setting is satisfied for all files.
 
 	# git annex fsck
-	unused  (checking for unused data...) ok
+	fsck some_file (checksum...) ok
 	fsck my_cool_big_file (checksum...) ok
 	...
 
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index 2f8edcd388..fb84193034 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -9,7 +9,7 @@ preserving it. So from time to time, you may want to check for such data and
 eliminate it to save space.
 
 	# git annex unused
-	unused  (checking for unused data...) 
+	unused . (checking for unused data...) 
 	  Some annexed data is no longer used by any files in the repository.
 	    NUMBER  KEY
 	    1       WORM-s3-m1289672605--file

From 66a99d3740d5996d328424c9492ed83fc36a77e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 22:24:48 -0400
Subject: [PATCH 1802/2835] tweak

---
 Command/Unused.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 7e5549c093..08f0148dcf 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -48,9 +48,9 @@ start = notBareRepo $ do
 checkUnused :: CommandPerform
 checkUnused = do
 	(unused, stalebad, staletmp) <- unusedKeys
-	n  <- list "" unusedMsg unused 0
-	n' <- list "bad" staleBadMsg stalebad n
-	_  <- list "tmp" staleTmpMsg staletmp n'
+	_ <- list "" unusedMsg unused 0 >>=
+		list "bad" staleBadMsg stalebad >>=
+			list "tmp" staleTmpMsg staletmp
 	next $ return True
 	where
 		list file msg l c = do

From 14ffb5d47bdb07e8263a2112b1634b403a299005 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 22:28:14 -0400
Subject: [PATCH 1803/2835] bugfix: fix unused list numbering

Introduced in 43f0a666f0f6cc152a2b778921831d6d7daedcaf
---
 Command/Unused.hs | 2 +-
 debian/changelog  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 08f0148dcf..96001683df 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -57,7 +57,7 @@ checkUnused = do
 			let unusedlist = number c l
 			when (not $ null l) $ showLongNote $ msg unusedlist
 			writeUnusedFile file unusedlist
-			return $ length l
+			return $ c + length l
 
 checkRemoteUnused :: String -> CommandPerform
 checkRemoteUnused name = do
diff --git a/debian/changelog b/debian/changelog
index 991244a885..13b53280b4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.20110523) UNRELEASED; urgency=low
+
+  * Minor bugfixes and error message improvements.
+
+ -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
+
 git-annex (0.20110522) unstable; urgency=low
 
   * Closer emulation of git's behavior when told to use "foo/.git" as a

From 86c5bd0327ba1ff0d6aed8ce47031a8497a3e2fb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 22:37:17 -0400
Subject: [PATCH 1804/2835] unused --from . checks local repo, for consistency

---
 Command/Unused.hs | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 96001683df..5422dad69f 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -37,13 +37,12 @@ seek = [withNothing start]
 start :: CommandStartNothing
 start = notBareRepo $ do
 	from <- Annex.getState Annex.fromremote
-	case from of
-		Nothing -> pass "." checkUnused
-		Just n -> pass n $ checkRemoteUnused n
-	where
-		pass n a = do
-			showStart "unused" n
-			next a
+	let (name, action) = case from of
+		Nothing -> (".", checkUnused)
+		Just "." -> (".", checkUnused)
+		Just n -> (n, checkRemoteUnused n)
+	showStart "unused" name
+	next action
 
 checkUnused :: CommandPerform
 checkUnused = do

From c4f848d29f0ee79a8a46bd27ff141d4c62257c76 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Tue, 31 May 2011 17:47:13 +0000
Subject: [PATCH 1805/2835]

---
 .../__34__git_annex_lock__34___very_slow_for_big_repo.mdwn   | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
new file mode 100644
index 0000000000..8ca3916847
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
@@ -0,0 +1,5 @@
+I found the command "git annex lock" very slow (much slower than the initial "git annex add" with SHA1), for a not so big directory, when run in a big repo.
+It seems that each underlying git command is not fast, so I thought it would be better to run them once with all files as arguments.
+I had to stop the lock command, and ran "git checkout ." (I did not change any file), is this a correct alternative?
+
+Thanks a LOT for this software, one that I missed since a long time (but wasn't able to write)!

From 6e6e77f9c05fe67a047e93b93bc198e6d9f431ce Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Tue, 31 May 2011 17:47:34 +0000
Subject: [PATCH 1806/2835]

---
 .../__34__git_annex_lock__34___very_slow_for_big_repo.mdwn      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
index 8ca3916847..9bacf28dc0 100644
--- a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
@@ -3,3 +3,5 @@ It seems that each underlying git command is not fast, so I thought it would be
 I had to stop the lock command, and ran "git checkout ." (I did not change any file), is this a correct alternative?
 
 Thanks a LOT for this software, one that I missed since a long time (but wasn't able to write)!
+
+Rafaël

From fafe60768f98ed33b8aae6cba147a15a6608eaaf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 31 May 2011 14:50:41 -0400
Subject: [PATCH 1807/2835] Massively sped up `git annex lock` by avoiding use
 of the uber-slow `git reset`, and only running `git checkout` once, even when
 many files are being locked.

---
 Command/Lock.hs  | 12 +++++-------
 debian/changelog |  3 +++
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/Command/Lock.hs b/Command/Lock.hs
index 1ae4882272..e21792143a 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -12,8 +12,7 @@ import System.Directory
 
 import Command
 import Messages
-import qualified Annex
-import qualified GitRepo as Git
+import qualified AnnexQueue
 import Utility
 	
 command :: [Command]
@@ -31,9 +30,8 @@ start (file, _) = do
 perform :: FilePath -> CommandPerform
 perform file = do
 	liftIO $ removeFile file
-	g <- Annex.gitRepo
-	-- first reset the file to drop any changes checked into the index
-	liftIO $ Git.run g "reset" [Params "-q --", File file]
-	-- checkout the symlink
-	liftIO $ Git.run g "checkout" [Param "--", File file]
+	-- Checkout from HEAD to get rid of any changes that might be 
+	-- staged in the index, and get back to the previous symlink to
+	-- the content.
+	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] file
 	next $ return True -- no cleanup needed
diff --git a/debian/changelog b/debian/changelog
index 13b53280b4..783496614f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,9 @@
 git-annex (0.20110523) UNRELEASED; urgency=low
 
   * Minor bugfixes and error message improvements.
+  * Massively sped up `git annex lock` by avoiding use of the uber-slow
+    `git reset`, and only running `git checkout` once, even when many files
+    are being locked.
 
  -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
 

From 181920fab9d352d74404d0e64e0728b6654922d9 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 31 May 2011 18:51:13 +0000
Subject: [PATCH 1808/2835] Added a comment: fixed

---
 ...t_1_044f1c5e5f7a939315c28087495a8ba8._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment
new file mode 100644
index 0000000000..0e2773bda3
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="fixed"
+ date="2011-05-31T18:51:13Z"
+ content="""
+Running `git checkout` by hand is fine, of course.
+
+Underlying problem is that git has some O(N) scalability of operations on the index with regards to the number of files in the repo. So a repo with a whole lot of files will have a big index, and any operation that changes the index, like the `git reset` this needs to do, has to read in the entire index, and write out a new, modified version. It seems that git could be much smarter about its index data structures here, but I confess I don't understand the index's data structures at all. I hope someone takes it on, as git's scalability to number of files in the repo is becoming a new pain point, now that scalability to large files is \"solved\". ;)
+
+Still, it is possible to speed this up at git-annex's level. Rather than doing a `git reset` followed by a git checkout, it can just `git checkout HEAD -- file`, and since that's one command, it can then be fed into the queueing machinery in git-annex (that exists mostly to work around this git malfescence), and so only a single git command will need to be run to lock multiple files.
+
+I've just implemented the above. In my music repo, this changed an lock of a CD's worth of files from taking ctrl-c long to 1.75 seconds. Enjoy!
+
+(Hey, this even speeds up the one file case greatly, since `git reset -- file` is slooooow -- it seems to scan the *entire* repository tree. Yipes.)
+"""]]

From fb259033d44cda1e2470d5029940d7b0725b4add Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 31 May 2011 14:58:06 -0400
Subject: [PATCH 1809/2835] Fix locking of files with staged changes.

Previously, lock would skip files that had staged changes, but that is
counterintuitive, I think.
---
 Command/Lock.hs  | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/Lock.hs b/Command/Lock.hs
index e21792143a..e55cd9e79a 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -19,7 +19,7 @@ command :: [Command]
 command = [repoCommand "lock" paramPath seek "undo unlock command"]
 
 seek :: [CommandSeek]
-seek = [withFilesUnlocked start]
+seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
 
 {- Undo unlock -}
 start :: CommandStartBackendFile
diff --git a/debian/changelog b/debian/changelog
index 783496614f..393a6161bd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (0.20110523) UNRELEASED; urgency=low
   * Massively sped up `git annex lock` by avoiding use of the uber-slow
     `git reset`, and only running `git checkout` once, even when many files
     are being locked.
+  * Fix locking of files with staged changes.
 
  -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
 

From 038da52bdd94311e4b9512a21952ce0f256643e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 31 May 2011 16:08:37 -0400
Subject: [PATCH 1810/2835] Somewhat sped up `git commit` of modifications to
 unlocked files.

Avoid git reset here too, so I no longer need to care that it's much more
expensive than seems wise (but I asked the git list about that anyway).

It's not necessary to reset the staged file content from the index, as
the `git add` of the the symlink will replace it anyway.

`git commit` of unlocked files is still slow, since git still has to shove
their entire content into the index, only to have it be thrown away. So it's
still better to use `git annex add`
---
 Command/PreCommit.hs | 12 +-----------
 debian/changelog     |  1 +
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index d7f2487137..6d8f7b9b3f 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -33,15 +33,5 @@ perform :: BackendFile -> CommandPerform
 perform pair@(file, _) = do
 	ok <- doCommand $ Command.Add.start pair
 	if ok
-		then next $ cleanup file
+		then next $ return True
 		else error $ "failed to add " ++ file ++ "; canceling commit"
-
-cleanup :: FilePath -> CommandCleanup
-cleanup file = do
-	-- git commit will have staged the file's content;
-	-- drop that and run command queued by Add.state to
-	-- stage the symlink
-	g <- Annex.gitRepo
-	liftIO $ Git.run g "reset" [Params "-q --", File file]
-	AnnexQueue.flush True
-	return True
diff --git a/debian/changelog b/debian/changelog
index 393a6161bd..ceb218de55 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (0.20110523) UNRELEASED; urgency=low
     `git reset`, and only running `git checkout` once, even when many files
     are being locked.
   * Fix locking of files with staged changes.
+  * Somewhat sped up `git commit` of modifications to unlocked files.
 
  -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
 

From 7760ba9d97dbade1701797714858df3d44587e1f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Tue, 31 May 2011 21:43:23 +0000
Subject: [PATCH 1811/2835] Added a comment

---
 ...ment_2_e854b93415d5ab80eda8e3be3b145ec2._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment
new file mode 100644
index 0000000000..9e9e778ce9
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 2"
+ date="2011-05-31T21:43:22Z"
+ content="""
+Nice!
+So if I understand correctly, 'git reset -- file' was there to discard staged (but not commited) changes made to 'file', before checking out, so that it is equivalent to directly 'git checkout HEAD -- file' ?
+I'm curious about the \"queueing machinery in git-annex\": does it end up calling the one git command with multiple files as arguments? does it correspond to the message \"(Recording state in git...)\" ?
+Thanks!
+
+
+"""]]

From b3f6621a5342f1c0ff373339ded491af9fd997ca Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 31 May 2011 21:54:23 +0000
Subject: [PATCH 1812/2835] Added a comment

---
 .../comment_3_95c110500bc54013bc1969c1a9c8f842._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment
new file mode 100644
index 0000000000..87da0c396d
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-05-31T21:54:23Z"
+ content="""
+@Rafaël , you're correct on all counts.
+"""]]

From 869cb82f4927adff636051645dd3c693ed22ff31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 11:53:43 -0400
Subject: [PATCH 1813/2835] remove unnecessary imports

---
 Command/PreCommit.hs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 6d8f7b9b3f..3046d35626 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -7,15 +7,9 @@
 
 module Command.PreCommit where
 
-import Control.Monad.State (liftIO)
-
 import Command
-import qualified Annex
-import qualified AnnexQueue
-import qualified GitRepo as Git
 import qualified Command.Add
 import qualified Command.Fix
-import Utility
 
 command :: [Command]
 command = [repoCommand "pre-commit" paramPath seek "run by git pre-commit hook"]

From dc92a788c767f14533147758044a019737707a52 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 12:00:25 -0400
Subject: [PATCH 1814/2835] releasing version 0.20110601

---
 debian/changelog | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ceb218de55..5e10b3d268 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20110523) UNRELEASED; urgency=low
+git-annex (0.20110601) unstable; urgency=low
 
   * Minor bugfixes and error message improvements.
   * Massively sped up `git annex lock` by avoiding use of the uber-slow
@@ -6,8 +6,9 @@ git-annex (0.20110523) UNRELEASED; urgency=low
     are being locked.
   * Fix locking of files with staged changes.
   * Somewhat sped up `git commit` of modifications to unlocked files.
+  * Build fix for older ghc.
 
- -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
+ -- Joey Hess   Wed, 01 Jun 2011 11:50:47 -0400
 
 git-annex (0.20110522) unstable; urgency=low
 

From e47de0f6cd3c7309e88b59e554e710779e4a6878 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 12:02:20 -0400
Subject: [PATCH 1815/2835] add news item for git-annex 0.20110601

---
 doc/news/version_0.20110427.mdwn | 8 --------
 doc/news/version_0.20110601.mdwn | 9 +++++++++
 2 files changed, 9 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_0.20110427.mdwn
 create mode 100644 doc/news/version_0.20110601.mdwn

diff --git a/doc/news/version_0.20110427.mdwn b/doc/news/version_0.20110427.mdwn
deleted file mode 100644
index 2764c6de56..0000000000
--- a/doc/news/version_0.20110427.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 0.20110427 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Switch back to haskell SHA library, so git-annex remains buildable on
-     Debian stable.
-   * Added rsync special remotes. This could be used, for example, to
-     store annexed content on rsync.net (encrypted naturally). Or anywhere else.
-   * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a
-     lot of files. Possibly only occured with ghc 7."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110601.mdwn b/doc/news/version_0.20110601.mdwn
new file mode 100644
index 0000000000..59079088da
--- /dev/null
+++ b/doc/news/version_0.20110601.mdwn
@@ -0,0 +1,9 @@
+git-annex 0.20110601 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Minor bugfixes and error message improvements.
+   * Massively sped up `git annex lock` by avoiding use of the uber-slow
+     `git reset`, and only running `git checkout` once, even when many files
+     are being locked.
+   * Fix locking of files with staged changes.
+   * Somewhat sped up `git commit` of modifications to unlocked files.
+   * Build fix for older ghc."""]]
\ No newline at end of file

From 5aec8b8f160834e3b11048c1de54b6f7dabd81ea Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 1 Jun 2011 17:08:15 +0000
Subject: [PATCH 1816/2835]

---
 doc/forum/new_microfeatures.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/forum/new_microfeatures.mdwn

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
new file mode 100644
index 0000000000..cf3ddfd628
--- /dev/null
+++ b/doc/forum/new_microfeatures.mdwn
@@ -0,0 +1,20 @@
+I'm soliciting ideas for new small features that let git-annex do things that currently have to be done manually or whatever.
+
+Here are a few I've been considering:
+
+--
+
+* --numcopies would be a useful command line switch.
+* A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 
+
+Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=repob` to remove files that have been replicated out to the other 2 repositories, which could be offline. (Slightly unsafe, but in this case the files are podcasts so not really.)
+
+---
+
+[[wishlist:_git-annex_replicate]] suggests some way for git-annex to have the smarts to copy content around on its own to ensure numcopies is satisfied. I'd be satisfied with a `git annex copy --to foo --if-needed-by-numcopies`
+
+---
+
+Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.
+
+--[[Joey]]

From af7a0692637382bb00d44aedd12124f58de62d2e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 1 Jun 2011 17:08:52 +0000
Subject: [PATCH 1817/2835]

---
 doc/forum/new_microfeatures.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index cf3ddfd628..4b4389e3d1 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -2,7 +2,7 @@ I'm soliciting ideas for new small features that let git-annex do things that cu
 
 Here are a few I've been considering:
 
---
+---
 
 * --numcopies would be a useful command line switch.
 * A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 

From 82e83528f41de789275435076806afe27b53332e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 1 Jun 2011 17:36:51 +0000
Subject: [PATCH 1818/2835] Added a comment

---
 .../comment_1_058bd517c6fffaf3446b1f5d5be63623._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment

diff --git a/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment b/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment
new file mode 100644
index 0000000000..84fdd325dc
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-06-01T17:36:50Z"
+ content="""
+I've been longing for an automated way of removing references to a remote assuming I know the exact uuid that I want to remove. i.e. I have lost a portable HDD due to a destructive process, I now want to delete all references to copies of data that was on that disk. Unless this feature exists, I would love to see it implemented.
+"""]]

From 12e0e95916b81352b6f0dfdf95d3e623d4a12738 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 1 Jun 2011 20:24:33 +0000
Subject: [PATCH 1819/2835] Added a comment

---
 .../comment_2_41ad904c68e89c85e1fc49c9e9106969._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment

diff --git a/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment b/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment
new file mode 100644
index 0000000000..4451e20baf
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-06-01T20:24:33Z"
+ content="""
+@jimmy [[walkthrough/what_to_do_when_you_lose_a_repository]].. I have not seen a convincing argument that removing the location tracking data entirely serves any purpose
+"""]]

From 3d567aa64f263c6ee55c92e8b962087de063ebc8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 16:49:17 -0400
Subject: [PATCH 1820/2835] Add --numcopies option.

---
 Annex.hs                         |  2 ++
 Backend/File.hs                  | 12 ++++++++----
 GitAnnex.hs                      | 12 ++++++++----
 debian/changelog                 |  6 ++++++
 doc/forum/new_microfeatures.mdwn |  3 +++
 doc/git-annex.mdwn               |  5 +++++
 6 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 2148dd6252..c5a098d986 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -39,6 +39,7 @@ data AnnexState = AnnexState
 	, force :: Bool
 	, fast :: Bool
 	, forcebackend :: Maybe String
+	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
@@ -57,6 +58,7 @@ newState gitrepo allbackends = AnnexState
 	, force = False
 	, fast = False
 	, forcebackend = Nothing
+	, forcenumcopies = Nothing
 	, defaultkey = Nothing
 	, toremote = Nothing
 	, fromremote = Nothing
diff --git a/Backend/File.hs b/Backend/File.hs
index b86413e400..543f02af76 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -152,12 +152,16 @@ showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
 		(join ", " $ map Remote.name remotes)
 
+{- If a value is specified, it is used; otherwise the default is looked up
+ - in git config. forcenumcopies overrides everything. -}
 getNumCopies :: Maybe Int -> Annex Int
-getNumCopies (Just n) = return n
-getNumCopies Nothing = do
-	g <- Annex.gitRepo
-	return $ read $ Git.configGet g config "1"
+getNumCopies v = 
+	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
 	where
+		use (Just n) = return n
+		use Nothing = do
+			g <- Annex.gitRepo
+			return $ read $ Git.configGet g config "1"
 		config = "annex.numcopies"
 
 {- Ideally, all keys have file size metadata. Old keys may not. -}
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 99aec187a9..37a73424af 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -13,6 +13,7 @@ import qualified GitRepo as Git
 import CmdLine
 import Command
 import Options
+import Utility
 import qualified Annex
 
 import qualified Command.Add
@@ -82,20 +83,23 @@ cmds = concat
 
 options :: [Option]
 options = commonOptions ++
-	[ Option ['k'] ["key"] (ReqArg setkey paramKey)
-		"specify a key to use"
-	, Option ['t'] ["to"] (ReqArg setto paramRemote)
+	[ Option ['t'] ["to"] (ReqArg setto paramRemote)
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
 	, Option ['x'] ["exclude"] (ReqArg addexclude paramGlob)
 		"skip files matching the glob pattern"
+	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
+		"override default number of copies"
+	, Option ['k'] ["key"] (ReqArg setkey paramKey)
+		"specify a key to use"
 	]
 	where
-		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:(Annex.exclude s) }
+		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
+		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/debian/changelog b/debian/changelog
index 5e10b3d268..0d531a3206 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.20110602) UNRELEASED; urgency=low
+
+  * Add --numcopies option.
+
+ -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
+
 git-annex (0.20110601) unstable; urgency=low
 
   * Minor bugfixes and error message improvements.
diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index 4b4389e3d1..fe91addca8 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -5,6 +5,9 @@ Here are a few I've been considering:
 ---
 
 * --numcopies would be a useful command line switch.
+  > Update: Added. Also allows for things like `git annex drop
+  > --numcopies=2` when in a repo that normally needs 3 copies, if you need
+  > to urgently free up space.
 * A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 
 
 Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=repob` to remove files that have been replicated out to the other 2 repositories, which could be offline. (Slightly unsafe, but in this case the files are podcasts so not really.)
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 7f2fce9d23..e4924d373d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -354,6 +354,11 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   This option can be specified multiple times.
 
+* --numcopies=n
+
+  Overrides the `annex.numcopies` setting, forcing git-annex to ensure the
+  specified number of copies exist.
+
 * --backend=name
 
   Specifies which key-value backend to use. This can be used when

From e280c0a4fca41eee68961db86446c91ed2d6adab Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 1 Jun 2011 21:26:39 +0000
Subject: [PATCH 1821/2835]

---
 doc/forum/new_microfeatures.mdwn | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index fe91addca8..9b1f8231c0 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -16,8 +16,35 @@ Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=r
 
 [[wishlist:_git-annex_replicate]] suggests some way for git-annex to have the smarts to copy content around on its own to ensure numcopies is satisfied. I'd be satisfied with a `git annex copy --to foo --if-needed-by-numcopies`
 
+  > Contrary to the "basic" solution, I would love to have a git annex distribute which is smart enough to simply distribute all data according to certain rules. My ideal, personal use case during the next holidays where I will have two external disks, several SD cards with 32 GB each and a local disk with 20 GB (yes....) would be:
+
+    cd ~/photos.annex # this repository does not have any objects!
+    git annex inject --bare /path/to/SD/card  # this adds softlinks, but does **not** add anything to the index. it would calculate checksums (if enabled) and have to add a temporary location list, though
+    git annex distribute # this checks the config. it would see that my two external disks have a low cost whereas the two remotes have a higher cost.
+     # check numcopies. it's 3
+     # copy to external disk one (cost x)
+     # copy to external disk two (cost x)
+     # copy to remote one (cost x * 2)
+     # remove file from temporary tracking list
+    git annex fsck # everything ok. yay!
+
+Come to think of it, the inject --bare thing is probably not a microfeature. Should I add a new wishlist item for that? -- RichiH
+
 ---
 
 Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.
 
+---
+
+If a remote could send on received files to another remote, I could use my own local bandwith efficiently while still having my git-annex repos replicate data. -- RichiH
+
+---
+
+Really micro:
+
+    % grep annex-push .git/config
+        annex-push = !git pull && git annex add . && git annex copy . --to origin --fast --quiet && git commit -a -m "$HOST $(date +%F--%H-%M-%S-%Z)" && git push
+    %
+
+-- RichiH
 --[[Joey]]

From 7a3d9d8c2e2bd53d0d4290e99186c6e37f18456d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 16:54:28 -0400
Subject: [PATCH 1822/2835] mention --numcopies

---
 doc/copies.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/copies.mdwn b/doc/copies.mdwn
index 39a714d3bb..16eba19c81 100644
--- a/doc/copies.mdwn
+++ b/doc/copies.mdwn
@@ -8,7 +8,8 @@ to keep N copies of a file's content available across all repositories.
 
 By default, N is 1; it is configured by annex.numcopies. This default
 can be overridden on a per-file-type basis by the annex.numcopies
-setting in `.gitattributes` files.
+setting in `.gitattributes` files. The --numcopies switch allows
+temporarily using a different value.
 
 `git annex drop` attempts to check with other git remotes, to check that N
 copies of the file exist. If enough repositories cannot be verified to have

From a8fb97d2ce8e75b36b8e1572a83efd341e67d43e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 17:49:37 -0400
Subject: [PATCH 1823/2835] Add --trust, --untrust, and --semitrust options.

---
 Annex.hs                         |  3 +++
 Backend/File.hs                  |  5 ++--
 Command/Move.hs                  |  5 ++--
 GitAnnex.hs                      | 14 ++++++++---
 Remote.hs                        | 28 +--------------------
 RemoteUtils.hs                   | 42 ++++++++++++++++++++++++++++++++
 Trust.hs                         | 30 +++++++++--------------
 TrustLevel.hs                    | 23 +++++++++++++++++
 debian/changelog                 |  1 +
 doc/forum/new_microfeatures.mdwn |  5 ++--
 doc/git-annex.mdwn               |  8 ++++++
 doc/trust.mdwn                   | 11 ++++++---
 12 files changed, 117 insertions(+), 58 deletions(-)
 create mode 100644 RemoteUtils.hs
 create mode 100644 TrustLevel.hs

diff --git a/Annex.hs b/Annex.hs
index c5a098d986..13505de468 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -24,6 +24,7 @@ import qualified GitQueue
 import qualified BackendClass
 import qualified RemoteClass
 import qualified CryptoTypes
+import TrustLevel
 
 -- git-annex's monad
 type Annex = StateT AnnexState IO
@@ -44,6 +45,7 @@ data AnnexState = AnnexState
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
 	, exclude :: [String]
+	, forcetrust :: [(String, TrustLevel)]
 	, cipher :: Maybe CryptoTypes.Cipher
 	}
 
@@ -63,6 +65,7 @@ newState gitrepo allbackends = AnnexState
 	, toremote = Nothing
 	, fromremote = Nothing
 	, exclude = []
+	, forcetrust = []
 	, cipher = Nothing
 	}
 
diff --git a/Backend/File.hs b/Backend/File.hs
index 543f02af76..58506c861b 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -21,6 +21,7 @@ import Data.String.Utils
 import BackendClass
 import LocationLog
 import qualified Remote
+import qualified RemoteUtils
 import qualified GitRepo as Git
 import Content
 import qualified Annex
@@ -53,7 +54,7 @@ dummyStore _ _ = return True
  - and copy it to here. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(remotes, _) <- Remote.keyPossibilities key
+	(remotes, _) <- RemoteUtils.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -96,7 +97,7 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- Remote.keyPossibilities key
+			(remotes, trusteduuids) <- RemoteUtils.keyPossibilities key
 			untrusteduuids <- trustGet UnTrusted
 			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
diff --git a/Command/Move.hs b/Command/Move.hs
index f49fe20e00..6a23aee92a 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -16,6 +16,7 @@ import LocationLog
 import Types
 import Content
 import qualified Remote
+import qualified RemoteUtils
 import UUID
 import Messages
 
@@ -89,7 +90,7 @@ toPerform dest move key = do
 	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)
 	isthere <- if fastcheck
 		then do
-			(remotes, _) <- Remote.keyPossibilities key
+			(remotes, _) <- RemoteUtils.keyPossibilities key
 			return $ Right $ dest `elem` remotes
 		else Remote.hasKey dest key
 	case isthere of
@@ -123,7 +124,7 @@ fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	(remotes, _) <- Remote.keyPossibilities key
+	(remotes, _) <- RemoteUtils.keyPossibilities key
 	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
 		then stop
 		else do
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 37a73424af..64b0888b05 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -14,6 +14,7 @@ import CmdLine
 import Command
 import Options
 import Utility
+import TrustLevel
 import qualified Annex
 
 import qualified Command.Add
@@ -83,7 +84,9 @@ cmds = concat
 
 options :: [Option]
 options = commonOptions ++
-	[ Option ['t'] ["to"] (ReqArg setto paramRemote)
+	[ Option ['k'] ["key"] (ReqArg setkey paramKey)
+		"specify a key to use"
+	, Option ['t'] ["to"] (ReqArg setto paramRemote)
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
@@ -91,8 +94,12 @@ options = commonOptions ++
 		"skip files matching the glob pattern"
 	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
-	, Option ['k'] ["key"] (ReqArg setkey paramKey)
-		"specify a key to use"
+	, Option [] ["trust"] (ReqArg (settrust Trusted) paramRemote)
+		"override trust setting"
+	, Option [] ["semitrust"] (ReqArg (settrust SemiTrusted) paramRemote)
+		"override trust setting back to default value"
+	, Option [] ["untrust"] (ReqArg (settrust UnTrusted) paramRemote)
+		"override trust setting to untrusted"
 	]
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
@@ -100,6 +107,7 @@ options = commonOptions ++
 		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:(Annex.exclude s) }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
+		settrust t v = Annex.changeState $ \s -> s { Annex.forcetrust = (v, t):(Annex.forcetrust s) }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/Remote.hs b/Remote.hs
index 7df84a5da6..51da5e4715 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -16,9 +16,9 @@ module Remote (
 	hasKeyCheap,
 
 	remoteTypes,
+	genList,
 	byName,
 	nameToUUID,
-	keyPossibilities,
 	remotesWithUUID,
 	remotesWithoutUUID,
 
@@ -42,8 +42,6 @@ import RemoteClass
 import Types
 import UUID
 import qualified Annex
-import Trust
-import LocationLog
 import Locations
 import Utility
 import Config
@@ -104,30 +102,6 @@ nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = liftM uuid (byName n)
 
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
- -
- - Also returns a list of UUIDs that are trusted to have the key
- - (some may not have configured remotes).
- -}
-keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
-keyPossibilities key = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-	trusted <- trustGet Trusted
-
-	-- get uuids of all remotes that are recorded to have the key
-	uuids <- liftIO $ keyLocations g key
-	let validuuids = filter (/= u) uuids
-
-	-- note that validuuids is assumed to not have dups
-	let validtrusteduuids = intersect validuuids trusted
-
-	-- remotes that match uuids that have the key
-	allremotes <- genList
-	let validremotes = remotesWithUUID allremotes validuuids
-
-	return (sort validremotes, validtrusteduuids)
-
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
diff --git a/RemoteUtils.hs b/RemoteUtils.hs
new file mode 100644
index 0000000000..d042780e46
--- /dev/null
+++ b/RemoteUtils.hs
@@ -0,0 +1,42 @@
+{- git-annex remotes overflow (can't go in there due to dependency cycles)
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module RemoteUtils where
+
+import Control.Monad.State (liftIO)
+import Data.List
+
+import Annex
+import Trust
+import Remote
+import UUID
+import LocationLog
+import Key
+
+{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+ -
+ - Also returns a list of UUIDs that are trusted to have the key
+ - (some may not have configured remotes).
+ -}
+keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
+keyPossibilities key = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+	trusted <- trustGet Trusted
+
+	-- get uuids of all remotes that are recorded to have the key
+	uuids <- liftIO $ keyLocations g key
+	let validuuids = filter (/= u) uuids
+
+	-- note that validuuids is assumed to not have dups
+	let validtrusteduuids = intersect validuuids trusted
+
+	-- remotes that match uuids that have the key
+	allremotes <- genList
+	let validremotes = remotesWithUUID allremotes validuuids
+
+	return (sort validremotes, validtrusteduuids)
diff --git a/Trust.hs b/Trust.hs
index 7b2cf9ff88..d6d0516abb 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -1,4 +1,4 @@
-{- git-annex trust levels
+{- git-annex trust
  -
  - Copyright 2010 Joey Hess 
  -
@@ -17,26 +17,15 @@ module Trust (
 import Control.Monad.State
 import qualified Data.Map as M
 
+import TrustLevel
 import qualified GitRepo as Git
 import Types
 import UUID
 import Locations
 import qualified Annex
+import qualified Remote
 import Utility
 
-data TrustLevel = SemiTrusted | UnTrusted | Trusted
-	deriving Eq
-
-instance Show TrustLevel where
-        show SemiTrusted = "?"
-        show UnTrusted = "0"
-        show Trusted = "1"
-
-instance Read TrustLevel where
-        readsPrec _ "1" = [(Trusted, "")]
-        readsPrec _ "0" = [(UnTrusted, "")]
-	readsPrec _ _ = [(SemiTrusted, "")]
-
 {- Filename of trust.log. -}
 trustLog :: Annex FilePath
 trustLog = do
@@ -49,18 +38,23 @@ trustGet level = do
 	m <- trustMap
 	return $ M.keys $ M.filter (== level) m
 
-{- Read the trustLog into a map. -}
+{- Read the trustLog into a map, overriding with any
+ - values from forcetrust -}
 trustMap :: Annex (M.Map UUID TrustLevel)
 trustMap = do
 	logfile <- trustLog
+	overrides <- Annex.getState Annex.forcetrust >>= mapM findoverride
 	s <- liftIO $ catch (readFile logfile) ignoreerror
-	return $ trustMapParse s
+	return $ M.fromList $ trustMapParse s ++ overrides
 	where
                 ignoreerror _ = return ""
+		findoverride (name, t) = do
+			uuid <- Remote.nameToUUID name
+			return (uuid, t)
 
 {- Trust map parser. -}
-trustMapParse :: String -> M.Map UUID TrustLevel
-trustMapParse s = M.fromList $ map pair $ filter (not . null) $ lines s
+trustMapParse :: String -> [(UUID, TrustLevel)]
+trustMapParse s = map pair $ filter (not . null) $ lines s
 	where
 		pair l
 			| length w > 1 = (w !! 0, read (w !! 1) :: TrustLevel)
diff --git a/TrustLevel.hs b/TrustLevel.hs
new file mode 100644
index 0000000000..5da142ca30
--- /dev/null
+++ b/TrustLevel.hs
@@ -0,0 +1,23 @@
+{- git-annex trust levels
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module TrustLevel (
+	TrustLevel(..),
+) where
+
+data TrustLevel = SemiTrusted | UnTrusted | Trusted
+	deriving Eq
+
+instance Show TrustLevel where
+        show SemiTrusted = "?"
+        show UnTrusted = "0"
+        show Trusted = "1"
+
+instance Read TrustLevel where
+        readsPrec _ "1" = [(Trusted, "")]
+        readsPrec _ "0" = [(UnTrusted, "")]
+	readsPrec _ _ = [(SemiTrusted, "")]
diff --git a/debian/changelog b/debian/changelog
index 0d531a3206..8cb256099e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (0.20110602) UNRELEASED; urgency=low
 
   * Add --numcopies option.
+  * Add --trust, --untrust, and --semitrust options.
 
  -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
 
diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index 9b1f8231c0..e992bfb4fe 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -5,13 +5,14 @@ Here are a few I've been considering:
 ---
 
 * --numcopies would be a useful command line switch.
-  > Update: Added. Also allows for things like `git annex drop
-  > --numcopies=2` when in a repo that normally needs 3 copies, if you need
+  > Update: Added. Also allows for things like `git annex drop --numcopies=2` when in a repo that normally needs 3 copies, if you need
   > to urgently free up space.
 * A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 
 
 Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=repob` to remove files that have been replicated out to the other 2 repositories, which could be offline. (Slightly unsafe, but in this case the files are podcasts so not really.)
 
+> Update: done --[[Joey]] 
+
 ---
 
 [[wishlist:_git-annex_replicate]] suggests some way for git-annex to have the smarts to copy content around on its own to ensure numcopies is satisfied. I'd be satisfied with a `git annex copy --to foo --if-needed-by-numcopies`
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e4924d373d..b15ce1a296 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -359,6 +359,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Overrides the `annex.numcopies` setting, forcing git-annex to ensure the
   specified number of copies exist.
 
+* --trust=repository
+* --semitrust=repository
+* --untrust=repository
+
+  Overrides trust settings for a repository. May be specified more than once.
+  
+  The repository should be specified using the name of a configured remote.
+
 * --backend=name
 
   Specifies which key-value backend to use. This can be used when
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index 317e4b541f..7505a7af65 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -20,7 +20,9 @@ depended on to retain a copy of the file content; possibly the only
 [[copy|copies]].
 
 (Being semitrusted is the default. The `git annex semitrust` command
-restores a repository to this default, when it has been overridden.)
+restores a repository to this default, when it has been overridden.
+The `--semitrust` option can temporarily restore a repository to this
+default.)
 
 ## untrusted
 
@@ -42,7 +44,8 @@ archival drive, from which you rarely or never remove content. Deciding
 when it makes sense to trust the tracking info is up to you.
 
 One way to handle this is just to use `--force` when a command cannot
-access a remote you trust.
+access a remote you trust. Or to use `--trust` to specify a repisitory to
+trust temporarily.
 
-To configure a repository as fully trusted, use the `git annex trust`
-command.
+To configure a repository as fully and permanently trusted,
+use the `git annex trust` command.

From 80efafe4960e0fb33d1e6783bd34eaf459febea1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 1 Jun 2011 22:15:58 +0000
Subject: [PATCH 1824/2835]

---
 doc/forum/new_microfeatures.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index e992bfb4fe..683cc69b8b 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -31,6 +31,8 @@ Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=r
 
 Come to think of it, the inject --bare thing is probably not a microfeature. Should I add a new wishlist item for that? -- RichiH
 
+> I've thought about such things before; does not seem really micro and I'm unsure how well it would work, but it would be worth a [[todo]]. --[[Joey]]
+
 ---
 
 Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.

From 971ab27e7820a3228f71dd42f3e870c0fc2f4345 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 19:10:38 -0400
Subject: [PATCH 1825/2835] better types allowed breaking module dep loop

---
 Annex.hs        | 27 ++++++++++++++-------------
 Backend/File.hs |  5 ++---
 Command/Move.hs |  5 ++---
 GitAnnex.hs     |  7 +++++--
 Remote.hs       | 28 ++++++++++++++++++++++++++++
 RemoteUtils.hs  | 42 ------------------------------------------
 Trust.hs        |  6 +-----
 UUID.hs         |  3 +--
 UUIDType.hs     | 11 +++++++++++
 9 files changed, 64 insertions(+), 70 deletions(-)
 delete mode 100644 RemoteUtils.hs
 create mode 100644 UUIDType.hs

diff --git a/Annex.hs b/Annex.hs
index 13505de468..92a4911ea1 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -20,11 +20,12 @@ import Control.Monad.State
 	(liftIO, StateT, runStateT, evalStateT, liftM, get, put)
 
 import qualified GitRepo as Git
-import qualified GitQueue
-import qualified BackendClass
-import qualified RemoteClass
-import qualified CryptoTypes
+import GitQueue
+import BackendClass
+import RemoteClass
+import CryptoTypes
 import TrustLevel
+import UUIDType
 
 -- git-annex's monad
 type Annex = StateT AnnexState IO
@@ -32,10 +33,10 @@ type Annex = StateT AnnexState IO
 -- internal state storage
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
-	, backends :: [BackendClass.Backend Annex]
-	, supportedBackends :: [BackendClass.Backend Annex]
-	, remotes :: [RemoteClass.Remote Annex]
-	, repoqueue :: GitQueue.Queue
+	, backends :: [Backend Annex]
+	, supportedBackends :: [Backend Annex]
+	, remotes :: [Remote Annex]
+	, repoqueue :: Queue
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
@@ -45,17 +46,17 @@ data AnnexState = AnnexState
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
 	, exclude :: [String]
-	, forcetrust :: [(String, TrustLevel)]
-	, cipher :: Maybe CryptoTypes.Cipher
+	, forcetrust :: [(UUID, TrustLevel)]
+	, cipher :: Maybe Cipher
 	}
 
-newState :: Git.Repo -> [BackendClass.Backend Annex] -> AnnexState
+newState :: Git.Repo -> [Backend Annex] -> AnnexState
 newState gitrepo allbackends = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
 	, supportedBackends = allbackends
-	, repoqueue = GitQueue.empty
+	, repoqueue = empty
 	, quiet = False
 	, force = False
 	, fast = False
@@ -70,7 +71,7 @@ newState gitrepo allbackends = AnnexState
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
-new :: Git.Repo -> [BackendClass.Backend Annex] -> IO AnnexState
+new :: Git.Repo -> [Backend Annex] -> IO AnnexState
 new gitrepo allbackends = do
 	gitrepo' <- liftIO $ Git.configRead gitrepo
 	return $ newState gitrepo' allbackends
diff --git a/Backend/File.hs b/Backend/File.hs
index 58506c861b..543f02af76 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -21,7 +21,6 @@ import Data.String.Utils
 import BackendClass
 import LocationLog
 import qualified Remote
-import qualified RemoteUtils
 import qualified GitRepo as Git
 import Content
 import qualified Annex
@@ -54,7 +53,7 @@ dummyStore _ _ = return True
  - and copy it to here. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(remotes, _) <- RemoteUtils.keyPossibilities key
+	(remotes, _) <- Remote.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -97,7 +96,7 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- RemoteUtils.keyPossibilities key
+			(remotes, trusteduuids) <- Remote.keyPossibilities key
 			untrusteduuids <- trustGet UnTrusted
 			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
diff --git a/Command/Move.hs b/Command/Move.hs
index 6a23aee92a..f49fe20e00 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -16,7 +16,6 @@ import LocationLog
 import Types
 import Content
 import qualified Remote
-import qualified RemoteUtils
 import UUID
 import Messages
 
@@ -90,7 +89,7 @@ toPerform dest move key = do
 	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)
 	isthere <- if fastcheck
 		then do
-			(remotes, _) <- RemoteUtils.keyPossibilities key
+			(remotes, _) <- Remote.keyPossibilities key
 			return $ Right $ dest `elem` remotes
 		else Remote.hasKey dest key
 	case isthere of
@@ -124,7 +123,7 @@ fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	(remotes, _) <- RemoteUtils.keyPossibilities key
+	(remotes, _) <- Remote.keyPossibilities key
 	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
 		then stop
 		else do
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 64b0888b05..2a9fcbe3e7 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -16,6 +16,7 @@ import Options
 import Utility
 import TrustLevel
 import qualified Annex
+import qualified Remote
 
 import qualified Command.Add
 import qualified Command.Unannex
@@ -104,10 +105,12 @@ options = commonOptions ++
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
-		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:(Annex.exclude s) }
+		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
-		settrust t v = Annex.changeState $ \s -> s { Annex.forcetrust = (v, t):(Annex.forcetrust s) }
+		settrust t v = do
+			r <- Remote.nameToUUID v
+			Annex.changeState $ \s -> s { Annex.forcetrust = (r, t):Annex.forcetrust s }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/Remote.hs b/Remote.hs
index 51da5e4715..9685b4612f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -14,6 +14,7 @@ module Remote (
 	removeKey,
 	hasKey,
 	hasKeyCheap,
+	keyPossibilities,
 
 	remoteTypes,
 	genList,
@@ -45,6 +46,8 @@ import qualified Annex
 import Locations
 import Utility
 import Config
+import Trust
+import LocationLog
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -110,6 +113,31 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
+{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+ -
+ - Also returns a list of UUIDs that are trusted to have the key
+ - (some may not have configured remotes).
+ -}
+keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
+keyPossibilities key = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+	trusted <- trustGet Trusted
+
+	-- get uuids of all remotes that are recorded to have the key
+	uuids <- liftIO $ keyLocations g key
+	let validuuids = filter (/= u) uuids
+
+	-- note that validuuids is assumed to not have dups
+	let validtrusteduuids = intersect validuuids trusted
+
+	-- remotes that match uuids that have the key
+	allremotes <- genList
+	let validremotes = remotesWithUUID allremotes validuuids
+
+	return (sort validremotes, validtrusteduuids)
+
+
 {- Filename of remote.log. -}
 remoteLog :: Annex FilePath
 remoteLog = do
diff --git a/RemoteUtils.hs b/RemoteUtils.hs
deleted file mode 100644
index d042780e46..0000000000
--- a/RemoteUtils.hs
+++ /dev/null
@@ -1,42 +0,0 @@
-{- git-annex remotes overflow (can't go in there due to dependency cycles)
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module RemoteUtils where
-
-import Control.Monad.State (liftIO)
-import Data.List
-
-import Annex
-import Trust
-import Remote
-import UUID
-import LocationLog
-import Key
-
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
- -
- - Also returns a list of UUIDs that are trusted to have the key
- - (some may not have configured remotes).
- -}
-keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
-keyPossibilities key = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-	trusted <- trustGet Trusted
-
-	-- get uuids of all remotes that are recorded to have the key
-	uuids <- liftIO $ keyLocations g key
-	let validuuids = filter (/= u) uuids
-
-	-- note that validuuids is assumed to not have dups
-	let validtrusteduuids = intersect validuuids trusted
-
-	-- remotes that match uuids that have the key
-	allremotes <- genList
-	let validremotes = remotesWithUUID allremotes validuuids
-
-	return (sort validremotes, validtrusteduuids)
diff --git a/Trust.hs b/Trust.hs
index d6d0516abb..aaca3b3706 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -23,7 +23,6 @@ import Types
 import UUID
 import Locations
 import qualified Annex
-import qualified Remote
 import Utility
 
 {- Filename of trust.log. -}
@@ -43,14 +42,11 @@ trustGet level = do
 trustMap :: Annex (M.Map UUID TrustLevel)
 trustMap = do
 	logfile <- trustLog
-	overrides <- Annex.getState Annex.forcetrust >>= mapM findoverride
+	overrides <- Annex.getState Annex.forcetrust
 	s <- liftIO $ catch (readFile logfile) ignoreerror
 	return $ M.fromList $ trustMapParse s ++ overrides
 	where
                 ignoreerror _ = return ""
-		findoverride (name, t) = do
-			uuid <- Remote.nameToUUID name
-			return (uuid, t)
 
 {- Trust map parser. -}
 trustMapParse :: String -> [(UUID, TrustLevel)]
diff --git a/UUID.hs b/UUID.hs
index 0d7aee1414..33835e261b 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -36,8 +36,7 @@ import qualified Annex
 import Utility
 import qualified SysConfig
 import Config
-
-type UUID = String
+import UUIDType
 
 configkey :: String
 configkey = "annex.uuid"
diff --git a/UUIDType.hs b/UUIDType.hs
new file mode 100644
index 0000000000..8e207b444e
--- /dev/null
+++ b/UUIDType.hs
@@ -0,0 +1,11 @@
+{- git-annex UUID type
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module UUIDType where
+
+-- might be nice to have a newtype, but lots of stuff treats uuids as strings
+type UUID = String

From 703c437bd9c6cb9e4675b65ac2b107f76b135d71 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 21:56:04 -0400
Subject: [PATCH 1826/2835] rename modules for data types into Types/ directory

---
 Annex.hs                            |  8 ++++----
 Backend.hs                          |  4 ++--
 Backend/File.hs                     |  4 ++--
 Backend/SHA.hs                      |  4 ++--
 Backend/URL.hs                      |  4 ++--
 Backend/WORM.hs                     |  4 ++--
 Command.hs                          |  2 +-
 Command/DropUnused.hs               |  2 +-
 Command/FromKey.hs                  |  2 +-
 Command/InitRemote.hs               | 16 ++++++++--------
 Command/Status.hs                   | 10 +++++-----
 Content.hs                          |  2 +-
 Crypto.hs                           |  6 +++---
 Locations.hs                        |  2 +-
 Remote.hs                           |  2 +-
 Remote/Bup.hs                       |  2 +-
 Remote/Directory.hs                 |  2 +-
 Remote/Encryptable.hs               |  2 +-
 Remote/Git.hs                       |  2 +-
 Remote/Hook.hs                      |  2 +-
 Remote/Rsync.hs                     |  2 +-
 Remote/S3real.hs                    |  4 ++--
 Remote/Special.hs                   |  2 +-
 Types.hs                            |  4 ++--
 BackendClass.hs => Types/Backend.hs |  4 ++--
 CryptoTypes.hs => Types/Crypto.hs   |  2 +-
 Key.hs => Types/Key.hs              |  4 +++-
 RemoteClass.hs => Types/Remote.hs   |  6 +++---
 UUIDType.hs => Types/UUID.hs        |  2 +-
 UUID.hs                             |  2 +-
 Upgrade/V1.hs                       |  2 +-
 test.hs                             |  4 ++--
 32 files changed, 61 insertions(+), 59 deletions(-)
 rename BackendClass.hs => Types/Backend.hs (96%)
 rename CryptoTypes.hs => Types/Crypto.hs (94%)
 rename Key.hs => Types/Key.hs (95%)
 rename RemoteClass.hs => Types/Remote.hs (96%)
 rename UUIDType.hs => Types/UUID.hs (90%)

diff --git a/Annex.hs b/Annex.hs
index 92a4911ea1..06d642b742 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -21,11 +21,11 @@ import Control.Monad.State
 
 import qualified GitRepo as Git
 import GitQueue
-import BackendClass
-import RemoteClass
-import CryptoTypes
+import Types.Backend
+import Types.Remote
+import Types.Crypto
 import TrustLevel
-import UUIDType
+import Types.UUID
 
 -- git-annex's monad
 type Annex = StateT AnnexState IO
diff --git a/Backend.hs b/Backend.hs
index 645bfdfc3f..78a53d02c7 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -42,8 +42,8 @@ import Locations
 import qualified GitRepo as Git
 import qualified Annex
 import Types
-import Key
-import qualified BackendClass as B
+import Types.Key
+import qualified Types.Backend as B
 import Messages
 import Content
 import DataUnits
diff --git a/Backend/File.hs b/Backend/File.hs
index 543f02af76..bf21224a92 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -18,7 +18,7 @@ import Control.Monad.State (liftIO)
 import Data.List
 import Data.String.Utils
 
-import BackendClass
+import Types.Backend
 import LocationLog
 import qualified Remote
 import qualified GitRepo as Git
@@ -28,7 +28,7 @@ import Types
 import UUID
 import Messages
 import Trust
-import Key
+import Types.Key
 
 backend :: Backend Annex
 backend = Backend {
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 6d721038c3..94ebe093ed 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -17,15 +17,15 @@ import System.Posix.Files
 import System.FilePath
 
 import qualified Backend.File
-import BackendClass
 import Messages
 import qualified Annex
 import Locations
 import Content
 import Types
+import Types.Backend
+import Types.Key
 import Utility
 import qualified SysConfig
-import Key
 
 type SHASize = Int
 
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 3068c30270..e41004dd46 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,10 +10,10 @@ module Backend.URL (backends) where
 import Control.Monad.State (liftIO)
 
 import Types
-import BackendClass
+import Types.Backend
 import Utility
 import Messages
-import Key
+import Types.Key
 
 backends :: [Backend Annex]
 backends = [backend]
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index b33c607632..dc2e48adce 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -12,9 +12,9 @@ import System.FilePath
 import System.Posix.Files
 
 import qualified Backend.File
-import BackendClass
+import Types.Backend
 import Types
-import Key
+import Types.Key
 
 backends :: [Backend Annex]
 backends = [backend]
diff --git a/Command.hs b/Command.hs
index 0da847d24d..228c1f40e9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -24,7 +24,7 @@ import qualified Annex
 import qualified GitRepo as Git
 import Locations
 import Utility
-import Key
+import Types.Key
 
 {- A command runs in four stages.
  -
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 1bb3b7f970..0f99814471 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -22,7 +22,7 @@ import qualified Command.Move
 import qualified Remote
 import qualified GitRepo as Git
 import Backend
-import Key
+import Types.Key
 import Utility
 
 type UnusedMap = M.Map String Key
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index ca61094eb4..34816d6574 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -18,7 +18,7 @@ import Utility
 import qualified Backend
 import Content
 import Messages
-import Key
+import Types.Key
 
 command :: [Command]
 command = [repoCommand "fromkey" paramPath seek
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 460f14de2c..41d3c37c71 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -16,7 +16,7 @@ import Data.String.Utils
 import Command
 import qualified Annex
 import qualified Remote
-import qualified RemoteClass
+import qualified Types.Remote as R
 import qualified GitRepo as Git
 import Utility
 import Types
@@ -54,12 +54,12 @@ start ws = notBareRepo $ do
 				else err $ "Either a new name, or one of these existing special remotes: " ++ join " " names
 			
 
-perform :: RemoteClass.RemoteType Annex -> UUID -> RemoteClass.RemoteConfig -> CommandPerform
+perform :: R.RemoteType Annex -> UUID -> R.RemoteConfig -> CommandPerform
 perform t u c = do
-	c' <- RemoteClass.setup t u c
+	c' <- R.setup t u c
 	next $ cleanup u c'
 
-cleanup :: UUID -> RemoteClass.RemoteConfig -> CommandCleanup
+cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
 cleanup u c = do
 	Remote.configSet u c
 	g <- Annex.gitRepo
@@ -73,7 +73,7 @@ cleanup u c = do
         return True
 
 {- Look up existing remote's UUID and config by name, or generate a new one -}
-findByName :: String -> Annex (UUID, RemoteClass.RemoteConfig)
+findByName :: String -> Annex (UUID, R.RemoteConfig)
 findByName name = do
 	m <- Remote.readRemoteLog
 	maybe generate return $ findByName' name m
@@ -82,7 +82,7 @@ findByName name = do
 			uuid <- liftIO $ genUUID
 			return (uuid, M.insert nameKey name M.empty)
 
-findByName' :: String ->  M.Map UUID RemoteClass.RemoteConfig -> Maybe (UUID, RemoteClass.RemoteConfig)
+findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
 findByName' n m = if null matches then Nothing else Just $ head matches
 	where
 		matches = filter (matching . snd) $ M.toList m
@@ -98,14 +98,14 @@ remoteNames = do
 	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m
 
 {- find the specified remote type -}
-findType :: RemoteClass.RemoteConfig -> Annex (RemoteClass.RemoteType Annex)
+findType :: R.RemoteConfig -> Annex (R.RemoteType Annex)
 findType config = maybe unspecified specified $ M.lookup typeKey config
 	where
 		unspecified = error "Specify the type of remote with type="
 		specified s = case filter (findtype s) Remote.remoteTypes of
 			[] -> error $ "Unknown remote type " ++ s
 			(t:_) -> return t
-		findtype s i = RemoteClass.typename i == s
+		findtype s i = R.typename i == s
 
 {- The name of a configured remote is stored in its config using this key. -}
 nameKey :: String
diff --git a/Command/Status.hs b/Command/Status.hs
index dd518416cf..1a7f694ba8 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -14,8 +14,8 @@ import Data.List
 import qualified Data.Map as M
 
 import qualified Annex
-import qualified BackendClass
-import qualified RemoteClass
+import qualified Types.Backend as B
+import qualified Types.Remote as R
 import qualified Remote
 import qualified Command.Unused
 import qualified GitRepo as Git
@@ -23,7 +23,7 @@ import Command
 import Types
 import DataUnits
 import Content
-import Key
+import Types.Key
 import Locations
 
 -- a named computation that produces a statistic
@@ -97,11 +97,11 @@ showStat s = calc =<< s
 supported_backends :: Stat
 supported_backends = stat "supported backends" $
 	lift (Annex.getState Annex.supportedBackends) >>=
-		return . unwords . (map BackendClass.name)
+		return . unwords . (map B.name)
 
 supported_remote_types :: Stat
 supported_remote_types = stat "supported remote types" $
-	return $ unwords $ map RemoteClass.typename Remote.remoteTypes
+	return $ unwords $ map R.typename Remote.remoteTypes
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
diff --git a/Content.hs b/Content.hs
index ec7a3776bf..57977ce344 100644
--- a/Content.hs
+++ b/Content.hs
@@ -41,7 +41,7 @@ import qualified Annex
 import qualified AnnexQueue
 import Utility
 import StatFS
-import Key
+import Types.Key
 import DataUnits
 import Config
 
diff --git a/Crypto.hs b/Crypto.hs
index 42f1389507..e84e397f2e 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -43,11 +43,11 @@ import System.Exit
 import System.Environment
 
 import Types
-import Key
-import RemoteClass
+import Types.Key
+import Types.Remote
 import Utility
 import Base64
-import CryptoTypes
+import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
  - is used as the GPG symmetric encryption passphrase.
diff --git a/Locations.hs b/Locations.hs
index 38a320a2b2..da781ac83a 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -36,7 +36,7 @@ import Word
 import Data.Hash.MD5
 
 import Types
-import Key
+import Types.Key
 import qualified GitRepo as Git
 
 {- Conventions:
diff --git a/Remote.hs b/Remote.hs
index 9685b4612f..e7ef5f1952 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -39,8 +39,8 @@ import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
 
-import RemoteClass
 import Types
+import Types.Remote
 import UUID
 import qualified Annex
 import Locations
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index c40826e5eb..c011c979ca 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -19,8 +19,8 @@ import System.FilePath
 import Data.List.Utils
 import System.Cmd.Utils
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index dedab473f3..7b5917dca8 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -16,8 +16,8 @@ import Control.Monad.State (liftIO)
 import System.Directory hiding (copyFile)
 import System.FilePath
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs
index 68ecfd01e6..443f5cf83d 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Encryptable.hs
@@ -11,7 +11,7 @@ import qualified Data.Map as M
 import Control.Monad.State (liftIO)
 
 import Types
-import RemoteClass
+import Types.Remote
 import Crypto
 import qualified Annex
 import Messages
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e6df6be46e..67d49df7d2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -12,8 +12,8 @@ import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import qualified AnnexQueue
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index dc4d392741..cc511965f8 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -18,8 +18,8 @@ import System.IO
 import System.IO.Error (try)
 import System.Exit
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9d32ad19b9..bf1bbd8707 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -16,8 +16,8 @@ import System.Directory
 import System.Posix.Files
 import System.Posix.Process
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index baf570593e..2479dfa023 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -24,8 +24,9 @@ import System.Environment
 import System.Posix.Files
 import System.Posix.Env (setEnv)
 
-import RemoteClass
 import Types
+import Types.Remote
+import Types.Key
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
@@ -35,7 +36,6 @@ import Config
 import Remote.Special
 import Remote.Encryptable
 import Crypto
-import Key
 import Content
 import Base64
 
diff --git a/Remote/Special.hs b/Remote/Special.hs
index 53ac2c6eed..7d2ea1d704 100644
--- a/Remote/Special.hs
+++ b/Remote/Special.hs
@@ -13,7 +13,7 @@ import Data.String.Utils
 import Control.Monad.State (liftIO)
 
 import Types
-import RemoteClass
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Types.hs b/Types.hs
index 503e27d312..6353f6da68 100644
--- a/Types.hs
+++ b/Types.hs
@@ -11,6 +11,6 @@ module Types (
 	Key
 ) where
 
-import BackendClass
 import Annex
-import Key
+import Types.Backend
+import Types.Key
diff --git a/BackendClass.hs b/Types/Backend.hs
similarity index 96%
rename from BackendClass.hs
rename to Types/Backend.hs
index b2d8879c2f..8100eaf285 100644
--- a/BackendClass.hs
+++ b/Types/Backend.hs
@@ -7,9 +7,9 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module BackendClass where
+module Types.Backend where
 
-import Key
+import Types.Key
 
 data Backend a = Backend {
 	-- name of this backend
diff --git a/CryptoTypes.hs b/Types/Crypto.hs
similarity index 94%
rename from CryptoTypes.hs
rename to Types/Crypto.hs
index ba22c4cbe8..a39a016b8b 100644
--- a/CryptoTypes.hs
+++ b/Types/Crypto.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module CryptoTypes where
+module Types.Crypto where
 
 import Data.String.Utils
 
diff --git a/Key.hs b/Types/Key.hs
similarity index 95%
rename from Key.hs
rename to Types/Key.hs
index e1d8ee34d0..1d9bf8e11c 100644
--- a/Key.hs
+++ b/Types/Key.hs
@@ -1,11 +1,13 @@
 {- git-annex Key data type
+ - 
+ - Most things should not need this, using Types instead
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Key (
+module Types.Key (
 	Key(..),
 	stubKey,
 	readKey,
diff --git a/RemoteClass.hs b/Types/Remote.hs
similarity index 96%
rename from RemoteClass.hs
rename to Types/Remote.hs
index f954e4ff8f..01ced04ae1 100644
--- a/RemoteClass.hs
+++ b/Types/Remote.hs
@@ -1,4 +1,4 @@
-{- git-annex remotes class
+{- git-annex remotes types
  -
  - Most things should not need this, using Remote instead
  -
@@ -7,13 +7,13 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module RemoteClass where
+module Types.Remote where
 
 import Control.Exception
 import Data.Map as M
 
 import qualified GitRepo as Git
-import Key
+import Types.Key
 
 type RemoteConfig = M.Map String String
 
diff --git a/UUIDType.hs b/Types/UUID.hs
similarity index 90%
rename from UUIDType.hs
rename to Types/UUID.hs
index 8e207b444e..eb3497fa94 100644
--- a/UUIDType.hs
+++ b/Types/UUID.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module UUIDType where
+module Types.UUID where
 
 -- might be nice to have a newtype, but lots of stuff treats uuids as strings
 type UUID = String
diff --git a/UUID.hs b/UUID.hs
index 33835e261b..f222f7a9d4 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -31,12 +31,12 @@ import Data.Maybe
 
 import qualified GitRepo as Git
 import Types
+import Types.UUID
 import Locations
 import qualified Annex
 import Utility
 import qualified SysConfig
 import Config
-import UUIDType
 
 configkey :: String
 configkey = "annex.uuid"
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 9278bce603..1e634e00e8 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -18,7 +18,7 @@ import System.Posix.Types
 import Data.Maybe
 import Data.Char
 
-import Key
+import Types.Key
 import Content
 import Types
 import Locations
diff --git a/test.hs b/test.hs
index 456c09060c..221607755a 100644
--- a/test.hs
+++ b/test.hs
@@ -31,7 +31,7 @@ import qualified Backend
 import qualified GitRepo as Git
 import qualified Locations
 import qualified Utility
-import qualified BackendClass
+import qualified Type.Backend
 import qualified Types
 import qualified GitAnnex
 import qualified LocationLog
@@ -40,7 +40,7 @@ import qualified Trust
 import qualified Remote
 import qualified Content
 import qualified Command.DropUnused
-import qualified Key
+import qualified Type.Key
 import qualified Config
 import qualified Crypto
 

From eb4278834bccb040f32d14c1f899b55250c634d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 00:51:56 -0400
Subject: [PATCH 1827/2835] add fedora install instructions

---
 doc/install.mdwn        |  1 +
 doc/install/Fedora.mdwn | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 doc/install/Fedora.mdwn

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 3d15eac604..c5324bddae 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -3,6 +3,7 @@
 * [[OSX]]
 * [[Debian]]
 * [[Ubuntu]]
+* [[Fedora]]
 
 ## Generic instructions
 
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
new file mode 100644
index 0000000000..0050295e86
--- /dev/null
+++ b/doc/install/Fedora.mdwn
@@ -0,0 +1,21 @@
+Installation recipe for Fedora 14.
+
+
+sudo yum install ghc cabal-install
+sudo cabal update
+sudo cabal install missingh
+sudo cabal install utf8-string
+sudo cabal install pcre-light
+sudo cabal install quickcheck
+sudo cabal install SHA
+sudo cabal install dataenc
+sudo cabal install hS3
+
+git clone git://git-annex.branchable.com/
+
+cd git-annex
+sudo make   # For some reason you need to use sudo here as otherwise the cabal installed packages doesn't seem to be there...
+sudo install git-annex
+

+
+Originally posted by Jon at  --[[Joey]]

From f94a0aed833b5a0f0311a25a3401a214f38edd81 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 00:52:25 -0400
Subject: [PATCH 1828/2835] adjust to use primary git repo

---
 doc/install/OSX.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index a6afd408bd..e7bb763937 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -11,7 +11,7 @@ sudo cabal install dataenc
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
-git clone  git://git.kitenet.net/git-annex
+git clone  git://git-annex.branchable.com/
 
 cd git-annex
 make

From 3d7d61e679a375abefeb9bdddd789dc84243bcb3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 00:54:25 -0400
Subject: [PATCH 1829/2835] update

---
 .../comment_2_cf0f829536744098d6846500db998b6a._comment      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
index 134024908e..81d5a2c629 100644
--- a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
+++ b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
@@ -9,4 +9,9 @@ Because I haven't learned Cabal yet.
 But also because I've had bad experiences with both a) tying a particular program to a particular language's pet build system and then having to add ugliness when I later need to do something in the build that has nothing to do with that language and b) as a user, needing to deal with the pet build systems of languages when I just need to make some small change to the build process that is trivial in a Makefile.
 
 With that said, I do have a configure program written in Haskell, so at least it doesn't use autotools. :)
+
+Update: I did try using cabal, but git-annex includes 3 programs, and they
+all link to a lot of git-annex modules, and cabal wanted to build nearly
+every module 3 times, which was too slow for me and I could not find a way
+around.
 """]]

From c3c127a667eba4b02562203329618a1c0e8ca453 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 00:56:22 -0400
Subject: [PATCH 1830/2835] update

---
 doc/install/Ubuntu.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/Ubuntu.mdwn b/doc/install/Ubuntu.mdwn
index ffc763ff31..afcf22b6d4 100644
--- a/doc/install/Ubuntu.mdwn
+++ b/doc/install/Ubuntu.mdwn
@@ -2,4 +2,4 @@ If using Ubuntu natty or newer:
 
 	sudo apt-get install git-annex
 
-Otherwise, see [[Debian]] manual installation instructions.
+Otherwise, see [[manual_installation_instructions|install]].

From ac6510b337026b50e0610f1e334e887e5dac2a6e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 01:01:14 -0400
Subject: [PATCH 1831/2835] reformat

---
 doc/install.mdwn | 37 +++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index c5324bddae..8c71b56a8b 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -9,24 +9,25 @@
 
 To build and use git-annex, you will need:
 
-* `git`: 
-* The Haskell Platform: 
-* MissingH: 
-* pcre-light: 
-* utf8-string: 
-* SHA: 
-* dataenc: 
-* TestPack 
-* QuickCheck 2 
-* hS3  (optional, but recommended)
-* `uuid`: 
+* Haskell stuff
+  * [The Haskell Platform](http://haskell.org/platform/)
+  * [MissingH](http://github.com/jgoerzen/missingh/wiki
+  * [pcre-light](http://hackage.haskell.org/package/pcre-light)
+  * [utf8-string](http://hackage.haskell.org/package/utf8-string)
+  * [SHA]()
+  * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
+  * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
+  * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
+* [git](http://git-scm.com/)
+* [uuid](http://www.ossp.org/pkg/lib/uuid/)
   (or `uuidgen` from util-linux)
-* `xargs`: 
-* `rsync`: 
-* `curl` :  (optional, but recommended)
-* `sha1sum`:  (optional, but recommended)
-* `gpg`:  (optional; needed for encryption)
-* [Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
-  but that will be skipped if it is not installed.
+* [xargs](http://savannah.gnu.org/projects/findutils/)
+* [rsync](http://rsync.samba.org/)
+* [curl](http://http://curl.haxx.se/) (optional, but recommended)
+* [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
+  a sha1 command will also do)
+* [gpg](http://gnupg.org/) (optional; needed for encryption)
+* [Ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`

From 28cb279765b42f6a4df2026858911df40c627e77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 01:01:59 -0400
Subject: [PATCH 1832/2835] fix

---
 doc/install.mdwn | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 8c71b56a8b..bb0942b257 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -11,11 +11,11 @@ To build and use git-annex, you will need:
 
 * Haskell stuff
   * [The Haskell Platform](http://haskell.org/platform/)
-  * [MissingH](http://github.com/jgoerzen/missingh/wiki
+  * [MissingH](http://github.com/jgoerzen/missingh/wiki)
   * [pcre-light](http://hackage.haskell.org/package/pcre-light)
   * [utf8-string](http://hackage.haskell.org/package/utf8-string)
-  * [SHA]()
+  * [SHA](http://hackage.haskell.org/package/SHA)
+  * [dataenc](http://hackage.haskell.org/package/dataenc)
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)

From 76be8c34acfacfc884b3fdd086bcf60b28570237 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 01:02:57 -0400
Subject: [PATCH 1833/2835] update

---
 doc/install.mdwn | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index bb0942b257..7818aaf152 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -19,15 +19,16 @@ To build and use git-annex, you will need:
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
-* [git](http://git-scm.com/)
-* [uuid](http://www.ossp.org/pkg/lib/uuid/)
-  (or `uuidgen` from util-linux)
-* [xargs](http://savannah.gnu.org/projects/findutils/)
-* [rsync](http://rsync.samba.org/)
-* [curl](http://http://curl.haxx.se/) (optional, but recommended)
-* [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
-  a sha1 command will also do)
-* [gpg](http://gnupg.org/) (optional; needed for encryption)
-* [Ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
+* Shell commands
+  * [git](http://git-scm.com/)
+  * [uuid](http://www.ossp.org/pkg/lib/uuid/)
+    (or `uuidgen` from util-linux)
+  * [xargs](http://savannah.gnu.org/projects/findutils/)
+  * [rsync](http://rsync.samba.org/)
+  * [curl](http://http://curl.haxx.se/) (optional, but recommended)
+  * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
+    a sha1 command will also do)
+  * [gpg](http://gnupg.org/) (optional; needed for encryption)
+  * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`

From f2cc87860ccb4ccb4a51dd9d255717a0e749fe76 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 02:33:31 -0400
Subject: [PATCH 1834/2835] refactor

---
 GitAnnex.hs | 9 +++------
 Remote.hs   | 6 ++++++
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index 2a9fcbe3e7..b22313d3c5 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -95,11 +95,11 @@ options = commonOptions ++
 		"skip files matching the glob pattern"
 	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
-	, Option [] ["trust"] (ReqArg (settrust Trusted) paramRemote)
+	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)
 		"override trust setting"
-	, Option [] ["semitrust"] (ReqArg (settrust SemiTrusted) paramRemote)
+	, Option [] ["semitrust"] (ReqArg (Remote.forceTrust SemiTrusted) paramRemote)
 		"override trust setting back to default value"
-	, Option [] ["untrust"] (ReqArg (settrust UnTrusted) paramRemote)
+	, Option [] ["untrust"] (ReqArg (Remote.forceTrust UnTrusted) paramRemote)
 		"override trust setting to untrusted"
 	]
 	where
@@ -108,9 +108,6 @@ options = commonOptions ++
 		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
-		settrust t v = do
-			r <- Remote.nameToUUID v
-			Annex.changeState $ \s -> s { Annex.forcetrust = (r, t):Annex.forcetrust s }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/Remote.hs b/Remote.hs
index e7ef5f1952..2e956cb81c 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,6 +15,7 @@ module Remote (
 	hasKey,
 	hasKeyCheap,
 	keyPossibilities,
+	forceTrust,
 
 	remoteTypes,
 	genList,
@@ -137,6 +138,11 @@ keyPossibilities key = do
 
 	return (sort validremotes, validtrusteduuids)
 
+forceTrust :: TrustLevel -> String -> Annex ()
+forceTrust level remotename = do
+	r <- Remote.nameToUUID remotename
+	Annex.changeState $ \s ->
+		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
 
 {- Filename of remote.log. -}
 remoteLog :: Annex FilePath

From bcb72ce0f2623a77d18f8064738ba26661dad762 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 02:40:43 -0400
Subject: [PATCH 1835/2835] tweak

---
 CmdLine.hs         | 4 ++--
 GitAnnex.hs        | 4 +---
 git-annex-shell.hs | 7 +++----
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 684ebf979a..861a31be97 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -28,8 +28,8 @@ import Messages
 import UUID
 
 {- Runs the passed command line. -}
-dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO ()
-dispatch gitrepo args cmds options header = do
+dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
+dispatch args cmds options header gitrepo = do
 	setupConsole
 	state <- Annex.new gitrepo allBackends
 	(actions, state') <- Annex.run state $ parseCmd args header cmds options
diff --git a/GitAnnex.hs b/GitAnnex.hs
index b22313d3c5..103ee262f2 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -113,6 +113,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = do
-	gitrepo <- Git.repoFromCwd
-	dispatch gitrepo args cmds options header
+run args = dispatch args cmds options header =<< Git.repoFromCwd
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 940db71c34..55f34e1027 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -58,10 +58,9 @@ builtins :: [String]
 builtins = map cmdname cmds
 
 builtin :: String -> String -> [String] -> IO ()
-builtin cmd dir params = do
-	dir' <- Git.repoAbsPath dir
-	gitrepo <- Git.repoFromAbsPath dir'
-	dispatch gitrepo (cmd:(filterparams params)) cmds commonOptions header
+builtin cmd dir params =
+	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
+		dispatch (cmd:(filterparams params)) cmds commonOptions header
 
 external :: [String] -> IO ()
 external params = do

From 269e3627a14102ab65fc26283936c56b1ac549f5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Thu, 2 Jun 2011 11:34:44 +0000
Subject: [PATCH 1836/2835] Added a comment: git annex unlock --readonly

---
 ...nt_3_a1a9347b5bc517f2a89a8b292c3f8517._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment

diff --git a/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment b/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment
new file mode 100644
index 0000000000..4bb3aa684f
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unlock --readonly"
+ date="2011-06-02T11:34:42Z"
+ content="""
+This was already asked [here](http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=606577), but I have a use case where I need to unlock with the files being hardlinked instead of copied (my fs does not support CoW), even though 'git annex lock' is now much faster ;-) . The idea is that 1) I want the external world see my repo \"as if\" it wasn't annexed (because of its own limitation to deal with soft links), and 2) I know what I do, and am sure that files won't be written to but only read.
+
+My case is: the repo contains a snapshot A1 of a certain remote directory. Later I want to rsync this dir into a new snapshot A2. Of course, I want to transfer only new or changed files, with the --copy-dest=A1 (or --compare-dest) rsync's options. Unfortunately, rsync won't recognize soft-links from git-annex, and will re-transfer everything.
+
+
+Maybe I'm overusing git-annex ;-) but still, I find it is a legitimate use case, and even though there are workarounds (I don't even remember what I had to do), it would be much more straightforward to have 'git annex unlock --readonly' (or '--readonly-unsafe'?), ... or have rsync take soft-links into account, but I did not see the author ask for microfeatures ideas :) (it was discussed, and only some convoluted workarounds were proposed). Thanks.
+
+
+"""]]

From 24f2acd27258416420e5e4a17efffc94c95534e2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Thu, 2 Jun 2011 11:55:59 +0000
Subject: [PATCH 1837/2835] Added a comment: git annex unused

---
 ...4_5a6786dc52382fff5cc42fdb05770196._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment

diff --git a/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment b/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment
new file mode 100644
index 0000000000..cc98109e6b
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unused"
+ date="2011-06-02T11:55:58Z"
+ content="""
+Before dropping unsused items, sometimes I want to check the content of the files manually.
+But currently, from e.g. a sha1 key, I don't know how to find the corresponding file, except with
+'find .git/annex/objects -type f -name 'SHA1-s1678--70....', wich is too slow (I'm in the case where \"git log --stat -S'KEY'\"
+won't work, either because it is too slow or it was never commited). By the way,
+is it documented somewhere how to determine the 2 (nested) sub-directories in which a given
+(by name) object is located?
+
+So I would like 'git-annex unused' be able to give me the list of *paths* to the unused items.
+Also, I would really appreciate a command like 'git annex unused --log NUMBER [NUMBER2...]' which would do for me the suggested command
+\"git log --stat -S'KEY'\", where NUMBER is from the 'git annex unused' output.
+Thanks.
+"""]]

From 75a966534992eda522e9a0e9c0f1213c0d4a544e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Thu, 2 Jun 2011 19:51:50 +0000
Subject: [PATCH 1838/2835] Added a comment: git annex unused

---
 .../comment_5_3c627d275586ff499d928a8f8136babf._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment

diff --git a/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment b/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment
new file mode 100644
index 0000000000..f7361f5d1c
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unused"
+ date="2011-06-02T19:51:49Z"
+ content="""
+ps: concerning the command 'find .git/annex/objects -type f -name 'SHA1-s1678--70....' from my previous comment, it is \"significantly\" faster to search for the containing directory which have the same name: 'find .git/annex/objects -maxdepth 2 -mindepth 2 -type d -name 'SHA1-s1678--70....'. I am just curious: what is the need to have each file object in its own directory, itself nested under two more sub-directories?
+"""]]

From 0ee760fcedae58ede4684337d78ae451da23d8a8 Mon Sep 17 00:00:00 2001
From: ssqq 
Date: Sat, 4 Jun 2011 06:18:13 +0000
Subject: [PATCH 1839/2835]

---
 doc/install/OSX.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index e7bb763937..ba10071a16 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,6 +7,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck  
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install hS3   # stub S3 class (used if you don't have hS3 installed) has a bug, so you want to install this
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin

From cd60c364573918f37d70856442cf69a1e38654d6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnEpj9IoEzNagnLDTTc9ormo5orwHW5aaI"
 
Date: Wed, 8 Jun 2011 00:22:56 +0000
Subject: [PATCH 1840/2835]

---
 doc/forum/incompatible_versions__63__.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/incompatible_versions__63__.mdwn

diff --git a/doc/forum/incompatible_versions__63__.mdwn b/doc/forum/incompatible_versions__63__.mdwn
new file mode 100644
index 0000000000..13eb181491
--- /dev/null
+++ b/doc/forum/incompatible_versions__63__.mdwn
@@ -0,0 +1 @@
+Are versions 0.14 and 0.20110522 incompatible? I can't seem to copy files from a system running 0.14 to one running 20110522.

From c9d8b830c6db43a9c3140e7680a972e39663bfd6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 8 Jun 2011 00:40:54 +0000
Subject: [PATCH 1841/2835] Added a comment

---
 .../comment_1_629f28258746d413e452cbd42a1a43f4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment

diff --git a/doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment b/doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment
new file mode 100644
index 0000000000..3702fde6ea
--- /dev/null
+++ b/doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-06-08T00:40:54Z"
+ content="""
+They are not. See [[upgrades]]
+"""]]

From 90dd245522ccffee0e77eba3b79e32d6029977fc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 9 Jun 2011 18:54:49 -0400
Subject: [PATCH 1842/2835] get --from is the same as copy --from

get not honoring --from has surprised me a few times, so least surprise
suggests it should just behave like copy --from. This leaves the difference
between get and copy being that copy always requires the remote to copy
from, while get will decide whether to get a file from a key/value store or
a remote.
---
 Backend/File.hs                                       |  2 +-
 Command/Get.hs                                        | 11 +++++++++--
 debian/changelog                                      |  1 +
 doc/bugs/fsck_output.mdwn                             |  2 +-
 doc/git-annex.mdwn                                    |  2 +-
 doc/walkthrough/getting_file_content.mdwn             |  4 ++--
 .../transferring_files:_When_things_go_wrong.mdwn     |  2 +-
 doc/walkthrough/using_ssh_remotes.mdwn                |  2 +-
 8 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index bf21224a92..386af02663 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -81,7 +81,7 @@ copyKeyFile key file = do
 						Left _ -> return False
 				else return True
 		docopy r continue = do
-			showNote $ "copying from " ++ Remote.name r ++ "..."
+			showNote $ "from " ++ Remote.name r ++ "..."
 			copied <- Remote.retrieveKeyFile r key file
 			if copied
 				then return True
diff --git a/Command/Get.hs b/Command/Get.hs
index 90c0540960..50dc009feb 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -9,9 +9,12 @@ module Command.Get where
 
 import Command
 import qualified Backend
+import qualified Annex
+import qualified Remote
 import Types
 import Content
 import Messages
+import qualified Command.Move
 
 command :: [Command]
 command = [repoCommand "get" paramPath seek
@@ -20,7 +23,6 @@ command = [repoCommand "get" paramPath seek
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
-{- Gets an annexed file from one of the backends. -}
 start :: CommandStartString
 start file = isAnnexed file $ \(key, backend) -> do
 	inannex <- inAnnex key
@@ -28,7 +30,12 @@ start file = isAnnexed file $ \(key, backend) -> do
 		then stop
 		else do
 			showStart "get" file
-			next $ perform key backend
+			from <- Annex.getState Annex.fromremote
+			case from of
+				Nothing -> next $ perform key backend
+				Just name -> do
+					src <- Remote.byName name
+					next $ Command.Move.fromPerform src False key
 
 perform :: Key -> Backend Annex -> CommandPerform
 perform key backend = do
diff --git a/debian/changelog b/debian/changelog
index 8cb256099e..4fe1febd31 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (0.20110602) UNRELEASED; urgency=low
 
   * Add --numcopies option.
   * Add --trust, --untrust, and --semitrust options.
+  * get --from is the same as copy --from
 
  -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
 
diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 3ded1b409e..90af1600d8 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -30,7 +30,7 @@ The newline is in the wrong place and confuses the user. It should be printed _a
 > A related problem occurs if an error message is unexpetedly printed.
 > Dummying up an example:
 > 
-> O get test1 (copying from foo...) E git-annex: failed to run ssh
+> O get test1 (from foo...) E git-annex: failed to run ssh
 > failed
 > 
 > --[[Joey]] 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index b15ce1a296..25f053af69 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -41,7 +41,7 @@ content from the key-value store.
 	# sudo mount /media/usb
 	# git remote add usbdrive /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
-	get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok
+	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
 	# git commit -a -m "got a video I want to rewatch on the plane"
 	
 	# git annex add iso
diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index 5c899ee3c5..bf45fd97fd 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -8,8 +8,8 @@ USB drive.
 	# cd /media/usb/annex
 	# git pull laptop master
 	# git annex get .
-	get my_cool_big_file (copying from laptop...) ok
-	get iso/debian.iso (copying from laptop...) ok
+	get my_cool_big_file (from laptop...) ok
+	get iso/debian.iso (from laptop...) ok
 
 Notice that you had to git pull from laptop first, this lets git-annex know
 what has changed in laptop, and so it knows about the files present there and
diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
index d8f0a19bd6..936d088f1f 100644
--- a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
+++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
@@ -14,5 +14,5 @@ it:
 	failed
 	# sudo mount /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
-	get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok
+	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
 	# git commit -a -m "got a video I want to rewatch on the plane"
diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn
index 4c2f830de8..fbbbbe0701 100644
--- a/doc/walkthrough/using_ssh_remotes.mdwn
+++ b/doc/walkthrough/using_ssh_remotes.mdwn
@@ -12,7 +12,7 @@ to clone the laptop's annex to it:
 Now you can get files and they will be transferred (using `rsync` via `ssh`):
 
 	# git annex get my_cool_big_file
-	get my_cool_big_file (getting UUID for origin...) (copying from origin...)
+	get my_cool_big_file (getting UUID for origin...) (from origin...)
 	WORM-s2159-m1285650548--my_cool_big_file       100% 2159     2.1KB/s   00:00
 	ok
 

From 9a272815ddd33272008a4052ea74e7bbc4c842f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 11:43:28 -0400
Subject: [PATCH 1843/2835] Bugfix: Fix fsck to not think all SHAnE keys are
 bad.

---
 Backend/SHA.hs   | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 94ebe093ed..8ed00b7073 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -119,7 +119,7 @@ checkKeyChecksum size key = do
 		then return True
 		else do
 			s <- shaN size file
-			if s == keyName key
+			if s == dropExtension (keyName key)
 				then return True
 				else do
 					dest <- moveBad key
diff --git a/debian/changelog b/debian/changelog
index 4fe1febd31..25c00963c2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (0.20110602) UNRELEASED; urgency=low
   * Add --numcopies option.
   * Add --trust, --untrust, and --semitrust options.
   * get --from is the same as copy --from
+  * Bugfix: Fix fsck to not think all SHAnE keys are bad.
 
  -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
 

From 38e0100a6929b4538bbcd5ec82827b4bc996915d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 11:58:21 -0400
Subject: [PATCH 1844/2835] releasing version 0.20110610

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 25c00963c2..09655df65c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,11 @@
-git-annex (0.20110602) UNRELEASED; urgency=low
+git-annex (0.20110610) unstable; urgency=low
 
   * Add --numcopies option.
   * Add --trust, --untrust, and --semitrust options.
   * get --from is the same as copy --from
   * Bugfix: Fix fsck to not think all SHAnE keys are bad.
 
- -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
+ -- Joey Hess   Fri, 10 Jun 2011 11:48:40 -0400
 
 git-annex (0.20110601) unstable; urgency=low
 

From a4735605159e2f447d71f50aa10ca5094de8b8f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 11:58:36 -0400
Subject: [PATCH 1845/2835] add news item for git-annex 0.20110610

---
 doc/news/version_0.20110503.mdwn | 9 ---------
 doc/news/version_0.20110610.mdwn | 6 ++++++
 2 files changed, 6 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_0.20110503.mdwn
 create mode 100644 doc/news/version_0.20110610.mdwn

diff --git a/doc/news/version_0.20110503.mdwn b/doc/news/version_0.20110503.mdwn
deleted file mode 100644
index f24f0c48f6..0000000000
--- a/doc/news/version_0.20110503.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 0.20110503 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix hasKeyCheap setting for bup and rsync special remotes.
-   * Add hook special remotes.
-   * Avoid crashing when an existing key is readded to the annex.
-   * unused: Now also lists files fsck places in .git/annex/bad/
-   * S3: When encryption is enabled, the Amazon S3 login credentials
-     are stored, encrypted, in .git-annex/remotes.log, so environment
-     variables need not be set after the remote is initialized."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110610.mdwn b/doc/news/version_0.20110610.mdwn
new file mode 100644
index 0000000000..9ab9e09076
--- /dev/null
+++ b/doc/news/version_0.20110610.mdwn
@@ -0,0 +1,6 @@
+git-annex 0.20110610 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Add --numcopies option.
+   * Add --trust, --untrust, and --semitrust options.
+   * get --from is the same as copy --from
+   * Bugfix: Fix fsck to not think all SHAnE keys are bad."""]]
\ No newline at end of file

From 19428ea2f4cd1ef9f1c91b5670933bdf989a2af3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 12:06:40 -0400
Subject: [PATCH 1846/2835] fix building with S3 stub

---
 Remote/S3stub.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/S3stub.hs b/Remote/S3stub.hs
index 0d6ec47de3..d91a222e86 100644
--- a/Remote/S3stub.hs
+++ b/Remote/S3stub.hs
@@ -1,7 +1,7 @@
 -- stub for when hS3 is not available
 module Remote.S3 (remote) where
 
-import RemoteClass
+import Types.Remote
 import Types
 
 remote :: RemoteType Annex

From 0b85ffd6e7e875ee4ac671ca8d145586ef77ab38 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 12:11:55 -0400
Subject: [PATCH 1847/2835] S3 stub bug got fixed

---
 doc/install/OSX.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index ba10071a16..81ffe1d035 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,7 +7,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck  
 sudo cabal install SHA
 sudo cabal install dataenc
-sudo cabal install hS3   # stub S3 class (used if you don't have hS3 installed) has a bug, so you want to install this
+sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin

From ef6209db43d2dcea8a74805d5425988b0f7bf4ab Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 10 Jun 2011 16:39:45 +0000
Subject: [PATCH 1848/2835] Comment moderation

---
 ..._90a8a15bedd94480945a374f9d706b86._comment | 10 ++++++++++
 ..._7e328b970169fffb8bce373d1522743b._comment | 19 +++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment

diff --git a/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment b/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment
new file mode 100644
index 0000000000..722cbdd9e7
--- /dev/null
+++ b/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://ethan.betacantrips.com/"
+ nickname="ethan.glasser.camp"
+ subject="no symlinks"
+ date="2011-06-08T20:59:38Z"
+ content="""
+If you try to clone a git repo that has a symlink over to a VFAT filesystem, you get (in its place) a regular file that contains the name of the symlink target.  So why can't git-annex use that?  I could still do git annex get on this file, git annex would still \"know\" that it's a symlink, and could replace it with a copy of the real file (instead of putting it in .git/annex).
+
+I know if it were that simple, someone would have done it already, so what am I missing?  I guess trying to get the file FROM the repository would fail because it wouldn't find the file in .git/annex?  Couldn't you store a reverse mapping?  You wouldn't be able to move the file around, but you already lose that once you give up symlinks.  It would also be a little harder to tell which symlinks were \"dangling\"; I don't see an easy way to get around that.  It would still be better than a bare repo..
+"""]]
diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment
new file mode 100644
index 0000000000..8f0f5ef180
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="ssqq"
+ ip="208.70.196.4"
+ subject="Still a problem on 0.20110523"
+ date="2011-06-02T20:31:55Z"
+ content="""
+Hi,
+
+(I'm new to git and git annex, so please forgive any mistakes I make...)
+
+My repo is messed up right now.  The fact that I copied the repo with rsync -a back and forth from a case insensitive filesystem to a case sensitive one, probably didn't help.
+
+I believe the annexed files in .git/annex/objects/ are still using a mixed case directory hashing scheme.  That's the problem I'm having.  The symlinks point to the wrong case and are now broken.  I don't think the latest versions of git-annex changed that (it only changed the hashing under .git-annex, right?).
+
+Even if I clean up my repo, I think I'm still going to have a problem because I have one repo on an OS X case insensitive filesystem and my other repos on case sensitive Linux filesystems.  Potentially the directory name under .git/annex/objects will have a different case.  Then the symlink might have a different case than my Linux FS.  Does git-annex track changes in git by the contents of the symlink?  In which case the case difference would show up as a change even though there is no change?
+
+Is it possible to change the directory hashing scheme under .git/annex/objects to use lowercase names?
+
+"""]]

From 68ed12eab4e0f73008a2af152b534ff08d92a57b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 10 Jun 2011 16:41:43 +0000
Subject: [PATCH 1849/2835] Added a comment

---
 .../comment_5_64bbf89de0836673224b83fdefa0407b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment

diff --git a/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment b/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment
new file mode 100644
index 0000000000..1063b0f910
--- /dev/null
+++ b/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-06-10T16:41:43Z"
+ content="""
+@ethan the reason that wouldn't work is because git would then see a file that was checked in and had its one line symlinkish content replaced with a huge binary blob. And git commit would try to commit that etc. The potential for foot-shooting is too high.
+"""]]

From 8da84d8860e9530ad223fc7b81811f7cc97db121 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 10 Jun 2011 16:46:03 +0000
Subject: [PATCH 1850/2835] Added a comment

---
 ...omment_21_98f632652b0db9131b0173d3572f4d62._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment
new file mode 100644
index 0000000000..453a8be11b
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 21"
+ date="2011-06-10T16:46:03Z"
+ content="""
+@seqq git-annex always uses the same case when creating and accessing the files pointed to by the symlinks. So it will not matter if it's used on a case-insensative, or case-insensative but preserving system like OSX.
+
+You need to fix up the cases of the files in .git/annex/objects to what it expects. I'm not sure what would be the best way to do that. The method described in [[walkthrough/recover_data_from_lost+found]] might work well.
+"""]]

From 88bdf17e1a26f65cb760d7bf208b07ecc4651639 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmByD9tmR48HuYgS4qWEGDDaoVTTC3m4kc"
 
Date: Fri, 10 Jun 2011 18:08:37 +0000
Subject: [PATCH 1851/2835] Added a comment: Any chance to get git-annex going
 on windows?

---
 ...comment_3_cff163ea3e7cad926f4ed9e78b896598._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment

diff --git a/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment b/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment
new file mode 100644
index 0000000000..6b47ed0e3e
--- /dev/null
+++ b/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmByD9tmR48HuYgS4qWEGDDaoVTTC3m4kc"
+ nickname="Jonas"
+ subject="Any chance to get git-annex going on windows?"
+ date="2011-06-10T18:08:36Z"
+ content="""
+Would be great! :-)
+
+Jonas
+"""]]

From 76df8d5f86fa7c978ed6a695a5c2817d835a3307 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 10 Jun 2011 19:55:38 +0000
Subject: [PATCH 1852/2835] Added a comment: short answer: no

---
 ..._82a17eee4a076c6c79fddeda347e0c9a._comment | 69 +++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment

diff --git a/doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment b/doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment
new file mode 100644
index 0000000000..678847ecae
--- /dev/null
+++ b/doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment
@@ -0,0 +1,69 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="short answer: no"
+ date="2011-06-10T19:55:38Z"
+ content="""
+Long answer, quoting from a mail to someone else:
+
+Well, I can tell you that it assumes a POSIX system, both in available
+utilities and system calls, So you'd need to use cygwin or something
+like that. (Perhaps you already are for git, I think git also assumes a
+POSIX system.) So you need a Haskell that can target that. What this
+page refers to as \"GHC-Cygwin\":
+
+I don't know where to get one. Did find this:
+
+
+(There are probably also still some places where it assumes / as a path
+separator, although I fixed some.)
+
+FWIW, git-annex works fine on OS X and other fine proprietary unixen. ;P
+
+----
+
+Alternatively, windows versions of these functions could be found,
+which are all the ones that need POSIX, I think. A fair amount of this,
+the stuff to do with signals and users, could be empty stubs in windows.
+The file manipulation, particularly symlinks, would probably be the main
+challenge.
+
+
+addSignal
+blockSignals
+changeWorkingDirectory
+createLink
+createSymbolicLink
+emptySignalSet
+executeFile
+fileMode
+fileSize
+forkProcess
+getAnyProcessStatus
+getEffectiveUserID
+getEnvDefault
+getFileStatus
+getProcessID
+getProcessStatus
+getSignalMask
+getSymbolicLinkStatus
+getUserEntryForID
+getUserEntryForName
+groupWriteMode
+homeDirectory
+installHandler
+intersectFileModes
+isRegularFile
+isSymbolicLink
+modificationTime
+otherWriteMode
+ownerWriteMode
+readSymbolicLink
+setEnv
+setFileMode
+setSignalMask
+sigCHLD
+sigINT
+unionFileModes
+

+"""]]

From 57ff2e297a0a9cc610a3fd5e1b8c635fc44270b5 Mon Sep 17 00:00:00 2001
From: "https://lithitux.org/openidserver/users/pavel" 
Date: Mon, 13 Jun 2011 10:13:47 +0000
Subject: [PATCH 1853/2835]

---
 ...selecting_files_based_on_meta-information.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn

diff --git a/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn
new file mode 100644
index 0000000000..1de06f7cda
--- /dev/null
+++ b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn
@@ -0,0 +1,15 @@
+It would be extremely useful to have some additional ways to select files (for git annex copy/move/get and maybe others) based on the meta-information available to git-annex, rather than just by file or directory name.
+
+An example of what I'd like to do is this:
+
+    host1$ git annex copy --to usb-drive --missing-on host2
+
+This would check location tracking information and copy each file from host1's annex which is not present on host2 onto the usb-drive annex -- i.e. it's what I want when I need to do a sneakernet synchronisation of host1 and host2 (for backup purposes, for example). Note that of course I could copy --to host2, assuming network connectivity, but that would take a long time.
+
+There's probably other selectors that we can imagine; an obvious one could be --present-on  -- useful for judiciously dropping only those files that you have easily available in a local annex (as you may want to keep files that are hard to make available even if --numcopies would nominally be satisfied).
+
+Other similar ideas for file content selectors:
+
+ * Files that have less than n, exactly n or more than n copies -- for when you need to satisfy your --numcopies policy over sneakernet.
+ * Files that are present (or not present) on some trusted annex -- for making sure you have trusted copies of everything.
+ * Boolean combinations of these filters -- "git annex drop --present-on lanserver1 --or --present-on lanserver2" or similar syntax, although obviously doing this in full generality may be quite fiddly.

From a85e9b1276cd2b5bdcd9fee0828c3e6432580807 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 13 Jun 2011 16:57:19 +0000
Subject: [PATCH 1854/2835]

---
 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
new file mode 100644
index 0000000000..dc36b709ea
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -0,0 +1,5 @@
+What is says on the tin:
+
+git annex fsck is a no-op in bare repos
+
+See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html

From eb8ecd857d4a5796455b9a608753042241eb6165 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 13 Jun 2011 16:58:53 +0000
Subject: [PATCH 1855/2835] Added a comment

---
 .../comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment
new file mode 100644
index 0000000000..d50938a784
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-06-13T16:58:52Z"
+ content="""
+And, maybe, a way to start a fsck from remote? At least when the other side is a ssh or git annex shell, this would work.
+"""]]

From eb5fbc4cb97b73ad5d5d194600cc52c99edc89a0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 13:03:23 -0400
Subject: [PATCH 1856/2835] response

---
 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
index dc36b709ea..b6ff6bba11 100644
--- a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -3,3 +3,10 @@ What is says on the tin:
 git annex fsck is a no-op in bare repos
 
 See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html
+
+> Thinking about this some more, it would be difficult to do anything
+> when bad content is found, since it also cannot update the location log.
+> 
+> So this may be another thing blocked by [[todo/branching]], assuming
+> that is fixed in a way that makes `.git-annex` available to bare repos.
+> --[[Joey]] 

From 056a784f3d747809c6f256603ff0f8cfb9f6c042 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 13 Jun 2011 18:09:58 +0000
Subject: [PATCH 1857/2835]

---
 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
index b6ff6bba11..249a175055 100644
--- a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -10,3 +10,6 @@ See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html
 > So this may be another thing blocked by [[todo/branching]], assuming
 > that is fixed in a way that makes `.git-annex` available to bare repos.
 > --[[Joey]] 
+
+>> Even if there is nothing it can _do_, knowing that the data is intact,
+>> or not, is valuable in and as of itself. -- RichiH

From 30d7cce7ecb7dbcbabe83ab3f6cd5f32ce685992 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 20:23:47 -0400
Subject: [PATCH 1858/2835] rsync is now used when copying files from repos on
 other filesystems

cp is still used when copying file from repos on the same filesystem, since
--reflink=auto can make it significantly faster on filesystems such as
btrfs.

Directory special remotes still use cp, not rsync. It's not clear what
tmp file should be used when rsyncing to such a remote.
---
 Remote/Git.hs                        | 49 +++++++++++++++++++---------
 debian/changelog                     |  9 +++++
 doc/todo/file_copy_progress_bar.mdwn |  2 ++
 3 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 67d49df7d2..0d8e2425a3 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -11,6 +11,7 @@ import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
+import System.Posix.Files
 
 import Types
 import Types.Remote
@@ -130,10 +131,10 @@ dropKey r key =
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
-	| not $ Git.repoIsUrl r = liftIO $ copyFile (gitAnnexLocation r key) file
-	| Git.repoIsSsh r = rsynchelper r True key file
+	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
+	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
 	| otherwise = error "copying from non-ssh repo not supported"
-
+		
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
@@ -145,19 +146,18 @@ copyToRemote r key
 			a <- Annex.new r []
 			Annex.eval a $ do
 				ok <- Content.getViaTmp key $
-					\f -> liftIO $ copyFile keysrc f
+					rsyncOrCopyFile r keysrc
 				AnnexQueue.flush True
 				return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key
-		rsynchelper r False key keysrc
+		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
-rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool)
-rsynchelper r sending key file = do
+rsyncHelper :: [CommandParam] -> Annex (Bool)
+rsyncHelper p = do
 	showProgress -- make way for progress bar
-	p <- rsyncParams r sending key file
 	res <- liftIO $ rsync p
 	if res
 		then return res
@@ -165,10 +165,22 @@ rsynchelper r sending key file = do
 			showLongNote "rsync failed -- run git annex again to resume file transfer"
 			return res
 
+{- Copys a file with rsync unless both locations are on the same
+ - filesystem. Then cp could be faster. -}
+rsyncOrCopyFile :: Git.Repo -> FilePath -> FilePath -> Annex Bool
+rsyncOrCopyFile r src dest = do
+	ss <- liftIO $ getFileStatus src
+	ds <- liftIO $ getFileStatus dest
+	if deviceID ss == deviceID ds
+		then liftIO $ copyFile src dest
+		else do
+			params <- rsyncParams r
+			rsyncHelper $ params ++ [Param src, Param dest]
+
 {- Generates rsync parameters that ssh to the remote and asks it
  - to either receive or send the key's content. -}
-rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]
-rsyncParams r sending key file = do
+rsyncParamsRemote :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]
+rsyncParamsRemote r sending key file = do
 	Just (shellcmd, shellparams) <- git_annex_shell r
 		(if sending then "sendkey" else "recvkey")
 		[ Param $ show key
@@ -179,15 +191,20 @@ rsyncParams r sending key file = do
 		]
 	-- Convert the ssh command into rsync command line.
 	let eparam = rsyncShell (Param shellcmd:shellparams)
-	o <- getConfig r "rsync-options" ""
-	let base = options ++ map Param (words o) ++ eparam
+	o <- rsyncParams r
 	if sending
-		then return $ base ++ [dummy, File file]
-		else return $ base ++ [File file, dummy]
+		then return $ o ++ eparam ++ [dummy, File file]
+		else return $ o ++ eparam ++ [File file, dummy]
 	where
-		-- inplace makes rsync resume partial files
-		options = [Params "-p --progress --inplace"]
 		-- the rsync shell parameter controls where rsync
 		-- goes, so the source/dest parameter can be a dummy value,
 		-- that just enables remote rsync mode.
 		dummy = Param ":"
+
+rsyncParams :: Git.Repo -> Annex [CommandParam]
+rsyncParams r = do
+	o <- getConfig r "rsync-options" ""
+	return $ options ++ map Param (words o)
+	where
+ 		-- --inplace to resume partial files
+		options = [Params "-p --progress --inplace"]
diff --git a/debian/changelog b/debian/changelog
index 09655df65c..0837a9f345 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+git-annex (0.20110611) UNRELEASED; urgency=low
+
+  * rsync is now used when copying files from repos on other filesystems.
+    cp is still used when copying file from repos on the same filesystem,
+    since --reflink=auto can make it significantly faster on filesystems
+    such as btrfs.
+
+ -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
+
 git-annex (0.20110610) unstable; urgency=low
 
   * Add --numcopies option.
diff --git a/doc/todo/file_copy_progress_bar.mdwn b/doc/todo/file_copy_progress_bar.mdwn
index cd4ea33b78..847c1d1eb6 100644
--- a/doc/todo/file_copy_progress_bar.mdwn
+++ b/doc/todo/file_copy_progress_bar.mdwn
@@ -1,3 +1,5 @@
 Find a way to copy a file with a progress bar, while still preserving
 stat. Easiest way might be to use pv and fix up the permissions etc
 after?
+
+[[done]]

From d0482d4154e90e7e5c3d70113a760eb76e5d4e83 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 21:46:28 -0400
Subject: [PATCH 1859/2835] bigfix: stat parent dirs

---
 Remote/Git.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 0d8e2425a3..5b9d5d3dfe 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -169,8 +169,8 @@ rsyncHelper p = do
  - filesystem. Then cp could be faster. -}
 rsyncOrCopyFile :: Git.Repo -> FilePath -> FilePath -> Annex Bool
 rsyncOrCopyFile r src dest = do
-	ss <- liftIO $ getFileStatus src
-	ds <- liftIO $ getFileStatus dest
+	ss <- liftIO $ getFileStatus $ parentDir src
+	ds <- liftIO $ getFileStatus $ parentDir dest
 	if deviceID ss == deviceID ds
 		then liftIO $ copyFile src dest
 		else do

From 66f5b390fe761946280c8999648d13e917fba45b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 21:51:52 -0400
Subject: [PATCH 1860/2835] fix test suite

---
 test.hs | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/test.hs b/test.hs
index 221607755a..498c7b6806 100644
--- a/test.hs
+++ b/test.hs
@@ -31,7 +31,7 @@ import qualified Backend
 import qualified GitRepo as Git
 import qualified Locations
 import qualified Utility
-import qualified Type.Backend
+import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
 import qualified LocationLog
@@ -40,20 +40,20 @@ import qualified Trust
 import qualified Remote
 import qualified Content
 import qualified Command.DropUnused
-import qualified Type.Key
+import qualified Types.Key
 import qualified Config
 import qualified Crypto
 
 -- for quickcheck
-instance Arbitrary Key.Key where
+instance Arbitrary Types.Key.Key where
 	arbitrary = do
 		n <- arbitrary
 		b <- elements ['A'..'Z']
-		return $ Key.Key {
-			Key.keyName = n,
-			Key.keyBackendName = [b],
-			Key.keySize = Nothing,
-			Key.keyMtime = Nothing
+		return $ Types.Key.Key {
+			Types.Key.keyName = n,
+			Types.Key.keyBackendName = [b],
+			Types.Key.keySize = Nothing,
+			Types.Key.keyMtime = Nothing
 		}
 
 main :: IO ()
@@ -72,7 +72,7 @@ quickcheck :: Test
 quickcheck = TestLabel "quickcheck" $ TestList
 	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
-	, qctest "prop_idempotent_key_read_show" Key.prop_idempotent_key_read_show
+	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
 	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
 	, qctest "prop_idempotent_configEscape" Remote.prop_idempotent_configEscape
@@ -139,7 +139,7 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup]
 test_setkey :: Test
 test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
 	writeFile tmp $ content sha1annexedfile
-	r <- annexeval $ BackendClass.getKey backendSHA1 tmp
+	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
 	let key = show $ fromJust r
 	git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed"
 	git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed"

From f547277b751d81f516a638bc281735fa1946b17d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 22:19:44 -0400
Subject: [PATCH 1861/2835] Allow --trust etc to specify a repository by name,
 for temporarily  trusting repositories that are not configured remotes.

---
 Remote.hs          | 29 ++++++++++++++++++++++-------
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  3 ++-
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 2e956cb81c..d975c2404f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -34,7 +34,7 @@ module Remote (
 ) where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when, liftM, filterM)
+import Control.Monad (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
@@ -91,20 +91,35 @@ genList = do
 
 {- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex (Remote Annex)
-byName "" = error "no remote specified"
 byName n = do
+	res <- byName' n
+	case res of
+		Left e -> error e
+		Right r -> return r
+byName' :: String -> Annex (Either String (Remote Annex))
+byName' "" = return $ Left "no remote specified"
+byName' n = do
 	allremotes <- genList
 	let match = filter matching allremotes
-	when (null match) $ error $
-		"there is no git remote named \"" ++ n ++ "\""
-	return $ head match
+	if (null match)
+		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
+		else return $ Right $ head match
 	where
 		matching r = n == name r || n == uuid r
 
-{- Looks up a remote by name (or by UUID), and returns its UUID. -}
+{- Looks up a remote by name (or by UUID, or even by description),
+ - and returns its UUID. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
-nameToUUID n = liftM uuid (byName n)
+nameToUUID n = do
+	res <- byName' n
+	case res of
+		Left e -> return . (maybe (error e) id) =<< byDescription
+		Right r -> return $ uuid r
+	where
+		byDescription = return . M.lookup n . invertMap =<< uuidMap
+		invertMap = M.fromList . map swap . M.toList
+		swap (a, b) = (b, a)
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
diff --git a/debian/changelog b/debian/changelog
index 0837a9f345..de012de5bd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (0.20110611) UNRELEASED; urgency=low
     cp is still used when copying file from repos on the same filesystem,
     since --reflink=auto can make it significantly faster on filesystems
     such as btrfs.
+  * Allow --trust etc to specify a repository by name, for temporarily 
+    trusting repositories that are not configured remotes.
 
  -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 25f053af69..12756d8020 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -365,7 +365,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Overrides trust settings for a repository. May be specified more than once.
   
-  The repository should be specified using the name of a configured remote.
+  The repository should be specified using the name of a configured remote,
+  or the UUID or description of a repository.
 
 * --backend=name
 

From f4609a5d288525abbb47e361194601d21684df8e Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Tue, 14 Jun 2011 17:01:13 +0000
Subject: [PATCH 1862/2835] minor bug report

---
 doc/bugs/git_annex_initremote_walks_.git-annex.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git_annex_initremote_walks_.git-annex.mdwn

diff --git a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
new file mode 100644
index 0000000000..6e3a60e3bf
--- /dev/null
+++ b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
@@ -0,0 +1,10 @@
+a [[!taglink minor]]  issue: `git annex initremote` (in particular, adding
+a key as described in [[encryption]] -- `git annex initremote my_remote
+encryption=my_key`) seems to iterate over the `.git-annex/???/???/*.log` files
+with lstat (tested using strace).
+
+in a 50k key git-annex on a slow disk, this takes quite a while, while not
+seeming necessary (it's just re-encrypting the shared secret, is it?).
+
+could you verify the observed behavior?

From 64d6520704fde80ee1584cacd4a16a54d88970fd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Jun 2011 19:05:45 -0400
Subject: [PATCH 1863/2835] git is slow

---
 doc/bugs/git_annex_initremote_walks_.git-annex.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
index 6e3a60e3bf..2457057c81 100644
--- a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
+++ b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
@@ -8,3 +8,12 @@ in a 50k key git-annex on a slow disk, this takes quite a while, while not
 seeming necessary (it's just re-encrypting the shared secret, is it?).
 
 could you verify the observed behavior?
+
+> This is due to `git commit` being called. `git commit` exposes git's 
+> rather innefficient handling of the index; in order to make a commit
+> it has to write a new index file, and it does this by scanning every
+> file in the repository. I think that git generally needs its index
+> file handleing overhauled, particularly to deal with repositories with
+> large numbers of files. git-annex is seems to already be running
+> `git commit` in its most efficient mode, by specifying exactly what file
+> to commit. [[done]] --[[Joey]]

From 601b07196ee122b96c984133637b4e2c3debc4b6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 20 Jun 2011 17:43:05 +0000
Subject: [PATCH 1864/2835] Comment moderation

---
 .../comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment

diff --git a/doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment b/doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment
new file mode 100644
index 0000000000..f006f88a0a
--- /dev/null
+++ b/doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="status of other remotes?"
+ date="2011-06-15T08:39:24Z"
+ content="""
+using the location tracking information, it should be possible to show the status of other remotes as well. what about supporting `--from=...` or `--all`? (thus, among other things, one could determine if a remote has a complete checkout.)
+"""]]

From dd4de9deb464adb6631585abd3c5a7e1401c8c6f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Mon, 20 Jun 2011 21:03:00 +0000
Subject: [PATCH 1865/2835]

---
 doc/forum/git_annex_unlock_is_not_atomic.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/git_annex_unlock_is_not_atomic.mdwn

diff --git a/doc/forum/git_annex_unlock_is_not_atomic.mdwn b/doc/forum/git_annex_unlock_is_not_atomic.mdwn
new file mode 100644
index 0000000000..a7751f6b7d
--- /dev/null
+++ b/doc/forum/git_annex_unlock_is_not_atomic.mdwn
@@ -0,0 +1,5 @@
+Running a command like
+
+git annex unlock myfile
+
+is not atomic, that is if the execution is aborted you may end up with an incomplete version of myfile in the directory. If you don't notice this you may lock it again and then propagate this bad version of the file to your other repositories. A simple workaround is to simply name it something else while unlocking and then rename it to the correct filename once it's completely copied. I don't know Haskel yet so I can not fix this issue otherwise I would sure try. A part from this, I love git annex.

From bc731ba6e0b94c7d5d4be5ba3c5c13d6644183f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Jun 2011 18:27:01 -0400
Subject: [PATCH 1866/2835] pointless golfing

---
 Annex.hs | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 06d642b742..0d6b309cca 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -17,7 +17,6 @@ module Annex (
 ) where
 
 import Control.Monad.State
-	(liftIO, StateT, runStateT, evalStateT, liftM, get, put)
 
 import qualified GitRepo as Git
 import GitQueue
@@ -50,8 +49,8 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-newState :: Git.Repo -> [Backend Annex] -> AnnexState
-newState gitrepo allbackends = AnnexState
+newState :: [Backend Annex] -> Git.Repo -> AnnexState
+newState allbackends gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
@@ -72,27 +71,26 @@ newState gitrepo allbackends = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> [Backend Annex] -> IO AnnexState
-new gitrepo allbackends = do
-	gitrepo' <- liftIO $ Git.configRead gitrepo
-	return $ newState gitrepo' allbackends
+new gitrepo allbackends =
+	newState allbackends `liftM` (liftIO . Git.configRead) gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
-run state action = runStateT action state
+run = flip runStateT
 eval :: AnnexState -> Annex a -> IO a
-eval state action = evalStateT action state
+eval = flip evalStateT
 
 {- Gets a value from the internal state, selected by the passed value
  - constructor. -}
 getState :: (AnnexState -> a) -> Annex a
-getState c = liftM c get
+getState = gets
 
 {- Applies a state mutation function to change the internal state. 
  -
- - Example: changeState (\s -> s { quiet = True })
+ - Example: changeState $ \s -> s { quiet = True }
  -}
 changeState :: (AnnexState -> AnnexState) -> Annex ()
-changeState a = put . a =<< get
+changeState = modify
 
 {- Returns the git repository being acted on -}
 gitRepo :: Annex Git.Repo

From e593a81041f3c55e8b2803d4cc07adc9b206b46f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 13:19:08 -0400
Subject: [PATCH 1867/2835] update

---
 doc/todo/branching.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index 79f278480e..a035e53e80 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -47,6 +47,8 @@ Let's use one branch per uuid, named git-annex/$UUID.
   branches?
 - A given repo only ever writes to its UUID branch. So no conflicts.
   - **problem**: git annex move needs to update log info for other repos!
+    (possibly solvable by having git-annex-shell update the log info
+    when content is moved using it)
 - (BTW, UUIDs probably don't compress well, and this reduces the bloat of having
   them repeated lots of times in the tree.)
 - Per UUID branches mean that if it wants to find a file's location
@@ -77,7 +79,7 @@ with a indication of the presense/absense of the key is found.
   or transfered.
 - It could get pretty slow when digging deeper.
 - Only 3 places in git-annex will be affected by any slowdown: move --from,
-  get and drop.
+  get and drop. (Update: Now also unused, whereis, fsck) 
 
 ## alternate
 
@@ -109,7 +111,7 @@ which could get expensive.
 ## way outside the box approach
 
 Another approach I have been mulling over is keeping the log file
-branch checked out in .git-annex/logs/ -- this would be a checkout of a git
+branch checked out in .git/annex/logs/ -- this would be a checkout of a git
 repository inside a git repository, using "git fake bare" techniques. This
 would solve the merge problem, since git auto merge could be used. It would
 still mean all the log files are on-disk, which annoys some. It would

From 2fcb8e3b11cdbb9de253050c1fbb6f08ef1071e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 13:48:02 -0400
Subject: [PATCH 1868/2835] update

---
 doc/todo/branching.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index a035e53e80..3a7a2d3ca9 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -118,6 +118,13 @@ still mean all the log files are on-disk, which annoys some. It would
 require some tighter integration with git, so that after a pull, the log
 repo is updated with the data pulled. --[[Joey]] 
 
+> Seems I can't use git fake bare exactly. Instead, the best option
+> seems to be `git clone --shared` to make a clone that uses
+> `.git/annex/logs/.git` to hold its index etc, but (mostly) uses
+> objects from the main repo. There would be some bloat,
+> as commits to the logs made in there would not be shared with the main
+> repo. Using `GIT_OBJECT_DIRECTORY` might be a way to avoid that bloat.
+
 ## notes
 
 Another approach could be to use git-notes. It supports merging branches

From 91e50782ce6d634ffc8c2f809c80b6d4ff94a5ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 15:22:07 -0400
Subject: [PATCH 1869/2835] thought

---
 doc/todo/branching.mdwn | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index 3a7a2d3ca9..9c44c03feb 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -135,3 +135,25 @@ Problem: Notes are usually attached to git
 objects, and there are no git objects corresponding to git-annex keys.
 
 Problem: Notes are not normally copied when cloning.
+
+------
+
+## elminating the merge problem
+
+Most of the above options are complicated by the problem of how to merge
+changes from remotes. It should be possible to deal with the merge
+problem generically. Something like this:
+
+* We have a local branch `B`.
+* For remotes, there are also `origin/B`, `otherremote/B`, etc.
+* To merge two branches `B` and `foo/B`, construct a merge commit that
+  makes each file have all lines that were in either version of the file,
+  with duplicates removed (probably). Do this without checking out a tree,
+  or using a temporary directory. (One easy but expensive way is to just
+  check out the branch to a temp dir, union merge into it, and remove the
+  temp dir ... but it should be possible to do it without using a temp dir.)
+* As a `post-merge` hook, merge `*/B` into `B`. This will ensure `B`
+  is always up-to-date after a pull from a remote.
+* When pushing to a remote, nothing need to be done, except ensure
+  `B` is either successfully pushed, or the push fails (and a pull needs to
+  be done to get the remote's changes merged into `B`).

From c835166a7cebfa44d232bbed7c5b5e22bdfeb2bd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 19:44:45 -0400
Subject: [PATCH 1870/2835] add git-union-merge

This is a new git subcommand, that does a generic union merge operation
between two refs, storing the result in a branch. It operates efficiently
without touching the working tree. It does need to write out a temporary
index file, and may need to write out some other temp files as well.

This could be useful for anything that stores data in a branch,
and needs to merge changes into that branch without actually checking the
branch out. Since conflict handling can't be done without a working copy,
the merge type is always a union merge, which is fine for data stored in
log format (as git-annex does), or in non-conflicting files
(as pristine-tar does).

This probably belongs in git proper, but it will live in git-annex for now.

---

Plan is to move .git-annex/ to a git-annex branch, and use git-union-merge
to handle merging changes when pulling from remotes.

Some preliminary benchmarking using real .git-annex/ data indicates
that it's quite fast, except for the "git add" call, which is as slow
as "git add" tends to be with a big index.
---
 .gitignore               |   2 +
 GitRepo.hs               |   3 +-
 Makefile                 |   6 +-
 debian/changelog         |   2 +
 doc/git-union-merge.mdwn |  38 +++++++++++++
 doc/todo/branching.mdwn  |   6 +-
 git-union-merge.hs       | 120 +++++++++++++++++++++++++++++++++++++++
 7 files changed, 170 insertions(+), 7 deletions(-)
 create mode 100644 doc/git-union-merge.mdwn
 create mode 100644 git-union-merge.hs

diff --git a/.gitignore b/.gitignore
index b73167c925..9a4bc80de3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,8 +5,10 @@ configure
 SysConfig.hs
 git-annex
 git-annex-shell
+git-union-merge
 git-annex.1
 git-annex-shell.1
+git-union-merge.1
 doc/.ikiwiki
 html
 *.tix
diff --git a/GitRepo.hs b/GitRepo.hs
index 24bc9b5c2b..0bee2842a5 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -38,6 +38,7 @@ module GitRepo (
 	gitCommandLine,
 	run,
 	pipeRead,
+	pipeNullSplit,
 	attributes,
 	remotes,
 	remotesAdd,
@@ -412,7 +413,7 @@ typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
 		end = [Param "--"] ++ map File l
 
 {- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it into a list of files. -}
+ - parameter), and splits it into a list of files/lines/whatever. -}
 pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
 pipeNullSplit repo params = do
 	fs0 <- pipeRead repo params
diff --git a/Makefile b/Makefile
index 286c3a6e54..915b0bf0b2 100644
--- a/Makefile
+++ b/Makefile
@@ -6,8 +6,8 @@ GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE)
 endif
 GHCMAKE=ghc $(GHCFLAGS) --make
 
-bins=git-annex git-annex-shell
-mans=git-annex.1 git-annex-shell.1
+bins=git-annex git-annex-shell git-union-merge
+mans=git-annex.1 git-annex-shell.1 git-union-merge.1
 
 all: $(bins) $(mans) docs
 
@@ -33,6 +33,8 @@ git-annex.1: doc/git-annex.mdwn
 	./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1
 git-annex-shell.1: doc/git-annex-shell.mdwn
 	./mdwn2man git-annex-shell 1 doc/git-annex-shell.mdwn > git-annex-shell.1
+git-union-merge.1: doc/git-union-merge.mdwn
+	./mdwn2man git-union-merge 1 doc/git-union-merge.mdwn > git-union-merge.1
 
 install: all
 	install -d $(DESTDIR)$(PREFIX)/bin
diff --git a/debian/changelog b/debian/changelog
index de012de5bd..b96b9f43d6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (0.20110611) UNRELEASED; urgency=low
     such as btrfs.
   * Allow --trust etc to specify a repository by name, for temporarily 
     trusting repositories that are not configured remotes.
+  * git-union-merge: New git subcommand, that does a generic union merge
+    operation, and operates efficiently without touching the working tree.
 
  -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
 
diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
new file mode 100644
index 0000000000..ac8e8f7a99
--- /dev/null
+++ b/doc/git-union-merge.mdwn
@@ -0,0 +1,38 @@
+# NAME
+
+git-union-merge - Join branches together using a union merge
+
+# SYNOPSIS
+
+git union-merge branch ref ref
+
+# DESCRIPTION
+
+Does a union merge between two refs, storing the result in the
+specified branch.
+
+The union merge will always succeed, but assumes that files can be merged
+simply by concacenating together lines from all the oldrefs, in any order.
+So, this is useful only for branches containing log-type data.
+
+That this does not touch the checked out working copy. It operates
+entirely on git refs and branches.
+
+# EXAMPLE
+
+	git union-merge git-annex git-annex origin/git-annex
+
+Merges the current git-annex branch, and a version from origin,
+storing the result in the git-annex branch.
+
+# BUGS
+
+File modes are not currently merged.
+
+# AUTHOR
+
+Joey Hess 
+
+
+
+Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD).  Edit with care
diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index 9c44c03feb..37e7b6edd2 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -148,10 +148,8 @@ problem generically. Something like this:
 * For remotes, there are also `origin/B`, `otherremote/B`, etc.
 * To merge two branches `B` and `foo/B`, construct a merge commit that
   makes each file have all lines that were in either version of the file,
-  with duplicates removed (probably). Do this without checking out a tree,
-  or using a temporary directory. (One easy but expensive way is to just
-  check out the branch to a temp dir, union merge into it, and remove the
-  temp dir ... but it should be possible to do it without using a temp dir.)
+  with duplicates removed (probably). Do this without checking out a tree.
+  -- now implemented as git-union-merge
 * As a `post-merge` hook, merge `*/B` into `B`. This will ensure `B`
   is always up-to-date after a pull from a remote.
 * When pushing to a remote, nothing need to be done, except ensure
diff --git a/git-union-merge.hs b/git-union-merge.hs
new file mode 100644
index 0000000000..482f66daa0
--- /dev/null
+++ b/git-union-merge.hs
@@ -0,0 +1,120 @@
+{- git-union-merge program
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+import System.Environment
+import System.FilePath
+import System.Directory
+import System.Cmd
+import System.Cmd.Utils
+import System.Posix.Env (setEnv)
+import System.Posix.Directory (changeWorkingDirectory)
+import Control.Monad (when, unless)
+import Data.List
+
+import qualified GitRepo as Git
+import Utility
+
+header :: String
+header = "Usage: git-union-merge branch ref ref"
+
+usage :: IO a
+usage = error $ "bad parameters\n\n" ++ header
+
+main :: IO ()
+main = do
+	[branch, aref, bref] <- parseArgs
+	g <- setup
+	stage g aref bref
+	commit g branch aref bref
+	cleanup g
+
+parseArgs :: IO [String]
+parseArgs = do
+	args <- getArgs
+	if (length args /= 3)
+		then usage
+		else return args
+
+tmpDir :: Git.Repo -> FilePath
+tmpDir g = Git.workTree g  Git.gitDir g  "tmp"  "git-union-merge"
+
+tmpIndex :: Git.Repo -> FilePath
+tmpIndex g = Git.workTree g  Git.gitDir g  "tmp"  "git-union-merge.index"
+
+{- Moves to a temporary directory, and configures git to use it as its
+ - working tree, and to use a temporary index file as well. -}
+setup :: IO Git.Repo
+setup = do
+	g <- Git.configRead =<< Git.repoFromCwd
+	cleanup g -- idempotency
+	let tmp = tmpDir g
+	createDirectoryIfMissing True tmp
+	changeWorkingDirectory tmp
+	-- Note that due to these variables being set, Git.run and
+	-- similar helpers cannot be used, as they override the work tree.
+	-- It is only safe to use Git.run etc when doing things that do
+	-- not operate on the work tree.
+	setEnv "GIT_WORK_TREE" tmp True
+	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
+	return g
+
+cleanup :: Git.Repo -> IO ()
+cleanup g = do
+	e <- doesDirectoryExist (tmpDir g)
+	when e $ removeDirectoryRecursive (tmpDir g)
+	e' <- doesFileExist (tmpIndex g)
+	when e' $ removeFile (tmpIndex g)
+
+{- Stages the content of both refs into the index. -}
+stage :: Git.Repo -> String -> String -> IO ()
+stage g aref bref = do
+	-- populate index with the contents of aref, as a starting point
+	_ <- system $ "git ls-tree -r --full-name --full-tree " ++ aref ++ 
+		" | git update-index --index-info"
+	-- identify files that are different in bref, and stage merged files
+	diff <- Git.pipeNullSplit g $ map Param
+		["diff-tree", "--raw", "-z", "--no-renames", "-l0", aref, bref]
+	mapM_ genfile (pairs diff)
+	_ <- system "git add ."
+	return ()
+	where
+		pairs [] = []
+		pairs (_:[]) = error "parse error"
+		pairs (a:b:rest) = (a,b):pairs rest
+
+		nullsha = take 40 $ repeat '0'
+
+		genfile (info, file) = do
+			let [_colonamode, _bmode, asha, bsha, _status] = words info
+			let shas = 
+				if bsha == nullsha
+					then [] -- staged from aref
+					else
+						if asha == nullsha
+							then [bsha]
+							else [asha, bsha]
+			unless (null shas) $ do
+				content <- Git.pipeRead g $ map Param ("show":shas)
+				writeFile file $ unlines $ nub $ lines content
+
+{- Commits the index into the specified branch. -}
+commit :: Git.Repo -> String -> String -> String -> IO ()
+commit g branch aref bref = do
+	tree <- getsha $
+		pipeFrom "git" ["write-tree"]
+	sha <- getsha $ 
+		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
+			"union merge"
+	Git.run g "update-ref" [Param $ "refs/heads/" ++ branch, Param sha]
+	where
+		getsha a = do
+			(_, t) <- a
+			let t' = if last t == '\n'
+				then take (length t - 1) t
+				else t
+			when (null t') $ error "failed to read sha from git"
+			return t'

From d519bc71372fe1962a9e03fc5472f9fe870066f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 21:35:39 -0400
Subject: [PATCH 1871/2835] sped up git-union-merge

Avoided the slow git add, instead inject content directly into git and
populate the index all in one pass. Now this runs on my large real-world
repo in 10 seconds, which is acceptable.

Also lots of code cleanups.
---
 GitRepo.hs         |  16 ++++++-
 git-union-merge.hs | 111 ++++++++++++++++++++++++---------------------
 2 files changed, 74 insertions(+), 53 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 0bee2842a5..9f4a38a5fb 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -38,6 +38,8 @@ module GitRepo (
 	gitCommandLine,
 	run,
 	pipeRead,
+	pipeWrite,
+	pipeWriteRead,
 	pipeNullSplit,
 	attributes,
 	remotes,
@@ -348,7 +350,7 @@ run repo subcommand params = assertLocal repo $
 	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
 		>>! error $ "git " ++ show params ++ " failed"
 
-{- Runs a git subcommand and returns it output, lazily. 
+{- Runs a git subcommand and returns its output, lazily. 
  -
  - Note that this leaves the git process running, and so zombies will
  - result unless reap is called.
@@ -358,6 +360,18 @@ pipeRead repo params = assertLocal repo $ do
 	(_, s) <- pipeFrom "git" $ toCommand $ gitCommandLine repo params
 	return s
 
+{- Runs a git subcommand, feeding it input.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWrite :: Repo -> [CommandParam] -> String -> IO PipeHandle
+pipeWrite repo params s = assertLocal repo $
+	pipeTo "git" (toCommand $ gitCommandLine repo params) s
+
+{- Runs a git subcommand, feeding it input, and returning its output.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWriteRead :: Repo -> [CommandParam] -> String -> IO (PipeHandle, String)
+pipeWriteRead repo params s = assertLocal repo $
+	pipeBoth "git" (toCommand $ gitCommandLine repo params) s
+
 {- Reaps any zombie git processes. -}
 reap :: IO ()
 reap = do
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 482f66daa0..b0c59a6d36 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -8,12 +8,12 @@
 import System.Environment
 import System.FilePath
 import System.Directory
-import System.Cmd
 import System.Cmd.Utils
 import System.Posix.Env (setEnv)
-import System.Posix.Directory (changeWorkingDirectory)
-import Control.Monad (when, unless)
+import Control.Monad (when)
 import Data.List
+import Data.Maybe
+import Data.String.Utils
 
 import qualified GitRepo as Git
 import Utility
@@ -39,82 +39,89 @@ parseArgs = do
 		then usage
 		else return args
 
-tmpDir :: Git.Repo -> FilePath
-tmpDir g = Git.workTree g  Git.gitDir g  "tmp"  "git-union-merge"
-
 tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "tmp"  "git-union-merge.index"
+tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
 
-{- Moves to a temporary directory, and configures git to use it as its
- - working tree, and to use a temporary index file as well. -}
+{- Configures git to use a temporary index file. -}
 setup :: IO Git.Repo
 setup = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	cleanup g -- idempotency
-	let tmp = tmpDir g
-	createDirectoryIfMissing True tmp
-	changeWorkingDirectory tmp
-	-- Note that due to these variables being set, Git.run and
-	-- similar helpers cannot be used, as they override the work tree.
-	-- It is only safe to use Git.run etc when doing things that do
-	-- not operate on the work tree.
-	setEnv "GIT_WORK_TREE" tmp True
 	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
 	return g
 
 cleanup :: Git.Repo -> IO ()
 cleanup g = do
-	e <- doesDirectoryExist (tmpDir g)
-	when e $ removeDirectoryRecursive (tmpDir g)
 	e' <- doesFileExist (tmpIndex g)
 	when e' $ removeFile (tmpIndex g)
 
 {- Stages the content of both refs into the index. -}
 stage :: Git.Repo -> String -> String -> IO ()
 stage g aref bref = do
-	-- populate index with the contents of aref, as a starting point
-	_ <- system $ "git ls-tree -r --full-name --full-tree " ++ aref ++ 
-		" | git update-index --index-info"
-	-- identify files that are different in bref, and stage merged files
-	diff <- Git.pipeNullSplit g $ map Param
-		["diff-tree", "--raw", "-z", "--no-renames", "-l0", aref, bref]
-	mapM_ genfile (pairs diff)
-	_ <- system "git add ."
-	return ()
+	-- Get the contents of aref, as a starting point.
+	ls <- fromgit
+		["ls-tree", "-z", "-r", "--full-tree", aref]
+	-- Identify files that are different between aref and bref, and
+	-- inject merged versions into git.
+	diff <- fromgit
+		["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", aref, bref]
+	ls' <- mapM mergefile (pairs diff)
+	-- Populate the index file. Later lines override earlier ones.
+	togit ["update-index", "-z", "--index-info"]
+		(join "\0" $ ls++catMaybes ls')
 	where
+		fromgit l = Git.pipeNullSplit g (map Param l)
+		togit l content = Git.pipeWrite g (map Param l) content
+			>>= forceSuccess
+		tofromgit l content = do
+			(h, s) <- Git.pipeWriteRead g (map Param l) content
+			length s `seq` do
+				forceSuccess h
+				Git.reap
+				return ((), s)
+
 		pairs [] = []
 		pairs (_:[]) = error "parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
-
-		nullsha = take 40 $ repeat '0'
-
-		genfile (info, file) = do
+		
+		nullsha = take shaSize $ repeat '0'
+		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
+		unionmerge = unlines . nub . lines
+		
+		mergefile (info, file) = do
 			let [_colonamode, _bmode, asha, bsha, _status] = words info
-			let shas = 
-				if bsha == nullsha
-					then [] -- staged from aref
-					else
-						if asha == nullsha
-							then [bsha]
-							else [asha, bsha]
-			unless (null shas) $ do
-				content <- Git.pipeRead g $ map Param ("show":shas)
-				writeFile file $ unlines $ nub $ lines content
+			if bsha == nullsha
+				then return Nothing -- already staged from aref
+				else mergefile' file asha bsha
+		mergefile' file asha bsha = do
+			let shas = filter (/= nullsha) [asha, bsha]
+			content <- Git.pipeRead g $ map Param ("show":shas)
+			sha <- getSha "hash-object" $
+				tofromgit ["hash-object", "-w", "--stdin"] $
+					unionmerge content
+			return $ Just $ ls_tree_line sha file
 
 {- Commits the index into the specified branch. -}
 commit :: Git.Repo -> String -> String -> String -> IO ()
 commit g branch aref bref = do
-	tree <- getsha $
+	tree <- getSha "write-tree"  $
 		pipeFrom "git" ["write-tree"]
-	sha <- getsha $ 
+	sha <- getSha "commit-tree" $
 		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
 			"union merge"
 	Git.run g "update-ref" [Param $ "refs/heads/" ++ branch, Param sha]
-	where
-		getsha a = do
-			(_, t) <- a
-			let t' = if last t == '\n'
-				then take (length t - 1) t
-				else t
-			when (null t') $ error "failed to read sha from git"
-			return t'
+
+{- Runs an action that causes a git subcommand to emit a sha, and strips
+   any trailing newline, returning the sha. -}
+getSha :: String -> IO (a, String) -> IO String
+getSha subcommand a = do
+	(_, t) <- a
+	let t' = if last t == '\n'
+		then take (length t - 1) t
+		else t
+	when (length t' /= shaSize) $
+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
+	return t'
+
+shaSize :: Int
+shaSize = 40

From 01e8a0a9e5de13b4c1f5baa5d44e8898692f62e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 21:38:52 -0400
Subject: [PATCH 1872/2835] allow git-union-merge to write to any ref

Not just refs/heads/* branches.
---
 doc/git-union-merge.mdwn | 6 +++---
 git-union-merge.hs       | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
index ac8e8f7a99..0322641efa 100644
--- a/doc/git-union-merge.mdwn
+++ b/doc/git-union-merge.mdwn
@@ -4,12 +4,12 @@ git-union-merge - Join branches together using a union merge
 
 # SYNOPSIS
 
-git union-merge branch ref ref
+git union-merge newref ref ref
 
 # DESCRIPTION
 
 Does a union merge between two refs, storing the result in the
-specified branch.
+specified newref.
 
 The union merge will always succeed, but assumes that files can be merged
 simply by concacenating together lines from all the oldrefs, in any order.
@@ -20,7 +20,7 @@ entirely on git refs and branches.
 
 # EXAMPLE
 
-	git union-merge git-annex git-annex origin/git-annex
+	git union-merge refs/heads/git-annex git-annex origin/git-annex
 
 Merges the current git-annex branch, and a version from origin,
 storing the result in the git-annex branch.
diff --git a/git-union-merge.hs b/git-union-merge.hs
index b0c59a6d36..8e7e0367b7 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -19,7 +19,7 @@ import qualified GitRepo as Git
 import Utility
 
 header :: String
-header = "Usage: git-union-merge branch ref ref"
+header = "Usage: git-union-merge newref ref ref"
 
 usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
@@ -109,7 +109,7 @@ commit g branch aref bref = do
 	sha <- getSha "commit-tree" $
 		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
 			"union merge"
-	Git.run g "update-ref" [Param $ "refs/heads/" ++ branch, Param sha]
+	Git.run g "update-ref" [Param branch, Param sha]
 
 {- Runs an action that causes a git subcommand to emit a sha, and strips
    any trailing newline, returning the sha. -}

From 8b749d4bfdc81155e0dc17026f1575620bc132e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 21:42:17 -0400
Subject: [PATCH 1873/2835] reorder git-union-merge params

---
 doc/git-union-merge.mdwn | 4 ++--
 git-union-merge.hs       | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
index 0322641efa..495612f36d 100644
--- a/doc/git-union-merge.mdwn
+++ b/doc/git-union-merge.mdwn
@@ -4,7 +4,7 @@ git-union-merge - Join branches together using a union merge
 
 # SYNOPSIS
 
-git union-merge newref ref ref
+git union-merge ref ref newref
 
 # DESCRIPTION
 
@@ -20,7 +20,7 @@ entirely on git refs and branches.
 
 # EXAMPLE
 
-	git union-merge refs/heads/git-annex git-annex origin/git-annex
+	git union-merge git-annex origin/git-annex refs/heads/git-annex 
 
 Merges the current git-annex branch, and a version from origin,
 storing the result in the git-annex branch.
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 8e7e0367b7..2a2570cf4f 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -26,10 +26,10 @@ usage = error $ "bad parameters\n\n" ++ header
 
 main :: IO ()
 main = do
-	[branch, aref, bref] <- parseArgs
+	[aref, bref, newref] <- parseArgs
 	g <- setup
 	stage g aref bref
-	commit g branch aref bref
+	commit g aref bref newref
 	cleanup g
 
 parseArgs :: IO [String]
@@ -103,13 +103,13 @@ stage g aref bref = do
 
 {- Commits the index into the specified branch. -}
 commit :: Git.Repo -> String -> String -> String -> IO ()
-commit g branch aref bref = do
+commit g aref bref newref = do
 	tree <- getSha "write-tree"  $
 		pipeFrom "git" ["write-tree"]
 	sha <- getSha "commit-tree" $
 		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
 			"union merge"
-	Git.run g "update-ref" [Param branch, Param sha]
+	Git.run g "update-ref" [Param newref, Param sha]
 
 {- Runs an action that causes a git subcommand to emit a sha, and strips
    any trailing newline, returning the sha. -}

From a9a464914887b953d216fc2af8c691b0ee9b61d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 22:21:02 -0400
Subject: [PATCH 1874/2835] typo

---
 git-union-merge.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-union-merge.hs b/git-union-merge.hs
index 2a2570cf4f..e11f93701c 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -19,7 +19,7 @@ import qualified GitRepo as Git
 import Utility
 
 header :: String
-header = "Usage: git-union-merge newref ref ref"
+header = "Usage: git-union-merge ref ref newref"
 
 usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header

From 53706ad9bf9bf22359154edab0ca61acea9027e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 22:29:46 -0400
Subject: [PATCH 1875/2835] move bug report

---
 doc/{forum => bugs}/git_annex_unlock_is_not_atomic.mdwn | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/{forum => bugs}/git_annex_unlock_is_not_atomic.mdwn (100%)

diff --git a/doc/forum/git_annex_unlock_is_not_atomic.mdwn b/doc/bugs/git_annex_unlock_is_not_atomic.mdwn
similarity index 100%
rename from doc/forum/git_annex_unlock_is_not_atomic.mdwn
rename to doc/bugs/git_annex_unlock_is_not_atomic.mdwn

From 9f9e17aa0f4898063a58c88661bca01465b126a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 22:38:18 -0400
Subject: [PATCH 1876/2835] unlock: Made atomic.

---
 Command/Unlock.hs                            | 10 +++++++---
 debian/changelog                             |  1 +
 doc/bugs/git_annex_unlock_is_not_atomic.mdwn |  2 ++
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 161df2ddf9..ca8b62502a 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -44,11 +44,15 @@ perform dest key = do
 
 	g <- Annex.gitRepo
 	let src = gitAnnexLocation g key
-	liftIO $ removeFile dest
+	let tmpdest = gitAnnexTmpLocation g key
+	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showNote "copying..."
-	ok <- liftIO $ copyFile src dest
+	ok <- liftIO $ copyFile src tmpdest
         if ok
                 then do
-			liftIO $ allowWrite dest
+			liftIO $ do
+				removeFile dest
+				renameFile tmpdest dest
+				allowWrite dest
 			next $ return True
                 else error "copy failed!"
diff --git a/debian/changelog b/debian/changelog
index b96b9f43d6..6439eb89fa 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (0.20110611) UNRELEASED; urgency=low
     such as btrfs.
   * Allow --trust etc to specify a repository by name, for temporarily 
     trusting repositories that are not configured remotes.
+  * unlock: Made atomic.
   * git-union-merge: New git subcommand, that does a generic union merge
     operation, and operates efficiently without touching the working tree.
 
diff --git a/doc/bugs/git_annex_unlock_is_not_atomic.mdwn b/doc/bugs/git_annex_unlock_is_not_atomic.mdwn
index a7751f6b7d..6d324ff500 100644
--- a/doc/bugs/git_annex_unlock_is_not_atomic.mdwn
+++ b/doc/bugs/git_annex_unlock_is_not_atomic.mdwn
@@ -3,3 +3,5 @@ Running a command like
 git annex unlock myfile
 
 is not atomic, that is if the execution is aborted you may end up with an incomplete version of myfile in the directory. If you don't notice this you may lock it again and then propagate this bad version of the file to your other repositories. A simple workaround is to simply name it something else while unlocking and then rename it to the correct filename once it's completely copied. I don't know Haskel yet so I can not fix this issue otherwise I would sure try. A part from this, I love git annex.
+
+> [[fixed|done]] --[[Joey]] 

From e735d459b531246798622994718eaccfcf0086ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 14:09:06 -0400
Subject: [PATCH 1877/2835] moved to library

---
 GitUnionMerge.hs   | 115 +++++++++++++++++++++++++++++++++++++++++++++
 git-union-merge.hs | 107 ++---------------------------------------
 2 files changed, 120 insertions(+), 102 deletions(-)
 create mode 100644 GitUnionMerge.hs

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
new file mode 100644
index 0000000000..dde4b7a043
--- /dev/null
+++ b/GitUnionMerge.hs
@@ -0,0 +1,115 @@
+{- git-union-merge library
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module GitUnionMerge (
+	unionMerge
+) where
+
+import System.FilePath
+import System.Directory
+import System.Cmd.Utils
+import System.Posix.Env (setEnv, unsetEnv)
+import Control.Monad (when)
+import Data.List
+import Data.Maybe
+import Data.String.Utils
+
+import qualified GitRepo as Git
+import Utility
+
+unionMerge :: Git.Repo -> String -> String -> String -> IO ()
+unionMerge g aref bref newref = do
+	setup g
+	stage g aref bref
+	commit g aref bref newref
+	cleanup g
+
+tmpIndex :: Git.Repo -> FilePath
+tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
+
+{- Configures git to use a temporary index file. -}
+setup :: Git.Repo -> IO ()
+setup g = do
+	cleanup g -- idempotency
+	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
+
+cleanup :: Git.Repo -> IO ()
+cleanup g = do
+	unsetEnv "GIT_INDEX_FILE"
+	e' <- doesFileExist (tmpIndex g)
+	when e' $ removeFile (tmpIndex g)
+
+{- Stages the content of both refs into the index. -}
+stage :: Git.Repo -> String -> String -> IO ()
+stage g aref bref = do
+	-- Get the contents of aref, as a starting point.
+	ls <- fromgit
+		["ls-tree", "-z", "-r", "--full-tree", aref]
+	-- Identify files that are different between aref and bref, and
+	-- inject merged versions into git.
+	diff <- fromgit
+		["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", aref, bref]
+	ls' <- mapM mergefile (pairs diff)
+	-- Populate the index file. Later lines override earlier ones.
+	togit ["update-index", "-z", "--index-info"]
+		(join "\0" $ ls++catMaybes ls')
+	where
+		fromgit l = Git.pipeNullSplit g (map Param l)
+		togit l content = Git.pipeWrite g (map Param l) content
+			>>= forceSuccess
+		tofromgit l content = do
+			(h, s) <- Git.pipeWriteRead g (map Param l) content
+			length s `seq` do
+				forceSuccess h
+				Git.reap
+				return ((), s)
+
+		pairs [] = []
+		pairs (_:[]) = error "parse error"
+		pairs (a:b:rest) = (a,b):pairs rest
+		
+		nullsha = take shaSize $ repeat '0'
+		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
+		unionmerge = unlines . nub . lines
+		
+		mergefile (info, file) = do
+			let [_colonamode, _bmode, asha, bsha, _status] = words info
+			if bsha == nullsha
+				then return Nothing -- already staged from aref
+				else mergefile' file asha bsha
+		mergefile' file asha bsha = do
+			let shas = filter (/= nullsha) [asha, bsha]
+			content <- Git.pipeRead g $ map Param ("show":shas)
+			sha <- getSha "hash-object" $
+				tofromgit ["hash-object", "-w", "--stdin"] $
+					unionmerge content
+			return $ Just $ ls_tree_line sha file
+
+{- Commits the index into the specified branch. -}
+commit :: Git.Repo -> String -> String -> String -> IO ()
+commit g aref bref newref = do
+	tree <- getSha "write-tree"  $
+		pipeFrom "git" ["write-tree"]
+	sha <- getSha "commit-tree" $
+		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
+			"union merge"
+	Git.run g "update-ref" [Param newref, Param sha]
+
+{- Runs an action that causes a git subcommand to emit a sha, and strips
+   any trailing newline, returning the sha. -}
+getSha :: String -> IO (a, String) -> IO String
+getSha subcommand a = do
+	(_, t) <- a
+	let t' = if last t == '\n'
+		then take (length t - 1) t
+		else t
+	when (length t' /= shaSize) $
+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
+	return t'
+
+shaSize :: Int
+shaSize = 40
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e11f93701c..62a79a4c22 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -6,17 +6,9 @@
  -}
 
 import System.Environment
-import System.FilePath
-import System.Directory
-import System.Cmd.Utils
-import System.Posix.Env (setEnv)
-import Control.Monad (when)
-import Data.List
-import Data.Maybe
-import Data.String.Utils
 
+import GitUnionMerge
 import qualified GitRepo as Git
-import Utility
 
 header :: String
 header = "Usage: git-union-merge ref ref newref"
@@ -24,14 +16,6 @@ header = "Usage: git-union-merge ref ref newref"
 usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
-main :: IO ()
-main = do
-	[aref, bref, newref] <- parseArgs
-	g <- setup
-	stage g aref bref
-	commit g aref bref newref
-	cleanup g
-
 parseArgs :: IO [String]
 parseArgs = do
 	args <- getArgs
@@ -39,89 +23,8 @@ parseArgs = do
 		then usage
 		else return args
 
-tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
-
-{- Configures git to use a temporary index file. -}
-setup :: IO Git.Repo
-setup = do
+main :: IO ()
+main = do
+	[aref, bref, newref] <- parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
-	cleanup g -- idempotency
-	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
-	return g
-
-cleanup :: Git.Repo -> IO ()
-cleanup g = do
-	e' <- doesFileExist (tmpIndex g)
-	when e' $ removeFile (tmpIndex g)
-
-{- Stages the content of both refs into the index. -}
-stage :: Git.Repo -> String -> String -> IO ()
-stage g aref bref = do
-	-- Get the contents of aref, as a starting point.
-	ls <- fromgit
-		["ls-tree", "-z", "-r", "--full-tree", aref]
-	-- Identify files that are different between aref and bref, and
-	-- inject merged versions into git.
-	diff <- fromgit
-		["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", aref, bref]
-	ls' <- mapM mergefile (pairs diff)
-	-- Populate the index file. Later lines override earlier ones.
-	togit ["update-index", "-z", "--index-info"]
-		(join "\0" $ ls++catMaybes ls')
-	where
-		fromgit l = Git.pipeNullSplit g (map Param l)
-		togit l content = Git.pipeWrite g (map Param l) content
-			>>= forceSuccess
-		tofromgit l content = do
-			(h, s) <- Git.pipeWriteRead g (map Param l) content
-			length s `seq` do
-				forceSuccess h
-				Git.reap
-				return ((), s)
-
-		pairs [] = []
-		pairs (_:[]) = error "parse error"
-		pairs (a:b:rest) = (a,b):pairs rest
-		
-		nullsha = take shaSize $ repeat '0'
-		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
-		unionmerge = unlines . nub . lines
-		
-		mergefile (info, file) = do
-			let [_colonamode, _bmode, asha, bsha, _status] = words info
-			if bsha == nullsha
-				then return Nothing -- already staged from aref
-				else mergefile' file asha bsha
-		mergefile' file asha bsha = do
-			let shas = filter (/= nullsha) [asha, bsha]
-			content <- Git.pipeRead g $ map Param ("show":shas)
-			sha <- getSha "hash-object" $
-				tofromgit ["hash-object", "-w", "--stdin"] $
-					unionmerge content
-			return $ Just $ ls_tree_line sha file
-
-{- Commits the index into the specified branch. -}
-commit :: Git.Repo -> String -> String -> String -> IO ()
-commit g aref bref newref = do
-	tree <- getSha "write-tree"  $
-		pipeFrom "git" ["write-tree"]
-	sha <- getSha "commit-tree" $
-		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
-			"union merge"
-	Git.run g "update-ref" [Param newref, Param sha]
-
-{- Runs an action that causes a git subcommand to emit a sha, and strips
-   any trailing newline, returning the sha. -}
-getSha :: String -> IO (a, String) -> IO String
-getSha subcommand a = do
-	(_, t) <- a
-	let t' = if last t == '\n'
-		then take (length t - 1) t
-		else t
-	when (length t' /= shaSize) $
-		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return t'
-
-shaSize :: Int
-shaSize = 40
+	unionMerge g aref bref newref

From c03af0ed0c6530742206740e8b3fc75f5f959325 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 14:29:09 -0400
Subject: [PATCH 1878/2835] code to update a git-annex branch

There is no suitable git hook to run code when pulling changes that
might need to be merged into the git-annex branch. The post-merge hook
is only run when changes are merged into HEAD, and it's possible,
and indeed likely that many pulls will only have changes in git-annex,
but not in HEAD, and not trigger it.

So, git-annex will have to take care to update the branch before reading
from it, to make sure it has merged in current info from remotes. Happily,
this can be done quite inexpensively, just a git-show-ref to list
branches, and a minimalized git-log to see if there are unmerged changes
on the branches. To further speed up, it will be done only once per
git-annex run, max.
---
 Annex.hs  |  2 ++
 Branch.hs | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 Branch.hs

diff --git a/Annex.hs b/Annex.hs
index 0d6b309cca..bede0cbfb7 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -39,6 +39,7 @@ data AnnexState = AnnexState
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
+	, updated :: Bool
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
@@ -59,6 +60,7 @@ newState allbackends gitrepo = AnnexState
 	, quiet = False
 	, force = False
 	, fast = False
+	, updated = False
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, defaultkey = Nothing
diff --git a/Branch.hs b/Branch.hs
new file mode 100644
index 0000000000..db8dc23da8
--- /dev/null
+++ b/Branch.hs
@@ -0,0 +1,50 @@
+{- git-annex branch management
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Branch where
+
+import Control.Monad (unless)
+import Control.Monad.State (liftIO)
+
+import GitUnionMerge
+import GitRepo as Git
+import qualified Annex
+import Utility
+import Types
+import Messages
+
+name :: String
+name = "git-annex"
+
+fullname :: String
+fullname = "refs/heads/" ++ name
+
+{- Ensures that the branch is up-to-date; should be called before
+ - data is read from it. Runs only once per git-annex run. -}
+update :: Annex ()
+update = do
+	updated <- Annex.getState Annex.updated
+	unless updated $ do
+		g <- Annex.gitRepo
+		refs <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+		mapM_ updateRef $ map (last . words) (lines refs)
+		Annex.changeState $ \s -> s { Annex.updated = True }
+
+{- Ensures that a given ref has been merged into the local git-annex branch. -}
+updateRef :: String -> Annex ()
+updateRef ref
+	| ref == fullname = return ()
+	| otherwise = do
+		g <- Annex.gitRepo
+		diffs <- liftIO $ Git.pipeRead g [
+			Param "log",
+			Param (name++".."++ref),
+			Params "--oneline -n1"
+			]
+		unless (null diffs) $ do
+			showSideAction "merging " ++ ref ++ " into " ++ name ++ "..."
+			liftIO $ unionMerge g fullname ref fullname

From 9a1f0fcee2bbd96e2cebbc3df6cac8d64d38b8a5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 14:34:08 -0400
Subject: [PATCH 1879/2835] start v3

---
 Version.hs        | 4 ++--
 doc/upgrades.mdwn | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Version.hs b/Version.hs
index 947af8cef5..72d06f6639 100644
--- a/Version.hs
+++ b/Version.hs
@@ -20,13 +20,13 @@ import Config
 type Version = String
 
 defaultVersion :: Version
-defaultVersion = "2"
+defaultVersion = "3"
 
 supportedVersions :: [Version]
 supportedVersions = [defaultVersion]
 
 upgradableVersions :: [Version]
-upgradableVersions = ["0", "1"]
+upgradableVersions = ["0", "1", "2"]
 
 versionField :: String
 versionField = "annex.version"
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 0a07ef7aa8..516e5b3bb8 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -45,6 +45,10 @@ Example upgrade process:
 
 ## Upgrade events, so far
 
+### v2 -> v3 (git-annex version 0.20110610 to version 0.20110622)
+
+Involved moving the .git-annex/ directory into a separate git-annex branch.
+
 ### v1 -> v2 (git-annex version 0.23 to version 0.20110316)
 
 Involved adding hashing to .git/annex/ and changing the names of all keys.

From a5e6802b5b6f9354e065936998d9882e8ceecb5b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 14:44:56 -0400
Subject: [PATCH 1880/2835] typos in comments

---
 Locations.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index da781ac83a..8f7b11a5c2 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -94,7 +94,7 @@ gitAnnexObjectDir r
 	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  objectDir
 	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  objectDir
 
-{- .git-annex/tmp/ is used for temp files -}
+{- .git/annex/tmp/ is used for temp files -}
 gitAnnexTmpDir :: Git.Repo -> FilePath
 gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 
@@ -102,7 +102,7 @@ gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 gitAnnexTmpLocation :: Git.Repo -> Key -> FilePath
 gitAnnexTmpLocation r key = gitAnnexTmpDir r  keyFile key
 
-{- .git-annex/bad/ is used for bad files found during fsck -}
+{- .git/annex/bad/ is used for bad files found during fsck -}
 gitAnnexBadDir :: Git.Repo -> FilePath
 gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
 

From 7e7428f173ba1b72b4de69fd482f44161ee84420 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 16:08:09 -0400
Subject: [PATCH 1881/2835] refactor

---
 Branch.hs          | 14 ++++++++++++--
 GitRepo.hs         | 10 ++++++++++
 GitUnionMerge.hs   | 22 +++-------------------
 git-union-merge.hs | 20 +++++++++++++++++++-
 4 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index db8dc23da8..4b62fd645e 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -1,11 +1,14 @@
-{- git-annex branch management
+{- management of the git-annex branch
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Branch where
+module Branch (
+	update,
+	change
+) where
 
 import Control.Monad (unless)
 import Control.Monad.State (liftIO)
@@ -48,3 +51,10 @@ updateRef ref
 		unless (null diffs) $ do
 			showSideAction "merging " ++ ref ++ " into " ++ name ++ "..."
 			liftIO $ unionMerge g fullname ref fullname
+
+{- Stages the content of a file to be committed to the branch. -}
+change :: FilePath -> String -> Annex ()
+change file content = do
+	update
+
+{- Commits staged changes to the branch. -}
diff --git a/GitRepo.hs b/GitRepo.hs
index 9f4a38a5fb..11511f77d6 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -58,6 +58,7 @@ module GitRepo (
 	typeChangedStagedFiles,
 	repoAbsPath,
 	reap,
+	withIndex,
 
 	prop_idempotent_deencode
 ) where
@@ -82,6 +83,7 @@ import Codec.Binary.UTF8.String (encode)
 import Text.Printf
 import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
+import System.Posix.Env (setEnv, unsetEnv)
 
 import Utility
 
@@ -379,6 +381,14 @@ reap = do
 	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
 	maybe (return ()) (const reap) r
 
+{- Runs an action using a specified index file. -}
+withIndex :: FilePath -> IO a -> IO a
+withIndex index a = do
+	setEnv "GIT_INDEX_FILE" index True
+	r <- a
+	unsetEnv "GIT_INDEX_FILE"
+	return r
+
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo $
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index dde4b7a043..8aa04f53a3 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -12,7 +12,6 @@ module GitUnionMerge (
 import System.FilePath
 import System.Directory
 import System.Cmd.Utils
-import System.Posix.Env (setEnv, unsetEnv)
 import Control.Monad (when)
 import Data.List
 import Data.Maybe
@@ -21,27 +20,12 @@ import Data.String.Utils
 import qualified GitRepo as Git
 import Utility
 
+{- Performs a union merge. Should be run with a temporary index file
+ - configured by Git.withIndex. -}
 unionMerge :: Git.Repo -> String -> String -> String -> IO ()
 unionMerge g aref bref newref = do
-	setup g
 	stage g aref bref
 	commit g aref bref newref
-	cleanup g
-
-tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
-
-{- Configures git to use a temporary index file. -}
-setup :: Git.Repo -> IO ()
-setup g = do
-	cleanup g -- idempotency
-	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
-
-cleanup :: Git.Repo -> IO ()
-cleanup g = do
-	unsetEnv "GIT_INDEX_FILE"
-	e' <- doesFileExist (tmpIndex g)
-	when e' $ removeFile (tmpIndex g)
 
 {- Stages the content of both refs into the index. -}
 stage :: Git.Repo -> String -> String -> IO ()
@@ -89,7 +73,7 @@ stage g aref bref = do
 					unionmerge content
 			return $ Just $ ls_tree_line sha file
 
-{- Commits the index into the specified branch. -}
+{- Commits the index into the specified branch, as a merge commit. -}
 commit :: Git.Repo -> String -> String -> String -> IO ()
 commit g aref bref newref = do
 	tree <- getSha "write-tree"  $
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 62a79a4c22..12f49adc6f 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -6,6 +6,9 @@
  -}
 
 import System.Environment
+import System.FilePath
+import System.Directory
+import Control.Monad (when)
 
 import GitUnionMerge
 import qualified GitRepo as Git
@@ -16,6 +19,18 @@ header = "Usage: git-union-merge ref ref newref"
 usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
+tmpIndex :: Git.Repo -> FilePath
+tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
+
+setup :: Git.Repo -> IO ()
+setup g = do
+	cleanup g -- idempotency
+
+cleanup :: Git.Repo -> IO ()
+cleanup g = do
+	e' <- doesFileExist (tmpIndex g)
+	when e' $ removeFile (tmpIndex g)
+
 parseArgs :: IO [String]
 parseArgs = do
 	args <- getArgs
@@ -27,4 +42,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
-	unionMerge g aref bref newref
+	Git.withIndex (tmpIndex g) $ do
+		setup g
+		unionMerge g aref bref newref
+		cleanup g

From 40ec8a9726586f24357a5ae2057a092a971c1046 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 17:39:45 -0400
Subject: [PATCH 1882/2835] Branch module complete

Refactored some code that it needs into GitRepo.
---
 Branch.hs          | 76 +++++++++++++++++++++++++++++++++++++++++-----
 GitRepo.hs         | 50 ++++++++++++++++++++++++------
 GitUnionMerge.hs   | 60 +++++++++++++-----------------------
 git-union-merge.hs |  8 ++---
 4 files changed, 135 insertions(+), 59 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 4b62fd645e..9152a03250 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -12,26 +12,63 @@ module Branch (
 
 import Control.Monad (unless)
 import Control.Monad.State (liftIO)
+import System.FilePath
+import System.Directory
+import Data.String.Utils
+import System.Cmd.Utils
 
 import GitUnionMerge
-import GitRepo as Git
+import qualified GitRepo as Git
 import qualified Annex
 import Utility
 import Types
 import Messages
 
+{- Name of the branch that is used to store git-annex's information. -}
 name :: String
 name = "git-annex"
 
+{- Fully qualified name of the branch. -}
 fullname :: String
 fullname = "refs/heads/" ++ name
 
+{- A separate index file for the branch. -}
+index :: Git.Repo -> FilePath
+index g = Git.workTree g  Git.gitDir g  "index." ++ name
+
+{- Populates the branch's index file with the current branch contents.
+ - 
+ - Usually, this is only done when the index doesn't yet exist, and
+ - the index is used to build up changes to be commited to the branch.
+ -}
+genIndex :: FilePath -> Git.Repo -> IO ()
+genIndex f g = do
+	ls <- Git.pipeNullSplit g $
+		map Param ["ls-tree", "-z", "-r", "--full-tree", fullname]
+	forceSuccess =<< Git.pipeWrite g
+		(map Param ["update-index", "-z", "--index-info"])
+		(join "\0" ls)
+
+{- Runs an action using the branch's index file. -}
+withIndex :: Annex a -> Annex a
+withIndex a = do
+	g <- Annex.gitRepo
+	let f = index g
+	liftIO $ Git.useIndex f
+
+	e <- liftIO $ doesFileExist f
+	unless e $ liftIO $ genIndex f g
+
+	r <- a
+	liftIO $ Git.useDefaultIndex
+	return r
+
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
 	updated <- Annex.getState Annex.updated
-	unless updated $ do
+	unless updated $ withIndex $ do
 		g <- Annex.gitRepo
 		refs <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 		mapM_ updateRef $ map (last . words) (lines refs)
@@ -49,12 +86,37 @@ updateRef ref
 			Params "--oneline -n1"
 			]
 		unless (null diffs) $ do
-			showSideAction "merging " ++ ref ++ " into " ++ name ++ "..."
-			liftIO $ unionMerge g fullname ref fullname
+			showSideAction $ "merging " ++ ref ++ " into " ++ name ++ "..."
+			liftIO $ unionMerge g fullname ref fullname True
 
-{- Stages the content of a file to be committed to the branch. -}
+{- Stages the content of a file into the branch's index. -}
 change :: FilePath -> String -> Annex ()
-change file content = do
-	update
+change file content = update >> do
+	g <- Annex.gitRepo
+	sha <- liftIO $ Git.hashObject g content
+	withIndex $ liftIO $ Git.run g "update-index"
+		[ Params "--add --cacheinfo 100644 ",
+		  Param sha, File file]
 
 {- Commits staged changes to the branch. -}
+commit :: String -> Annex ()
+commit message = withIndex $ do
+	g <- Annex.gitRepo
+	-- It would be expensive to check if anything needs to be
+	-- committed, so --allow-empty is used.
+	liftIO $ Git.run g "commit"
+		[Param "--allow-empty", Param "-m", Param message]
+
+{- Gets the content of a file on the branch, or content staged in the index
+ - if it's newer. Returns an empty string if the file didn't exist yet. -}
+get :: FilePath -> Annex String
+get file = withIndex $ do
+	g <- Annex.gitRepo
+	liftIO $ catch (cat g) (const $ return "")
+	where
+		-- To avoid stderr from cat-file when file does not exist,
+		-- first run it with -e to check that it exists.
+		cat g = do
+			Git.run g "cat-file" [Param "-e", catfile]
+			Git.pipeRead g [Param "cat-file", Param "blob", catfile]
+		catfile = Param $ ':':file
diff --git a/GitRepo.hs b/GitRepo.hs
index 11511f77d6..91ddf6dca9 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -58,12 +58,16 @@ module GitRepo (
 	typeChangedStagedFiles,
 	repoAbsPath,
 	reap,
-	withIndex,
+	useIndex,
+	useDefaultIndex,
+	hashObject,
+	getSha,
+	shaSize,
 
 	prop_idempotent_deencode
 ) where
 
-import Control.Monad (unless)
+import Control.Monad (unless, when)
 import System.Directory
 import System.FilePath
 import System.Posix.Directory
@@ -381,13 +385,41 @@ reap = do
 	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
 	maybe (return ()) (const reap) r
 
-{- Runs an action using a specified index file. -}
-withIndex :: FilePath -> IO a -> IO a
-withIndex index a = do
-	setEnv "GIT_INDEX_FILE" index True
-	r <- a
-	unsetEnv "GIT_INDEX_FILE"
-	return r
+{- Forces git to use the specified index file. -}
+useIndex :: FilePath -> IO ()
+useIndex index = setEnv "GIT_INDEX_FILE" index True
+
+{- Undoes useIndex -}
+useDefaultIndex :: IO ()
+useDefaultIndex = unsetEnv "GIT_INDEX_FILE"
+
+{- Injects some content into git, returning its hash. -}
+hashObject :: Repo -> String -> IO String
+hashObject repo content = getSha subcmd $ do
+	(h, s) <- pipeWriteRead repo (map Param params) content
+	length s `seq` do
+		forceSuccess h
+		reap -- XXX unsure why this is needed
+		return s
+	where
+		subcmd = "hash-object"
+		params = [subcmd, "-w", "--stdin"]
+
+{- Runs an action that causes a git subcommand to emit a sha, and strips
+   any trailing newline, returning the sha. -}
+getSha :: String -> IO String -> IO String
+getSha subcommand a = do
+	t <- a
+	let t' = if last t == '\n'
+		then take (length t - 1) t
+		else t
+	when (length t' /= shaSize) $
+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
+	return t'
+
+{- Size of a git sha. -}
+shaSize :: Int
+shaSize = 40
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 8aa04f53a3..ba9ea79e4f 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -9,10 +9,7 @@ module GitUnionMerge (
 	unionMerge
 ) where
 
-import System.FilePath
-import System.Directory
 import System.Cmd.Utils
-import Control.Monad (when)
 import Data.List
 import Data.Maybe
 import Data.String.Utils
@@ -21,18 +18,24 @@ import qualified GitRepo as Git
 import Utility
 
 {- Performs a union merge. Should be run with a temporary index file
- - configured by Git.withIndex. -}
-unionMerge :: Git.Repo -> String -> String -> String -> IO ()
-unionMerge g aref bref newref = do
-	stage g aref bref
+ - configured by Git.useIndex.
+ -
+ - Use indexpopulated only if the index file already contains exactly the
+ - contents of aref.
+ -}
+unionMerge :: Git.Repo -> String -> String -> String -> Bool -> IO ()
+unionMerge g aref bref newref indexpopulated = do
+	stage g aref bref indexpopulated
 	commit g aref bref newref
 
 {- Stages the content of both refs into the index. -}
-stage :: Git.Repo -> String -> String -> IO ()
-stage g aref bref = do
-	-- Get the contents of aref, as a starting point.
-	ls <- fromgit
-		["ls-tree", "-z", "-r", "--full-tree", aref]
+stage :: Git.Repo -> String -> String -> Bool -> IO ()
+stage g aref bref indexpopulated = do
+	-- Get the contents of aref, as a starting point, unless
+	-- the index is already populated with it.
+	ls <- if indexpopulated
+		then return []
+		else fromgit ["ls-tree", "-z", "-r", "--full-tree", aref]
 	-- Identify files that are different between aref and bref, and
 	-- inject merged versions into git.
 	diff <- fromgit
@@ -45,18 +48,12 @@ stage g aref bref = do
 		fromgit l = Git.pipeNullSplit g (map Param l)
 		togit l content = Git.pipeWrite g (map Param l) content
 			>>= forceSuccess
-		tofromgit l content = do
-			(h, s) <- Git.pipeWriteRead g (map Param l) content
-			length s `seq` do
-				forceSuccess h
-				Git.reap
-				return ((), s)
 
 		pairs [] = []
 		pairs (_:[]) = error "parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
 		
-		nullsha = take shaSize $ repeat '0'
+		nullsha = take Git.shaSize $ repeat '0'
 		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 		unionmerge = unlines . nub . lines
 		
@@ -68,32 +65,17 @@ stage g aref bref = do
 		mergefile' file asha bsha = do
 			let shas = filter (/= nullsha) [asha, bsha]
 			content <- Git.pipeRead g $ map Param ("show":shas)
-			sha <- getSha "hash-object" $
-				tofromgit ["hash-object", "-w", "--stdin"] $
-					unionmerge content
+			sha <- Git.hashObject g $ unionmerge content
 			return $ Just $ ls_tree_line sha file
 
 {- Commits the index into the specified branch, as a merge commit. -}
 commit :: Git.Repo -> String -> String -> String -> IO ()
 commit g aref bref newref = do
-	tree <- getSha "write-tree"  $
+	tree <- Git.getSha "write-tree" $ ignorehandle $
 		pipeFrom "git" ["write-tree"]
-	sha <- getSha "commit-tree" $
+	sha <- Git.getSha "commit-tree" $ ignorehandle $ 
 		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
 			"union merge"
 	Git.run g "update-ref" [Param newref, Param sha]
-
-{- Runs an action that causes a git subcommand to emit a sha, and strips
-   any trailing newline, returning the sha. -}
-getSha :: String -> IO (a, String) -> IO String
-getSha subcommand a = do
-	(_, t) <- a
-	let t' = if last t == '\n'
-		then take (length t - 1) t
-		else t
-	when (length t' /= shaSize) $
-		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return t'
-
-shaSize :: Int
-shaSize = 40
+	where
+		ignorehandle a = return . snd =<< a
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 12f49adc6f..e8ac0a0c54 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -42,7 +42,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
-	Git.withIndex (tmpIndex g) $ do
-		setup g
-		unionMerge g aref bref newref
-		cleanup g
+	Git.useIndex (tmpIndex g)
+	setup g
+	unionMerge g aref bref newref False
+	cleanup g

From 5d20ac5800f6577fb40ebfa7c8d68df43757a3e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 19:09:20 -0400
Subject: [PATCH 1883/2835] export the commit function and generalize

---
 GitUnionMerge.hs   | 21 ++++++++++++---------
 git-union-merge.hs |  4 ++--
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index ba9ea79e4f..bc12cbe275 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -6,7 +6,9 @@
  -}
 
 module GitUnionMerge (
-	unionMerge
+	merge,
+	stage,
+	commit
 ) where
 
 import System.Cmd.Utils
@@ -23,10 +25,10 @@ import Utility
  - Use indexpopulated only if the index file already contains exactly the
  - contents of aref.
  -}
-unionMerge :: Git.Repo -> String -> String -> String -> Bool -> IO ()
-unionMerge g aref bref newref indexpopulated = do
+merge :: Git.Repo -> String -> String -> String -> Bool -> IO ()
+merge g aref bref newref indexpopulated = do
 	stage g aref bref indexpopulated
-	commit g aref bref newref
+	commit g "union merge" newref [aref, bref]
 
 {- Stages the content of both refs into the index. -}
 stage :: Git.Repo -> String -> String -> Bool -> IO ()
@@ -68,14 +70,15 @@ stage g aref bref indexpopulated = do
 			sha <- Git.hashObject g $ unionmerge content
 			return $ Just $ ls_tree_line sha file
 
-{- Commits the index into the specified branch, as a merge commit. -}
-commit :: Git.Repo -> String -> String -> String -> IO ()
-commit g aref bref newref = do
+{- Commits the index into the specified branch. If refs are specified,
+ - commits a merge. -}
+commit :: Git.Repo -> String -> String -> [String] -> IO ()
+commit g message newref mergedrefs = do
 	tree <- Git.getSha "write-tree" $ ignorehandle $
 		pipeFrom "git" ["write-tree"]
 	sha <- Git.getSha "commit-tree" $ ignorehandle $ 
-		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
-			"union merge"
+		pipeBoth "git" (["commit-tree", tree] ++ ps) message
 	Git.run g "update-ref" [Param newref, Param sha]
 	where
 		ignorehandle a = return . snd =<< a
+		ps = concatMap (\r -> ["-p", r]) mergedrefs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e8ac0a0c54..f02db6be3d 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -10,7 +10,7 @@ import System.FilePath
 import System.Directory
 import Control.Monad (when)
 
-import GitUnionMerge
+import qualified GitUnionMerge
 import qualified GitRepo as Git
 
 header :: String
@@ -44,5 +44,5 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	Git.useIndex (tmpIndex g)
 	setup g
-	unionMerge g aref bref newref False
+	GitUnionMerge.merge g aref bref newref False
 	cleanup g

From 5e0adb26375413aaeef7c41dd72e8761e3cc1ada Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 19:11:55 -0400
Subject: [PATCH 1884/2835] fixes

make commit commit to the right branch

when getting content from the branch, update first
---
 Branch.hs | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 9152a03250..82ff5fcad7 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -7,7 +7,9 @@
 
 module Branch (
 	update,
-	change
+	get,
+	change,
+	commit
 ) where
 
 import Control.Monad (unless)
@@ -17,8 +19,8 @@ import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
 
-import GitUnionMerge
 import qualified GitRepo as Git
+import qualified GitUnionMerge
 import qualified Annex
 import Utility
 import Types
@@ -91,7 +93,7 @@ updateRef ref
 
 {- Stages the content of a file into the branch's index. -}
 change :: FilePath -> String -> Annex ()
-change file content = update >> do
+change file content = do
 	g <- Annex.gitRepo
 	sha <- liftIO $ Git.hashObject g content
 	withIndex $ liftIO $ Git.run g "update-index"
@@ -102,17 +104,15 @@ change file content = update >> do
 commit :: String -> Annex ()
 commit message = withIndex $ do
 	g <- Annex.gitRepo
-	-- It would be expensive to check if anything needs to be
-	-- committed, so --allow-empty is used.
-	liftIO $ Git.run g "commit"
-		[Param "--allow-empty", Param "-m", Param message]
+	liftIO $ GitUnionMerge.commit g message branch []
 
 {- Gets the content of a file on the branch, or content staged in the index
  - if it's newer. Returns an empty string if the file didn't exist yet. -}
 get :: FilePath -> Annex String
-get file = withIndex $ do
-	g <- Annex.gitRepo
-	liftIO $ catch (cat g) (const $ return "")
+get file = update >> do
+	withIndex $ do
+		g <- Annex.gitRepo
+		liftIO $ catch (cat g) (const $ return "")
 	where
 		-- To avoid stderr from cat-file when file does not exist,
 		-- first run it with -e to check that it exists.

From 7a693394f4acd05d1dedbaffec0762007a954d9e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 19:52:40 -0400
Subject: [PATCH 1885/2835] allow for union merges between a tree and the
 content in the index

This is needed for robust handling of the git-annex branch. Since changes
are staged to its index as git-annex runs, and committed at the end,
it's possible that git-annex is interrupted, and leaves a dirty index.

When it next runs, it needs to be able to merge the git-annex branch
as necessary, without losing the existing changes in the index.

Note that this assumes that the git-annex branch is only modified by
git-annex. Any changes to it will be lost when git-annex updates the
branch. I don't see a good, inexpensive way to find changes in
the git-annex branch that arn't in the index, and union merging the
git-annex branch into the index every time would likewise be expensive.
---
 Branch.hs          | 31 ++++++++++++------
 GitUnionMerge.hs   | 79 +++++++++++++++++++++++++++-------------------
 git-union-merge.hs |  3 +-
 3 files changed, 70 insertions(+), 43 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 82ff5fcad7..9d7b1b0941 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -12,12 +12,13 @@ module Branch (
 	commit
 ) where
 
-import Control.Monad (unless)
+import Control.Monad (unless, liftM)
 import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
+import Data.Maybe
 
 import qualified GitRepo as Git
 import qualified GitUnionMerge
@@ -72,14 +73,18 @@ update = do
 	updated <- Annex.getState Annex.updated
 	unless updated $ withIndex $ do
 		g <- Annex.gitRepo
-		refs <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-		mapM_ updateRef $ map (last . words) (lines refs)
+		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+		let refs = map (last . words) (lines r)
+		updated <- catMaybes `liftM` mapM updateRef refs
+		unless (null updated) $ liftIO $
+			GitUnionMerge.commit g "update" fullname
+				(fullname:updated)
 		Annex.changeState $ \s -> s { Annex.updated = True }
 
-{- Ensures that a given ref has been merged into the local git-annex branch. -}
-updateRef :: String -> Annex ()
+{- Ensures that a given ref has been merged into the index. -}
+updateRef :: String -> Annex (Maybe String)
 updateRef ref
-	| ref == fullname = return ()
+	| ref == fullname = return Nothing
 	| otherwise = do
 		g <- Annex.gitRepo
 		diffs <- liftIO $ Git.pipeRead g [
@@ -87,9 +92,15 @@ updateRef ref
 			Param (name++".."++ref),
 			Params "--oneline -n1"
 			]
-		unless (null diffs) $ do
-			showSideAction $ "merging " ++ ref ++ " into " ++ name ++ "..."
-			liftIO $ unionMerge g fullname ref fullname True
+		if (null diffs)
+			then return Nothing
+			else do
+				showSideAction $ "merging " ++ ref ++ " into " ++ name ++ "..."
+				-- By passing only one ref, it is actually
+				-- merged into the index, preserving any
+				-- changes that may already be staged.
+				liftIO $ GitUnionMerge.merge g [ref]
+				return $ Just ref
 
 {- Stages the content of a file into the branch's index. -}
 change :: FilePath -> String -> Annex ()
@@ -104,7 +115,7 @@ change file content = do
 commit :: String -> Annex ()
 commit message = withIndex $ do
 	g <- Annex.gitRepo
-	liftIO $ GitUnionMerge.commit g message branch []
+	liftIO $ GitUnionMerge.commit g message fullname []
 
 {- Gets the content of a file on the branch, or content staged in the index
  - if it's newer. Returns an empty string if the file didn't exist yet. -}
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index bc12cbe275..82f01cc0ff 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -7,7 +7,6 @@
 
 module GitUnionMerge (
 	merge,
-	stage,
 	commit
 ) where
 
@@ -19,38 +18,54 @@ import Data.String.Utils
 import qualified GitRepo as Git
 import Utility
 
-{- Performs a union merge. Should be run with a temporary index file
- - configured by Git.useIndex.
+{- Performs a union merge between two branches, staging it in the index.
+ - Any previously staged changes in the index will be lost.
  -
- - Use indexpopulated only if the index file already contains exactly the
- - contents of aref.
+ - When only one branch is specified, it is merged into the index.
+ - In this case, previously staged changes in the index are preserved.
+ -
+ - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: Git.Repo -> String -> String -> String -> Bool -> IO ()
-merge g aref bref newref indexpopulated = do
-	stage g aref bref indexpopulated
-	commit g "union merge" newref [aref, bref]
+merge :: Git.Repo -> [String] -> IO ()
+merge g (x:y:[]) = do
+	a <- ls_tree g x
+	b <- merge_trees g x y
+	update_index g (a++b)
+merge g [x] = merge_tree_index g x >>= update_index g
+merge _ _ = error "wrong number of branches to merge"
 
-{- Stages the content of both refs into the index. -}
-stage :: Git.Repo -> String -> String -> Bool -> IO ()
-stage g aref bref indexpopulated = do
-	-- Get the contents of aref, as a starting point, unless
-	-- the index is already populated with it.
-	ls <- if indexpopulated
-		then return []
-		else fromgit ["ls-tree", "-z", "-r", "--full-tree", aref]
-	-- Identify files that are different between aref and bref, and
-	-- inject merged versions into git.
-	diff <- fromgit
-		["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", aref, bref]
-	ls' <- mapM mergefile (pairs diff)
-	-- Populate the index file. Later lines override earlier ones.
-	togit ["update-index", "-z", "--index-info"]
-		(join "\0" $ ls++catMaybes ls')
+{- Feeds a list into update-index. Later items in the list can override
+ - earlier ones, so the list can be generated from any combination of
+ - ls_tree, merge_trees, and merge_tree. -}
+update_index :: Git.Repo -> [String] -> IO ()
+update_index g l =  togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
-		fromgit l = Git.pipeNullSplit g (map Param l)
-		togit l content = Git.pipeWrite g (map Param l) content
+		togit ps content = Git.pipeWrite g (map Param ps) content
 			>>= forceSuccess
 
+{- Gets the contents of a tree in a format suitable for update_index. -}
+ls_tree :: Git.Repo -> String -> IO [String]
+ls_tree g x = Git.pipeNullSplit g $ 
+	map Param ["ls-tree", "-z", "-r", "--full-tree", x]
+
+{- For merging two trees. -}
+merge_trees :: Git.Repo -> String -> String -> IO [String]
+merge_trees g x y = calc_merge g
+	["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", x, y]
+
+{- For merging a single tree into the index. -}
+merge_tree_index :: Git.Repo -> String -> IO [String]
+merge_tree_index g x = calc_merge g
+	["diff-index", "--raw", "-z", "-r", "--no-renames", "-l0", x]
+
+{- Calculates how to perform a merge, using git to get a raw diff,
+ - and returning a list suitable for update_index. -}
+calc_merge :: Git.Repo -> [String] -> IO [String]
+calc_merge g differ = do
+	diff <- Git.pipeNullSplit g $ map Param differ
+	l <- mapM mergefile (pairs diff)
+	return $ catMaybes l
+	where
 		pairs [] = []
 		pairs (_:[]) = error "parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
@@ -62,7 +77,7 @@ stage g aref bref indexpopulated = do
 		mergefile (info, file) = do
 			let [_colonamode, _bmode, asha, bsha, _status] = words info
 			if bsha == nullsha
-				then return Nothing -- already staged from aref
+				then return Nothing -- already staged
 				else mergefile' file asha bsha
 		mergefile' file asha bsha = do
 			let shas = filter (/= nullsha) [asha, bsha]
@@ -70,10 +85,10 @@ stage g aref bref indexpopulated = do
 			sha <- Git.hashObject g $ unionmerge content
 			return $ Just $ ls_tree_line sha file
 
-{- Commits the index into the specified branch. If refs are specified,
- - commits a merge. -}
+{- Commits the index into the specified branch, 
+ - with the specified parent refs. -}
 commit :: Git.Repo -> String -> String -> [String] -> IO ()
-commit g message newref mergedrefs = do
+commit g message newref parentrefs = do
 	tree <- Git.getSha "write-tree" $ ignorehandle $
 		pipeFrom "git" ["write-tree"]
 	sha <- Git.getSha "commit-tree" $ ignorehandle $ 
@@ -81,4 +96,4 @@ commit g message newref mergedrefs = do
 	Git.run g "update-ref" [Param newref, Param sha]
 	where
 		ignorehandle a = return . snd =<< a
-		ps = concatMap (\r -> ["-p", r]) mergedrefs
+		ps = concatMap (\r -> ["-p", r]) parentrefs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index f02db6be3d..7c0c1cd843 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -44,5 +44,6 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	Git.useIndex (tmpIndex g)
 	setup g
-	GitUnionMerge.merge g aref bref newref False
+	GitUnionMerge.merge g [aref, bref]
+	GitUnionMerge.commit g "union merge" newref [aref, bref]
 	cleanup g

From 818ae0c6da1d74e02d74e15b8832a58ca9514fef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 20:21:33 -0400
Subject: [PATCH 1886/2835] docs for v3

---
 debian/NEWS      | 16 ++++++++++++++++
 debian/changelog |  2 ++
 2 files changed, 18 insertions(+)

diff --git a/debian/NEWS b/debian/NEWS
index a8b258bfe4..4e085bb002 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,3 +1,19 @@
+git-annex (0.20110622) unstable; urgency=low
+
+  There has been another change to the git-annex data store.
+  Use `git annex upgrade` to migrate your repositories to the new
+  layout.
+
+  The significant change this time is that the .git-annex/ directory
+  is gone; instead there is a git-annex branch that is automatically
+  maintained by git-annex, and encapsulates all its state nicely out
+  of your way.
+
+  You should make sure you include the git-annex branch when
+  git pushing and pulling.
+
+ -- Joey Hess   Tue, 21 Jun 2011 20:18:00 -0400
+
 git-annex (0.20110316) experimental; urgency=low
 
   This version reorganises the layout of git-annex's files in your repository.
diff --git a/debian/changelog b/debian/changelog
index 6439eb89fa..360121cbf7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,7 @@
 git-annex (0.20110611) UNRELEASED; urgency=low
 
+  * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
+
   * rsync is now used when copying files from repos on other filesystems.
     cp is still used when copying file from repos on the same filesystem,
     since --reflink=auto can make it significantly faster on filesystems

From 1cca8b4edb963b980e64ed0b7de7814b5380e214 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 13:59:42 -0400
Subject: [PATCH 1887/2835] rework core merge code

More likely to be 100% correct now, I think.
---
 GitUnionMerge.hs | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 82f01cc0ff..267376ed57 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -36,7 +36,7 @@ merge _ _ = error "wrong number of branches to merge"
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
- - ls_tree, merge_trees, and merge_tree. -}
+ - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Git.Repo -> [String] -> IO ()
 update_index g l =  togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
@@ -63,27 +63,29 @@ merge_tree_index g x = calc_merge g
 calc_merge :: Git.Repo -> [String] -> IO [String]
 calc_merge g differ = do
 	diff <- Git.pipeNullSplit g $ map Param differ
-	l <- mapM mergefile (pairs diff)
+	l <- mapM (mergeFile g) (pairs diff)
 	return $ catMaybes l
 	where
 		pairs [] = []
-		pairs (_:[]) = error "parse error"
+		pairs (_:[]) = error "calc_merge parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
-		
+
+{- Given an info line from a git raw diff, and the filename, generates
+ - a line suitable for update_index that union merges the two sides of the
+ - diff. -}
+mergeFile :: Git.Repo -> (String, FilePath) -> IO (Maybe String)
+mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
+	[] -> return Nothing
+	(sha:[]) -> return $ Just $ ls_tree_line sha
+	shas -> do
+		content <- Git.pipeRead g $ map Param ("show":shas)
+		sha <- Git.hashObject g $ unionmerge content
+		return $ Just $ ls_tree_line sha
+	where
+		[_colonamode, _bmode, asha, bsha, _status] = words info
+		ls_tree_line sha = "100644 blob " ++ sha ++ "\t" ++ file
 		nullsha = take Git.shaSize $ repeat '0'
-		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 		unionmerge = unlines . nub . lines
-		
-		mergefile (info, file) = do
-			let [_colonamode, _bmode, asha, bsha, _status] = words info
-			if bsha == nullsha
-				then return Nothing -- already staged
-				else mergefile' file asha bsha
-		mergefile' file asha bsha = do
-			let shas = filter (/= nullsha) [asha, bsha]
-			content <- Git.pipeRead g $ map Param ("show":shas)
-			sha <- Git.hashObject g $ unionmerge content
-			return $ Just $ ls_tree_line sha file
 
 {- Commits the index into the specified branch, 
  - with the specified parent refs. -}

From 78a325b09315efd593e6b729de18f15871a0d643 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 14:18:49 -0400
Subject: [PATCH 1888/2835] add a small cache of the most recently accessed
 item from the git-annex branch

This will speed up typical cases like git-annex get, which currently
has to read the location log once, then read it a second time in order to
add a line to it. Since these reads now involve more than just reading
in a file, it seemed good to add a cache layer.

Only the most recent thing needs to be cached, because git-annex has
good locality; it operates on one file at a time, and only cares
about one item from the branch per file.
---
 Annex.hs        |  7 +++++--
 Branch.hs       | 21 +++++++++++++++++----
 Types/Branch.hs | 16 ++++++++++++++++
 3 files changed, 38 insertions(+), 6 deletions(-)
 create mode 100644 Types/Branch.hs

diff --git a/Annex.hs b/Annex.hs
index bede0cbfb7..b6834d6dd7 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -23,6 +23,7 @@ import GitQueue
 import Types.Backend
 import Types.Remote
 import Types.Crypto
+import Types.Branch
 import TrustLevel
 import Types.UUID
 
@@ -39,7 +40,8 @@ data AnnexState = AnnexState
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
-	, updated :: Bool
+	, branchupdated :: Bool
+	, branchcache :: BranchCache
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
@@ -60,7 +62,8 @@ newState allbackends gitrepo = AnnexState
 	, quiet = False
 	, force = False
 	, fast = False
-	, updated = False
+	, branchupdated = False
+	, branchcache = emptyBranchCache
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, defaultkey = Nothing
diff --git a/Branch.hs b/Branch.hs
index 9d7b1b0941..442f47ed53 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -20,6 +20,7 @@ import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
 
+import Types.Branch
 import qualified GitRepo as Git
 import qualified GitUnionMerge
 import qualified Annex
@@ -66,11 +67,19 @@ withIndex a = do
 	liftIO $ Git.useDefaultIndex
 	return r
 
+{- There is a small cache of the most recently accessed item from the
+ - branch. git-annex has good locality, so that is enough. -}
+setCache :: FilePath -> String -> Annex ()
+setCache file content = Annex.changeState $ \s -> s { Annex.branchcache = BranchCache (Just file) content }
+
+invalidateCache :: Annex ()
+invalidateCache = Annex.changeState $ \s -> s { Annex.branchcache = emptyBranchCache }
+
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
-	updated <- Annex.getState Annex.updated
+	updated <- Annex.getState Annex.branchupdated
 	unless updated $ withIndex $ do
 		g <- Annex.gitRepo
 		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
@@ -79,7 +88,8 @@ update = do
 		unless (null updated) $ liftIO $
 			GitUnionMerge.commit g "update" fullname
 				(fullname:updated)
-		Annex.changeState $ \s -> s { Annex.updated = True }
+		Annex.changeState $ \s -> s { Annex.branchupdated = True }
+		invalidateCache
 
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: String -> Annex (Maybe String)
@@ -108,8 +118,9 @@ change file content = do
 	g <- Annex.gitRepo
 	sha <- liftIO $ Git.hashObject g content
 	withIndex $ liftIO $ Git.run g "update-index"
-		[ Params "--add --cacheinfo 100644 ",
+		[ Param "--add", Param "--cacheinfo", Param "100644",
 		  Param sha, File file]
+	setCache file content
 
 {- Commits staged changes to the branch. -}
 commit :: String -> Annex ()
@@ -123,7 +134,9 @@ get :: FilePath -> Annex String
 get file = update >> do
 	withIndex $ do
 		g <- Annex.gitRepo
-		liftIO $ catch (cat g) (const $ return "")
+		content <- liftIO $ catch (cat g) (const $ return "")
+		setCache file content
+		return content
 	where
 		-- To avoid stderr from cat-file when file does not exist,
 		-- first run it with -e to check that it exists.
diff --git a/Types/Branch.hs b/Types/Branch.hs
new file mode 100644
index 0000000000..c0ccb5ca03
--- /dev/null
+++ b/Types/Branch.hs
@@ -0,0 +1,16 @@
+{- git-annex branch data types
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.Branch where
+
+data BranchCache = BranchCache {
+	cachedFile :: Maybe FilePath,
+	cachedContent :: String
+}
+
+emptyBranchCache :: BranchCache
+emptyBranchCache = BranchCache Nothing ""

From d3f0106f2ed15a4e4abbc09cc3e985a27dfee662 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 14:27:50 -0400
Subject: [PATCH 1889/2835] move LocationLog into Annex monad from IO

It will need to run in Annex so it can use Branch
---
 Backend/File.hs    |  5 ++---
 Command/Fsck.hs    |  2 +-
 Command/Unused.hs  |  4 ++--
 Command/Whereis.hs |  4 +---
 Content.hs         |  2 +-
 LocationLog.hs     | 25 +++++++++++++------------
 Remote.hs          |  2 +-
 Upgrade/V1.hs      |  6 +++---
 test.hs            |  2 +-
 9 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 386af02663..20cb3e95ad 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -14,7 +14,6 @@
 
 module Backend.File (backend, checkKey) where
 
-import Control.Monad.State (liftIO)
 import Data.List
 import Data.String.Utils
 
@@ -132,7 +131,7 @@ showLocations :: Key -> [UUID] -> Annex ()
 showLocations key exclude = do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	uuids <- liftIO $ keyLocations g key
+	uuids <- keyLocations g key
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
@@ -190,7 +189,7 @@ checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
 	g <- Annex.gitRepo
-	locations <- liftIO $ keyLocations g key
+	locations <- keyLocations g key
 	untrusted <- trustGet UnTrusted
 	let untrustedlocations = intersect untrusted locations
 	let safelocations = filter (`notElem` untrusted) locations
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index adfd702de7..7c840d5288 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -60,7 +60,7 @@ verifyLocationLog key file = do
 		preventWrite (parentDir f)
 
 	u <- getUUID g
-        uuids <- liftIO $ keyLocations g key
+        uuids <- keyLocations g key
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5422dad69f..4389b2209e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -68,7 +68,7 @@ checkRemoteUnused' r = do
 	showNote $ "checking for unused data..."
 	g <- Annex.gitRepo
 	referenced <- getKeysReferenced
-	logged <- liftIO $ loggedKeys g
+	logged <- loggedKeys g
 	remotehas <- filterM isthere logged
 	let remoteunused = remotehas `exclude` referenced
 	let list = number 0 remoteunused
@@ -79,7 +79,7 @@ checkRemoteUnused' r = do
 	where
 		isthere k = do
 			g <- Annex.gitRepo
-			us <- liftIO $ keyLocations g k
+			us <- keyLocations g k
 			return $ uuid `elem` us
 		uuid = Remote.uuid r
 
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 2e0fa15f6f..bcd4a2e228 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,8 +7,6 @@
 
 module Command.Whereis where
 
-import Control.Monad.State (liftIO)
-
 import qualified Annex
 import LocationLog
 import Command
@@ -31,7 +29,7 @@ start file = isAnnexed file $ \(key, _) -> do
 perform :: Key -> CommandPerform
 perform key = do
 	g <- Annex.gitRepo
-	uuids <- liftIO $ keyLocations g key
+	uuids <- keyLocations g key
 	let num = length uuids
 	showNote $ show num ++ " " ++ copiesplural num
 	if null $ uuids
diff --git a/Content.hs b/Content.hs
index 57977ce344..ccd51a553e 100644
--- a/Content.hs
+++ b/Content.hs
@@ -81,7 +81,7 @@ logStatusFor :: UUID -> Key -> LogStatus -> Annex ()
 logStatusFor u key status = do
 	g <- Annex.gitRepo
 	unless (Git.repoIsLocalBare g) $ do
-		logfile <- liftIO $ logChange g key u status
+		logfile <- logChange g key u status
 		rellogfile <- liftIO $ Git.workTreeFile g logfile
 		AnnexQueue.add "add" [Param "--"] rellogfile
 
diff --git a/LocationLog.hs b/LocationLog.hs
index b2d423cf99..1b55abfb29 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -36,6 +36,7 @@ import System.FilePath
 import qualified Data.Map as Map
 import Control.Monad (when)
 import Data.Maybe
+import Control.Monad.State (liftIO)
 
 import qualified GitRepo as Git
 import Utility
@@ -86,7 +87,7 @@ instance Read LogLine where
 
 {- Log a change in the presence of a key's value in a repository,
  - and returns the filename of the logfile. -}
-logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath
+logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex FilePath
 logChange repo key u s = do
 	when (null u) $
 		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
@@ -100,8 +101,8 @@ logChange repo key u s = do
 
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
-readLog :: FilePath -> IO [LogLine]
-readLog file = catch (return . parseLog =<< readFileStrict file) (const $ return [])
+readLog :: FilePath -> Annex [LogLine]
+readLog file = liftIO $ catch (return . parseLog =<< readFileStrict file) (const $ return [])
 
 parseLog :: String -> [LogLine]
 parseLog s = filter parsable $ map read $ lines s
@@ -110,18 +111,18 @@ parseLog s = filter parsable $ map read $ lines s
 		parsable l = status l /= Undefined
 
 {- Writes a set of lines to a log file -}
-writeLog :: FilePath -> [LogLine] -> IO ()
-writeLog file ls = safeWriteFile file (unlines $ map show ls)
+writeLog :: FilePath -> [LogLine] -> Annex ()
+writeLog file ls = liftIO $ safeWriteFile file (unlines $ map show ls)
 
 {- Generates a new LogLine with the current date. -}
-logNow :: LogStatus -> UUID -> IO LogLine
+logNow :: LogStatus -> UUID -> Annex LogLine
 logNow s u = do
-	now <- getPOSIXTime
+	now <- liftIO $ getPOSIXTime
 	return $ LogLine now s u
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
-keyLocations :: Git.Repo -> Key -> IO [UUID]
+keyLocations :: Git.Repo -> Key -> Annex [UUID]
 keyLocations thisrepo key = do
 	ls <- readLog $ logFile thisrepo key
 	ls' <- readLog $ logFileOld thisrepo key
@@ -155,18 +156,18 @@ mapLog m l =
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
-loggedKeys :: Git.Repo -> IO [Key]
+loggedKeys :: Git.Repo -> Annex [Key]
 loggedKeys repo = do
-	exists <- doesDirectoryExist dir
+	exists <- liftIO $ doesDirectoryExist dir
 	if exists
 		then do
 			-- 2 levels of hashing
-			levela <- dirContents dir
+			levela <- liftIO $ dirContents dir
 			levelb <- mapM tryDirContents levela
 			files <- mapM tryDirContents (concat levelb)
 			return $ catMaybes $
 				map (logFileKey . takeFileName) (concat files)
 		else return []
 	where
-		tryDirContents d = catch (dirContents d) (return . const [])
+		tryDirContents d = liftIO $ catch (dirContents d) (return . const [])
 		dir = gitStateDir repo
diff --git a/Remote.hs b/Remote.hs
index d975c2404f..2706bf20b2 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -141,7 +141,7 @@ keyPossibilities key = do
 	trusted <- trustGet Trusted
 
 	-- get uuids of all remotes that are recorded to have the key
-	uuids <- liftIO $ keyLocations g key
+	uuids <- keyLocations g key
 	let validuuids = filter (/= u) uuids
 
 	-- note that validuuids is assumed to not have dups
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 1e634e00e8..c09bd74c1c 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -134,9 +134,9 @@ moveLocationLogs = do
 				-- log files that are not checked into git,
 				-- as well as merging with already upgraded
 				-- logs that have been pulled from elsewhere
-				old <- liftIO $ readLog f
-				new <- liftIO $ readLog dest
-				liftIO $ writeLog dest (old++new)
+				old <- readLog f
+				new <- readLog dest
+				writeLog dest (old++new)
 				AnnexQueue.add "add" [Param "--"] dest
 				AnnexQueue.add "add" [Param "--"] f
 				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] f
diff --git a/test.hs b/test.hs
index 498c7b6806..1eac942b1c 100644
--- a/test.hs
+++ b/test.hs
@@ -611,7 +611,7 @@ checklocationlog f expected = do
 		Just (k, _) -> do
 			uuids <- annexeval $ do
 				g <- Annex.gitRepo
-				liftIO $ LocationLog.keyLocations g k
+				LocationLog.keyLocations g k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
 				expected (thisuuid `elem` uuids)
 

From 8166facaef8357a6e74b1038c082bd86386c2ecd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 15:58:30 -0400
Subject: [PATCH 1890/2835] Branch handling improvements

Support creating the branch.

Unified branch state into a single data type.

Only commit changes when the index has been changed.
---
 Annex.hs             |   8 ++--
 Branch.hs            | 104 +++++++++++++++++++++++++++++++------------
 Types/Branch.hs      |  16 -------
 Types/BranchState.hs |  18 ++++++++
 4 files changed, 97 insertions(+), 49 deletions(-)
 delete mode 100644 Types/Branch.hs
 create mode 100644 Types/BranchState.hs

diff --git a/Annex.hs b/Annex.hs
index b6834d6dd7..2bd090e906 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -23,7 +23,7 @@ import GitQueue
 import Types.Backend
 import Types.Remote
 import Types.Crypto
-import Types.Branch
+import Types.BranchState
 import TrustLevel
 import Types.UUID
 
@@ -40,8 +40,7 @@ data AnnexState = AnnexState
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
-	, branchupdated :: Bool
-	, branchcache :: BranchCache
+	, branchstate :: BranchState
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
@@ -62,8 +61,7 @@ newState allbackends gitrepo = AnnexState
 	, quiet = False
 	, force = False
 	, fast = False
-	, branchupdated = False
-	, branchcache = emptyBranchCache
+	, branchstate = startBranchState
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, defaultkey = Nothing
diff --git a/Branch.hs b/Branch.hs
index 442f47ed53..85928765d8 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -6,13 +6,14 @@
  -}
 
 module Branch (
+	create,
 	update,
 	get,
 	change,
 	commit
 ) where
 
-import Control.Monad (unless, liftM)
+import Control.Monad (unless, when, liftM)
 import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
@@ -20,7 +21,7 @@ import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
 
-import Types.Branch
+import Types.BranchState
 import qualified GitRepo as Git
 import qualified GitUnionMerge
 import qualified Annex
@@ -45,8 +46,8 @@ index g = Git.workTree g  Git.gitDir g  "index." ++ name
  - Usually, this is only done when the index doesn't yet exist, and
  - the index is used to build up changes to be commited to the branch.
  -}
-genIndex :: FilePath -> Git.Repo -> IO ()
-genIndex f g = do
+genIndex :: Git.Repo -> IO ()
+genIndex g = do
 	ls <- Git.pipeNullSplit g $
 		map Param ["ls-tree", "-z", "-r", "--full-tree", fullname]
 	forceSuccess =<< Git.pipeWrite g
@@ -61,26 +62,71 @@ withIndex a = do
 	liftIO $ Git.useIndex f
 
 	e <- liftIO $ doesFileExist f
-	unless e $ liftIO $ genIndex f g
+	unless e $ liftIO $ genIndex g
 
 	r <- a
 	liftIO $ Git.useDefaultIndex
 	return r
 
-{- There is a small cache of the most recently accessed item from the
- - branch. git-annex has good locality, so that is enough. -}
+withIndexUpdate :: Annex a -> Annex a
+withIndexUpdate a = update >> withIndex a
+
+getState :: Annex BranchState
+getState = Annex.getState Annex.branchstate
+
+setState :: BranchState -> Annex ()
+setState state = Annex.changeState $ \s -> s { Annex.branchstate = state }
+
 setCache :: FilePath -> String -> Annex ()
-setCache file content = Annex.changeState $ \s -> s { Annex.branchcache = BranchCache (Just file) content }
+setCache file content = do
+	state <- getState
+	setState state { cachedFile = Just file, cachedContent = content }
+
+setCacheChanged :: FilePath -> String -> Annex ()
+setCacheChanged file content = do
+	state <- getState
+	setState state { cachedFile = Just file, cachedContent = content, branchChanged = True }
 
 invalidateCache :: Annex ()
-invalidateCache = Annex.changeState $ \s -> s { Annex.branchcache = emptyBranchCache }
+invalidateCache = do
+	state <- getState
+	setState state { cachedFile = Nothing, cachedContent = "" }
+
+getCache :: FilePath -> Annex (Maybe String)
+getCache file = getState >>= handle
+	where
+		handle state
+			| cachedFile state == Just file =
+				return $ Just $ cachedContent state
+			| otherwise = return Nothing
+
+{- Creates the branch, if it does not already exist. -}
+create :: Annex ()
+create = do
+	exists <- refexists fullname
+	unless exists $ do
+		g <- Annex.gitRepo
+		inorigin <- refexists origin
+		if inorigin
+			then liftIO $ Git.run g "branch" [Param name, Param origin]
+			else liftIO $ do
+				let f = index g
+				liftIO $ Git.useIndex f
+				GitUnionMerge.commit g "branch created" fullname []
+				liftIO $ Git.useDefaultIndex
+	where
+		origin = "origin/" ++ name
+		refexists ref = do
+			g <- Annex.gitRepo
+			liftIO $ Git.runBool g "show-ref"
+				[Param "--verify", Param "-q", Param ref]
 
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
-	updated <- Annex.getState Annex.branchupdated
-	unless updated $ withIndex $ do
+	state <- Annex.getState Annex.branchstate
+	unless (branchUpdated state) $ withIndex $ do
 		g <- Annex.gitRepo
 		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 		let refs = map (last . words) (lines r)
@@ -88,7 +134,7 @@ update = do
 		unless (null updated) $ liftIO $
 			GitUnionMerge.commit g "update" fullname
 				(fullname:updated)
-		Annex.changeState $ \s -> s { Annex.branchupdated = True }
+		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
 {- Ensures that a given ref has been merged into the index. -}
@@ -120,27 +166,29 @@ change file content = do
 	withIndex $ liftIO $ Git.run g "update-index"
 		[ Param "--add", Param "--cacheinfo", Param "100644",
 		  Param sha, File file]
-	setCache file content
+	setCacheChanged file content
 
-{- Commits staged changes to the branch. -}
+{- Commits any staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = withIndex $ do
-	g <- Annex.gitRepo
-	liftIO $ GitUnionMerge.commit g message fullname []
+commit message = do
+	state <- getState
+	when (branchChanged state) $ do
+		g <- Annex.gitRepo
+		withIndex $ liftIO $
+			GitUnionMerge.commit g message fullname [fullname]
 
 {- Gets the content of a file on the branch, or content staged in the index
  - if it's newer. Returns an empty string if the file didn't exist yet. -}
 get :: FilePath -> Annex String
-get file = update >> do
-	withIndex $ do
-		g <- Annex.gitRepo
-		content <- liftIO $ catch (cat g) (const $ return "")
-		setCache file content
-		return content
+get file = do
+	cached <- getCache file
+	case cached of
+		Just content -> return content
+		Nothing -> withIndexUpdate $ do
+			g <- Annex.gitRepo
+			content <- liftIO $ catch (cat g) (const $ return "")
+			setCache file content
+			return content
 	where
-		-- To avoid stderr from cat-file when file does not exist,
-		-- first run it with -e to check that it exists.
-		cat g = do
-			Git.run g "cat-file" [Param "-e", catfile]
-			Git.pipeRead g [Param "cat-file", Param "blob", catfile]
+		cat g = Git.pipeRead g [Param "cat-file", Param "blob", catfile]
 		catfile = Param $ ':':file
diff --git a/Types/Branch.hs b/Types/Branch.hs
deleted file mode 100644
index c0ccb5ca03..0000000000
--- a/Types/Branch.hs
+++ /dev/null
@@ -1,16 +0,0 @@
-{- git-annex branch data types
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Types.Branch where
-
-data BranchCache = BranchCache {
-	cachedFile :: Maybe FilePath,
-	cachedContent :: String
-}
-
-emptyBranchCache :: BranchCache
-emptyBranchCache = BranchCache Nothing ""
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
new file mode 100644
index 0000000000..65d0642a14
--- /dev/null
+++ b/Types/BranchState.hs
@@ -0,0 +1,18 @@
+{- git-annex BranchState data type
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.BranchState where
+
+data BranchState = BranchState {
+	branchUpdated :: Bool,
+	branchChanged :: Bool,
+	cachedFile :: Maybe FilePath,
+	cachedContent :: String
+}
+
+startBranchState :: BranchState
+startBranchState = BranchState False False Nothing ""

From 17a09fccad3cf2f958967081326b61c403b995cd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:00:04 -0400
Subject: [PATCH 1891/2835] commit changes to git-annex branch on shutdown

---
 CmdLine.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CmdLine.hs b/CmdLine.hs
index 861a31be97..a5bda695ab 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,6 +19,7 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified GitRepo as Git
+import qualified Branch
 import Types
 import Command
 import BackendList
@@ -102,6 +103,7 @@ startup = do
 {- Cleanup actions. -}
 shutdown :: Annex Bool
 shutdown = do
+	Branch.commit "update"
 	AnnexQueue.flush False
 
 	liftIO $ Git.reap

From 06c58922bdb26da34a4070c84968407350f1e3c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:00:32 -0400
Subject: [PATCH 1892/2835] stop changing gitattributes on update from v1

gitattributes changes are not needed, and will be removed in the v2 upgrade
---
 Upgrade/V1.hs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c09bd74c1c..b139e2820b 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -30,7 +30,6 @@ import Backend
 import Messages
 import Version
 import Utility
-import qualified Command.Init
 
 -- v2 adds hashing of filenames of content and location log files.
 -- Key information is encoded in filenames differently, so
@@ -72,10 +71,6 @@ upgrade = do
 			AnnexQueue.flush True
 			setVersion
 
-			-- add new line to auto-merge hashed location logs
-			-- this commits, so has to come after the upgrade
-			liftIO $ Command.Init.gitAttributesWrite g
-
 	return True
 
 moveContent :: Annex ()

From 2e5c8ca6bf016058f2aba4eaba5b89955e3e0e95 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:01:32 -0400
Subject: [PATCH 1893/2835] use git-annex branch for location log

---
 Content.hs     |  4 +---
 LocationLog.hs | 24 +++++++++---------------
 2 files changed, 10 insertions(+), 18 deletions(-)

diff --git a/Content.hs b/Content.hs
index ccd51a553e..fc8ebeb214 100644
--- a/Content.hs
+++ b/Content.hs
@@ -81,9 +81,7 @@ logStatusFor :: UUID -> Key -> LogStatus -> Annex ()
 logStatusFor u key status = do
 	g <- Annex.gitRepo
 	unless (Git.repoIsLocalBare g) $ do
-		logfile <- logChange g key u status
-		rellogfile <- liftIO $ Git.workTreeFile g logfile
-		AnnexQueue.add "add" [Param "--"] rellogfile
+		logChange g key u status
 
 {- Runs an action, passing it a temporary filename to download,
  - and if the action succeeds, moves the temp file into 
diff --git a/LocationLog.hs b/LocationLog.hs
index 1b55abfb29..68a1eb7907 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -3,16 +3,12 @@
  - git-annex keeps track of which repositories have the contents of annexed
  - files.
  -
- - Location tracking information is stored in `.git-annex/key.log`.
  - Repositories record their UUID and the date when they --get or --drop
  - a value.
  -
  - A line of the log will look like: "date N UUID"
  - Where N=1 when the repo has the file, and 0 otherwise.
  - 
- - Git is configured to use a union merge for this file,
- - so the lines may be in arbitrary order, but it will never conflict.
- -
  - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
@@ -39,6 +35,7 @@ import Data.Maybe
 import Control.Monad.State (liftIO)
 
 import qualified GitRepo as Git
+import qualified Branch
 import Utility
 import UUID
 import Types
@@ -85,24 +82,21 @@ instance Read LogLine where
 			bad = ret $ LogLine 0 Undefined ""
 			ret v = [(v, "")]
 
-{- Log a change in the presence of a key's value in a repository,
- - and returns the filename of the logfile. -}
-logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex FilePath
+{- Log a change in the presence of a key's value in a repository. -}
+logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
 logChange repo key u s = do
 	when (null u) $
 		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
 			" (have you run git annex init there?)"
 	line <- logNow s u
 	let f = logFile repo key
-	ls' <- readLog $ logFileOld repo key
 	ls <- readLog f
-	writeLog f (compactLog $ line:ls'++ls)
-	return f
+	writeLog f (compactLog $ line:ls)
 
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
-readLog file = liftIO $ catch (return . parseLog =<< readFileStrict file) (const $ return [])
+readLog file = return . parseLog =<< Branch.get file
 
 parseLog :: String -> [LogLine]
 parseLog s = filter parsable $ map read $ lines s
@@ -110,9 +104,9 @@ parseLog s = filter parsable $ map read $ lines s
 		-- some lines may be unparseable, avoid them
 		parsable l = status l /= Undefined
 
-{- Writes a set of lines to a log file -}
+{- Stores a set of lines in a log file -}
 writeLog :: FilePath -> [LogLine] -> Annex ()
-writeLog file ls = liftIO $ safeWriteFile file (unlines $ map show ls)
+writeLog file ls = Branch.change file (unlines $ map show ls)
 
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> UUID -> Annex LogLine
@@ -125,8 +119,7 @@ logNow s u = do
 keyLocations :: Git.Repo -> Key -> Annex [UUID]
 keyLocations thisrepo key = do
 	ls <- readLog $ logFile thisrepo key
-	ls' <- readLog $ logFileOld thisrepo key
-	return $ map uuid $ filterPresent $ ls'++ls
+	return $ map uuid $ filterPresent ls
 
 {- Filters the list of LogLines to find ones where the value
  - is (or should still be) present. -}
@@ -158,6 +151,7 @@ mapLog m l =
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Git.Repo -> Annex [Key]
 loggedKeys repo = do
+	error "FIXME.. does not look in git-annex branch yet"
 	exists <- liftIO $ doesDirectoryExist dir
 	if exists
 		then do

From ae2be332d463bb2942c5951ffa4acf4d32d63ce2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:02:07 -0400
Subject: [PATCH 1894/2835] add runBool

---
 GitRepo.hs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 91ddf6dca9..d1f122fba5 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -37,6 +37,7 @@ module GitRepo (
 	configTrue,
 	gitCommandLine,
 	run,
+	runBool,
 	pipeRead,
 	pipeWrite,
 	pipeWriteRead,
@@ -350,10 +351,15 @@ gitCommandLine repo@(Repo { location = Dir d} ) params =
 	] ++ params
 gitCommandLine repo _ = assertLocal repo $ error "internal"
 
+{- Runs git in the specified repo. -}
+runBool :: Repo -> String -> [CommandParam] -> IO Bool
+runBool repo subcommand params = assertLocal repo $
+	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
+
 {- Runs git in the specified repo, throwing an error if it fails. -}
 run :: Repo -> String -> [CommandParam] -> IO ()
 run repo subcommand params = assertLocal repo $
-	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
+	runBool repo subcommand params
 		>>! error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns its output, lazily. 

From 80274f4c92397a88c62bf82459fe0c1a9bf03bf7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:02:22 -0400
Subject: [PATCH 1895/2835] use git-annex branch for uuid.log

---
 UUID.hs | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/UUID.hs b/UUID.hs
index f222f7a9d4..c02f51480c 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -30,11 +30,10 @@ import qualified Data.Map as M
 import Data.Maybe
 
 import qualified GitRepo as Git
+import qualified Branch
 import Types
 import Types.UUID
-import Locations
 import qualified Annex
-import Utility
 import qualified SysConfig
 import Config
 
@@ -103,26 +102,21 @@ describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
 	m <- uuidMap
 	let m' = M.insert uuid desc m
-	logfile <- uuidLog
-	liftIO $ safeWriteFile logfile (serialize m')
+	Branch.change uuidLog (serialize m')
 	where
 		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
 
 {- Read and parse the uuidLog into a Map -}
 uuidMap :: Annex (M.Map UUID String)
 uuidMap = do
-	logfile <- uuidLog
-	s <- liftIO $ catch (readFile logfile) ignoreerror
+	s <- Branch.get uuidLog
 	return $ M.fromList $ map pair $ lines s
 	where
 		pair l =
 			if 1 < length (words l)
 				then (head $ words l, unwords $ drop 1 $ words l)
 				else ("", "")
-		ignoreerror _ = return ""
 
 {- Filename of uuid.log. -}
-uuidLog :: Annex FilePath
-uuidLog = do
-	g <- Annex.gitRepo
-	return $ gitStateDir g ++ "uuid.log"
+uuidLog :: FilePath
+uuidLog = "uuid.log"

From 5c706d1ec48172f98e1826684ab380a69079b66a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:02:43 -0400
Subject: [PATCH 1896/2835] stop undoing gitattributes on uninit

v2 upgrade will undo them
---
 Command/Uninit.hs | 11 -----------
 Upgrade/V2.hs     | 48 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 11 deletions(-)
 create mode 100644 Upgrade/V2.hs

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 1e96e1e6f7..9698ed8200 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -34,10 +34,7 @@ start = do
 perform :: CommandPerform
 perform = do
 	g <- Annex.gitRepo
-
 	gitPreCommitHookUnWrite g
-	liftIO $ gitAttributesUnWrite g
-
 	next $ return True
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
@@ -50,11 +47,3 @@ gitPreCommitHookUnWrite repo = do
 			else warning $ "pre-commit hook (" ++ hook ++ 
 				") contents modified; not deleting." ++
 				" Edit it to remove call to git annex."
-
-gitAttributesUnWrite :: Git.Repo -> IO ()
-gitAttributesUnWrite repo = do
-	let attributes = Git.attributes repo
-	whenM (doesFileExist attributes) $ do
-		c <- readFileStrict attributes
-		safeWriteFile attributes $ unlines $
-			filter (\l -> not $ l `elem` Command.Init.attrLines) $ lines c
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
new file mode 100644
index 0000000000..deb231d529
--- /dev/null
+++ b/Upgrade/V2.hs
@@ -0,0 +1,48 @@
+{- git-annex v2 -> v2 upgrade support
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Upgrade.V1 where
+
+import System.IO.Error (try)
+import System.Directory
+import Control.Monad.State (liftIO)
+import Control.Monad (filterM, forM_, unless)
+import System.Posix.Files
+import System.FilePath
+import Data.String.Utils
+import System.Posix.Types
+import Data.Maybe
+import Data.Char
+
+import Types.Key
+import Content
+import Types
+import Locations
+import LocationLog
+import qualified Annex
+import qualified AnnexQueue
+import qualified GitRepo as Git
+import Backend
+import Messages
+import Version
+import Utility
+import qualified Command.Init
+
+{- Old .gitattributes contents, not needed anymore. -}
+attrLines :: [String]
+attrLines =
+	[ stateDir  "*.log merge=union"
+	, stateDir  "*/*/*.log merge=union"
+	]
+
+gitAttributesUnWrite :: Git.Repo -> IO ()
+gitAttributesUnWrite repo = do
+	let attributes = Git.attributes repo
+	whenM (doesFileExist attributes) $ do
+		c <- readFileStrict attributes
+		safeWriteFile attributes $ unlines $
+			filter (\l -> not $ l `elem` attrLines) $ lines c

From e0bd9d43a21bae8193cb0a56be2246ee8cdafdaa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:03:26 -0400
Subject: [PATCH 1897/2835] update for git-annex branch

stop changing gitattributes on init

create git-annex branch on init

ugly special case for init in a bare repository goes away, yay!

git annex init is also faster, at least in a large existing repo, as
it does not need to run the slow 'git add'
---
 Command/Describe.hs |  3 +--
 Command/Init.hs     | 62 ++++-----------------------------------------
 2 files changed, 6 insertions(+), 59 deletions(-)

diff --git a/Command/Describe.hs b/Command/Describe.hs
index 57f884e037..2ad3e01ee5 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -11,7 +11,6 @@ import Command
 import qualified Remote
 import UUID
 import Messages
-import qualified Command.Init
 
 command :: [Command]
 command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
@@ -34,4 +33,4 @@ start ws = notBareRepo $ do
 perform :: UUID -> String -> CommandPerform
 perform u description = do
 	describeUUID u description
-	next $ Command.Init.cleanup
+	next $ return True
diff --git a/Command/Init.hs b/Command/Init.hs
index b7a0787991..1e1eb527a5 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -15,6 +15,7 @@ import System.FilePath
 import Command
 import qualified Annex
 import qualified GitRepo as Git
+import qualified Branch
 import UUID
 import Version
 import Messages
@@ -29,7 +30,6 @@ command = [repoCommand "init" paramDesc seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-{- Stores description for the repository etc. -}
 start :: CommandStartWords
 start ws = do
 	when (null description) $
@@ -41,65 +41,13 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
+	Branch.create
 	g <- Annex.gitRepo
 	u <- getUUID g
 	setVersion
-	if Git.repoIsLocalBare g
-		then do
-			showLongNote $
-				"This is a bare repository, so its description cannot be committed.\n" ++
-				"To record the description, run this command in a clone of this repository:\n" ++
-				"   git annex describe " ++ show u ++ " " ++ show description ++ "\n\n"
-			next $ return True
-		else do
-			describeUUID u description
-			liftIO $ gitAttributesWrite g
-			gitPreCommitHookWrite g
-			next cleanup
-
-cleanup :: CommandCleanup
-cleanup = do
-	g <- Annex.gitRepo
-	logfile <- uuidLog
-	liftIO $ Git.run g "add" [File logfile]
-	liftIO $ Git.run g "commit" 
-		[ Params "-q -m"
-		, Param "git annex repository description"
-		, File logfile
-		]
-	return True
-
-{- configure git to use union merge driver on state files, if it is not
- - already -}
-gitAttributesWrite :: Git.Repo -> IO ()
-gitAttributesWrite repo = do
-	exists <- doesFileExist attributes
-	if not exists
-		then do
-			safeWriteFile attributes $ unlines attrLines
-			commit
-		else do
-			content <- readFile attributes
-			let present = lines content
-			let missing = filter (\l -> not $ l `elem` present) attrLines
-			unless (null missing) $ do
-				appendFile attributes $ unlines missing
-				commit
-	where
-		attributes = Git.attributes repo
-		commit = do
-			Git.run repo "add" [Param attributes]
-			Git.run repo "commit"
-				[ Params "-q -m"
-				, Param "git-annex setup"
-				, Param attributes
-				]
-
-attrLines :: [String]
-attrLines =
-	[ stateDir  "*.log merge=union"
-	, stateDir  "*/*/*.log merge=union"
-	]
+	describeUUID u description
+	gitPreCommitHookWrite g
+	next $ return True
 
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()

From 1870186632e3d4f99e9b87f71f0ddea83ad04568 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:13:43 -0400
Subject: [PATCH 1898/2835] fixed logFile

---
 Backend/File.hs    |  5 ++---
 Command/Fsck.hs    |  2 +-
 Command/Init.hs    |  4 +---
 Command/Unused.hs  |  3 +--
 Command/Whereis.hs |  4 +---
 Content.hs         |  1 -
 LocationLog.hs     | 13 ++++++-------
 Locations.hs       | 16 ++--------------
 Remote.hs          |  2 +-
 Upgrade/V1.hs      |  2 +-
 Upgrade/V2.hs      | 10 ++++++++++
 11 files changed, 26 insertions(+), 36 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 20cb3e95ad..eab987ef81 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -131,7 +131,7 @@ showLocations :: Key -> [UUID] -> Annex ()
 showLocations key exclude = do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	uuids <- keyLocations g key
+	uuids <- keyLocations key
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
@@ -188,8 +188,7 @@ checkKeyOnly = checkKey (\_ -> return True)
 checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
-	g <- Annex.gitRepo
-	locations <- keyLocations g key
+	locations <- keyLocations key
 	untrusted <- trustGet UnTrusted
 	let untrustedlocations = intersect untrusted locations
 	let safelocations = filter (`notElem` untrusted) locations
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 7c840d5288..0e3df03dd0 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -60,7 +60,7 @@ verifyLocationLog key file = do
 		preventWrite (parentDir f)
 
 	u <- getUUID g
-        uuids <- keyLocations g key
+        uuids <- keyLocations key
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
diff --git a/Command/Init.hs b/Command/Init.hs
index 1e1eb527a5..dbf5666cd9 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -8,9 +8,8 @@
 module Command.Init where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when, unless)
+import Control.Monad (when)
 import System.Directory
-import System.FilePath
 
 import Command
 import qualified Annex
@@ -19,7 +18,6 @@ import qualified Branch
 import UUID
 import Version
 import Messages
-import Locations
 import Types
 import Utility
 	
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 4389b2209e..5d4e433ad8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -78,8 +78,7 @@ checkRemoteUnused' r = do
 		showLongNote $ "\n"
 	where
 		isthere k = do
-			g <- Annex.gitRepo
-			us <- keyLocations g k
+			us <- keyLocations k
 			return $ uuid `elem` us
 		uuid = Remote.uuid r
 
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index bcd4a2e228..3a7213217a 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,7 +7,6 @@
 
 module Command.Whereis where
 
-import qualified Annex
 import LocationLog
 import Command
 import Messages
@@ -28,8 +27,7 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: Key -> CommandPerform
 perform key = do
-	g <- Annex.gitRepo
-	uuids <- keyLocations g key
+	uuids <- keyLocations key
 	let num = length uuids
 	showNote $ show num ++ " " ++ copiesplural num
 	if null $ uuids
diff --git a/Content.hs b/Content.hs
index fc8ebeb214..5d77cc9795 100644
--- a/Content.hs
+++ b/Content.hs
@@ -38,7 +38,6 @@ import LocationLog
 import UUID
 import qualified GitRepo as Git
 import qualified Annex
-import qualified AnnexQueue
 import Utility
 import StatFS
 import Types.Key
diff --git a/LocationLog.hs b/LocationLog.hs
index 68a1eb7907..8dbeb729ca 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -20,8 +20,7 @@ module LocationLog (
 	readLog,
 	writeLog,
 	keyLocations,
-	loggedKeys,
-	logFile
+	loggedKeys
 ) where
 
 import Data.Time.Clock.POSIX
@@ -89,7 +88,7 @@ logChange repo key u s = do
 		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
 			" (have you run git annex init there?)"
 	line <- logNow s u
-	let f = logFile repo key
+	let f = logFile key
 	ls <- readLog f
 	writeLog f (compactLog $ line:ls)
 
@@ -116,9 +115,9 @@ logNow s u = do
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
-keyLocations :: Git.Repo -> Key -> Annex [UUID]
-keyLocations thisrepo key = do
-	ls <- readLog $ logFile thisrepo key
+keyLocations :: Key -> Annex [UUID]
+keyLocations key = do
+	ls <- readLog $ logFile key
 	return $ map uuid $ filterPresent ls
 
 {- Filters the list of LogLines to find ones where the value
@@ -151,7 +150,7 @@ mapLog m l =
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Git.Repo -> Annex [Key]
 loggedKeys repo = do
-	error "FIXME.. does not look in git-annex branch yet"
+	_ <- error "FIXME.. does not look in git-annex branch yet"
 	exists <- liftIO $ doesDirectoryExist dir
 	if exists
 		then do
diff --git a/Locations.hs b/Locations.hs
index 8f7b11a5c2..d2241636ef 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -21,7 +21,6 @@ module Locations (
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
 	logFile,
-	logFileOld,
 	logFileKey,
 	hashDirMixed,
 
@@ -119,19 +118,8 @@ isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
 {- The filename of the log file for a given key. -}
-logFile :: Git.Repo -> Key -> String
-logFile = logFile' hashDirLower
-
-{- The old filename of the log file for a key. These can have mixed
- - case, which turned out to be a bad idea for directories whose contents
- - are checked into git. There was no conversion, so these have to be checked
- - for and merged in at runtime. -}
-logFileOld :: Git.Repo -> Key -> String
-logFileOld = logFile' hashDirMixed
-
-logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
-logFile' hasher repo key =
-	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
+logFile :: Key -> String
+logFile key = hashDirLower key ++ keyFile key ++ ".log"
 
 {- Converts a log filename into a key. -}
 logFileKey :: FilePath -> Maybe Key
diff --git a/Remote.hs b/Remote.hs
index 2706bf20b2..804c0ef5af 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -141,7 +141,7 @@ keyPossibilities key = do
 	trusted <- trustGet Trusted
 
 	-- get uuids of all remotes that are recorded to have the key
-	uuids <- keyLocations g key
+	uuids <- keyLocations key
 	let validuuids = filter (/= u) uuids
 
 	-- note that validuuids is assumed to not have dups
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index b139e2820b..1f327f77b9 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -121,7 +121,7 @@ moveLocationLogs = do
 					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
-				let dest = logFile g k
+				let dest = logFile k
 				let dir = gitStateDir g
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index deb231d529..03ef7ba69d 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -46,3 +46,13 @@ gitAttributesUnWrite repo = do
 		c <- readFileStrict attributes
 		safeWriteFile attributes $ unlines $
 			filter (\l -> not $ l `elem` attrLines) $ lines c
+
+oldlogFile :: Git.Repo -> Key -> String
+oldlogFile = logFile' hashDirLower
+
+oldlogFileOld :: Git.Repo -> Key -> String
+oldlogFileOld = logFile' hashDirMixed
+
+logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
+logFile' hasher repo key =
+	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"

From 235e2e63a13c629dcca1aa1638f5f47a8d3983ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:30:34 -0400
Subject: [PATCH 1899/2835] move

---
 Locations.hs | 5 +++++
 UUID.hs      | 6 +-----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index d2241636ef..9175c4f613 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,6 +20,7 @@ module Locations (
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
+	uuidLog,
 	logFile,
 	logFileKey,
 	hashDirMixed,
@@ -117,6 +118,10 @@ gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
+{- Filename of uuid.log. -}
+uuidLog :: FilePath
+uuidLog = "uuid.log"
+
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
 logFile key = hashDirLower key ++ keyFile key ++ ".log"
diff --git a/UUID.hs b/UUID.hs
index c02f51480c..78667f235e 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -19,7 +19,6 @@ module UUID (
 	genUUID,
 	prettyPrintUUIDs,
 	describeUUID,
-	uuidLog,
 	uuidMap
 ) where
 
@@ -36,6 +35,7 @@ import Types.UUID
 import qualified Annex
 import qualified SysConfig
 import Config
+import Locations
 
 configkey :: String
 configkey = "annex.uuid"
@@ -116,7 +116,3 @@ uuidMap = do
 			if 1 < length (words l)
 				then (head $ words l, unwords $ drop 1 $ words l)
 				else ("", "")
-
-{- Filename of uuid.log. -}
-uuidLog :: FilePath
-uuidLog = "uuid.log"

From 4c4ebf2d7570030a70fdbd313b8b60e9fa727eee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:08:51 -0400
Subject: [PATCH 1900/2835] store trust.log and remote.log in the git-annex
 branch

.. and I think that's everything that will use the branch
---
 Branch.hs             | 18 +++++++++---------
 CmdLine.hs            |  2 +-
 Command/InitRemote.hs | 11 -----------
 Locations.hs          |  5 -----
 Remote.hs             | 20 +++++---------------
 Trust.hs              | 27 ++++++---------------------
 UUID.hs               |  8 ++++++--
 7 files changed, 27 insertions(+), 64 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 85928765d8..ad9b805d0e 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -168,15 +168,6 @@ change file content = do
 		  Param sha, File file]
 	setCacheChanged file content
 
-{- Commits any staged changes to the branch. -}
-commit :: String -> Annex ()
-commit message = do
-	state <- getState
-	when (branchChanged state) $ do
-		g <- Annex.gitRepo
-		withIndex $ liftIO $
-			GitUnionMerge.commit g message fullname [fullname]
-
 {- Gets the content of a file on the branch, or content staged in the index
  - if it's newer. Returns an empty string if the file didn't exist yet. -}
 get :: FilePath -> Annex String
@@ -192,3 +183,12 @@ get file = do
 	where
 		cat g = Git.pipeRead g [Param "cat-file", Param "blob", catfile]
 		catfile = Param $ ':':file
+
+{- Commits any staged changes to the branch. -}
+commit :: String -> Annex ()
+commit message = do
+	state <- getState
+	when (branchChanged state) $ do
+		g <- Annex.gitRepo
+		withIndex $ liftIO $
+			GitUnionMerge.commit g message fullname [fullname]
diff --git a/CmdLine.hs b/CmdLine.hs
index a5bda695ab..d10516bb9c 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -103,8 +103,8 @@ startup = do
 {- Cleanup actions. -}
 shutdown :: Annex Bool
 shutdown = do
-	Branch.commit "update"
 	AnnexQueue.flush False
+	Branch.commit "update"
 
 	liftIO $ Git.reap
 	
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 41d3c37c71..67030689da 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -14,11 +14,8 @@ import Data.Maybe
 import Data.String.Utils
 
 import Command
-import qualified Annex
 import qualified Remote
 import qualified Types.Remote as R
-import qualified GitRepo as Git
-import Utility
 import Types
 import UUID
 import Messages
@@ -62,14 +59,6 @@ perform t u c = do
 cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
 cleanup u c = do
 	Remote.configSet u c
-	g <- Annex.gitRepo
-	logfile <- Remote.remoteLog
-	liftIO $ Git.run g "add" [File logfile]
-        liftIO $ Git.run g "commit"
-                [ Params "-q --allow-empty -m"
-                , Param "git annex initremote"
-                , File logfile
-                ]
         return True
 
 {- Look up existing remote's UUID and config by name, or generate a new one -}
diff --git a/Locations.hs b/Locations.hs
index 9175c4f613..d2241636ef 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,7 +20,6 @@ module Locations (
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
-	uuidLog,
 	logFile,
 	logFileKey,
 	hashDirMixed,
@@ -118,10 +117,6 @@ gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
-{- Filename of uuid.log. -}
-uuidLog :: FilePath
-uuidLog = "uuid.log"
-
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
 logFile key = hashDirLower key ++ keyFile key ++ ".log"
diff --git a/Remote.hs b/Remote.hs
index 804c0ef5af..5122423421 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -33,19 +33,17 @@ module Remote (
 	prop_idempotent_configEscape
 ) where
 
-import Control.Monad.State (liftIO)
 import Control.Monad (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
 
+import qualified Branch
 import Types
 import Types.Remote
 import UUID
 import qualified Annex
-import Locations
-import Utility
 import Config
 import Trust
 import LocationLog
@@ -160,29 +158,21 @@ forceTrust level remotename = do
 		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
 
 {- Filename of remote.log. -}
-remoteLog :: Annex FilePath
-remoteLog = do
-	g <- Annex.gitRepo
-	return $ gitStateDir g ++ "remote.log"
+remoteLog :: FilePath
+remoteLog = "remote.log"
 
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
 configSet u c = do
 	m <- readRemoteLog
-	l <- remoteLog
-	liftIO $ safeWriteFile l $ unlines $ sort $
+	Branch.change remoteLog $ unlines $ sort $
 		map toline $ M.toList $ M.insert u c m
 	where
 		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = do
-	l <- remoteLog
-	s <- liftIO $ catch (readFile l) ignoreerror
-	return $ remoteLogParse s
-	where
-		ignoreerror _ = return ""
+readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
diff --git a/Trust.hs b/Trust.hs
index aaca3b3706..d328235bfa 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -18,18 +18,14 @@ import Control.Monad.State
 import qualified Data.Map as M
 
 import TrustLevel
-import qualified GitRepo as Git
+import qualified Branch
 import Types
 import UUID
-import Locations
 import qualified Annex
-import Utility
 
 {- Filename of trust.log. -}
-trustLog :: Annex FilePath
-trustLog = do
-	g <- Annex.gitRepo
-	return $ gitStateDir g ++ "trust.log"
+trustLog :: FilePath
+trustLog = "trust.log"
 
 {- Returns a list of UUIDs at the specified trust level. -}
 trustGet :: TrustLevel -> Annex [UUID]
@@ -41,12 +37,9 @@ trustGet level = do
  - values from forcetrust -}
 trustMap :: Annex (M.Map UUID TrustLevel)
 trustMap = do
-	logfile <- trustLog
 	overrides <- Annex.getState Annex.forcetrust
-	s <- liftIO $ catch (readFile logfile) ignoreerror
+	s <- Branch.get trustLog
 	return $ M.fromList $ trustMapParse s ++ overrides
-	where
-                ignoreerror _ = return ""
 
 {- Trust map parser. -}
 trustMapParse :: String -> [(UUID, TrustLevel)]
@@ -60,7 +53,7 @@ trustMapParse s = map pair $ filter (not . null) $ lines s
 			where
 				w = words l
 
-{- Changes the trust level for a uuid in the trustLog, and commits it. -}
+{- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
 	when (null uuid) $
@@ -68,15 +61,7 @@ trustSet uuid level = do
         m <- trustMap
 	when (M.lookup uuid m /= Just level) $ do
 		let m' = M.insert uuid level m
-	        logfile <- trustLog
-	        liftIO $ safeWriteFile logfile (serialize m')
-		g <- Annex.gitRepo
-		liftIO $ Git.run g "add" [File logfile]
-		liftIO $ Git.run g "commit"
-			[ Params "-q -m"
-			, Param "git annex trust change"
-			, File logfile
-			]
+		Branch.change trustLog (serialize m')
         where
                 serialize m = unlines $ map showpair $ M.toList m
 		showpair (u, t) = u ++ " " ++ show t
diff --git a/UUID.hs b/UUID.hs
index 78667f235e..5d8304f83d 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -19,7 +19,8 @@ module UUID (
 	genUUID,
 	prettyPrintUUIDs,
 	describeUUID,
-	uuidMap
+	uuidMap,
+	uuidLog
 ) where
 
 import Control.Monad.State
@@ -35,11 +36,14 @@ import Types.UUID
 import qualified Annex
 import qualified SysConfig
 import Config
-import Locations
 
 configkey :: String
 configkey = "annex.uuid"
 
+{- Filename of uuid.log. -}
+uuidLog :: FilePath
+uuidLog = "uuid.log"
+
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
 genUUID :: IO UUID

From d70e9a945b4ac44ff42872b08dcf09051759eb9c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:15:33 -0400
Subject: [PATCH 1901/2835] remove some tests that no longer make sense

---
 test.hs | 24 +-----------------------
 1 file changed, 1 insertion(+), 23 deletions(-)

diff --git a/test.hs b/test.hs
index 1eac942b1c..528f78063a 100644
--- a/test.hs
+++ b/test.hs
@@ -105,12 +105,7 @@ blackbox = TestLabel "blackbox" $ TestList
 test_init :: Test
 test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 	git_annex "init" ["-q", reponame] @? "init failed"
-	e <- doesFileExist annexlog
-	e @? (annexlog ++ " not created")
-	c <- readFile annexlog
-	reponame `isInfixOf` c @? annexlog ++ " does not contain repo name"
 	where
-		annexlog = ".git-annex/uuid.log"
 		reponame = "test repo"
 
 test_add :: Test
@@ -609,26 +604,9 @@ checklocationlog f expected = do
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do
-			uuids <- annexeval $ do
-				g <- Annex.gitRepo
-				LocationLog.keyLocations g k
+			uuids <- annexeval $ LocationLog.keyLocations k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
 				expected (thisuuid `elem` uuids)
-
-			-- Location log files should always be checked
-			-- into git, and any modifications staged for
-			-- commit. This is a regression test, as some
-			-- commands forgot to.
-			(fs, ufs) <- annexeval $ do
-				g <- Annex.gitRepo
-				let lf = LocationLog.logFile g k
-				fs <- liftIO $ Git.inRepo g [lf]
-				ufs <- liftIO $ Git.changedUnstagedFiles g [lf]
-				return (fs, ufs)
-			when (null fs) $
-				assertFailure $ f ++ " logfile not added to git repo"
-			when (not $ null ufs) $
-				assertFailure $ f ++ " logfile changes not staged"
 		_ -> assertFailure $ f ++ " failed to look up key"
 
 inlocationlog :: FilePath -> Assertion

From b1acf41036a1eddea29cc69c6b2a595582378465 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:26:34 -0400
Subject: [PATCH 1902/2835] update documentation that mentioned .git-annex/

---
 doc/bare_repositories.mdwn                   |  4 ----
 doc/design/encryption.mdwn                   |  2 +-
 doc/git-annex.mdwn                           | 12 ------------
 doc/internals.mdwn                           | 16 ++++++++++++----
 doc/location_tracking.mdwn                   |  2 +-
 doc/walkthrough/modifying_annexed_files.mdwn |  3 +--
 6 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index a9ccab8d1b..5cbad5c2d8 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -17,10 +17,6 @@ Known to work ok:
 
 There are a few caveats to keep in mind when using bare repositories:
 
-* `git annex init` can be run in a bare repository, but it cannot
-  store the name you gave the repository in .git-annex/uuid.log (because
-  the bare repository has no such file to commit to). Instead, it will
-  tell you a command to run in some non-bare clone of the repository.
 * Some subcommands, like `fsck`, `trust`, `unused` and `fromkey`, 
   cannot be run in a bare repository. Those subcommands will
   refuse to do anything.
diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn
index 11056478bd..e5053134ee 100644
--- a/doc/design/encryption.mdwn
+++ b/doc/design/encryption.mdwn
@@ -36,7 +36,7 @@ There does not seem to be much benefit to using the same cipher for
 two different encrypted remotes.
 
 So, the encrypted cipher could just be stored with the rest of a remote's
-configuration in `.git-annex/remotes.log` (see [[internals]]). When `git
+configuration in `remotes.log` (see [[internals]]). When `git
 annex intiremote` makes a remote, it can generate a random symmetric
 cipher, and encrypt it with the specified gpg key. To allow another gpg
 public key access, update the encrypted cipher to be encrypted to both gpg
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 12756d8020..ced6fc1b3a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -497,18 +497,6 @@ These files are used by git-annex, in your git repository:
 `.git/annex/objects/` contains the annexed file contents that are currently
 available. Annexed files in your git repository symlink to that content.
 
-`.git-annex/uuid.log` is used to map between repository UUID and
-descriptions.
-
-`.git-annex/trust.log` is used to indicate which repositories are trusted
-and untrusted.
-
-`.git-annex/*.log` is where git-annex records its content tracking
-information. These files should be committed to git.
-
-`.gitattributes` is configured to use git's union merge driver
-to avoid conflicts when merging files in the `.git-annex` directory.
-
 # SEE ALSO
 
 Most of git-annex's documentation is available on its web site, 
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index b362e68e1e..419096744d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -17,7 +17,15 @@ This two-level structure is used because it allows the write bit to be removed
 from the subdirectories as well as from the files. That prevents accidentially
 deleting or changing the file contents.
 
-## `.git-annex/uuid.log`
+## The git-annex branch
+
+This branch is managed by git-annex, with the contents listed below.
+
+Note that it assumes only it will modify the branch. If you go in and make
+changes, be sure to remove `.git/index.git-annex` before running git-annex,
+otherwise it will probably revert your changes in its next commit to the branch.
+
+### `uuid.log`
 
 Records the UUIDs of known repositories, and associates them with a
 description of the repository. This allows git-annex to display something
@@ -30,7 +38,7 @@ space and then the description through to the end of the line. Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 laptop
 	26339d22-446b-11e0-9101-002170d25c55 usb disk
 
-## `git-annex/remotes.log`
+## `remotes.log`
 
 Holds persistent configuration settings for [[special_remotes]] such as
 Amazon S3.
@@ -39,7 +47,7 @@ The file format is one line per remote, starting with the uuid of the
 remote, followed by a space, and then a series of key=value pairs,
 each separated by whitespace.
 
-## `.git-annex/trust.log`
+## `trust.log`
 
 Records the [[trust]] information for repositories. Does not exist unless
 [[trust]] values are configured.
@@ -53,7 +61,7 @@ Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 1
 	26339d22-446b-11e0-9101-002170d25c55 ?
 
-## `.git-annex/aaa/bbb/*.log`
+## `aaa/bbb/*.log`
 
 The remainder of the log files record [[location_tracking]] information
 for file contents. Again these are placed in two levels of subdirectories
diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn
index 301282b6ff..85bb3d1b55 100644
--- a/doc/location_tracking.mdwn
+++ b/doc/location_tracking.mdwn
@@ -1,5 +1,5 @@
 git-annex keeps track of in which repositories it last saw a file's content.
-This location tracking information is stored in `.git-annex/$key.log`.
+This location tracking information is stored in the git-annex branch.
 Repositories record their UUID and the date when they get or drop 
 a file's content. (Git is configured to use a union merge for this file,
 so the lines may be in arbitrary order, but it will never conflict.)
diff --git a/doc/walkthrough/modifying_annexed_files.mdwn b/doc/walkthrough/modifying_annexed_files.mdwn
index f75b73a24c..1f7a7efb77 100644
--- a/doc/walkthrough/modifying_annexed_files.mdwn
+++ b/doc/walkthrough/modifying_annexed_files.mdwn
@@ -26,8 +26,7 @@ and this symlink is what gets committed to git in the end.
 	# git commit my_cool_big_file -m "changed an annexed file"
 	add my_cool_big_file ok
 	[master 64cda67] changed an annexed file
-	 2 files changed, 2 insertions(+), 1 deletions(-)
-	 create mode 100644 .git-annex/WORM-s30-m1289672605--file.log
+	 1 files changed, 1 insertions(+), 1 deletions(-)
 
 There is one problem with using `git commit` like this: Git wants to first
 stage the entire contents of the file in its index. That can be slow for

From 2035b22a01ba45505ce6367cdbeb1f6e86e9bd70 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:47:06 -0400
Subject: [PATCH 1903/2835] better branch display

---
 Branch.hs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index ad9b805d0e..e6896aa849 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -10,7 +10,8 @@ module Branch (
 	update,
 	get,
 	change,
-	commit
+	commit,
+	shortref
 ) where
 
 import Control.Monad (unless, when, liftM)
@@ -20,6 +21,7 @@ import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
+import Data.List
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -37,6 +39,13 @@ name = "git-annex"
 fullname :: String
 fullname = "refs/heads/" ++ name
 
+shortref :: String -> String
+shortref = remove "refs/heads/" . remove "refs/remotes/"
+	where
+		remove prefix s
+			| prefix `isPrefixOf` s = drop (length prefix) s
+			| otherwise = s
+
 {- A separate index file for the branch. -}
 index :: Git.Repo -> FilePath
 index g = Git.workTree g  Git.gitDir g  "index." ++ name
@@ -151,7 +160,7 @@ updateRef ref
 		if (null diffs)
 			then return Nothing
 			else do
-				showSideAction $ "merging " ++ ref ++ " into " ++ name ++ "..."
+				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name ++ "..."
 				-- By passing only one ref, it is actually
 				-- merged into the index, preserving any
 				-- changes that may already be staged.

From c3d96ee38a07e2cd9b27241155f80c5020a814f1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:47:14 -0400
Subject: [PATCH 1904/2835] adjust walkthrough for git-annex branch

don't just pull master.. pull everything
---
 doc/walkthrough/getting_file_content.mdwn | 2 +-
 doc/walkthrough/using_Amazon_S3.mdwn      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index bf45fd97fd..71f95f79e0 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -6,7 +6,7 @@ We can use this to copy everything in the laptop's annex to the
 USB drive.
 
 	# cd /media/usb/annex
-	# git pull laptop master
+	# git pull laptop
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 7f972afe11..b59ca9b4f8 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -23,7 +23,7 @@ The configuration for the S3 remote is stored in git. So to make another
 repository use the same S3 remote is easy:
 
 	# cd /media/usb/annex
-	# git pull laptop master
+	# git pull laptop
 	# git annex initremote cloud
 	initremote cloud (gpg) (checking bucket) ok
 

From 1a182d4d047c24e217663dbccfa39aae907cbbc0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:51:48 -0400
Subject: [PATCH 1905/2835] stub in v2 upgrade

---
 Locations.hs  |  1 +
 Upgrade.hs    |  2 ++
 Upgrade/V2.hs | 24 +++++++-----------------
 3 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index d2241636ef..7f9626a11b 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -23,6 +23,7 @@ module Locations (
 	logFile,
 	logFileKey,
 	hashDirMixed,
+	hashDirLower,
 
 	prop_idempotent_fileKey
 ) where
diff --git a/Upgrade.hs b/Upgrade.hs
index 08481755f3..6cd75cf3e8 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -11,6 +11,7 @@ import Types
 import Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
+import qualified Upgrade.V2
 
 {- Uses the annex.version git config setting to automate upgrades. -}
 upgrade :: Annex Bool
@@ -19,4 +20,5 @@ upgrade = do
 	case version of
 		"0" -> Upgrade.V0.upgrade
 		"1" -> Upgrade.V1.upgrade
+		"2" -> Upgrade.V2.upgrade
 		_ -> return True
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 03ef7ba69d..df1f62b8c8 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -5,32 +5,22 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Upgrade.V1 where
+module Upgrade.V2 where
 
-import System.IO.Error (try)
 import System.Directory
-import Control.Monad.State (liftIO)
-import Control.Monad (filterM, forM_, unless)
-import System.Posix.Files
 import System.FilePath
-import Data.String.Utils
-import System.Posix.Types
-import Data.Maybe
-import Data.Char
 
 import Types.Key
-import Content
 import Types
-import Locations
-import LocationLog
-import qualified Annex
-import qualified AnnexQueue
 import qualified GitRepo as Git
-import Backend
 import Messages
-import Version
 import Utility
-import qualified Command.Init
+import Locations
+
+upgrade :: Annex Bool
+upgrade = do
+	showNote "v2 to v3"
+	error "TODO"
 
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]

From c7a1690f0247fefedc9b735bee1273660fc94e77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:56:07 -0400
Subject: [PATCH 1906/2835] update

---
 doc/bugs/bare_git_repos.mdwn | 2 +-
 doc/todo/branching.mdwn      | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn
index f219840e71..5e9100acfe 100644
--- a/doc/bugs/bare_git_repos.mdwn
+++ b/doc/bugs/bare_git_repos.mdwn
@@ -11,7 +11,7 @@ However, that is not currently supported. Problems include:
   a git repo at all!)
 * `.git-annex/` needs to have state recorded to it and committed, and that
   is not possible with a bare repo. (If [[todo/branching]] were done,
-  that might be fixed.)
+  that might be fixed.) (now fixed)
 
 ----
 
diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index 37e7b6edd2..ad7ece6f10 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -1,3 +1,5 @@
+[[done]] !!!
+
 The use of `.git-annex` to store logs means that if a repo has branches 
 and the user switched between them, git-annex will see different logs in
 the different branches, and so may miss info about what remotes have which

From 944c51ba26efc39416c5f148b6ec36151dc7f42e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 18:07:45 -0400
Subject: [PATCH 1907/2835] improve version checking for v3

Do not set annex.version whenever any command is run. Just do it in init.
This ensures that, if a repo has annex.version=3, it has a git-annex
branch, so we don't have to run a command every time to check for the
branch.

Remove the old ad-hoc logic for v0 and v1, to simplify version checking.
---
 Command/Init.hs    |  2 +-
 Command/Version.hs |  3 ++-
 Upgrade.hs         |  6 +++---
 Version.hs         | 37 ++++++++++---------------------------
 4 files changed, 16 insertions(+), 32 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index dbf5666cd9..df416eed36 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -22,7 +22,7 @@ import Types
 import Utility
 	
 command :: [Command]
-command = [repoCommand "init" paramDesc seek
+command = [standaloneCommand "init" paramDesc seek
 		"initialize git-annex with repository description"]
 
 seek :: [CommandSeek]
diff --git a/Command/Version.hs b/Command/Version.hs
index 755b95acca..bb7acd12dd 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -9,6 +9,7 @@ module Command.Version where
 
 import Control.Monad.State (liftIO)
 import Data.String.Utils
+import Data.Maybe
 
 import Command
 import qualified SysConfig
@@ -24,7 +25,7 @@ start :: CommandStartNothing
 start = do
 	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion
 	v <- getVersion
-	liftIO $ putStrLn $ "local repository version: " ++ v
+	liftIO $ putStrLn $ "local repository version: " ++ fromMaybe "unknown" v
 	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion
 	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions
 	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
diff --git a/Upgrade.hs b/Upgrade.hs
index 6cd75cf3e8..a724ecce31 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -18,7 +18,7 @@ upgrade :: Annex Bool
 upgrade = do
 	version <- getVersion
 	case version of
-		"0" -> Upgrade.V0.upgrade
-		"1" -> Upgrade.V1.upgrade
-		"2" -> Upgrade.V2.upgrade
+		Just "0" -> Upgrade.V0.upgrade
+		Just "1" -> Upgrade.V1.upgrade
+		Just "2" -> Upgrade.V2.upgrade
 		_ -> return True
diff --git a/Version.hs b/Version.hs
index 72d06f6639..690e693e23 100644
--- a/Version.hs
+++ b/Version.hs
@@ -7,14 +7,11 @@
 
 module Version where
 
-import Control.Monad.State (liftIO)
 import Control.Monad (unless)
-import System.Directory
 
 import Types
 import qualified Annex
 import qualified GitRepo as Git
-import Locations
 import Config
 
 type Version = String
@@ -31,40 +28,26 @@ upgradableVersions = ["0", "1", "2"]
 versionField :: String
 versionField = "annex.version"
 
-getVersion :: Annex Version
+getVersion :: Annex (Maybe Version)
 getVersion = do
 	g <- Annex.gitRepo
 	let v = Git.configGet g versionField ""
 	if not $ null v
-		then return v
-		else do
-			-- version 0 was not recorded in .git/config;
-			-- such a repo should have an gitAnnexDir but no
-			-- gitAnnexObjectDir.
-			--
-			-- version 1 may not be recorded if the user
-			-- forgot to init. Such a repo should have a
-			-- gitAnnexObjectDir already.
-			d <- liftIO $ doesDirectoryExist $ gitAnnexDir g
-			o <- liftIO $ doesDirectoryExist $ gitAnnexObjectDir g
-			case (d, o) of
-				(True, False) -> return "0"
-				(True, True) -> return "1"
-				_ -> do
-					setVersion
-					return defaultVersion
+		then return $ Just v
+		else return Nothing
 
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
 checkVersion :: Annex ()
-checkVersion = do
-	v <- getVersion
-	unless (v `elem` supportedVersions) $ do
-		error $ "Repository version " ++ v ++ 
-			" is not supported. " ++
-			msg v
+checkVersion = getVersion >>= handle
 	where
+		handle Nothing = error "First run: git-annex init"
+		handle (Just v) = do
+			unless (v `elem` supportedVersions) $ do
+			error $ "Repository version " ++ v ++ 
+				" is not supported. " ++
+				msg v
 		msg v
 			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"
 			| otherwise = "Upgrade git-annex."

From 80302d0b46c5d45df1cf290796e0e27d9264ece8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 18:32:41 -0400
Subject: [PATCH 1908/2835] improve bare repo handing

Many more commands can work in bare repos now, thanks to the git-annex
branch.
---
 Command/Describe.hs        |  2 +-
 Command/Fsck.hs            |  8 ++++----
 Command/Init.hs            |  5 +++--
 Command/InitRemote.hs      |  2 +-
 Command/Move.hs            |  6 +++---
 Command/Semitrust.hs       |  2 +-
 Command/Trust.hs           |  2 +-
 Command/Untrust.hs         |  2 +-
 Content.hs                 | 14 +++-----------
 debian/changelog           |  2 ++
 doc/bare_repositories.mdwn |  9 +++------
 11 files changed, 23 insertions(+), 31 deletions(-)

diff --git a/Command/Describe.hs b/Command/Describe.hs
index 2ad3e01ee5..453e4ebafc 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -20,7 +20,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	let (name, description) =
 		case ws of
 			(n:d) -> (n,unwords d)
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0e3df03dd0..cb062342d7 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -64,11 +64,11 @@ verifyLocationLog key file = do
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
-				fix u ValuePresent
+				fix g u ValuePresent
 				-- There is no data loss, so do not fail.
 				return True
 		(False, True) -> do
-				fix u ValueMissing
+				fix g u ValueMissing
 				warning $
 					"** Based on the location log, " ++ file
 					++ "\n** was expected to be present, " ++
@@ -77,6 +77,6 @@ verifyLocationLog key file = do
 		_ -> return True
 	
 	where
-		fix u s = do
+		fix g u s = do
 			showNote "fixing location log"
-			logStatusFor u key s
+			logChange g key u s
diff --git a/Command/Init.hs b/Command/Init.hs
index df416eed36..7f5773117a 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -8,7 +8,7 @@
 module Command.Init where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when)
+import Control.Monad (when, unless)
 import System.Directory
 
 import Command
@@ -44,7 +44,8 @@ perform description = do
 	u <- getUUID g
 	setVersion
 	describeUUID u description
-	gitPreCommitHookWrite g
+	unless (Git.repoIsLocalBare g) $
+		gitPreCommitHookWrite g
 	next $ return True
 
 {- set up a git pre-commit hook, if one is not already present -}
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 67030689da..a3054630c3 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -30,7 +30,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	when (null ws) $ needname
 
 	(u, c) <- findByName name
diff --git a/Command/Move.hs b/Command/Move.hs
index f49fe20e00..7fa195bab5 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -51,12 +51,12 @@ showAction False file = showStart "copy" file
 {- Used to log a change in a remote's having a key. The change is logged
  - in the local repo, not on the remote. The process of transferring the
  - key to the remote, or removing the key from it *may* log the change
- - on the remote, but this cannot be relied on. For example, it's not done
- - for bare repos. -}
+ - on the remote, but this cannot be relied on. -}
 remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()
 remoteHasKey remote key present	= do
 	let remoteuuid = Remote.uuid remote
-	logStatusFor remoteuuid key status
+	g <- Annex.gitRepo
+	logChange g key remoteuuid status
 	where
 		status = if present then ValuePresent else ValueMissing
 
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 11742137f1..b467861bf9 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -21,7 +21,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	let name = unwords ws
 	showStart "semitrust" name
 	u <- Remote.nameToUUID name
diff --git a/Command/Trust.hs b/Command/Trust.hs
index d5444affe6..41eb17ccdd 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -21,7 +21,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	let name = unwords ws
 	showStart "trust" name
 	u <- Remote.nameToUUID name
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 174c395066..ea23208006 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -21,7 +21,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	let name = unwords ws
 	showStart "untrust" name
 	u <- Remote.nameToUUID name
diff --git a/Content.hs b/Content.hs
index 5d77cc9795..45653fc9e7 100644
--- a/Content.hs
+++ b/Content.hs
@@ -9,7 +9,6 @@ module Content (
 	inAnnex,
 	calcGitLink,
 	logStatus,
-	logStatusFor,
 	getViaTmp,
 	getViaTmpUnchecked,
 	withTmp,
@@ -27,7 +26,7 @@ import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import System.Path
-import Control.Monad (when, unless, filterM)
+import Control.Monad (when, filterM)
 import System.Posix.Files
 import System.FilePath
 import Data.Maybe
@@ -71,16 +70,9 @@ calcGitLink file key = do
  - updated instead. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
-	u <- getUUID =<< Annex.gitRepo
-	logStatusFor u key status
-
-{- Updates the LocationLog when a key's presence changes in a repository
- - identified by UUID. -}
-logStatusFor :: UUID -> Key -> LogStatus -> Annex ()
-logStatusFor u key status = do
 	g <- Annex.gitRepo
-	unless (Git.repoIsLocalBare g) $ do
-		logChange g key u status
+	u <- getUUID g
+	logChange g key u status
 
 {- Runs an action, passing it a temporary filename to download,
  - and if the action succeeds, moves the temp file into 
diff --git a/debian/changelog b/debian/changelog
index 360121cbf7..a76a0534e2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (0.20110611) UNRELEASED; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
 
+  * Improved handling of bare git repos with annexes. Many more commands will
+    work in them.
   * rsync is now used when copying files from repos on other filesystems.
     cp is still used when copying file from repos on the same filesystem,
     since --reflink=auto can make it significantly faster on filesystems
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index 5cbad5c2d8..c5663d84cf 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -14,12 +14,9 @@ Known to work ok:
 * `git annex drop` can check that a bare repository has a copy of data
   that is being dropped.
 * `git annex get` can transfer data from a bare repository.
+* Most other stuff (ie, init, describe, trust, etc.)
 
 There are a few caveats to keep in mind when using bare repositories:
 
-* Some subcommands, like `fsck`, `trust`, `unused` and `fromkey`, 
-  cannot be run in a bare repository. Those subcommands will
-  refuse to do anything.
-* `git annex setkey` is a plumbing-level command; using it manually
-  to add content to a bare repository is not recommended, since there
-  will be no record that the content is stored there.
+* A few subcommands, like `unused` cannot be run in a bare repository.
+  Those subcommands will refuse to do anything.

From ad3770e0b203b32a07fa142d6d83c980b23310ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 18:46:45 -0400
Subject: [PATCH 1909/2835] add merge subcommand

---
 Command/Merge.hs   | 29 +++++++++++++++++++++++++++++
 Command/Status.hs  |  2 +-
 GitAnnex.hs        |  2 ++
 debian/changelog   |  3 ++-
 doc/git-annex.mdwn | 11 ++++++++---
 5 files changed, 42 insertions(+), 5 deletions(-)
 create mode 100644 Command/Merge.hs

diff --git a/Command/Merge.hs b/Command/Merge.hs
new file mode 100644
index 0000000000..04328e8c5a
--- /dev/null
+++ b/Command/Merge.hs
@@ -0,0 +1,29 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Merge where
+
+import Command
+import qualified Branch
+import Messages
+
+command :: [Command]
+command = [repoCommand "merge" paramNothing seek
+		"auto-merges remote changes into the git-annex branch"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStartNothing
+start = do
+	showStart "merge" "."
+	next perform
+
+perform :: CommandPerform
+perform = do
+	Branch.update
+	next $ return True
diff --git a/Command/Status.hs b/Command/Status.hs
index 1a7f694ba8..3b096d9793 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -47,7 +47,7 @@ sizeList :: [a] -> SizeList a
 sizeList l = (l, genericLength l)
 
 command :: [Command]
-command = [repoCommand "status" (paramNothing) seek
+command = [repoCommand "status" paramNothing seek
 	"shows status information about the annex"]
 
 seek :: [CommandSeek]
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 103ee262f2..727e0c3961 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -39,6 +39,7 @@ import qualified Command.Lock
 import qualified Command.PreCommit
 import qualified Command.Find
 import qualified Command.Whereis
+import qualified Command.Merge
 import qualified Command.Status
 import qualified Command.Migrate
 import qualified Command.Uninit
@@ -76,6 +77,7 @@ cmds = concat
 	, Command.DropUnused.command
 	, Command.Find.command
 	, Command.Whereis.command
+	, Command.Merge.command
 	, Command.Status.command
 	, Command.Migrate.command
 	, Command.Map.command
diff --git a/debian/changelog b/debian/changelog
index a76a0534e2..03ccbe03f0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,8 @@
 git-annex (0.20110611) UNRELEASED; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
-
+  * git-annex now stores its logs in a git-annex branch.
+  * merge: New subcommand. Auto-merges the new git-annex branch.
   * Improved handling of bare git repos with annexes. Many more commands will
     work in them.
   * rsync is now used when copying files from repos on other filesystems.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index ced6fc1b3a..1cb079ae9a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -182,6 +182,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Displays a list of repositories known to contain the content of the
   specified file or files.
 
+* merge
+
+  Automatically merges any changes from remotes into the git-annex branch.
+  While git-annex mostly handles keeping the git-annex branch merged
+  automatically, if you find you are unable to push the git-annex branch
+  due non-fast-forward, this will fix it.
+
 * status
 
   Displays some statistics and other information, including how much data
@@ -301,9 +308,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * upgrade
 
-  Upgrades the repository to current layout. Upgrades are done automatically
-  whenever a newer git annex encounters an old repository; this command
-  allows explcitly starting an upgrade.
+  Upgrades the repository to current layout.
 
 * version
 

From 1285763015bb357297f573031cc3793d17fa702c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 18:56:08 -0400
Subject: [PATCH 1910/2835] decruft

---
 .gitattributes | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitattributes b/.gitattributes
index b83edcae09..5d425843f2 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,3 +1 @@
 debian/changelog merge=dpkg-mergechangelogs
-.git-annex/*.log merge=union
-.git-annex/*/*/*.log merge=union

From 36109a286e867a6a70b5f0194332f78cd64ca277 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 19:48:04 -0400
Subject: [PATCH 1911/2835] squelched git-cat-file's error message when file
 DNE

This seemed much too hard to do. I just wanted to close stderr
when running it.
---
 Branch.hs | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index e6896aa849..2cd658fe8e 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -22,6 +22,9 @@ import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
 import Data.List
+import System.IO
+import System.Posix.IO
+import System.Posix.Process
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -130,6 +133,15 @@ create = do
 			liftIO $ Git.runBool g "show-ref"
 				[Param "--verify", Param "-q", Param ref]
 
+{- Commits any staged changes to the branch. -}
+commit :: String -> Annex ()
+commit message = do
+	state <- getState
+	when (branchChanged state) $ do
+		g <- Annex.gitRepo
+		withIndex $ liftIO $
+			GitUnionMerge.commit g message fullname [fullname]
+
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
@@ -190,14 +202,23 @@ get file = do
 			setCache file content
 			return content
 	where
-		cat g = Git.pipeRead g [Param "cat-file", Param "blob", catfile]
-		catfile = Param $ ':':file
+		cat g = cmdOutput "git" $ toCommand $ Git.gitCommandLine g
+			[Param "cat-file", Param "blob", Param $ ':':file]
 
-{- Commits any staged changes to the branch. -}
-commit :: String -> Annex ()
-commit message = do
-	state <- getState
-	when (branchChanged state) $ do
-		g <- Annex.gitRepo
-		withIndex $ liftIO $
-			GitUnionMerge.commit g message fullname [fullname]
+{- Runs a command, returning its output, ignoring nonzero exit
+ - status, and discarding stderr. -}
+cmdOutput :: FilePath -> [String] -> IO String
+cmdOutput cmd params = do
+	pipepair <- createPipe
+	let callfunc _ = do
+		closeFd (snd pipepair)
+		h <- fdToHandle (fst pipepair)
+		x <- hGetContentsStrict h
+		hClose h
+		return $! x
+	pid <- pOpen3Raw Nothing (Just (snd pipepair)) Nothing cmd params
+		(closeFd (fst pipepair) >> closeFd stdError)
+	retval <- callfunc $! pid
+	let rv = seq retval retval
+	_ <- getProcessStatus True False pid
+	return rv

From 4c8770c646c41598b73b3c86280514f1068e0d36 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 19:52:13 -0400
Subject: [PATCH 1912/2835] reove 2 tests that no longer make sense (state is
 not autocommitted)

---
 test.hs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/test.hs b/test.hs
index 528f78063a..9729818211 100644
--- a/test.hs
+++ b/test.hs
@@ -19,7 +19,6 @@ import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
 import Control.Exception (throw)
-import Control.Monad.State (liftIO)
 import Maybe
 import qualified Data.Map as M
 import System.Path (recurseDir)
@@ -152,8 +151,6 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy]
 			annexed_notpresent annexedfile
 		withcopy = "with content" ~: intmpclonerepo $ do
 			git_annex "get" ["-q", annexedfile] @? "get failed"
-			Utility.boolSystem "git" [Utility.Params "commit -q -a -m statechanged"]
-				@? "git commit of state failed"
 			annexed_present annexedfile
 			git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed"
 			unannexed annexedfile
@@ -167,8 +164,6 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do
 			git_annex "get" ["-q", annexedfile] @? "get failed"
-			Utility.boolSystem "git" [Utility.Params "commit -q -a -m statechanged"]
-				@? "git commit of state failed"
 			Utility.boolSystem "git" [Utility.Params "remote rm origin"]
 				@? "git remote rm origin failed"
 			r <- git_annex "drop" ["-q", annexedfile]

From cfe0894736aaabf320d080ac098ce0a65279fe27 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 20:35:09 -0400
Subject: [PATCH 1913/2835] merge bugfix

Use GitRepo functions to call git, the bug occurred when it was run
in a git repo that was not the same as the repo being acted on.
---
 GitUnionMerge.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 267376ed57..096a153a48 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -91,10 +91,10 @@ mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
  - with the specified parent refs. -}
 commit :: Git.Repo -> String -> String -> [String] -> IO ()
 commit g message newref parentrefs = do
-	tree <- Git.getSha "write-tree" $ ignorehandle $
-		pipeFrom "git" ["write-tree"]
-	sha <- Git.getSha "commit-tree" $ ignorehandle $ 
-		pipeBoth "git" (["commit-tree", tree] ++ ps) message
+	tree <- Git.getSha "write-tree" $
+		Git.pipeRead g [Param "write-tree"]
+	sha <- Git.getSha "commit-tree" $ ignorehandle $
+		Git.pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
 	Git.run g "update-ref" [Param newref, Param sha]
 	where
 		ignorehandle a = return . snd =<< a

From c0fbd3017f215cd61d79d24e110ce177e4823089 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 20:42:00 -0400
Subject: [PATCH 1914/2835] ssh

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 915b0bf0b2..49a80eb4b4 100644
--- a/Makefile
+++ b/Makefile
@@ -47,7 +47,7 @@ install: all
 	fi
 
 test: $(bins)
-	if ! $(GHCMAKE) -O0 test; then \
+	@if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
 		./test; \

From c4e6730042e64e3b2f92626aee4a6b38a8f9c70c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 21:19:52 -0400
Subject: [PATCH 1915/2835] commit git-annex branch when copying to a remote
 (locally)

Otherwise, the location log changes are only staged in its index,
and this can confuse matters if pulling or cloning from the remote.

The test suite was failing because this wasn't done.
---
 CmdLine.hs    |  7 ++-----
 Content.hs    | 11 ++++++++++-
 Remote/Git.hs |  3 +--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index d10516bb9c..ab7236847f 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,7 +19,7 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified GitRepo as Git
-import qualified Branch
+import Content
 import Types
 import Command
 import BackendList
@@ -103,9 +103,6 @@ startup = do
 {- Cleanup actions. -}
 shutdown :: Annex Bool
 shutdown = do
-	AnnexQueue.flush False
-	Branch.commit "update"
-
+	saveState
 	liftIO $ Git.reap
-	
 	return True
diff --git a/Content.hs b/Content.hs
index 45653fc9e7..d733ad8b36 100644
--- a/Content.hs
+++ b/Content.hs
@@ -19,7 +19,8 @@ module Content (
 	removeAnnex,
 	fromAnnex,
 	moveBad,
-	getKeysPresent
+	getKeysPresent,
+	saveState
 ) where
 
 import System.IO.Error (try)
@@ -37,6 +38,8 @@ import LocationLog
 import UUID
 import qualified GitRepo as Git
 import qualified Annex
+import qualified AnnexQueue
+import qualified Branch
 import Utility
 import StatFS
 import Types.Key
@@ -263,3 +266,9 @@ getKeysPresent' dir = do
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
+
+{- Things to do to record changes to content. -}
+saveState :: Annex ()
+saveState = do
+	AnnexQueue.flush False
+	Branch.commit "update"
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 5b9d5d3dfe..c8290c9a79 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -17,7 +17,6 @@ import Types
 import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
-import qualified AnnexQueue
 import Locations
 import UUID
 import Utility
@@ -147,7 +146,7 @@ copyToRemote r key
 			Annex.eval a $ do
 				ok <- Content.getViaTmp key $
 					rsyncOrCopyFile r keysrc
-				AnnexQueue.flush True
+				Content.saveState
 				return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo

From a4ef0e4da4431755c98fa104204af5254727e7a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 22:56:27 -0400
Subject: [PATCH 1916/2835] bugfix: restore index file env var

This fixes precommit, since in that hook, git sets the env var to write
to the lock file, which avoids git add failing due to the presence of the
lock file. (Took me a good hour and a half of confusion to figure this out.)

Test suite now passes 100%! Only the upgrade code still remains to be
written.
---
 Branch.hs  | 20 ++++++++++----------
 GitRepo.hs | 23 ++++++++++++++---------
 2 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 2cd658fe8e..4f204929ea 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -68,16 +68,19 @@ genIndex g = do
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
-withIndex a = do
+withIndex = withIndex' False
+withIndex' :: Bool -> Annex a -> Annex a
+withIndex' bootstrapping a = do
 	g <- Annex.gitRepo
 	let f = index g
-	liftIO $ Git.useIndex f
+	reset <- liftIO $ Git.useIndex f
 
-	e <- liftIO $ doesFileExist f
-	unless e $ liftIO $ genIndex g
+	unless bootstrapping $ do
+		e <- liftIO $ doesFileExist f
+		unless e $ liftIO $ genIndex g
 
 	r <- a
-	liftIO $ Git.useDefaultIndex
+	liftIO reset
 	return r
 
 withIndexUpdate :: Annex a -> Annex a
@@ -121,11 +124,8 @@ create = do
 		inorigin <- refexists origin
 		if inorigin
 			then liftIO $ Git.run g "branch" [Param name, Param origin]
-			else liftIO $ do
-				let f = index g
-				liftIO $ Git.useIndex f
-				GitUnionMerge.commit g "branch created" fullname []
-				liftIO $ Git.useDefaultIndex
+			else withIndex' True $
+				liftIO $ GitUnionMerge.commit g "branch created" fullname []
 	where
 		origin = "origin/" ++ name
 		refexists ref = do
diff --git a/GitRepo.hs b/GitRepo.hs
index d1f122fba5..4a6c4d7637 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -60,7 +60,6 @@ module GitRepo (
 	repoAbsPath,
 	reap,
 	useIndex,
-	useDefaultIndex,
 	hashObject,
 	getSha,
 	shaSize,
@@ -79,6 +78,7 @@ import System.Cmd.Utils
 import IO (bracket_)
 import Data.String.Utils
 import System.IO
+import IO (try)
 import qualified Data.Map as Map hiding (map, split)
 import Network.URI
 import Data.Maybe
@@ -88,7 +88,7 @@ import Codec.Binary.UTF8.String (encode)
 import Text.Printf
 import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
-import System.Posix.Env (setEnv, unsetEnv)
+import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
 import Utility
 
@@ -391,13 +391,18 @@ reap = do
 	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
 	maybe (return ()) (const reap) r
 
-{- Forces git to use the specified index file. -}
-useIndex :: FilePath -> IO ()
-useIndex index = setEnv "GIT_INDEX_FILE" index True
-
-{- Undoes useIndex -}
-useDefaultIndex :: IO ()
-useDefaultIndex = unsetEnv "GIT_INDEX_FILE"
+{- Forces git to use the specified index file.
+ - Returns an action that will reset back to the default
+ - index file. -}
+useIndex :: FilePath -> IO (IO ())
+useIndex index = do
+	res <- try $ getEnv var
+	setEnv var index True
+	return $ reset res
+	where
+		var = "GIT_INDEX_FILE"
+		reset (Right (Just v)) = setEnv var v True
+		reset _ = unsetEnv var
 
 {- Injects some content into git, returning its hash. -}
 hashObject :: Repo -> String -> IO String

From 68783fd5e0e23ce698f2c2c2e2bd28c54cadf9c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 23:02:58 -0400
Subject: [PATCH 1917/2835] let's have the major version number be
 annex.version

---
 debian/NEWS      | 2 +-
 debian/changelog | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/NEWS b/debian/NEWS
index 4e085bb002..00531d41bc 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,4 +1,4 @@
-git-annex (0.20110622) unstable; urgency=low
+git-annex (3.20110622) unstable; urgency=low
 
   There has been another change to the git-annex data store.
   Use `git annex upgrade` to migrate your repositories to the new
diff --git a/debian/changelog b/debian/changelog
index 03ccbe03f0..4543ef271a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20110611) UNRELEASED; urgency=low
+git-annex (3.20110611) UNRELEASED; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
   * git-annex now stores its logs in a git-annex branch.

From aad73c5721490a5679820ab9c16a8b462fa0e0f2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 23:24:14 -0400
Subject: [PATCH 1918/2835] rewrite loggedkeys to use git-annex branch

That sucking sound is a whole page of code vanishing to be replaced with
  return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files
What can I say, git is my database, and haskell my copilot.
---
 Branch.hs         |  9 ++++++++-
 Command/Unused.hs |  4 +---
 LocationLog.hs    | 21 +++------------------
 3 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 4f204929ea..a43ee227b0 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -11,7 +11,7 @@ module Branch (
 	get,
 	change,
 	commit,
-	shortref
+	files
 ) where
 
 import Control.Monad (unless, when, liftM)
@@ -222,3 +222,10 @@ cmdOutput cmd params = do
 	let rv = seq retval retval
 	_ <- getProcessStatus True False pid
 	return rv
+
+{- Lists all files on the branch. -}
+files :: Annex [FilePath]
+files = withIndexUpdate $ do
+	g <- Annex.gitRepo
+	liftIO $ Git.pipeNullSplit g
+		[Params "ls-tree --name-only -r -z", Param fullname]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5d4e433ad8..51964cc57c 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -66,10 +66,8 @@ checkRemoteUnused name = do
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
 	showNote $ "checking for unused data..."
-	g <- Annex.gitRepo
 	referenced <- getKeysReferenced
-	logged <- loggedKeys g
-	remotehas <- filterM isthere logged
+	remotehas <- filterM isthere =<< loggedKeys
 	let remoteunused = remotehas `exclude` referenced
 	let list = number 0 remoteunused
 	writeUnusedFile "" list
diff --git a/LocationLog.hs b/LocationLog.hs
index 8dbeb729ca..4e2caca959 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -26,7 +26,6 @@ module LocationLog (
 import Data.Time.Clock.POSIX
 import Data.Time
 import System.Locale
-import System.Directory
 import System.FilePath
 import qualified Data.Map as Map
 import Control.Monad (when)
@@ -35,7 +34,6 @@ import Control.Monad.State (liftIO)
 
 import qualified GitRepo as Git
 import qualified Branch
-import Utility
 import UUID
 import Types
 import Locations
@@ -148,19 +146,6 @@ mapLog m l =
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
-loggedKeys :: Git.Repo -> Annex [Key]
-loggedKeys repo = do
-	_ <- error "FIXME.. does not look in git-annex branch yet"
-	exists <- liftIO $ doesDirectoryExist dir
-	if exists
-		then do
-			-- 2 levels of hashing
-			levela <- liftIO $ dirContents dir
-			levelb <- mapM tryDirContents levela
-			files <- mapM tryDirContents (concat levelb)
-			return $ catMaybes $
-				map (logFileKey . takeFileName) (concat files)
-		else return []
-	where
-		tryDirContents d = liftIO $ catch (dirContents d) (return . const [])
-		dir = gitStateDir repo
+loggedKeys :: Annex [Key]
+loggedKeys =
+	return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files

From 66ceb9270266be677bdb0731a9c95569bad37d28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 23:37:46 -0400
Subject: [PATCH 1919/2835] docs

---
 Upgrade/V2.hs    | 15 +++++++++++++++
 debian/changelog |  4 ++++
 2 files changed, 19 insertions(+)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index df1f62b8c8..c249e340bf 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -17,6 +17,21 @@ import Messages
 import Utility
 import Locations
 
+{- .git-annex/ moved to a git-annex branch.
+ - 
+ - Strategy:
+ - 
+ - * Create the git-annex branch.
+ - * Find each location log file in .git-annex/, and inject its content
+ -   into the git-annex branch, unioning with any content already in
+ -   there. (in passing, this deals with the semi transition that left
+ -   some location logs hashed two different ways; both are found and
+ -   merged).
+ - * Also inject remote.log, trust.log, and uuid.log.
+ - * git rm -rf .git-annex
+ - * Remove stuff that used to be needed in .gitattributes.
+ - * Commit changes.
+ -}
 upgrade :: Annex Bool
 upgrade = do
 	showNote "v2 to v3"
diff --git a/debian/changelog b/debian/changelog
index 4543ef271a..3fc7216e20 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,10 @@ git-annex (3.20110611) UNRELEASED; urgency=low
   * merge: New subcommand. Auto-merges the new git-annex branch.
   * Improved handling of bare git repos with annexes. Many more commands will
     work in them.
+  * Sped up many commands.
+  * git-annex is now more robust; it will never leave state files
+    uncommitted when some other git process comes along and locks the index
+    at an inconvenient time.
   * rsync is now used when copying files from repos on other filesystems.
     cp is still used when copying file from repos on the same filesystem,
     since --reflink=auto can make it significantly faster on filesystems

From af10b2854a199ed9985cde938d46b252f4d5e503 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 02:30:20 -0400
Subject: [PATCH 1920/2835] v3 upgrade code works

but write the index file a lot, so slow
---
 Command/Upgrade.hs |  4 +--
 Upgrade/V1.hs      |  5 +--
 Upgrade/V2.hs      | 54 +++++++++++++++++++++++--------
 doc/upgrades.mdwn  | 81 +++++++++++++++++++++++++++-------------------
 4 files changed, 94 insertions(+), 50 deletions(-)

diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index b3c0468039..b79b13cd3c 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -21,7 +21,7 @@ seek = [withNothing start]
 
 start :: CommandStartNothing
 start = do
-	showStart "upgrade" ""
+	showStart "upgrade" "."
 	r <- upgrade
-	checkVersion
+	setVersion
 	next $ next $ return r
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 1f327f77b9..61a8018590 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -30,6 +30,7 @@ import Backend
 import Messages
 import Version
 import Utility
+import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
 -- Key information is encoded in filenames differently, so
@@ -70,8 +71,8 @@ upgrade = do
 	
 			AnnexQueue.flush True
 			setVersion
-
-	return True
+	
+	Upgrade.V2.upgrade
 
 moveContent :: Annex ()
 moveContent = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index c249e340bf..36ba1a0f21 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -1,4 +1,4 @@
-{- git-annex v2 -> v2 upgrade support
+{- git-annex v2 -> v3 upgrade support
  -
  - Copyright 2011 Joey Hess 
  -
@@ -9,14 +9,22 @@ module Upgrade.V2 where
 
 import System.Directory
 import System.FilePath
+import Control.Monad.State (liftIO)
+import List
+import Data.Maybe
 
 import Types.Key
 import Types
+import qualified Annex
 import qualified GitRepo as Git
+import qualified Branch
 import Messages
 import Utility
 import Locations
 
+olddir :: FilePath
+olddir = ".git-annex"
+
 {- .git-annex/ moved to a git-annex branch.
  - 
  - Strategy:
@@ -35,7 +43,36 @@ import Locations
 upgrade :: Annex Bool
 upgrade = do
 	showNote "v2 to v3"
-	error "TODO"
+	g <- Annex.gitRepo
+	Branch.create
+	mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
+	mapM_ (\f -> inject f f) =<< logFiles olddir
+	liftIO $ do
+		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File olddir]
+		gitAttributesUnWrite g
+	return True
+
+locationLogs :: Git.Repo -> Annex [(Key, FilePath)]
+locationLogs repo = liftIO $ do
+	levela <- dirContents dir
+	levelb <- mapM tryDirContents levela
+	files <- mapM tryDirContents (concat levelb)
+	return $ catMaybes $ map islogfile (concat files)
+	where
+		tryDirContents d = catch (dirContents d) (return . const [])
+		dir = gitStateDir repo
+		islogfile f = maybe Nothing (\k -> Just $ (k, f)) $
+				logFileKey $ takeFileName f
+
+inject :: FilePath -> FilePath -> Annex ()
+inject source dest = do
+	new <- liftIO (readFile $ olddir  source)
+	prev <- Branch.get dest
+	Branch.change dest $ unlines $ nub $ lines prev ++ lines new
+
+logFiles :: FilePath -> Annex [FilePath]
+logFiles dir = return . filter (".log" `isSuffixOf`)
+		=<< liftIO (getDirectoryContents dir)
 
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]
@@ -49,15 +86,6 @@ gitAttributesUnWrite repo = do
 	let attributes = Git.attributes repo
 	whenM (doesFileExist attributes) $ do
 		c <- readFileStrict attributes
-		safeWriteFile attributes $ unlines $
+		liftIO $ safeWriteFile attributes $ unlines $
 			filter (\l -> not $ l `elem` attrLines) $ lines c
-
-oldlogFile :: Git.Repo -> Key -> String
-oldlogFile = logFile' hashDirLower
-
-oldlogFileOld :: Git.Repo -> Key -> String
-oldlogFileOld = logFile' hashDirMixed
-
-logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
-logFile' hasher repo key =
-	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
+		Git.run repo "add" [File attributes]
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 516e5b3bb8..bf12f7e438 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -7,27 +7,63 @@ There's a committment that git-annex will always support upgrades from all
 past versions. After all, you may have offline drives from an earlier
 git-annex, and might want to use them with a newer git-annex.
 
-## Upgrade process
-
 git-annex will notice if it is run in a repository that
 needs an upgrade, and refuse to do anything. To upgrade,
 use the "git annex upgrade" command. The upgrade can tend
 to take a while, if you have a lot of files.
 
-Each clone of a repository should be individually upgraded.
-Until a repository's remotes have been upgraded, git-annex
-will refuse to communicate with them.
-
-Generally, start by upgrading one repository, and then you can commit
-the changes git-annex staged during upgrade, and push them out to other
-repositories. And then upgrade those other repositories. Doing it this
-way avoids git-annex doing some duplicate work during the upgrade.
-
 The upgrade process is guaranteed to be conflict-free. Unless you
 already have git conflicts in your repository or between repositories.
 Upgrading a repository with conflicts is not recommended; resolve the
 conflicts first before upgrading git-annex.
 
+## Upgrade events, so far
+
+### v2 -> v3 (git-annex version 3.x)
+
+Involved moving the .git-annex/ directory into a separate git-annex branch.
+
+### tips for this upgrade
+
+This upgrade is easier than the previous upgrades. You don't need to
+upgrade every repository at once; it's sufficient to upgrade each
+repository only when you next use it.
+
+This upgrade can be sped up by, before you start, making
+.git/index.git-annex into a symlink to a file on a ramdisk.
+For example: `ln -s /run/shm/index.git-annex.$(git config annex.uuid) .git/index.git-annex`
+but, if you do that, be sure to remove the symlink after the upgrade!
+
+After the upgrade is complete, commit the changes it staged.
+	
+	git commit -m "upgrade v2 to v3"
+
+### v1 -> v2 (git-annex version 0.20110316)
+
+Involved adding hashing to .git/annex/ and changing the names of all keys.
+Symlinks changed.
+
+Also, hashing was added to location log files in .git-annex/.
+And .gitattributes needed to have another line added to it.
+
+Previously, files added to the SHA [[backends]] did not have their file
+size tracked, while files added to the WORM backend did. Files added to
+the SHA backends after the conversion will have their file size tracked,
+and that information will be used by git-annex for disk free space checking.
+To ensure that information is available for all your annexed files, see
+[[upgrades/SHA_size]].
+
+### tips for this upgrade
+
+Each clone of a repository should be individually upgraded.
+Until a repository's remotes have been upgraded, git-annex
+will refuse to communicate with them.
+
+Start by upgrading one repository, and then you can commit
+the changes git-annex staged during upgrade, and push them out to other
+repositories. And then upgrade those other repositories. Doing it this
+way avoids git-annex doing some duplicate work during the upgrade.
+
 Example upgrade process:
 
 	cd localrepo
@@ -43,28 +79,7 @@ Example upgrade process:
 	git annex upgrade
 	...
 
-## Upgrade events, so far
-
-### v2 -> v3 (git-annex version 0.20110610 to version 0.20110622)
-
-Involved moving the .git-annex/ directory into a separate git-annex branch.
-
-### v1 -> v2 (git-annex version 0.23 to version 0.20110316)
-
-Involved adding hashing to .git/annex/ and changing the names of all keys.
-Symlinks changed.
-
-Also, hashing was added to location log files in .git-annex/.
-And .gitattributes needed to have another line added to it.
-
-Previously, files added to the SHA [[backends]] did not have their file
-size tracked, while files added to the WORM backend did. Files added to
-the SHA backends after the conversion will have their file size tracked,
-and that information will be used by git-annex for disk free space checking.
-To ensure that information is available for all your annexed files, see
-[[upgrades/SHA_size]].
-
-### v0 -> v1 (git-annex version 0.03 to version 0.04)
+### v0 -> v1 (git-annex version 0.04)
 
 Involved a reorganisation of the layout of .git/annex/. Symlinks changed.
 

From 9e37898e2186c8f7bb71a5d2bd7a02303410b363 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 08:48:13 -0400
Subject: [PATCH 1921/2835] remove stateDir

---
 Command.hs        | 11 ++++-------
 Locations.hs      |  9 ---------
 Upgrade/V1.hs     |  6 +++---
 Upgrade/V2.hs     |  6 ++++++
 doc/upgrades.mdwn |  1 -
 5 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/Command.hs b/Command.hs
index 228c1f40e9..129233d740 100644
--- a/Command.hs
+++ b/Command.hs
@@ -22,7 +22,6 @@ import qualified Backend
 import Messages
 import qualified Annex
 import qualified GitRepo as Git
-import Locations
 import Utility
 import Types.Key
 
@@ -181,17 +180,15 @@ withNothing _ _ = error "This command takes no parameters."
 backendPairs :: CommandSeekBackendFiles
 backendPairs a files = liftM (map a) $ Backend.chooseBackends files
 
-{- Filter out files from the state directory, and those matching the
- - exclude glob pattern, if it was specified. -}
+{- Filter out files those matching the exclude glob pattern,
+ - if it was specified. -}
 filterFiles :: [FilePath] -> Annex [FilePath]
 filterFiles l = do
-	let l' = filter notState l
 	exclude <- Annex.getState Annex.exclude
 	if null exclude
-		then return l'
-		else return $ filter (notExcluded $ wildsRegex exclude) l'
+		then return l
+		else return $ filter (notExcluded $ wildsRegex exclude) l
 	where
-		notState f = not $ stateDir `isPrefixOf` f
 		notExcluded r f = isNothing $ match r f []
 
 wildsRegex :: [String] -> Regex
diff --git a/Locations.hs b/Locations.hs
index 7f9626a11b..df4957f3e1 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -6,8 +6,6 @@
  -}
 
 module Locations (
-	gitStateDir,
-	stateDir,
 	keyFile,
 	fileKey,
 	gitAnnexLocation,
@@ -52,13 +50,6 @@ import qualified GitRepo as Git
  - Everything else should use relative paths.
  -}
 
-{- Long-term, cross-repo state is stored in files inside the .git-annex
- - directory, in the git repository's working tree. -}
-stateDir :: FilePath
-stateDir = addTrailingPathSeparator $ ".git-annex"
-gitStateDir :: Git.Repo -> FilePath
-gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
-
 {- The directory git annex uses for local state, relative to the .git
  - directory -}
 annexDir :: FilePath
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 61a8018590..b06f00d34d 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -113,7 +113,7 @@ moveLocationLogs = do
 		where
 			oldlocationlogs = do
 				g <- Annex.gitRepo
-				let dir = gitStateDir g
+				let dir = Upgrade.V2.gitStateDir g
 				exists <- liftIO $ doesDirectoryExist dir
 				if exists
 					then do
@@ -123,7 +123,7 @@ moveLocationLogs = do
 			move (l, k) = do
 				g <- Annex.gitRepo
 				let dest = logFile k
-				let dir = gitStateDir g
+				let dir = Upgrade.V2.gitStateDir g
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
 				-- could just git mv, but this way deals with
@@ -186,7 +186,7 @@ fileKey1 file = readKey1 $
 	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file
 
 logFile1 :: Git.Repo -> Key -> String
-logFile1 repo key = gitStateDir repo ++ keyFile1 key ++ ".log"
+logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 36ba1a0f21..7e4cfb13a7 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -89,3 +89,9 @@ gitAttributesUnWrite repo = do
 		liftIO $ safeWriteFile attributes $ unlines $
 			filter (\l -> not $ l `elem` attrLines) $ lines c
 		Git.run repo "add" [File attributes]
+
+stateDir :: FilePath
+stateDir = addTrailingPathSeparator $ ".git-annex"
+gitStateDir :: Git.Repo -> FilePath
+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
+
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index bf12f7e438..63fbcf75b7 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -69,7 +69,6 @@ Example upgrade process:
 	cd localrepo
 	git pull
 	git annex upgrade
-	(Upgrading object directory layout v1 to v2...)
 	git commit -m "upgrade v1 to v2"
 	git push
 

From 9672496a9357c84a4436ead109ba2dc7bc010e8c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 08:49:27 -0400
Subject: [PATCH 1922/2835] update

---
 doc/bugs/annex_add_in_annex.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/annex_add_in_annex.mdwn b/doc/bugs/annex_add_in_annex.mdwn
index 80084cec12..e12826f00e 100644
--- a/doc/bugs/annex_add_in_annex.mdwn
+++ b/doc/bugs/annex_add_in_annex.mdwn
@@ -2,3 +2,5 @@ I accidentally annexed some files in the .git-annex directory and it cause git-a
 
 > There is a guard against `git annex add .git-annex/foo`, but it doesn't
 > notice `cd .git-annex; git annex add foo`. --[[Joey]]
+
+> Now fixed, by removing the .git-annex directory. [[done]] --[[Joey]] 

From 23e765b67c38a9f02b3b5152e7e123819bb696de Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 09:56:04 -0400
Subject: [PATCH 1923/2835] update re git-annex branch direct modification

---
 Branch.hs          |  8 ++++++++
 Locations.hs       |  5 +++++
 doc/internals.mdwn | 11 ++++++++---
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index a43ee227b0..00f406135e 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -164,6 +164,8 @@ updateRef ref
 	| ref == fullname = return Nothing
 	| otherwise = do
 		g <- Annex.gitRepo
+		-- checking with log to see if there have been changes
+		-- is less expensive than always merging
 		diffs <- liftIO $ Git.pipeRead g [
 			Param "log",
 			Param (name++".."++ref),
@@ -176,6 +178,12 @@ updateRef ref
 				-- By passing only one ref, it is actually
 				-- merged into the index, preserving any
 				-- changes that may already be staged.
+				--
+				-- However, any changes in the git-annex
+				-- branch that are *not* reflected in the
+				-- index will be removed. So, documentation
+				-- advises users not to directly modify the
+				-- branch.
 				liftIO $ GitUnionMerge.merge g [ref]
 				return $ Just ref
 
diff --git a/Locations.hs b/Locations.hs
index df4957f3e1..f93b0cc50d 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -105,6 +105,11 @@ gitAnnexBadLocation r key = gitAnnexBadDir r  keyFile key
 gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath
 gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 
+{- .git/annex/journal/ is used to journal changes made to the git-annex
+ - branch -}
+gitAnnexJournalDir :: Git.Repo -> FilePath
+gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r  "journal"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 419096744d..aaa125599d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -21,9 +21,14 @@ deleting or changing the file contents.
 
 This branch is managed by git-annex, with the contents listed below.
 
-Note that it assumes only it will modify the branch. If you go in and make
-changes, be sure to remove `.git/index.git-annex` before running git-annex,
-otherwise it will probably revert your changes in its next commit to the branch.
+Note that git-annex assumes only it will modify this branch. If you go in
+and make changes directly, it will probably revert your changes in its next
+commit to the branch.
+
+The best way to make changes to the git-annex branch is instead
+to create a branch of it, with a name like "my/git-annex", and then
+use "git annex merge" to automerge your branch into the main git-annex
+branch.
 
 ### `uuid.log`
 

From 5f494154a34bad7cc54915a2a408b830e8ca77be Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 11:37:26 -0400
Subject: [PATCH 1924/2835] add journaling to speed up changes to the git-annex
 branch

git is slow when the index file is large and has to be rewritten each time
a file is changed. To speed this up, added a journal where changes are
recorded before being fed into the index file and committed to the
git-annex branch. The entire journal can be fed into git with just 2
commands, and only one write of the index file.
---
 Branch.hs            | 137 +++++++++++++++++++++++++++++++++++--------
 GitUnionMerge.hs     |  14 +++--
 Locations.hs         |   1 +
 Types/BranchState.hs |   3 +-
 doc/internals.mdwn   |  10 +++-
 doc/upgrades.mdwn    |   5 --
 git-union-merge.hs   |   2 +-
 7 files changed, 132 insertions(+), 40 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 00f406135e..f0d97bfc36 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -33,6 +33,7 @@ import qualified Annex
 import Utility
 import Types
 import Messages
+import Locations
 
 {- Name of the branch that is used to store git-annex's information. -}
 name :: String
@@ -42,6 +43,8 @@ name = "git-annex"
 fullname :: String
 fullname = "refs/heads/" ++ name
 
+{- Converts a fully qualified git ref into a short version for human
+ - consumptiom. -}
 shortref :: String -> String
 shortref = remove "refs/heads/" . remove "refs/remotes/"
 	where
@@ -56,7 +59,8 @@ index g = Git.workTree g  Git.gitDir g  "index." ++ name
 {- Populates the branch's index file with the current branch contents.
  - 
  - Usually, this is only done when the index doesn't yet exist, and
- - the index is used to build up changes to be commited to the branch.
+ - the index is used to build up changes to be commited to the branch,
+ - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
 genIndex g = do
@@ -97,11 +101,6 @@ setCache file content = do
 	state <- getState
 	setState state { cachedFile = Just file, cachedContent = content }
 
-setCacheChanged :: FilePath -> String -> Annex ()
-setCacheChanged file content = do
-	state <- getState
-	setState state { cachedFile = Just file, cachedContent = content, branchChanged = True }
-
 invalidateCache :: Annex ()
 invalidateCache = do
 	state <- getState
@@ -133,11 +132,11 @@ create = do
 			liftIO $ Git.runBool g "show-ref"
 				[Param "--verify", Param "-q", Param ref]
 
-{- Commits any staged changes to the branch. -}
+{- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = do
-	state <- getState
-	when (branchChanged state) $ do
+	staged <- stageJournalFiles
+	when staged $ do
 		g <- Annex.gitRepo
 		withIndex $ liftIO $
 			GitUnionMerge.commit g message fullname [fullname]
@@ -187,28 +186,32 @@ updateRef ref
 				liftIO $ GitUnionMerge.merge g [ref]
 				return $ Just ref
 
-{- Stages the content of a file into the branch's index. -}
+{- Records changed content of a file into the journal. -}
 change :: FilePath -> String -> Annex ()
 change file content = do
-	g <- Annex.gitRepo
-	sha <- liftIO $ Git.hashObject g content
-	withIndex $ liftIO $ Git.run g "update-index"
-		[ Param "--add", Param "--cacheinfo", Param "100644",
-		  Param sha, File file]
-	setCacheChanged file content
+	setJournalFile file content
+	setCache file content
 
-{- Gets the content of a file on the branch, or content staged in the index
- - if it's newer. Returns an empty string if the file didn't exist yet. -}
+{- Gets the content of a file on the branch, or content from the journal, or
+ - staged in the index.
+ -
+ - Returns an empty string if the file doesn't exist yet. -}
 get :: FilePath -> Annex String
 get file = do
 	cached <- getCache file
 	case cached of
 		Just content -> return content
-		Nothing -> withIndexUpdate $ do
-			g <- Annex.gitRepo
-			content <- liftIO $ catch (cat g) (const $ return "")
-			setCache file content
-			return content
+		Nothing -> do
+			j <- getJournalFile file
+			case j of
+				Just content -> do
+					setCache file content
+					return content
+				Nothing -> withIndexUpdate $ do
+					g <- Annex.gitRepo
+					content <- liftIO $ catch (cat g) (const $ return "")
+					setCache file content
+					return content
 	where
 		cat g = cmdOutput "git" $ toCommand $ Git.gitCommandLine g
 			[Param "cat-file", Param "blob", Param $ ':':file]
@@ -231,9 +234,93 @@ cmdOutput cmd params = do
 	_ <- getProcessStatus True False pid
 	return rv
 
-{- Lists all files on the branch. -}
+{- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
 	g <- Annex.gitRepo
-	liftIO $ Git.pipeNullSplit g
+	bfiles <- liftIO $ Git.pipeNullSplit g
 		[Params "ls-tree --name-only -r -z", Param fullname]
+	jfiles <- getJournalFiles
+	return $ jfiles ++ bfiles
+
+{- Records content for a file in the branch to the journal.
+ -
+ - Using the journal, rather than immediatly staging content to the index
+ - avoids git needing to rewrite the index after every change. -}
+setJournalFile :: FilePath -> String -> Annex ()
+setJournalFile file content = do
+	g <- Annex.gitRepo
+	liftIO $ catch (write g) $ const $ do
+		createDirectoryIfMissing True $ gitAnnexJournalDir g
+		createDirectoryIfMissing True $ gitAnnexTmpDir g
+		write g
+	where
+		-- journal file is written atomically
+		write g = do
+			let jfile = journalFile g file
+			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
+			writeFile tmpfile content
+			renameFile tmpfile jfile
+
+{- Gets journalled content for a file in the branch. -}
+getJournalFile :: FilePath -> Annex (Maybe String)
+getJournalFile file = do
+	g <- Annex.gitRepo
+	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)
+		(const $ return Nothing)
+
+{- List of journal files. -}
+getJournalFiles :: Annex [FilePath]
+getJournalFiles = getJournalFilesRaw >>= return . map fileJournal
+
+getJournalFilesRaw :: Annex [FilePath]
+getJournalFilesRaw = do
+	g <- Annex.gitRepo
+	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
+		(const $ return [])
+	return $ filter (\f -> f /= "." && f /= "..") fs
+
+{- Stages all journal files into the index, and returns True if the index
+ - was modified. -}
+stageJournalFiles :: Annex Bool
+stageJournalFiles = do
+	l <- getJournalFilesRaw
+	if null l
+		then return False
+		else do
+			g <- Annex.gitRepo
+			withIndex $ liftIO $ stage g l
+			return True
+	where
+		stage g fs = do
+			let dir = gitAnnexJournalDir g
+			let paths = map (dir ) fs
+			-- inject all the journal files directly into git
+			-- in one quick command
+			(h, s) <- Git.pipeWriteRead g [Param "hash-object",
+				Param "-w", Param "--stdin-paths"] $ unlines paths
+			-- update the index, also in just one command
+			GitUnionMerge.update_index g $
+				index_lines (lines s) $ map fileJournal fs
+			forceSuccess h
+			mapM_ removeFile paths
+		index_lines shas fs = map genline $ zip shas fs
+		genline (sha, file) = GitUnionMerge.update_index_line sha file
+
+{- Produces a filename to use in the journal for a file on the branch.
+ -
+ - The journal typically won't have a lot of files in it, so the hashing
+ - used in the branch is not necessary, and all the files are put directly
+ - in the journal directory.
+ -}
+journalFile :: Git.Repo -> FilePath -> FilePath
+journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
+	where
+		mangle '/' = "_"
+		mangle '_' = "__"
+		mangle c = [c]
+
+{- Converts a journal file (relative to the journal dir) back to the
+ - filename on the branch. -}
+fileJournal :: FilePath -> FilePath
+fileJournal = replace "//" "_" . replace "_" "/"
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 096a153a48..fa14a6bc3f 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -7,7 +7,9 @@
 
 module GitUnionMerge (
 	merge,
-	commit
+	commit,
+	update_index,
+	update_index_line
 ) where
 
 import System.Cmd.Utils
@@ -43,6 +45,11 @@ update_index g l =  togit ["update-index", "-z", "--index-info"] (join "\0" l)
 		togit ps content = Git.pipeWrite g (map Param ps) content
 			>>= forceSuccess
 
+{- Generates a line suitable to be fed into update-index, to add
+ - a given file with a given sha. -}
+update_index_line :: String -> FilePath -> String
+update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
+
 {- Gets the contents of a tree in a format suitable for update_index. -}
 ls_tree :: Git.Repo -> String -> IO [String]
 ls_tree g x = Git.pipeNullSplit g $ 
@@ -76,14 +83,13 @@ calc_merge g differ = do
 mergeFile :: Git.Repo -> (String, FilePath) -> IO (Maybe String)
 mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
-	(sha:[]) -> return $ Just $ ls_tree_line sha
+	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
 		content <- Git.pipeRead g $ map Param ("show":shas)
 		sha <- Git.hashObject g $ unionmerge content
-		return $ Just $ ls_tree_line sha
+		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
-		ls_tree_line sha = "100644 blob " ++ sha ++ "\t" ++ file
 		nullsha = take Git.shaSize $ repeat '0'
 		unionmerge = unlines . nub . lines
 
diff --git a/Locations.hs b/Locations.hs
index f93b0cc50d..bfb0d3af9e 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -17,6 +17,7 @@ module Locations (
 	gitAnnexBadDir,
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
+	gitAnnexJournalDir,
 	isLinkToAnnex,
 	logFile,
 	logFileKey,
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
index 65d0642a14..40d7f5c2c7 100644
--- a/Types/BranchState.hs
+++ b/Types/BranchState.hs
@@ -9,10 +9,9 @@ module Types.BranchState where
 
 data BranchState = BranchState {
 	branchUpdated :: Bool,
-	branchChanged :: Bool,
 	cachedFile :: Maybe FilePath,
 	cachedContent :: String
 }
 
 startBranchState :: BranchState
-startBranchState = BranchState False False Nothing ""
+startBranchState = BranchState False Nothing ""
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index aaa125599d..27b5bb1f29 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -21,9 +21,13 @@ deleting or changing the file contents.
 
 This branch is managed by git-annex, with the contents listed below.
 
-Note that git-annex assumes only it will modify this branch. If you go in
-and make changes directly, it will probably revert your changes in its next
-commit to the branch.
+The file `.git/index.git-annex` is a separate git index file it uses
+to accumlate changes for the branch. Also, `.git/annex/journal/` is used
+to record changes before they are added to git.
+
+Note that for speed reasons, git-annex assumes only it will modify this
+branch. If you go in and make changes directly, it will probably revert
+your changes in its next commit to the branch.
 
 The best way to make changes to the git-annex branch is instead
 to create a branch of it, with a name like "my/git-annex", and then
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 63fbcf75b7..2e8f201fb0 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -29,11 +29,6 @@ This upgrade is easier than the previous upgrades. You don't need to
 upgrade every repository at once; it's sufficient to upgrade each
 repository only when you next use it.
 
-This upgrade can be sped up by, before you start, making
-.git/index.git-annex into a symlink to a file on a ramdisk.
-For example: `ln -s /run/shm/index.git-annex.$(git config annex.uuid) .git/index.git-annex`
-but, if you do that, be sure to remove the symlink after the upgrade!
-
 After the upgrade is complete, commit the changes it staged.
 	
 	git commit -m "upgrade v2 to v3"
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 7c0c1cd843..57232be67b 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -42,7 +42,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
-	Git.useIndex (tmpIndex g)
+	_ <- Git.useIndex (tmpIndex g)
 	setup g
 	GitUnionMerge.merge g [aref, bref]
 	GitUnionMerge.commit g "union merge" newref [aref, bref]

From 224a8098b5cc27a78b31094026e545137781b27e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 11:46:55 -0400
Subject: [PATCH 1925/2835] v3 upgrade is fast!

The journal sped this up approximatly 100-fold; it runs in just a few
minutes for a large repository with 30 thousand log files.
---
 doc/upgrades.mdwn | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 2e8f201fb0..509e296956 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -9,8 +9,7 @@ git-annex, and might want to use them with a newer git-annex.
 
 git-annex will notice if it is run in a repository that
 needs an upgrade, and refuse to do anything. To upgrade,
-use the "git annex upgrade" command. The upgrade can tend
-to take a while, if you have a lot of files.
+use the "git annex upgrade" command.
 
 The upgrade process is guaranteed to be conflict-free. Unless you
 already have git conflicts in your repository or between repositories.
@@ -25,9 +24,9 @@ Involved moving the .git-annex/ directory into a separate git-annex branch.
 
 ### tips for this upgrade
 
-This upgrade is easier than the previous upgrades. You don't need to
-upgrade every repository at once; it's sufficient to upgrade each
-repository only when you next use it.
+This upgrade is easier (and faster!) than the previous upgrades.
+You don't need to upgrade every repository at once; it's sufficient
+to upgrade each repository only when you next use it.
 
 After the upgrade is complete, commit the changes it staged.
 	
@@ -50,6 +49,8 @@ To ensure that information is available for all your annexed files, see
 
 ### tips for this upgrade
 
+This upgrade can tend to take a while, if you have a lot of files.
+
 Each clone of a repository should be individually upgraded.
 Until a repository's remotes have been upgraded, git-annex
 will refuse to communicate with them.

From d05fd113925bd3528a3b8643c58536c96d7151bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 12:11:03 -0400
Subject: [PATCH 1926/2835] updates

---
 doc/internals.mdwn | 4 ++--
 doc/upgrades.mdwn  | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 27b5bb1f29..1096845e5e 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -21,8 +21,8 @@ deleting or changing the file contents.
 
 This branch is managed by git-annex, with the contents listed below.
 
-The file `.git/index.git-annex` is a separate git index file it uses
-to accumlate changes for the branch. Also, `.git/annex/journal/` is used
+The file `.git/annex/index` is a separate git index file it uses
+to accumlate changes for the git-annex. Also, `.git/annex/journal/` is used
 to record changes before they are added to git.
 
 Note that for speed reasons, git-annex assumes only it will modify this
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 509e296956..1813d079a1 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -32,6 +32,9 @@ After the upgrade is complete, commit the changes it staged.
 	
 	git commit -m "upgrade v2 to v3"
 
+Running `git gc` after this upgrade will likely free up significant disk 
+space. (Tens to hundreds of megabytes.)
+
 ### v1 -> v2 (git-annex version 0.20110316)
 
 Involved adding hashing to .git/annex/ and changing the names of all keys.

From 1b21dd99c58c40ddd5bce9d575a47c66f121a29c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 12:11:09 -0400
Subject: [PATCH 1927/2835] rename git-annex index file

---
 Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Branch.hs b/Branch.hs
index f0d97bfc36..6b2e8c23eb 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -54,7 +54,7 @@ shortref = remove "refs/heads/" . remove "refs/remotes/"
 
 {- A separate index file for the branch. -}
 index :: Git.Repo -> FilePath
-index g = Git.workTree g  Git.gitDir g  "index." ++ name
+index g = gitAnnexDir g  "index"
 
 {- Populates the branch's index file with the current branch contents.
  - 

From 89fd7b34ce815fc5816de5c71f07382e30f50bd5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 12:23:25 -0400
Subject: [PATCH 1928/2835] unused command updates for branches

Now that branches are more likely, unused needs to more explicitly warn
that it does not look in them.
---
 Command/Unused.hs  | 18 +++++++++++-------
 doc/git-annex.mdwn |  8 ++++----
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 51964cc57c..5744f84fd4 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -54,7 +54,9 @@ checkUnused = do
 	where
 		list file msg l c = do
 			let unusedlist = number c l
-			when (not $ null l) $ showLongNote $ msg unusedlist
+			when (not $ null l) $ do
+				showLongNote $ msg unusedlist
+				showLongNote $ "\n"
 			writeUnusedFile file unusedlist
 			return $ c + length l
 
@@ -108,16 +110,18 @@ staleBadMsg t = unlines $
 
 unusedMsg :: [(Int, Key)] -> String
 unusedMsg u = unusedMsg' u
-	["Some annexed data is no longer used by any files in the repository:"]
-	[dropMsg Nothing]
+	["Some annexed data is no longer used by any files in the current branch:"]
+	[dropMsg Nothing,
+	"Please be cautious -- are you sure that another branch, or another",
+	"repository does not still use this data?"]
 
 remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String
 remoteUnusedMsg r u = unusedMsg' u
 	["Some annexed data on " ++ name ++ 
-	 " is not used by any files in this repository."]
+	 " is not used by any files in the current branch:"]
 	[dropMsg $ Just r,
-	 "Please be cautious -- are you sure that the remote repository",
-	 "does not use this data?"]
+	 "Please be cautious -- Are you sure that the remote repository",
+	 "does not use this data? Or that it's not used by another branch?"]
 	where
 		name = Remote.name r 
 
@@ -132,7 +136,7 @@ dropMsg :: Maybe (Remote.Remote Annex) -> String
 dropMsg Nothing = dropMsg' ""
 dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r
 dropMsg' :: String -> String
-dropMsg' s = "(To remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER)\n"
+dropMsg' s = "\nTo remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER\n"
 
 {- Finds keys whose content is present, but that do not seem to be used
  - by any files in the git repo, or that are only present as bad or tmp
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 1cb079ae9a..4f02db12f4 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -155,13 +155,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * unused
 
-  Checks the annex for data that does not correspond to any files currently
-  in the respository, and prints a numbered list of the data.
+  Checks the annex for data that does not correspond to any files present
+  in the currently checked out branch, and prints a numbered list of the data.
 
   To only show unused temp and bad files, specify --fast
 
-  To check data on a remote that does not correspond to any files currently
-  in the local repository, specify --from.
+  To check data on a remote that does not correspond to any files present
+  on the locally checked out branch, specify --from.
 
 * dropunused [number ...]
 

From aec4709c3f96fa17fef2fde812ed75167ddfbc60 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 13:32:29 -0400
Subject: [PATCH 1929/2835] fix gotcha with closed stderr and --debug

---
 Branch.hs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 6b2e8c23eb..cb2feea6a0 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -25,6 +25,7 @@ import Data.List
 import System.IO
 import System.Posix.IO
 import System.Posix.Process
+import System.Log.Logger
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -227,8 +228,16 @@ cmdOutput cmd params = do
 		x <- hGetContentsStrict h
 		hClose h
 		return $! x
-	pid <- pOpen3Raw Nothing (Just (snd pipepair)) Nothing cmd params
-		(closeFd (fst pipepair) >> closeFd stdError)
+	let child = do
+		closeFd (fst pipepair)
+ 		-- disable stderr output by this child,
+		-- and since the logger uses it, also disable it
+		liftIO $ updateGlobalLogger rootLoggerName $ setLevel EMERGENCY
+		closeFd stdError
+
+	debugM "Utility.executeFile" $ cmd ++ " " ++ show params
+	
+	pid <- pOpen3Raw Nothing (Just (snd pipepair)) Nothing cmd params child
 	retval <- callfunc $! pid
 	let rv = seq retval retval
 	_ <- getProcessStatus True False pid

From 7ee636f6ddca4b872dde36383077875563b0b369 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 13:39:04 -0400
Subject: [PATCH 1930/2835] avoid unnecessary read of trust.log

---
 Backend/File.hs  |  4 ++--
 Command/Move.hs  |  4 ++--
 Remote.hs        | 22 ++++++++++++++++++++--
 debian/changelog |  1 -
 4 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index eab987ef81..675b483076 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -52,7 +52,7 @@ dummyStore _ _ = return True
  - and copy it to here. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(remotes, _) <- Remote.keyPossibilities key
+	remotes <- Remote.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -95,7 +95,7 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- Remote.keyPossibilities key
+			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
 			untrusteduuids <- trustGet UnTrusted
 			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
diff --git a/Command/Move.hs b/Command/Move.hs
index 7fa195bab5..03b605ce57 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -89,7 +89,7 @@ toPerform dest move key = do
 	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)
 	isthere <- if fastcheck
 		then do
-			(remotes, _) <- Remote.keyPossibilities key
+			remotes <- Remote.keyPossibilities key
 			return $ Right $ dest `elem` remotes
 		else Remote.hasKey dest key
 	case isthere of
@@ -123,7 +123,7 @@ fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	(remotes, _) <- Remote.keyPossibilities key
+	remotes <- Remote.keyPossibilities key
 	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
 		then stop
 		else do
diff --git a/Remote.hs b/Remote.hs
index 5122423421..6295fc9476 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,6 +15,7 @@ module Remote (
 	hasKey,
 	hasKeyCheap,
 	keyPossibilities,
+	keyPossibilitiesTrusted,
 	forceTrust,
 
 	remoteTypes,
@@ -127,13 +128,30 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
+{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+ -}
+keyPossibilities :: Key -> Annex [Remote Annex]
+keyPossibilities key = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+
+	-- get uuids of all remotes that are recorded to have the key
+	uuids <- keyLocations key
+	let validuuids = filter (/= u) uuids
+
+	-- remotes that match uuids that have the key
+	allremotes <- genList
+	let validremotes = remotesWithUUID allremotes validuuids
+
+	return $ sort validremotes
+
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
  -}
-keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
-keyPossibilities key = do
+keyPossibilitiesTrusted :: Key -> Annex ([Remote Annex], [UUID])
+keyPossibilitiesTrusted key = do
 	g <- Annex.gitRepo
 	u <- getUUID g
 	trusted <- trustGet Trusted
diff --git a/debian/changelog b/debian/changelog
index 3fc7216e20..69b28c4551 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,6 @@ git-annex (3.20110611) UNRELEASED; urgency=low
   * merge: New subcommand. Auto-merges the new git-annex branch.
   * Improved handling of bare git repos with annexes. Many more commands will
     work in them.
-  * Sped up many commands.
   * git-annex is now more robust; it will never leave state files
     uncommitted when some other git process comes along and locks the index
     at an inconvenient time.

From 068703c40565cf8312c103b40d8c992b9d20a113 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 14:49:21 -0400
Subject: [PATCH 1931/2835] improve post-upgrade push instructions

---
 Upgrade/V2.hs     |  5 +++++
 debian/NEWS       |  3 ++-
 doc/upgrades.mdwn | 12 +++++++-----
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 7e4cfb13a7..9d32764e76 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -50,6 +50,11 @@ upgrade = do
 	liftIO $ do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File olddir]
 		gitAttributesUnWrite g
+
+	showLongNote $
+		"git-annex branch created\n" ++
+		"Now you should push the new branch: git push origin git-annex"
+
 	return True
 
 locationLogs :: Git.Repo -> Annex [(Key, FilePath)]
diff --git a/debian/NEWS b/debian/NEWS
index 00531d41bc..4218bf5681 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -2,7 +2,8 @@ git-annex (3.20110622) unstable; urgency=low
 
   There has been another change to the git-annex data store.
   Use `git annex upgrade` to migrate your repositories to the new
-  layout.
+  layout. See  or
+  /usr/share/doc/git-annex/html/upgrades.html
 
   The significant change this time is that the .git-annex/ directory
   is gone; instead there is a git-annex branch that is automatically
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 1813d079a1..f83a8c9477 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -27,13 +27,15 @@ Involved moving the .git-annex/ directory into a separate git-annex branch.
 This upgrade is easier (and faster!) than the previous upgrades.
 You don't need to upgrade every repository at once; it's sufficient
 to upgrade each repository only when you next use it.
-
-After the upgrade is complete, commit the changes it staged.
 	
-	git commit -m "upgrade v2 to v3"
+Example upgrade process:
 
-Running `git gc` after this upgrade will likely free up significant disk 
-space. (Tens to hundreds of megabytes.)
+	cd localrepo
+	git pull
+	git annex upgrade
+	git commit -m "upgrade v2 to v3"
+	git push origin git-annex master
+	git gc
 
 ### v1 -> v2 (git-annex version 0.20110316)
 

From c5531046bc6cd62cedfc528b844e0df9cdade177 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 15:30:04 -0400
Subject: [PATCH 1932/2835] refactor

---
 Remote.hs | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 6295fc9476..1accabf6d9 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -131,19 +131,7 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote Annex]
-keyPossibilities key = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-
-	-- get uuids of all remotes that are recorded to have the key
-	uuids <- keyLocations key
-	let validuuids = filter (/= u) uuids
-
-	-- remotes that match uuids that have the key
-	allremotes <- genList
-	let validremotes = remotesWithUUID allremotes validuuids
-
-	return $ sort validremotes
+keyPossibilities key = return . fst =<< keyPossibilities' False key
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
@@ -151,10 +139,13 @@ keyPossibilities key = do
  - (some may not have configured remotes).
  -}
 keyPossibilitiesTrusted :: Key -> Annex ([Remote Annex], [UUID])
-keyPossibilitiesTrusted key = do
+keyPossibilitiesTrusted = keyPossibilities' True
+
+keyPossibilities' :: Bool -> Key -> Annex ([Remote Annex], [UUID])
+keyPossibilities' withtrusted key = do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	trusted <- trustGet Trusted
+	trusted <- if withtrusted then trustGet Trusted else return []
 
 	-- get uuids of all remotes that are recorded to have the key
 	uuids <- keyLocations key

From 780ee5ff6d3fdcbbe69450d40260c1083315ca47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 15:38:52 -0400
Subject: [PATCH 1933/2835] fix bootstrapping, broken by move of
 .git/annex/index

---
 Branch.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index cb2feea6a0..10a7906314 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -80,9 +80,10 @@ withIndex' bootstrapping a = do
 	let f = index g
 	reset <- liftIO $ Git.useIndex f
 
-	unless bootstrapping $ do
-		e <- liftIO $ doesFileExist f
-		unless e $ liftIO $ genIndex g
+	e <- liftIO $ doesFileExist f
+	unless e $ liftIO $ do
+		createDirectoryIfMissing True $ takeDirectory f
+		unless bootstrapping $ genIndex g
 
 	r <- a
 	liftIO reset

From 7981eb4cb512fbe3c49a3dd165c31be14ae4bc49 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 15:51:07 -0400
Subject: [PATCH 1934/2835] fix consistency, and partially close a race during
 merge

Only "partially" because the journal is not locked during the merge, so
there's a small window where a different git-annex process could write info
to the journal that overwrites info taken from the merge.

That could be dealt with by locking, but the lock would really need to be
around the whole git-annex, to only let one run at a time. Otherwise, even
with the journal locked during the merge, another git-annex could already
be running, generate an overwriting change, and only store it in the journal
after the merge was complete. And similarly, two git-annex processes could
fight and overwrite each other's information independant of any merging.

So, a toplevel lock for git-annex may get added; it's something I've
considered before, as these potential, unlikely problems are not new.
(OTOH, fsck will deal with such problems.)
---
 Branch.hs | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/Branch.hs b/Branch.hs
index 10a7906314..e79ca3f60a 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -149,6 +149,21 @@ update :: Annex ()
 update = do
 	state <- Annex.getState Annex.branchstate
 	unless (branchUpdated state) $ withIndex $ do
+		{- Since branches get merged into the index, it's important to
+		 - first stage the journal into the index. Otherwise, any
+		 - changes in the journal would later get staged, and might
+		 - overwrite changes made during the merge.
+		 -
+		 - It would be cleaner to handle the merge by updating the
+		 - journal, not the index, with changes from the branches.
+		 -
+		 - XXX Anything written to the journal during the merge,
+		 - by another process could still race the merge. The
+		 - journal should really be blocking locked during the
+		 - merge.
+		 -}
+		_ <- stageJournalFiles
+
 		g <- Annex.gitRepo
 		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 		let refs = map (last . words) (lines r)

From 1686f60f8481e996291a5970b2217bc1d64f635d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 16:44:26 -0400
Subject: [PATCH 1935/2835] commit after merge if any journal files were staged

---
 Branch.hs | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index e79ca3f60a..6e9dae9b9a 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -156,19 +156,14 @@ update = do
 		 -
 		 - It would be cleaner to handle the merge by updating the
 		 - journal, not the index, with changes from the branches.
-		 -
-		 - XXX Anything written to the journal during the merge,
-		 - by another process could still race the merge. The
-		 - journal should really be blocking locked during the
-		 - merge.
 		 -}
-		_ <- stageJournalFiles
+		staged <- stageJournalFiles
 
 		g <- Annex.gitRepo
 		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 		let refs = map (last . words) (lines r)
 		updated <- catMaybes `liftM` mapM updateRef refs
-		unless (null updated) $ liftIO $
+		unless (null updated && not staged) $ liftIO $
 			GitUnionMerge.commit g "update" fullname
 				(fullname:updated)
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }

From 22243b87d2a43d6fd82581ef70a0537cb8ae8661 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 17:08:11 -0400
Subject: [PATCH 1936/2835] layout

---
 Upgrade/V2.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 9d32764e76..57a2ba7893 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -53,7 +53,7 @@ upgrade = do
 
 	showLongNote $
 		"git-annex branch created\n" ++
-		"Now you should push the new branch: git push origin git-annex"
+		"Now you should push the new branch: git push origin git-annex\n"
 
 	return True
 

From ab9b971f8f772cf7e89a904d665bc88ddb3afd47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 17:37:23 -0400
Subject: [PATCH 1937/2835] simplified to use existing functions

---
 Branch.hs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 6e9dae9b9a..fad024550f 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -64,12 +64,7 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = do
-	ls <- Git.pipeNullSplit g $
-		map Param ["ls-tree", "-z", "-r", "--full-tree", fullname]
-	forceSuccess =<< Git.pipeWrite g
-		(map Param ["update-index", "-z", "--index-info"])
-		(join "\0" ls)
+genIndex g = GitUnionMerge.ls_tree g fullname >>= GitUnionMerge.update_index g
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a

From c4cc6ee42f63bb8196f44608aaf35f7f9f411fe1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 17:38:27 -0400
Subject: [PATCH 1938/2835] fix merge_tree_index

--cached is needed when calling git-diff-index, as it is not diffing
against currently checked out branch.
---
 GitUnionMerge.hs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index fa14a6bc3f..74579ebcc9 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -9,7 +9,8 @@ module GitUnionMerge (
 	merge,
 	commit,
 	update_index,
-	update_index_line
+	update_index_line,
+	ls_tree
 ) where
 
 import System.Cmd.Utils
@@ -40,7 +41,7 @@ merge _ _ = error "wrong number of branches to merge"
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Git.Repo -> [String] -> IO ()
-update_index g l =  togit ["update-index", "-z", "--index-info"] (join "\0" l)
+update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
 		togit ps content = Git.pipeWrite g (map Param ps) content
 			>>= forceSuccess
@@ -57,13 +58,14 @@ ls_tree g x = Git.pipeNullSplit g $
 
 {- For merging two trees. -}
 merge_trees :: Git.Repo -> String -> String -> IO [String]
-merge_trees g x y = calc_merge g
-	["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", x, y]
+merge_trees g x y = calc_merge g $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
 merge_tree_index :: Git.Repo -> String -> IO [String]
-merge_tree_index g x = calc_merge g
-	["diff-index", "--raw", "-z", "-r", "--no-renames", "-l0", x]
+merge_tree_index g x = calc_merge g $ "diff-index":diff_opts ++ ["--cached", x]
+
+diff_opts :: [String]
+diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}

From e3384eb4764787daef926c307004d001e224a9fd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 19:56:24 -0400
Subject: [PATCH 1939/2835] tweak fsck wording so file is at the end of the
 line

---
 Backend/File.hs                                 | 4 ++--
 doc/walkthrough/fsck:_when_things_go_wrong.mdwn | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 675b483076..1c25f89db2 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -205,14 +205,14 @@ checkKeyNumCopies key file numcopies = do
 
 missingNote :: String -> Int -> Int -> String -> String
 missingNote file 0 _ [] = 
-		"** No known copies of " ++ file ++ " exist!"
+		"** No known copies exist of " ++ file
 missingNote file 0 _ untrusted =
 		"Only these untrusted locations may have copies of " ++ file ++
 		"\n" ++ untrusted ++
 		"Back it up to trusted locations with git-annex copy."
 missingNote file present needed [] =
 		"Only " ++ show present ++ " of " ++ show needed ++ 
-		" trustworthy copies of " ++ file ++ " exist." ++
+		" trustworthy copies exist of " ++ file ++
 		"\nBack it up with git-annex copy."
 missingNote file present needed untrusted = 
 		missingNote file present needed [] ++
diff --git a/doc/walkthrough/fsck:_when_things_go_wrong.mdwn b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn
index 05b9f385c0..85d9f20fe0 100644
--- a/doc/walkthrough/fsck:_when_things_go_wrong.mdwn
+++ b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn
@@ -5,7 +5,7 @@ might say about a badly messed up annex:
 	# git annex fsck
 	fsck my_cool_big_file (checksum...)
 	git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2
-	git-annex: ** No known copies of the file exist!
+	git-annex: ** No known copies exist of my_cool_big_file
 	failed
 	fsck important_file
 	git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up.

From a61154baf5e205af74766f44fe99cbbd63411f57 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 20:52:56 -0400
Subject: [PATCH 1940/2835] add

---
 doc/todo/speed_up_fsck.mdwn | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/todo/speed_up_fsck.mdwn

diff --git a/doc/todo/speed_up_fsck.mdwn b/doc/todo/speed_up_fsck.mdwn
new file mode 100644
index 0000000000..aceb5868cc
--- /dev/null
+++ b/doc/todo/speed_up_fsck.mdwn
@@ -0,0 +1,18 @@
+moving to the git-annex branch has slowed down fsck worse than most
+commands. Actually, some commands have sped up, while others like get
+are slightly slower but are swamped by the normal runtime. 
+
+For fsck though, it has to pull each file's location log info out of git.
+And, it's typically run on the entire tree.
+
+It would be possible to run a single `git cat-file --batch` and pass it
+sha1s of location logs for file that is going to be fsked (gotten via
+`read-tree`). Then just read its output until the next requested sha1 to
+chunk it, and pass this in to fsck in a closure.
+
+The difficulty, besides writing that is that everything that works with
+location logs now reads them out of git, would need to find a way to
+provide the info on a side channel of some sort.
+
+If this is implemented, the same infrastructure could be used for other
+commands like whereis and add. --[[Joey]]

From 69d3c1cec9f6be1dba1ffb391bf69464c52f5936 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 21:25:39 -0400
Subject: [PATCH 1941/2835] cache the trustmap

Doubles the speed of fsck, and speeds up drop as well.
---
 Annex.hs                             |  4 +++-
 GitAnnex.hs                          |  2 +-
 Trust.hs                             | 19 ++++++++++++-------
 TrustLevel.hs => Types/TrustLevel.hs |  9 ++++++++-
 4 files changed, 24 insertions(+), 10 deletions(-)
 rename TrustLevel.hs => Types/TrustLevel.hs (79%)

diff --git a/Annex.hs b/Annex.hs
index 2bd090e906..82908881da 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -24,7 +24,7 @@ import Types.Backend
 import Types.Remote
 import Types.Crypto
 import Types.BranchState
-import TrustLevel
+import Types.TrustLevel
 import Types.UUID
 
 -- git-annex's monad
@@ -48,6 +48,7 @@ data AnnexState = AnnexState
 	, fromremote :: Maybe String
 	, exclude :: [String]
 	, forcetrust :: [(UUID, TrustLevel)]
+	, trustmap :: Maybe TrustMap
 	, cipher :: Maybe Cipher
 	}
 
@@ -69,6 +70,7 @@ newState allbackends gitrepo = AnnexState
 	, fromremote = Nothing
 	, exclude = []
 	, forcetrust = []
+	, trustmap = Nothing
 	, cipher = Nothing
 	}
 
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 727e0c3961..49ba63b8ab 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -14,7 +14,7 @@ import CmdLine
 import Command
 import Options
 import Utility
-import TrustLevel
+import Types.TrustLevel
 import qualified Annex
 import qualified Remote
 
diff --git a/Trust.hs b/Trust.hs
index d328235bfa..365186da22 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -9,15 +9,13 @@ module Trust (
 	TrustLevel(..),
 	trustLog,
 	trustGet,
-	trustMap,
-	trustMapParse,
 	trustSet
 ) where
 
 import Control.Monad.State
 import qualified Data.Map as M
 
-import TrustLevel
+import Types.TrustLevel
 import qualified Branch
 import Types
 import UUID
@@ -35,11 +33,17 @@ trustGet level = do
 
 {- Read the trustLog into a map, overriding with any
  - values from forcetrust -}
-trustMap :: Annex (M.Map UUID TrustLevel)
+trustMap :: Annex TrustMap
 trustMap = do
-	overrides <- Annex.getState Annex.forcetrust
-	s <- Branch.get trustLog
-	return $ M.fromList $ trustMapParse s ++ overrides
+	cached <- Annex.getState Annex.trustmap
+	case cached of
+		Just m -> return m
+		Nothing -> do
+			overrides <- Annex.getState Annex.forcetrust
+			l <- Branch.get trustLog
+			let m = M.fromList $ trustMapParse l ++ overrides
+			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
+			return m
 
 {- Trust map parser. -}
 trustMapParse :: String -> [(UUID, TrustLevel)]
@@ -62,6 +66,7 @@ trustSet uuid level = do
 	when (M.lookup uuid m /= Just level) $ do
 		let m' = M.insert uuid level m
 		Branch.change trustLog (serialize m')
+		Annex.changeState $ \s -> s { Annex.trustmap = Just m' }
         where
                 serialize m = unlines $ map showpair $ M.toList m
 		showpair (u, t) = u ++ " " ++ show t
diff --git a/TrustLevel.hs b/Types/TrustLevel.hs
similarity index 79%
rename from TrustLevel.hs
rename to Types/TrustLevel.hs
index 5da142ca30..058ce4595c 100644
--- a/TrustLevel.hs
+++ b/Types/TrustLevel.hs
@@ -5,10 +5,15 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module TrustLevel (
+module Types.TrustLevel (
 	TrustLevel(..),
+	TrustMap
 ) where
 
+import qualified Data.Map as M
+
+import Types.UUID
+
 data TrustLevel = SemiTrusted | UnTrusted | Trusted
 	deriving Eq
 
@@ -21,3 +26,5 @@ instance Read TrustLevel where
         readsPrec _ "1" = [(Trusted, "")]
         readsPrec _ "0" = [(UnTrusted, "")]
 	readsPrec _ _ = [(SemiTrusted, "")]
+
+type TrustMap = M.Map UUID TrustLevel

From 59b2e4ec1d7de3cfc9c477a68ee418d403bc21d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 01:13:33 -0400
Subject: [PATCH 1942/2835] fixes for upgrading bare repos

---
 Upgrade/V2.hs | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 57a2ba7893..7f0062cf34 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -10,6 +10,7 @@ module Upgrade.V2 where
 import System.Directory
 import System.FilePath
 import Control.Monad.State (liftIO)
+import Control.Monad.State (unless)
 import List
 import Data.Maybe
 
@@ -22,8 +23,10 @@ import Messages
 import Utility
 import Locations
 
-olddir :: FilePath
-olddir = ".git-annex"
+olddir :: Git.Repo -> FilePath
+olddir g
+	| Git.repoIsLocalBare g = ""
+	| otherwise = ".git-annex"
 
 {- .git-annex/ moved to a git-annex branch.
  - 
@@ -46,14 +49,15 @@ upgrade = do
 	g <- Annex.gitRepo
 	Branch.create
 	mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
-	mapM_ (\f -> inject f f) =<< logFiles olddir
+	mapM_ (\f -> inject f f) =<< logFiles (olddir g)
 	liftIO $ do
-		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File olddir]
-		gitAttributesUnWrite g
+		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
+		unless (Git.repoIsLocalBare g) $ gitAttributesUnWrite g
 
 	showLongNote $
 		"git-annex branch created\n" ++
 		"Now you should push the new branch: git push origin git-annex\n"
+	showProgress
 
 	return True
 
@@ -71,7 +75,8 @@ locationLogs repo = liftIO $ do
 
 inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
-	new <- liftIO (readFile $ olddir  source)
+	g <- Annex.gitRepo
+	new <- liftIO (readFile $ olddir g  source)
 	prev <- Branch.get dest
 	Branch.change dest $ unlines $ nub $ lines prev ++ lines new
 

From 49d77156ac75dd9459f3e2f2be3baa3ff3b172bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 01:15:12 -0400
Subject: [PATCH 1943/2835] more upgrades fixes for bare repos

---
 Upgrade/V2.hs | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 7f0062cf34..af8f15919e 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -47,17 +47,20 @@ upgrade :: Annex Bool
 upgrade = do
 	showNote "v2 to v3"
 	g <- Annex.gitRepo
+	let bare = Git.repoIsLocalBare g
+
 	Branch.create
 	mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
 	mapM_ (\f -> inject f f) =<< logFiles (olddir g)
 	liftIO $ do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
-		unless (Git.repoIsLocalBare g) $ gitAttributesUnWrite g
+		unless bare $ gitAttributesUnWrite g
 
-	showLongNote $
-		"git-annex branch created\n" ++
-		"Now you should push the new branch: git push origin git-annex\n"
-	showProgress
+	unless bare $ do
+		showLongNote $
+			"git-annex branch created\n" ++
+			"Now you should push the new branch: git push origin git-annex\n"
+		showProgress
 
 	return True
 

From 50354a49163b8841af2024876a62e78b6caaa2b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 02:33:44 -0400
Subject: [PATCH 1944/2835] save state before message

to avoid long delay after
---
 Upgrade/V2.hs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index af8f15919e..2372edb3db 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -22,6 +22,7 @@ import qualified Branch
 import Messages
 import Utility
 import Locations
+import Content
 
 olddir :: Git.Repo -> FilePath
 olddir g
@@ -56,6 +57,8 @@ upgrade = do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
 		unless bare $ gitAttributesUnWrite g
 
+	saveState
+
 	unless bare $ do
 		showLongNote $
 			"git-annex branch created\n" ++

From ad38c0dfadc0fc68ed6213c75541fe06c7caca2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 11:59:34 -0400
Subject: [PATCH 1945/2835] better setup of git-annex branch pushing on upgrade

---
 Branch.hs         | 34 +++++++++++++++++++++++-----------
 Upgrade/V2.hs     | 41 ++++++++++++++++++++++++++++++++---------
 doc/upgrades.mdwn |  1 -
 3 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index fad024550f..3f5e0ee1ec 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -11,7 +11,10 @@ module Branch (
 	get,
 	change,
 	commit,
-	files
+	files,
+	refExists,
+	hasOrigin,
+	name	
 ) where
 
 import Control.Monad (unless, when, liftM)
@@ -44,6 +47,10 @@ name = "git-annex"
 fullname :: String
 fullname = "refs/heads/" ++ name
 
+{- Branch's name in origin. -}
+originname :: String
+originname = "origin/" ++ name
+
 {- Converts a fully qualified git ref into a short version for human
  - consumptiom. -}
 shortref :: String -> String
@@ -114,20 +121,14 @@ getCache file = getState >>= handle
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
 create = do
-	exists <- refexists fullname
+	exists <- refExists fullname
 	unless exists $ do
 		g <- Annex.gitRepo
-		inorigin <- refexists origin
-		if inorigin
-			then liftIO $ Git.run g "branch" [Param name, Param origin]
+		e <- hasOrigin
+		if e
+			then liftIO $ Git.run g "branch" [Param name, Param originname]
 			else withIndex' True $
 				liftIO $ GitUnionMerge.commit g "branch created" fullname []
-	where
-		origin = "origin/" ++ name
-		refexists ref = do
-			g <- Annex.gitRepo
-			liftIO $ Git.runBool g "show-ref"
-				[Param "--verify", Param "-q", Param ref]
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
@@ -164,6 +165,17 @@ update = do
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
+{- Does origin/git-annex exist? -}
+hasOrigin :: Annex Bool
+hasOrigin = refExists originname
+
+{- Checks if a git ref exists. -}
+refExists :: String -> Annex Bool
+refExists ref = do
+	g <- Annex.gitRepo
+	liftIO $ Git.runBool g "show-ref"
+		[Param "--verify", Param "-q", Param ref]
+
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: String -> Annex (Maybe String)
 updateRef ref
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 2372edb3db..8eab236ff5 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -9,8 +9,7 @@ module Upgrade.V2 where
 
 import System.Directory
 import System.FilePath
-import Control.Monad.State (liftIO)
-import Control.Monad.State (unless)
+import Control.Monad.State (unless, liftIO)
 import List
 import Data.Maybe
 
@@ -58,12 +57,7 @@ upgrade = do
 		unless bare $ gitAttributesUnWrite g
 
 	saveState
-
-	unless bare $ do
-		showLongNote $
-			"git-annex branch created\n" ++
-			"Now you should push the new branch: git push origin git-annex\n"
-		showProgress
+	unless bare $ push
 
 	return True
 
@@ -90,6 +84,36 @@ logFiles :: FilePath -> Annex [FilePath]
 logFiles dir = return . filter (".log" `isSuffixOf`)
 		=<< liftIO (getDirectoryContents dir)
 
+push :: Annex ()
+push = do
+	origin_master <- Branch.refExists "origin/master"
+	origin_gitannex <- Branch.hasOrigin
+	case (origin_master, origin_gitannex) of
+		(_, True) -> do
+			-- Merge in the origin's git-annex branch,
+			-- so that pushing the git-annex branch
+			-- will immediately work. Not pushed here,
+			-- because it's less obnoxious to let the user
+			-- push.
+			Branch.update
+		(True, False) -> do
+			-- push git-annex to origin, so that
+			-- "git push" will from then on
+			-- automatically push it
+			Branch.update -- just in case
+			showNote "pushing new git-annex branch to origin"
+			showProgress
+			g <- Annex.gitRepo
+			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]
+		_ -> do
+			-- no origin exists, so just let the user
+			-- know about the new branch
+			Branch.update
+			showLongNote $
+				"git-annex branch created\n" ++
+				"Be sure to push this branch when pushing to remotes.\n"
+			showProgress
+
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]
 attrLines =
@@ -110,4 +134,3 @@ stateDir :: FilePath
 stateDir = addTrailingPathSeparator $ ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
 gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
-
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index f83a8c9477..337c65767a 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -34,7 +34,6 @@ Example upgrade process:
 	git pull
 	git annex upgrade
 	git commit -m "upgrade v2 to v3"
-	git push origin git-annex master
 	git gc
 
 ### v1 -> v2 (git-annex version 0.20110316)

From d7018500fac3f496489888e2f5fd124c66c0b746 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 12:09:04 -0400
Subject: [PATCH 1946/2835] fix upgrade when .git-annex has already been
 entirely converted

---
 Upgrade/V2.hs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 8eab236ff5..8537a50226 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -9,7 +9,7 @@ module Upgrade.V2 where
 
 import System.Directory
 import System.FilePath
-import Control.Monad.State (unless, liftIO)
+import Control.Monad.State (unless, when, liftIO)
 import List
 import Data.Maybe
 
@@ -50,11 +50,13 @@ upgrade = do
 	let bare = Git.repoIsLocalBare g
 
 	Branch.create
-	mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
-	mapM_ (\f -> inject f f) =<< logFiles (olddir g)
-	liftIO $ do
-		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
-		unless bare $ gitAttributesUnWrite g
+	e <- liftIO $ doesDirectoryExist (olddir g)
+	when e $ do
+		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
+		mapM_ (\f -> inject f f) =<< logFiles (olddir g)
+		liftIO $ do
+			Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
+			unless bare $ gitAttributesUnWrite g
 
 	saveState
 	unless bare $ push

From 874fc044c1dec05fc398b44ccd69a2f501a52de2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 14:58:07 -0400
Subject: [PATCH 1947/2835] releasing version 3.20110624

---
 debian/NEWS      | 2 +-
 debian/changelog | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/NEWS b/debian/NEWS
index 4218bf5681..22835ace84 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,4 +1,4 @@
-git-annex (3.20110622) unstable; urgency=low
+git-annex (3.20110624) exerimental; urgency=low
 
   There has been another change to the git-annex data store.
   Use `git annex upgrade` to migrate your repositories to the new
diff --git a/debian/changelog b/debian/changelog
index 69b28c4551..47652241a0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110611) UNRELEASED; urgency=low
+git-annex (3.20110624) experimental; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
   * git-annex now stores its logs in a git-annex branch.
@@ -18,7 +18,7 @@ git-annex (3.20110611) UNRELEASED; urgency=low
   * git-union-merge: New git subcommand, that does a generic union merge
     operation, and operates efficiently without touching the working tree.
 
- -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
+ -- Joey Hess   Fri, 24 Jun 2011 14:32:18 -0400
 
 git-annex (0.20110610) unstable; urgency=low
 

From 14e2765ba8c29041575a13f9b06ce6de8ebc26ef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 14:58:20 -0400
Subject: [PATCH 1948/2835] add news item for git-annex 3.20110624

---
 doc/news/version_0.20110516.mdwn | 11 -----------
 doc/news/version_3.20110624.mdwn | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 11 deletions(-)
 delete mode 100644 doc/news/version_0.20110516.mdwn
 create mode 100644 doc/news/version_3.20110624.mdwn

diff --git a/doc/news/version_0.20110516.mdwn b/doc/news/version_0.20110516.mdwn
deleted file mode 100644
index aa350a4848..0000000000
--- a/doc/news/version_0.20110516.mdwn
+++ /dev/null
@@ -1,11 +0,0 @@
-git-annex 0.20110516 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Add a few tweaks to make it easy to use the Internet Archive's variant
-     of S3. In particular, munge key filenames to comply with the IA's filename
-     limits, disable encryption, support their nonstandard way of creating
-     buckets, and allow x-archive-* headers to be specified in initremote to
-     set item metadata.
-   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
-   * migrate: Use current filename when generating new key, for backends
-     where the filename affects the key name.
-   * Work around a bug in Network.URI's handling of bracketed ipv6 addresses."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110624.mdwn b/doc/news/version_3.20110624.mdwn
new file mode 100644
index 0000000000..5a91fa4295
--- /dev/null
+++ b/doc/news/version_3.20110624.mdwn
@@ -0,0 +1,32 @@
+News for git-annex 3.20110624:
+
+   There has been another change to the git-annex data store.
+   Use `git annex upgrade` to migrate your repositories to the new
+   layout. See <http://git-annex.branchable.com/upgrades/> or
+   /usr/share/doc/git-annex/html/upgrades.html
+   The significant change this time is that the .git-annex/ directory
+   is gone; instead there is a git-annex branch that is automatically
+   maintained by git-annex, and encapsulates all its state nicely out
+   of your way.
+   You should make sure you include the git-annex branch when
+   git pushing and pulling.
+
+git-annex 3.20110624 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
+   * git-annex now stores its logs in a git-annex branch.
+   * merge: New subcommand. Auto-merges the new git-annex branch.
+   * Improved handling of bare git repos with annexes. Many more commands will
+     work in them.
+   * git-annex is now more robust; it will never leave state files
+     uncommitted when some other git process comes along and locks the index
+     at an inconvenient time.
+   * rsync is now used when copying files from repos on other filesystems.
+     cp is still used when copying file from repos on the same filesystem,
+     since --reflink=auto can make it significantly faster on filesystems
+     such as btrfs.
+   * Allow --trust etc to specify a repository by name, for temporarily
+     trusting repositories that are not configured remotes.
+   * unlock: Made atomic.
+   * git-union-merge: New git subcommand, that does a generic union merge
+     operation, and operates efficiently without touching the working tree."""]]
\ No newline at end of file

From 45bce3a3a8706c12b8e1eb98a6fe0d62e378d10b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 26 Jun 2011 19:00:46 -0400
Subject: [PATCH 1949/2835] freebsd port now available

---
 doc/install.mdwn         | 1 +
 doc/install/FreeBSD.mdwn | 2 ++
 2 files changed, 3 insertions(+)
 create mode 100644 doc/install/FreeBSD.mdwn

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 7818aaf152..1c038d0cf9 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -4,6 +4,7 @@
 * [[Debian]]
 * [[Ubuntu]]
 * [[Fedora]]
+* [[FreeBSD]]
 
 ## Generic instructions
 
diff --git a/doc/install/FreeBSD.mdwn b/doc/install/FreeBSD.mdwn
new file mode 100644
index 0000000000..23152ac204
--- /dev/null
+++ b/doc/install/FreeBSD.mdwn
@@ -0,0 +1,2 @@
+git-annex is in FreeBSD ports in
+[devel/git-annex](http://www.freshports.org/devel/git-annex/)

From c90652f015b370e270da8174b4ac61e454e06ffd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 26 Jun 2011 21:01:19 -0400
Subject: [PATCH 1950/2835] Always ensure git-annex branch exists.

---
 Branch.hs        | 7 ++++---
 debian/changelog | 6 ++++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 3f5e0ee1ec..916261a9c9 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -83,9 +83,10 @@ withIndex' bootstrapping a = do
 	reset <- liftIO $ Git.useIndex f
 
 	e <- liftIO $ doesFileExist f
-	unless e $ liftIO $ do
-		createDirectoryIfMissing True $ takeDirectory f
-		unless bootstrapping $ genIndex g
+	unless e $ do
+		unless bootstrapping $ create
+		liftIO $ createDirectoryIfMissing True $ takeDirectory f
+		liftIO $ unless bootstrapping $ genIndex g
 
 	r <- a
 	liftIO reset
diff --git a/debian/changelog b/debian/changelog
index 47652241a0..854314856d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110625) UNRELEASED; urgency=low
+
+  * Always ensure git-annex branch exists.
+
+ -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
+
 git-annex (3.20110624) experimental; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.

From e8068f2ffbeb25bc094ecb3763da6ace278586cc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 28 Jun 2011 14:14:49 -0400
Subject: [PATCH 1951/2835] tweaks

---
 Branch.hs | 42 ++++++++++++++++++++----------------------
 1 file changed, 20 insertions(+), 22 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 916261a9c9..26aad44070 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -17,7 +17,7 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (unless, when, liftM)
+import Control.Monad (unless, liftM)
 import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
@@ -39,21 +39,23 @@ import Types
 import Messages
 import Locations
 
+type GitRef = String
+
 {- Name of the branch that is used to store git-annex's information. -}
-name :: String
+name :: GitRef
 name = "git-annex"
 
 {- Fully qualified name of the branch. -}
-fullname :: String
+fullname :: GitRef
 fullname = "refs/heads/" ++ name
 
 {- Branch's name in origin. -}
-originname :: String
+originname :: GitRef
 originname = "origin/" ++ name
 
 {- Converts a fully qualified git ref into a short version for human
  - consumptiom. -}
-shortref :: String -> String
+shortref :: GitRef -> String
 shortref = remove "refs/heads/" . remove "refs/remotes/"
 	where
 		remove prefix s
@@ -121,24 +123,20 @@ getCache file = getState >>= handle
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = do
-	exists <- refExists fullname
-	unless exists $ do
-		g <- Annex.gitRepo
-		e <- hasOrigin
-		if e
-			then liftIO $ Git.run g "branch" [Param name, Param originname]
-			else withIndex' True $
-				liftIO $ GitUnionMerge.commit g "branch created" fullname []
+create = unlessM (refExists fullname) $ do
+	g <- Annex.gitRepo
+	e <- hasOrigin
+	if e
+		then liftIO $ Git.run g "branch" [Param name, Param originname]
+		else withIndex' True $
+			liftIO $ GitUnionMerge.commit g "branch created" fullname []
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = do
-	staged <- stageJournalFiles
-	when staged $ do
-		g <- Annex.gitRepo
-		withIndex $ liftIO $
-			GitUnionMerge.commit g message fullname [fullname]
+commit message = whenM stageJournalFiles $ do
+	g <- Annex.gitRepo
+	withIndex $ liftIO $
+		GitUnionMerge.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
@@ -171,14 +169,14 @@ hasOrigin :: Annex Bool
 hasOrigin = refExists originname
 
 {- Checks if a git ref exists. -}
-refExists :: String -> Annex Bool
+refExists :: GitRef -> Annex Bool
 refExists ref = do
 	g <- Annex.gitRepo
 	liftIO $ Git.runBool g "show-ref"
 		[Param "--verify", Param "-q", Param ref]
 
 {- Ensures that a given ref has been merged into the index. -}
-updateRef :: String -> Annex (Maybe String)
+updateRef :: GitRef -> Annex (Maybe String)
 updateRef ref
 	| ref == fullname = return Nothing
 	| otherwise = do

From 5034d8c2985dafeb141bba383ab70d75729b3cb6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 28 Jun 2011 16:15:50 -0400
Subject: [PATCH 1952/2835] Modify location log parser to allow future
 expansion.

Since the logs have just been moved into the git-annex branch, don't need
to worry about backwards compatability with old versions of git-annex that
would fail to parse location logs with extra fields tacked on.
---
 LocationLog.hs                    | 4 +++-
 debian/changelog                  | 1 +
 doc/todo/tahoe_lfs_for_reals.mdwn | 7 -------
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index 4e2caca959..2a2bc63013 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -8,6 +8,8 @@
  -
  - A line of the log will look like: "date N UUID"
  - Where N=1 when the repo has the file, and 0 otherwise.
+ - (After the UUID can optionally come a white space and other data, 
+ - for future expansion.)
  - 
  - Copyright 2010-2011 Joey Hess 
  -
@@ -65,7 +67,7 @@ instance Read LogLine where
 	-- read without an exception being thrown.
 	-- Such lines have a status of Undefined.
 	readsPrec _ string = 
-		if length w == 3
+		if length w >= 3
 			then maybe bad good pdate
 			else bad
 		where
diff --git a/debian/changelog b/debian/changelog
index 854314856d..688a4adb85 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110625) UNRELEASED; urgency=low
 
   * Always ensure git-annex branch exists.
+  * Modify location log parser to allow future expansion.
 
  -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
 
diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn
index 4e0bf2fb16..9019767eb9 100644
--- a/doc/todo/tahoe_lfs_for_reals.mdwn
+++ b/doc/todo/tahoe_lfs_for_reals.mdwn
@@ -19,10 +19,3 @@ location log:
 
 This way, each remote can store its own key-specfic data in the same place
 as other key-specific data, with minimal overhead.
-
-Unfortunatly, the current location log parser throws out lines that it 
-cannot parse, so making this change would involve something of a flag day
-upgrade. Also unfortunatly, the location log and other .git-annex/ data
-does not have its own version that can be checked to force an upgrade
-across all clones. It might be best to deal with this at the same time
-the ideas in [[branching]] are done. --[[Joey]]

From fb7663ceb86b309dd223ebc113c8b586955a0e03 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Wed, 29 Jun 2011 14:46:41 +0000
Subject: [PATCH 1953/2835]

---
 ..._annex_should_use___39__git_add_-f__39___internally.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn

diff --git a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
new file mode 100644
index 0000000000..66b3aa36a3
--- /dev/null
+++ b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
@@ -0,0 +1,6 @@
+I have this line in the .gitignore file of one of my repos:
+*log
+
+So the command 'git annex init name' fails to add the file ".git-annex/uuid.log", and the same problem happens when git-annex-add'ing files.
+
+Also, when a file is git-ignored, it should be possible to 'git annex add' it with a -f/--force option, the same way git does it.

From b3aaf980e460c2287fc1ef2b262685b1879e6ed0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 11:42:00 -0400
Subject: [PATCH 1954/2835] --force will cause add, etc, to operate on ignored
 files.

---
 Command.hs                                         |  3 ++-
 Command/Add.hs                                     |  6 +++++-
 GitRepo.hs                                         | 14 ++++++++------
 debian/changelog                                   |  1 +
 ...ould_use___39__git_add_-f__39___internally.mdwn |  5 +++++
 doc/git-annex.mdwn                                 |  3 ++-
 6 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/Command.hs b/Command.hs
index 228c1f40e9..ee81bd0f6f 100644
--- a/Command.hs
+++ b/Command.hs
@@ -143,7 +143,8 @@ withFilesMissing a params = do
 withFilesNotInGit :: CommandSeekBackendFiles
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
-	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo) params
+	force <- Annex.getState Annex.force
+	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo force) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
 withWords :: CommandSeekWords
diff --git a/Command/Add.hs b/Command/Add.hs
index 29a1518e84..5133ee1fd5 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -11,6 +11,7 @@ import Control.Monad.State (liftIO)
 import System.Posix.Files
 
 import Command
+import qualified Annex
 import qualified AnnexQueue
 import qualified Backend
 import LocationLog
@@ -60,5 +61,8 @@ cleanup file key = do
 	let mtime = modificationTime s
 	liftIO $ touch file (TimeSpec mtime) False
 
-	AnnexQueue.add "add" [Param "--"] file
+	force <- Annex.getState Annex.force
+	if force
+		then AnnexQueue.add "add" [Param "-f", Param "--"] file
+		else AnnexQueue.add "add" [Param "--"] file
 	return True
diff --git a/GitRepo.hs b/GitRepo.hs
index 9f4a38a5fb..844917b880 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -382,13 +382,15 @@ reap = do
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo $
-	[Params "ls-files --cached --exclude-standard -z --"] ++ map File l
+	[Params "ls-files --cached -z --"] ++ map File l
 
-{- Scans for files at the specified locations that are not checked into git,
- - and not gitignored. -}
-notInRepo :: Repo -> [FilePath] -> IO [FilePath]
-notInRepo repo l = pipeNullSplit repo $
-	[Params "ls-files --others --exclude-standard -z --"] ++ map File l
+{- Scans for files at the specified locations that are not checked into
+ - git. -}
+notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
+notInRepo repo include_ignored l =
+	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l
+	where
+		exclude = if include_ignored then [] else [Param "--exclude-standard"]
 
 {- Returns a list of all files that are staged for commit. -}
 stagedFiles :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/debian/changelog b/debian/changelog
index 6439eb89fa..8848cf8611 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,7 @@ git-annex (0.20110611) UNRELEASED; urgency=low
   * unlock: Made atomic.
   * git-union-merge: New git subcommand, that does a generic union merge
     operation, and operates efficiently without touching the working tree.
+  * --force will cause add, etc, to operate on ignored files.
 
  -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
 
diff --git a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
index 66b3aa36a3..a92f5871b5 100644
--- a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
+++ b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
@@ -3,4 +3,9 @@ I have this line in the .gitignore file of one of my repos:
 
 So the command 'git annex init name' fails to add the file ".git-annex/uuid.log", and the same problem happens when git-annex-add'ing files.
 
+> This is avoided on the v3 branch, which does not store these files in the
+> same branch as your repository.
+
 Also, when a file is git-ignored, it should be possible to 'git annex add' it with a -f/--force option, the same way git does it.
+
+> Reasonable, [[done]] --[[Joey]] 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 12756d8020..177df09527 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -314,7 +314,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * --force
 
   Force unsafe actions, such as dropping a file's content when no other
-  source of it can be verified to still exist. Use with care.
+  source of it can be verified to still exist, or adding ignored files.
+  Use with care.
 
 * --fast
 

From 06a1f5f74286795708b219de8fb080077ff134a7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 11:55:16 -0400
Subject: [PATCH 1955/2835] factor out file list stuff from GitRepo

GitRepo is getting too large an interface; these all fit nicely into a
submodule.
---
 Command.hs         | 15 +++++-----
 Command/Unannex.hs |  3 +-
 Command/Unused.hs  |  3 +-
 GitRepo.hs         | 56 --------------------------------------
 GitRepo/LsFiles.hs | 68 ++++++++++++++++++++++++++++++++++++++++++++++
 Upgrade/V1.hs      |  3 +-
 6 files changed, 82 insertions(+), 66 deletions(-)
 create mode 100644 GitRepo/LsFiles.hs

diff --git a/Command.hs b/Command.hs
index a8cc6a132a..45a4cc70fd 100644
--- a/Command.hs
+++ b/Command.hs
@@ -22,6 +22,7 @@ import qualified Backend
 import Messages
 import qualified Annex
 import qualified GitRepo as Git
+import qualified GitRepo.LsFiles as LsFiles
 import Utility
 import Types.Key
 
@@ -118,17 +119,17 @@ notBareRepo a = do
 withFilesInGit :: CommandSeekStrings
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	liftM (map a) $ filterFiles files
 withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	liftM (map a) $ liftIO $ Git.checkAttr repo attr files
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	files' <- filterFiles files
 	backendPairs a files'
 withFilesMissing :: CommandSeekStrings
@@ -143,7 +144,7 @@ withFilesNotInGit :: CommandSeekBackendFiles
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
 	force <- Annex.getState Annex.force
-	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo force) params
+	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
 withWords :: CommandSeekWords
@@ -153,12 +154,12 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ runPreserveOrder (Git.stagedFilesNotDeleted repo) params
+	tocommit <- liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
 	liftM (map a) $ filterFiles tocommit
 withFilesUnlocked :: CommandSeekBackendFiles
-withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles
+withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
 withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles
-withFilesUnlockedToBeCommitted = withFilesUnlocked' Git.typeChangedStagedFiles
+withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBackendFiles
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 0a5381d562..0de98b1d3f 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -20,6 +20,7 @@ import LocationLog
 import Types
 import Content
 import qualified GitRepo as Git
+import qualified GitRepo.LsFiles as LsFiles
 import Messages
 
 command :: [Command]
@@ -37,7 +38,7 @@ start file = isAnnexed file $ \(key, backend) -> do
 			force <- Annex.getState Annex.force
 			unless force $ do
 				g <- Annex.gitRepo
-				staged <- liftIO $ Git.stagedFiles g [Git.workTree g]
+				staged <- liftIO $ LsFiles.staged g [Git.workTree g]
 				unless (null staged) $
 					error "This command cannot be run when there are already files staged for commit."
 				Annex.changeState $ \s -> s { Annex.force = True }
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5744f84fd4..c0a3471796 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -23,6 +23,7 @@ import Utility
 import LocationLog
 import qualified Annex
 import qualified GitRepo as Git
+import qualified GitRepo.LsFiles as LsFiles
 import qualified Backend
 import qualified Remote
 
@@ -175,7 +176,7 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 getKeysReferenced :: Annex [Key]
 getKeysReferenced = do
 	g <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo g [Git.workTree g]
+	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
 
diff --git a/GitRepo.hs b/GitRepo.hs
index cfe949d5e6..cc4636868d 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -47,16 +47,9 @@ module GitRepo (
 	remotesAdd,
 	repoRemoteName,
 	repoRemoteNameSet,
-	inRepo,
-	notInRepo,
-	stagedFiles,
-	stagedFilesNotDeleted,
-	changedUnstagedFiles,
 	checkAttr,
 	decodeGitFile,
 	encodeGitFile,
-	typeChangedFiles,
-	typeChangedStagedFiles,
 	repoAbsPath,
 	reap,
 	useIndex,
@@ -432,55 +425,6 @@ getSha subcommand a = do
 shaSize :: Int
 shaSize = 40
 
-{- Scans for files that are checked into git at the specified locations. -}
-inRepo :: Repo -> [FilePath] -> IO [FilePath]
-inRepo repo l = pipeNullSplit repo $
-	[Params "ls-files --cached -z --"] ++ map File l
-
-{- Scans for files at the specified locations that are not checked into
- - git. -}
-notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
-notInRepo repo include_ignored l =
-	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l
-	where
-		exclude = if include_ignored then [] else [Param "--exclude-standard"]
-
-{- Returns a list of all files that are staged for commit. -}
-stagedFiles :: Repo -> [FilePath] -> IO [FilePath]
-stagedFiles repo l = stagedFiles' repo l []
-
-{- Returns a list of the files, staged for commit, that are being added,
- - moved, or changed (but not deleted), from the specified locations. -}
-stagedFilesNotDeleted :: Repo -> [FilePath] -> IO [FilePath]
-stagedFilesNotDeleted repo l = stagedFiles' repo l [Param "--diff-filter=ACMRT"]
-
-stagedFiles' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
-stagedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
-	where
-		start = [Params "diff --cached --name-only -z"]
-		end = [Param "--"] ++ map File l
-
-{- Returns a list of files that have unstaged changes. -}
-changedUnstagedFiles :: Repo -> [FilePath] -> IO [FilePath]
-changedUnstagedFiles repo l = pipeNullSplit repo $
-	[Params "diff --name-only -z --"] ++ map File l
-
-{- Returns a list of the files in the specified locations that are staged
- - for commit, and whose type has changed. -}
-typeChangedStagedFiles :: Repo -> [FilePath] -> IO [FilePath]
-typeChangedStagedFiles repo l = typeChangedFiles' repo l [Param "--cached"]
-
-{- Returns a list of the files in the specified locations whose type has
- - changed.  Files only staged for commit will not be included. -}
-typeChangedFiles :: Repo -> [FilePath] -> IO [FilePath]
-typeChangedFiles repo l = typeChangedFiles' repo l []
-
-typeChangedFiles' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
-typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
-	where
-		start = [Params "diff --name-only --diff-filter=T -z"]
-		end = [Param "--"] ++ map File l
-
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it into a list of files/lines/whatever. -}
 pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
diff --git a/GitRepo/LsFiles.hs b/GitRepo/LsFiles.hs
new file mode 100644
index 0000000000..16604c319a
--- /dev/null
+++ b/GitRepo/LsFiles.hs
@@ -0,0 +1,68 @@
+{- git ls-files interface
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module GitRepo.LsFiles (
+	inRepo,
+	notInRepo,
+	staged,
+	stagedNotDeleted,
+	changedUnstaged,
+	typeChanged,
+	typeChangedStaged,
+) where
+
+import GitRepo
+import Utility
+
+{- Scans for files that are checked into git at the specified locations. -}
+inRepo :: Repo -> [FilePath] -> IO [FilePath]
+inRepo repo l = pipeNullSplit repo $
+	[Params "ls-files --cached -z --"] ++ map File l
+
+{- Scans for files at the specified locations that are not checked into
+ - git. -}
+notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
+notInRepo repo include_ignored l =
+	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l
+	where
+		exclude = if include_ignored then [] else [Param "--exclude-standard"]
+
+{- Returns a list of all files that are staged for commit. -}
+staged :: Repo -> [FilePath] -> IO [FilePath]
+staged repo l = staged' repo l []
+
+{- Returns a list of the files, staged for commit, that are being added,
+ - moved, or changed (but not deleted), from the specified locations. -}
+stagedNotDeleted :: Repo -> [FilePath] -> IO [FilePath]
+stagedNotDeleted repo l = staged' repo l [Param "--diff-filter=ACMRT"]
+
+staged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
+staged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+	where
+		start = [Params "diff --cached --name-only -z"]
+		end = [Param "--"] ++ map File l
+
+{- Returns a list of files that have unstaged changes. -}
+changedUnstaged :: Repo -> [FilePath] -> IO [FilePath]
+changedUnstaged repo l = pipeNullSplit repo $
+	[Params "diff --name-only -z --"] ++ map File l
+
+{- Returns a list of the files in the specified locations that are staged
+ - for commit, and whose type has changed. -}
+typeChangedStaged :: Repo -> [FilePath] -> IO [FilePath]
+typeChangedStaged repo l = typeChanged' repo l [Param "--cached"]
+
+{- Returns a list of the files in the specified locations whose type has
+ - changed.  Files only staged for commit will not be included. -}
+typeChanged :: Repo -> [FilePath] -> IO [FilePath]
+typeChanged repo l = typeChanged' repo l []
+
+typeChanged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
+typeChanged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+	where
+		start = [Params "diff --name-only --diff-filter=T -z"]
+		end = [Param "--"] ++ map File l
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index b06f00d34d..cb6928fcee 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -26,6 +26,7 @@ import LocationLog
 import qualified Annex
 import qualified AnnexQueue
 import qualified GitRepo as Git
+import qualified GitRepo.LsFiles as LsFiles
 import Backend
 import Messages
 import Version
@@ -92,7 +93,7 @@ updateSymlinks :: Annex ()
 updateSymlinks = do
 	showNote "updating symlinks..."
 	g <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo g [Git.workTree g]
+	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	forM_ files $ fixlink
 	where
 		fixlink f = do

From 8725fde5c66984d9769558a07612361b112be58f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 20:22:19 -0400
Subject: [PATCH 1956/2835] new plan

---
 doc/todo/speed_up_fsck.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/doc/todo/speed_up_fsck.mdwn b/doc/todo/speed_up_fsck.mdwn
index aceb5868cc..e22c01766d 100644
--- a/doc/todo/speed_up_fsck.mdwn
+++ b/doc/todo/speed_up_fsck.mdwn
@@ -5,6 +5,8 @@ are slightly slower but are swamped by the normal runtime.
 For fsck though, it has to pull each file's location log info out of git.
 And, it's typically run on the entire tree.
 
+Another slow one in `git annex copy --from`.
+
 It would be possible to run a single `git cat-file --batch` and pass it
 sha1s of location logs for file that is going to be fsked (gotten via
 `read-tree`). Then just read its output until the next requested sha1 to
@@ -16,3 +18,21 @@ provide the info on a side channel of some sort.
 
 If this is implemented, the same infrastructure could be used for other
 commands like whereis and add. --[[Joey]]
+
+> Updated plan:
+> 
+> Run `git ls-file --batch`, and cache its stdin and out handles in Branch
+> state.
+> 
+> To see a git-annex branch file, send it something like
+> "git-annex:uuid.log", and read the content fron stdout handle.
+> 
+> To detect the end of content, send "TOKEN\n", and look for 
+> "TOKEN missing" in its output. A good choice for TOKEN is anything
+> that will never exist in the repo; 40 0's would be a fairly good choice,
+> but even better seems to be something completely invalid and impossible
+> to have as a sha1 or filename or ref: "".
+> 
+> Hmm, except that's actually an error message sent to stderr. Unless
+> stderr is connected to stdout, it might be better to look for a known,
+> empty object. Could just add a git-annex:empty file to that end.

From e1c18ddec455e5d1259ab46ccccbe6a9c7079de6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 21:23:40 -0400
Subject: [PATCH 1957/2835] Sped back up fsck, copy --from etc

All commands that often have to read a lot of information from
the git-annex branch should now be nearly as fast as before
the branch was introduced.

Before fsck was taking approximatly 3 hours, now it's running in 8 minutes.

The code is very nasty. It should be rewritten to read the header line
from git cat-file, and then read the specified number of bytes of content.
---
 Branch.hs                   | 74 ++++++++++++++++++++-----------------
 Types/BranchState.hs        | 11 +++++-
 debian/changelog            |  3 ++
 doc/todo/speed_up_fsck.mdwn |  2 +
 4 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 26aad44070..4f568e36bc 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -26,9 +26,6 @@ import System.Cmd.Utils
 import Data.Maybe
 import Data.List
 import System.IO
-import System.Posix.IO
-import System.Posix.Process
-import System.Log.Logger
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -142,7 +139,7 @@ commit message = whenM stageJournalFiles $ do
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
-	state <- Annex.getState Annex.branchstate
+	state <- getState
 	unless (branchUpdated state) $ withIndex $ do
 		{- Since branches get merged into the index, it's important to
 		 - first stage the journal into the index. Otherwise, any
@@ -226,39 +223,48 @@ get file = do
 					setCache file content
 					return content
 				Nothing -> withIndexUpdate $ do
-					g <- Annex.gitRepo
-					content <- liftIO $ catch (cat g) (const $ return "")
+					content <- catFile file
 					setCache file content
 					return content
+
+{- Uses git cat-file in batch mode to read the content of a file.
+ -
+ - Only one process is run, and it persists and is used for all accesses. -}
+catFile :: FilePath -> Annex String
+catFile file = do
+	state <- getState
+	maybe (startup state) ask (catFileHandles state)
 	where
-		cat g = cmdOutput "git" $ toCommand $ Git.gitCommandLine g
-			[Param "cat-file", Param "blob", Param $ ':':file]
-
-{- Runs a command, returning its output, ignoring nonzero exit
- - status, and discarding stderr. -}
-cmdOutput :: FilePath -> [String] -> IO String
-cmdOutput cmd params = do
-	pipepair <- createPipe
-	let callfunc _ = do
-		closeFd (snd pipepair)
-		h <- fdToHandle (fst pipepair)
-		x <- hGetContentsStrict h
-		hClose h
-		return $! x
-	let child = do
-		closeFd (fst pipepair)
- 		-- disable stderr output by this child,
-		-- and since the logger uses it, also disable it
-		liftIO $ updateGlobalLogger rootLoggerName $ setLevel EMERGENCY
-		closeFd stdError
-
-	debugM "Utility.executeFile" $ cmd ++ " " ++ show params
-	
-	pid <- pOpen3Raw Nothing (Just (snd pipepair)) Nothing cmd params child
-	retval <- callfunc $! pid
-	let rv = seq retval retval
-	_ <- getProcessStatus True False pid
-	return rv
+		startup state = do
+			g <- Annex.gitRepo
+			let cmd = Git.gitCommandLine g
+				[Param "cat-file", Param "--batch"]
+			let gitcmd = join " " $ "git" : toCommand cmd
+			(_, from, to) <- liftIO $ hPipeBoth "sh"
+				-- want stderr on stdin for sentinal, and
+				-- to ignore other error messages
+				["-c", gitcmd ++ " 2>&1"]
+			setState state { catFileHandles = Just (from, to) }
+			ask (from, to)
+		ask (from, to) = do
+			_ <- liftIO $ do
+				hPutStr to $
+					fullname ++ ":" ++ file ++ "\n" ++
+					sentinal ++ "\n"
+				hFlush to
+			return . unlines =<< readContent from []
+		readContent from ls = do
+			l <- liftIO $ hGetLine from
+			if l == sentinal_line
+				-- first line is blob info,
+				-- or maybe an error message
+				then return $ drop 1 $ reverse ls
+				else readContent from (l:ls)
+		-- To find the end of a catted file, ask for a sentinal
+		-- value that is always missing, and look for the error
+		-- message. Utterly nasty, probably will break one day.
+		sentinal = ":"
+		sentinal_line = sentinal ++ " missing"
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
index 40d7f5c2c7..bc1d32e693 100644
--- a/Types/BranchState.hs
+++ b/Types/BranchState.hs
@@ -7,11 +7,18 @@
 
 module Types.BranchState where
 
+import System.IO
+
 data BranchState = BranchState {
-	branchUpdated :: Bool,
+	branchUpdated :: Bool, -- has the branch been updated this run?
+
+	-- (from, to) handles used to talk to a git-cat-file process
+	catFileHandles :: Maybe (Handle, Handle),
+
+	-- the content of one file is cached
 	cachedFile :: Maybe FilePath,
 	cachedContent :: String
 }
 
 startBranchState :: BranchState
-startBranchState = BranchState False Nothing ""
+startBranchState = BranchState False Nothing Nothing ""
diff --git a/debian/changelog b/debian/changelog
index 5dff0bbe56..a87a89860a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (3.20110625) UNRELEASED; urgency=low
   * Always ensure git-annex branch exists.
   * Modify location log parser to allow future expansion.
   * --force will cause add, etc, to operate on ignored files.
+  * Sped back up fsck, copy --from, and other commands that often
+    have to read a lot of information from the git-annex branch. Should
+    now be nearly as fast as before the branch was introduced.
 
  -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
 
diff --git a/doc/todo/speed_up_fsck.mdwn b/doc/todo/speed_up_fsck.mdwn
index e22c01766d..5d5e867f80 100644
--- a/doc/todo/speed_up_fsck.mdwn
+++ b/doc/todo/speed_up_fsck.mdwn
@@ -36,3 +36,5 @@ commands like whereis and add. --[[Joey]]
 > Hmm, except that's actually an error message sent to stderr. Unless
 > stderr is connected to stdout, it might be better to look for a known,
 > empty object. Could just add a git-annex:empty file to that end.
+
+[[done]] --[[Joey]] 

From 899ecbfba1c015c2c80f729c7e0d5544d7bcc415 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 22:19:40 -0400
Subject: [PATCH 1958/2835] improve git cat-file code

Now it reads the size specified, rather than using the sentinal hack to
determine EOF.

It still depends on error messages to handle files that are not present.
---
 Branch.hs | 58 ++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 4f568e36bc..033b7c6d02 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -17,7 +17,7 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (unless, liftM)
+import Control.Monad (when, unless, liftM)
 import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
@@ -26,6 +26,9 @@ import System.Cmd.Utils
 import Data.Maybe
 import Data.List
 import System.IO
+import System.IO.Unsafe
+import Foreign
+import Data.Char
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -239,32 +242,39 @@ catFile file = do
 			g <- Annex.gitRepo
 			let cmd = Git.gitCommandLine g
 				[Param "cat-file", Param "--batch"]
-			let gitcmd = join " " $ "git" : toCommand cmd
+			let gitcmd = join " " ("git" : toCommand cmd)
 			(_, from, to) <- liftIO $ hPipeBoth "sh"
-				-- want stderr on stdin for sentinal, and
-				-- to ignore other error messages
-				["-c", gitcmd ++ " 2>&1"]
+				-- want stderr on stdin to handle error messages
+				["-c", "LANG=C exec " ++ gitcmd ++ " 2>&1"]
 			setState state { catFileHandles = Just (from, to) }
 			ask (from, to)
-		ask (from, to) = do
-			_ <- liftIO $ do
-				hPutStr to $
-					fullname ++ ":" ++ file ++ "\n" ++
-					sentinal ++ "\n"
-				hFlush to
-			return . unlines =<< readContent from []
-		readContent from ls = do
-			l <- liftIO $ hGetLine from
-			if l == sentinal_line
-				-- first line is blob info,
-				-- or maybe an error message
-				then return $ drop 1 $ reverse ls
-				else readContent from (l:ls)
-		-- To find the end of a catted file, ask for a sentinal
-		-- value that is always missing, and look for the error
-		-- message. Utterly nasty, probably will break one day.
-		sentinal = ":"
-		sentinal_line = sentinal ++ " missing"
+		ask (from, to) = liftIO $ do
+			let want = fullname ++ ":" ++ file
+			hPutStrLn to want
+			hFlush to
+			header <- hGetLine from
+			if header == want ++ " missing"
+				then return ""
+				else do
+					let [_sha, _type, size] = words header
+					let bytes = read size
+					fp <- mallocForeignPtrBytes (fromIntegral bytes)
+					len <- withForeignPtr fp $ \buf -> hGetBuf from buf (fromIntegral bytes)
+					when (len /= bytes) $
+						error "short read from git cat-file"
+					content <- lazySlurp fp 0 len
+					c <- hGetChar from
+					when (c /= '\n') $
+						error "missing newline from git cat-file"
+					return content
+
+lazySlurp :: ForeignPtr Word8 -> Int -> Int -> IO String
+lazySlurp fp ix len
+	| ix == len = return []
+	| otherwise = do
+		c <- withForeignPtr fp $ \p -> peekElemOff p ix
+		cs <- unsafeInterleaveIO (lazySlurp fp (ix+1) len)
+		return $ chr (fromIntegral c) : cs
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]

From b089fba7b4e68c4482909319f562d32a3f00f379 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 23:56:47 -0400
Subject: [PATCH 1959/2835] use ByteString for hGet

Avoids the crazy low-level hGetBuf stuff. Also slightly faster.
---
 Branch.hs | 20 +++-----------------
 1 file changed, 3 insertions(+), 17 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 033b7c6d02..9cdb096fd1 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -26,9 +26,7 @@ import System.Cmd.Utils
 import Data.Maybe
 import Data.List
 import System.IO
-import System.IO.Unsafe
-import Foreign
-import Data.Char
+import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -258,23 +256,11 @@ catFile file = do
 				else do
 					let [_sha, _type, size] = words header
 					let bytes = read size
-					fp <- mallocForeignPtrBytes (fromIntegral bytes)
-					len <- withForeignPtr fp $ \buf -> hGetBuf from buf (fromIntegral bytes)
-					when (len /= bytes) $
-						error "short read from git cat-file"
-					content <- lazySlurp fp 0 len
+					content <- B.hGet from bytes
 					c <- hGetChar from
 					when (c /= '\n') $
 						error "missing newline from git cat-file"
-					return content
-
-lazySlurp :: ForeignPtr Word8 -> Int -> Int -> IO String
-lazySlurp fp ix len
-	| ix == len = return []
-	| otherwise = do
-		c <- withForeignPtr fp $ \p -> peekElemOff p ix
-		cs <- unsafeInterleaveIO (lazySlurp fp (ix+1) len)
-		return $ chr (fromIntegral c) : cs
+					return $ B.unpack content
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]

From d72fb5acc28af3ea6380dd09518f7d1382dea8d2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 00:35:51 -0400
Subject: [PATCH 1960/2835] Fix encoding of utf-8 etc when storing the
 description of repository and other content.

Write files in raw mode, to avoid mangling the encoding of content
provided.

Note: This was a longstanding problem, it was not introduced in v3.
---
 Branch.hs        | 3 ++-
 debian/changelog | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Branch.hs b/Branch.hs
index 9cdb096fd1..ab24e47525 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -26,6 +26,7 @@ import System.Cmd.Utils
 import Data.Maybe
 import Data.List
 import System.IO
+import System.IO.Binary
 import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
@@ -287,7 +288,7 @@ setJournalFile file content = do
 		write g = do
 			let jfile = journalFile g file
 			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
-			writeFile tmpfile content
+			writeBinaryFile tmpfile content
 			renameFile tmpfile jfile
 
 {- Gets journalled content for a file in the branch. -}
diff --git a/debian/changelog b/debian/changelog
index a87a89860a..f84f8ab498 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (3.20110625) UNRELEASED; urgency=low
   * Sped back up fsck, copy --from, and other commands that often
     have to read a lot of information from the git-annex branch. Should
     now be nearly as fast as before the branch was introduced.
+  * Fix encoding of utf-8 etc when storing the description of repository
+    and other content.
 
  -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
 

From 2cda9d0a0fcdd1cc2aebc066ef19282fbe36e898 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 00:42:09 -0400
Subject: [PATCH 1961/2835] generalized safeWriteFile to viaTmp

---
 Command/Init.hs   |  2 +-
 Command/Unused.hs |  2 +-
 Upgrade/V2.hs     |  2 +-
 Utility.hs        | 11 ++++++-----
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 7f5773117a..8cde8bb9fb 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -55,7 +55,7 @@ gitPreCommitHookWrite repo = do
 	if exists
 		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
 		else liftIO $ do
-			safeWriteFile hook preCommitScript
+			viaTmp writeFile hook preCommitScript
 			p <- getPermissions hook
 			setPermissions hook $ p {executable = True}
 	where
diff --git a/Command/Unused.hs b/Command/Unused.hs
index c0a3471796..a9d5f90a10 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -86,7 +86,7 @@ checkRemoteUnused' r = do
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
 	g <- Annex.gitRepo
-	liftIO $ safeWriteFile (gitAnnexUnusedLog prefix g) $
+	liftIO $ viaTmp writeFile (gitAnnexUnusedLog prefix g) $
 		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
 
 table :: [(Int, Key)] -> [String]
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 8537a50226..ea68e78c95 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -128,7 +128,7 @@ gitAttributesUnWrite repo = do
 	let attributes = Git.attributes repo
 	whenM (doesFileExist attributes) $ do
 		c <- readFileStrict attributes
-		liftIO $ safeWriteFile attributes $ unlines $
+		liftIO $ viaTmp writeFile attributes $ unlines $
 			filter (\l -> not $ l `elem` attrLines) $ lines c
 		Git.run repo "add" [File attributes]
 
diff --git a/Utility.hs b/Utility.hs
index 47d10ed759..7831a4ab4b 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -22,7 +22,7 @@ module Utility (
 	shellUnEscape,
 	unsetFileMode,
 	readMaybe,
-	safeWriteFile,
+	viaTmp,
 	dirContains,
 	dirContents,
 	myHomeDir,
@@ -243,13 +243,14 @@ readMaybe s = case reads s of
 	((x,_):_) -> Just x
 	_ -> Nothing
 
-{- Writes a file using a temp file that is renamed atomically into place. -}
-safeWriteFile :: FilePath -> String -> IO ()
-safeWriteFile file content = do
+{- Runs an action like writeFile, writing to a tmp file first and
+ - then moving it into place. -}
+viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()
+viaTmp a file content = do
 	pid <- getProcessID
         let tmpfile = file ++ ".tmp" ++ show pid
 	createDirectoryIfMissing True (parentDir file)
-	writeFile tmpfile content
+	a tmpfile content
 	renameFile tmpfile file
 
 {- Lists the contents of a directory.

From 8562e6096ca9a6819c04b4fd1938202ccd68c701 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 01:16:53 -0400
Subject: [PATCH 1962/2835] v3 is now faster than v2

Rebenchmarked v2 vs v3, and v3 is now actually faster. Yes, storing data
in git, using git as a filesystem is actually faster than just using the
filesystem. If you do it just right. :)
---
 debian/changelog | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index f84f8ab498..9013abd135 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,12 +1,13 @@
 git-annex (3.20110625) UNRELEASED; urgency=low
 
+  * Sped back up fsck, copy --from, and other commands that often
+    have to read a lot of information from the git-annex branch. Such
+    commands are now faster than they were before introduction of the
+    git-annex branch.
   * Always ensure git-annex branch exists.
   * Modify location log parser to allow future expansion.
   * --force will cause add, etc, to operate on ignored files.
-  * Sped back up fsck, copy --from, and other commands that often
-    have to read a lot of information from the git-annex branch. Should
-    now be nearly as fast as before the branch was introduced.
-  * Fix encoding of utf-8 etc when storing the description of repository
+  * Avoid mangling encoding when storing the description of repository
     and other content.
 
  -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400

From 5fe02f280726442496303859e83f9ce1c48be0cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:12:51 -0400
Subject: [PATCH 1963/2835] more robust git cat-file output parser

Only remaining ugliness is the handling of error messages for files that
are not present on the branch.
---
 Branch.hs | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index ab24e47525..5ad89df202 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -243,8 +243,8 @@ catFile file = do
 				[Param "cat-file", Param "--batch"]
 			let gitcmd = join " " ("git" : toCommand cmd)
 			(_, from, to) <- liftIO $ hPipeBoth "sh"
-				-- want stderr on stdin to handle error messages
-				["-c", "LANG=C exec " ++ gitcmd ++ " 2>&1"]
+				-- want stderr on stdin to see error messages
+				["-c", "exec " ++ gitcmd ++ " 2>&1"]
 			setState state { catFileHandles = Just (from, to) }
 			ask (from, to)
 		ask (from, to) = liftIO $ do
@@ -252,16 +252,22 @@ catFile file = do
 			hPutStrLn to want
 			hFlush to
 			header <- hGetLine from
-			if header == want ++ " missing"
-				then return ""
-				else do
-					let [_sha, _type, size] = words header
-					let bytes = read size
-					content <- B.hGet from bytes
-					c <- hGetChar from
-					when (c /= '\n') $
-						error "missing newline from git cat-file"
-					return $ B.unpack content
+			case words header of
+				[sha, blob, size]
+					| length sha == Git.shaSize &&
+					  blob == "blob" -> handle from size
+					| otherwise -> empty
+				_ -> empty
+		handle from size = case reads size of
+			[(bytes, "")] -> readcontent from bytes
+			_ -> empty
+		readcontent from bytes = do
+			content <- B.hGet from bytes
+			c <- hGetChar from
+			when (c /= '\n') $
+				error "missing newline from git cat-file"
+			return $ B.unpack content
+		empty = return ""
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]

From f6063a094ec02caec314b42dc05f2f0595ae0ce4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:16:57 -0400
Subject: [PATCH 1964/2835] renamed GitRepo to Git

It was always imported qualified as Git anyway
---
 Annex.hs                    | 2 +-
 Backend.hs                  | 2 +-
 Backend/File.hs             | 2 +-
 Branch.hs                   | 2 +-
 CmdLine.hs                  | 2 +-
 Command.hs                  | 4 ++--
 Command/DropUnused.hs       | 2 +-
 Command/Init.hs             | 2 +-
 Command/Map.hs              | 2 +-
 Command/Status.hs           | 2 +-
 Command/Unannex.hs          | 4 ++--
 Command/Uninit.hs           | 2 +-
 Command/Unused.hs           | 4 ++--
 Config.hs                   | 2 +-
 Content.hs                  | 2 +-
 GitRepo.hs => Git.hs        | 4 ++--
 {GitRepo => Git}/LsFiles.hs | 4 ++--
 GitAnnex.hs                 | 2 +-
 GitQueue.hs                 | 2 +-
 GitUnionMerge.hs            | 2 +-
 LocationLog.hs              | 2 +-
 Locations.hs                | 2 +-
 Remote/Bup.hs               | 2 +-
 Remote/Directory.hs         | 2 +-
 Remote/Git.hs               | 2 +-
 Remote/Hook.hs              | 2 +-
 Remote/Rsync.hs             | 2 +-
 Remote/S3real.hs            | 2 +-
 Remote/Special.hs           | 2 +-
 Ssh.hs                      | 2 +-
 Types/Remote.hs             | 2 +-
 UUID.hs                     | 2 +-
 Upgrade/V1.hs               | 4 ++--
 Upgrade/V2.hs               | 2 +-
 Version.hs                  | 2 +-
 git-annex-shell.hs          | 2 +-
 git-annex.hs                | 4 +---
 git-union-merge.hs          | 2 +-
 test.hs                     | 2 +-
 39 files changed, 45 insertions(+), 47 deletions(-)
 rename GitRepo.hs => Git.hs (99%)
 rename {GitRepo => Git}/LsFiles.hs (98%)

diff --git a/Annex.hs b/Annex.hs
index 82908881da..e2f4a10206 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -18,7 +18,7 @@ module Annex (
 
 import Control.Monad.State
 
-import qualified GitRepo as Git
+import qualified Git
 import GitQueue
 import Types.Backend
 import Types.Remote
diff --git a/Backend.hs b/Backend.hs
index 78a53d02c7..b1cd4c8f0e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -39,7 +39,7 @@ import System.Posix.Files
 import System.Directory
 
 import Locations
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import Types
 import Types.Key
diff --git a/Backend/File.hs b/Backend/File.hs
index 1c25f89db2..b8d3bb65ad 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -20,7 +20,7 @@ import Data.String.Utils
 import Types.Backend
 import LocationLog
 import qualified Remote
-import qualified GitRepo as Git
+import qualified Git
 import Content
 import qualified Annex
 import Types
diff --git a/Branch.hs b/Branch.hs
index 5ad89df202..7650fe8619 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -30,7 +30,7 @@ import System.IO.Binary
 import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
-import qualified GitRepo as Git
+import qualified Git
 import qualified GitUnionMerge
 import qualified Annex
 import Utility
diff --git a/CmdLine.hs b/CmdLine.hs
index ab7236847f..85423e5e89 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -18,7 +18,7 @@ import Control.Monad (when)
 
 import qualified Annex
 import qualified AnnexQueue
-import qualified GitRepo as Git
+import qualified Git
 import Content
 import Types
 import Command
diff --git a/Command.hs b/Command.hs
index 45a4cc70fd..d36f675d24 100644
--- a/Command.hs
+++ b/Command.hs
@@ -21,8 +21,8 @@ import Types
 import qualified Backend
 import Messages
 import qualified Annex
-import qualified GitRepo as Git
-import qualified GitRepo.LsFiles as LsFiles
+import qualified Git
+import qualified Git.LsFiles as LsFiles
 import Utility
 import Types.Key
 
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 0f99814471..2125abdc3c 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -20,7 +20,7 @@ import qualified Annex
 import qualified Command.Drop
 import qualified Command.Move
 import qualified Remote
-import qualified GitRepo as Git
+import qualified Git
 import Backend
 import Types.Key
 import Utility
diff --git a/Command/Init.hs b/Command/Init.hs
index 8cde8bb9fb..71e87050d8 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -13,7 +13,7 @@ import System.Directory
 
 import Command
 import qualified Annex
-import qualified GitRepo as Git
+import qualified Git
 import qualified Branch
 import UUID
 import Version
diff --git a/Command/Map.hs b/Command/Map.hs
index 7a9121b69b..7bb435ff81 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -15,7 +15,7 @@ import Data.List.Utils
 
 import Command
 import qualified Annex
-import qualified GitRepo as Git
+import qualified Git
 import Messages
 import Types
 import Utility
diff --git a/Command/Status.hs b/Command/Status.hs
index 3b096d9793..53589030b3 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -18,7 +18,7 @@ import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
 import qualified Command.Unused
-import qualified GitRepo as Git
+import qualified Git
 import Command
 import Types
 import DataUnits
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 0de98b1d3f..fe413b25b4 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -19,8 +19,8 @@ import qualified Backend
 import LocationLog
 import Types
 import Content
-import qualified GitRepo as Git
-import qualified GitRepo.LsFiles as LsFiles
+import qualified Git
+import qualified Git.LsFiles as LsFiles
 import Messages
 
 command :: [Command]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 9698ed8200..c47ac0c3af 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -14,7 +14,7 @@ import Command
 import Messages
 import Types
 import Utility
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import qualified Command.Unannex
 import qualified Command.Init
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a9d5f90a10..3f51e2c2c2 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -22,8 +22,8 @@ import Locations
 import Utility
 import LocationLog
 import qualified Annex
-import qualified GitRepo as Git
-import qualified GitRepo.LsFiles as LsFiles
+import qualified Git
+import qualified Git.LsFiles as LsFiles
 import qualified Backend
 import qualified Remote
 
diff --git a/Config.hs b/Config.hs
index a324427d44..1016845e75 100644
--- a/Config.hs
+++ b/Config.hs
@@ -10,7 +10,7 @@ module Config where
 import Data.Maybe
 import Control.Monad.State (liftIO)
 
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import Types
 import Utility
diff --git a/Content.hs b/Content.hs
index d733ad8b36..d24fc9ac70 100644
--- a/Content.hs
+++ b/Content.hs
@@ -36,7 +36,7 @@ import Types
 import Locations
 import LocationLog
 import UUID
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
diff --git a/GitRepo.hs b/Git.hs
similarity index 99%
rename from GitRepo.hs
rename to Git.hs
index cc4636868d..2b0cdd06d5 100644
--- a/GitRepo.hs
+++ b/Git.hs
@@ -3,12 +3,12 @@
  - This is written to be completely independant of git-annex and should be
  - suitable for other uses.
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module GitRepo (
+module Git (
 	Repo,
 	repoFromCwd,
 	repoFromAbsPath,
diff --git a/GitRepo/LsFiles.hs b/Git/LsFiles.hs
similarity index 98%
rename from GitRepo/LsFiles.hs
rename to Git/LsFiles.hs
index 16604c319a..b88c9144e6 100644
--- a/GitRepo/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module GitRepo.LsFiles (
+module Git.LsFiles (
 	inRepo,
 	notInRepo,
 	staged,
@@ -15,7 +15,7 @@ module GitRepo.LsFiles (
 	typeChangedStaged,
 ) where
 
-import GitRepo
+import Git
 import Utility
 
 {- Scans for files that are checked into git at the specified locations. -}
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 49ba63b8ab..58b512f718 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -9,7 +9,7 @@ module GitAnnex where
 
 import System.Console.GetOpt
 
-import qualified GitRepo as Git
+import qualified Git
 import CmdLine
 import Command
 import Options
diff --git a/GitQueue.hs b/GitQueue.hs
index be0fcfc4af..5da3ba1d6e 100644
--- a/GitQueue.hs
+++ b/GitQueue.hs
@@ -21,7 +21,7 @@ import Data.String.Utils
 import Control.Monad (unless, forM_)
 import Utility
 
-import qualified GitRepo as Git
+import qualified Git
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 74579ebcc9..84c7b44eae 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -18,7 +18,7 @@ import Data.List
 import Data.Maybe
 import Data.String.Utils
 
-import qualified GitRepo as Git
+import qualified Git
 import Utility
 
 {- Performs a union merge between two branches, staging it in the index.
diff --git a/LocationLog.hs b/LocationLog.hs
index 2a2bc63013..b7deb3ed9a 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -34,7 +34,7 @@ import Control.Monad (when)
 import Data.Maybe
 import Control.Monad.State (liftIO)
 
-import qualified GitRepo as Git
+import qualified Git
 import qualified Branch
 import UUID
 import Types
diff --git a/Locations.hs b/Locations.hs
index bfb0d3af9e..6142b2393a 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -36,7 +36,7 @@ import Data.Hash.MD5
 
 import Types
 import Types.Key
-import qualified GitRepo as Git
+import qualified Git
 
 {- Conventions:
  -
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index c011c979ca..11c0ec4daf 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -21,7 +21,7 @@ import System.Cmd.Utils
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Locations
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 7b5917dca8..991ccbe481 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -18,7 +18,7 @@ import System.FilePath
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Locations
diff --git a/Remote/Git.hs b/Remote/Git.hs
index c8290c9a79..471417e345 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -15,7 +15,7 @@ import System.Posix.Files
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import Locations
 import UUID
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index cc511965f8..86a7bca560 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -20,7 +20,7 @@ import System.Exit
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Locations
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index bf1bbd8707..aa2507fff4 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -18,7 +18,7 @@ import System.Posix.Process
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Locations
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 2479dfa023..829c58ad06 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -27,7 +27,7 @@ import System.Posix.Env (setEnv)
 import Types
 import Types.Remote
 import Types.Key
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Messages
diff --git a/Remote/Special.hs b/Remote/Special.hs
index 7d2ea1d704..9a00dbd82e 100644
--- a/Remote/Special.hs
+++ b/Remote/Special.hs
@@ -14,7 +14,7 @@ import Control.Monad.State (liftIO)
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Utility
diff --git a/Ssh.hs b/Ssh.hs
index 0cf2919c25..21e72c0839 100644
--- a/Ssh.hs
+++ b/Ssh.hs
@@ -9,7 +9,7 @@ module Ssh where
 
 import Control.Monad.State (liftIO)
 
-import qualified GitRepo as Git
+import qualified Git
 import Utility
 import Types
 import Config
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 01ced04ae1..1d67ad5cd7 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -12,7 +12,7 @@ module Types.Remote where
 import Control.Exception
 import Data.Map as M
 
-import qualified GitRepo as Git
+import qualified Git
 import Types.Key
 
 type RemoteConfig = M.Map String String
diff --git a/UUID.hs b/UUID.hs
index 5d8304f83d..0723abeca4 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -29,7 +29,7 @@ import System.IO
 import qualified Data.Map as M
 import Data.Maybe
 
-import qualified GitRepo as Git
+import qualified Git
 import qualified Branch
 import Types
 import Types.UUID
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index cb6928fcee..39b8e47c5f 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -25,8 +25,8 @@ import Locations
 import LocationLog
 import qualified Annex
 import qualified AnnexQueue
-import qualified GitRepo as Git
-import qualified GitRepo.LsFiles as LsFiles
+import qualified Git
+import qualified Git.LsFiles as LsFiles
 import Backend
 import Messages
 import Version
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index ea68e78c95..ce0424b302 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -16,7 +16,7 @@ import Data.Maybe
 import Types.Key
 import Types
 import qualified Annex
-import qualified GitRepo as Git
+import qualified Git
 import qualified Branch
 import Messages
 import Utility
diff --git a/Version.hs b/Version.hs
index 690e693e23..7e7a4c7ce6 100644
--- a/Version.hs
+++ b/Version.hs
@@ -11,7 +11,7 @@ import Control.Monad (unless)
 
 import Types
 import qualified Annex
-import qualified GitRepo as Git
+import qualified Git
 import Config
 
 type Version = String
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 55f34e1027..c240b1c190 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -8,7 +8,7 @@
 import System.Environment
 import Data.List
 
-import qualified GitRepo as Git
+import qualified Git
 import CmdLine
 import Command
 import Utility
diff --git a/git-annex.hs b/git-annex.hs
index 9d6012f2c1..a53697cdbb 100644
--- a/git-annex.hs
+++ b/git-annex.hs
@@ -10,6 +10,4 @@ import System.Environment
 import GitAnnex
 
 main :: IO ()
-main = do
-	args <- getArgs
-	run args
+main = run =<< getArgs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 57232be67b..4939fb57d1 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -11,7 +11,7 @@ import System.Directory
 import Control.Monad (when)
 
 import qualified GitUnionMerge
-import qualified GitRepo as Git
+import qualified Git
 
 header :: String
 header = "Usage: git-union-merge ref ref newref"
diff --git a/test.hs b/test.hs
index 9729818211..44a792f14c 100644
--- a/test.hs
+++ b/test.hs
@@ -27,7 +27,7 @@ import System.IO.HVFS (SystemFS(..))
 import qualified Annex
 import qualified BackendList
 import qualified Backend
-import qualified GitRepo as Git
+import qualified Git
 import qualified Locations
 import qualified Utility
 import qualified Types.Backend

From 0f2859454cb1d1edd779fccd9b35e2b5d78b0861 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:22:10 -0400
Subject: [PATCH 1965/2835] tweak

---
 git-annex-shell.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index c240b1c190..a64552c721 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -35,9 +35,7 @@ header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
 
 main :: IO ()
-main = do
-	args <- getArgs
-	main' args
+main = main' =<< getArgs
 
 main' :: [String] -> IO ()
 main' [] = failure

From f0497312a77d59f24c8273245ac324b02bb1eb13 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:25:37 -0400
Subject: [PATCH 1966/2835] rename GitQueue to Git.Queue

---
 Annex.hs                    |  2 +-
 AnnexQueue.hs               | 12 ++++++------
 GitQueue.hs => Git/Queue.hs | 10 +++++-----
 3 files changed, 12 insertions(+), 12 deletions(-)
 rename GitQueue.hs => Git/Queue.hs (93%)

diff --git a/Annex.hs b/Annex.hs
index e2f4a10206..c21cfb37ca 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -19,7 +19,7 @@ module Annex (
 import Control.Monad.State
 
 import qualified Git
-import GitQueue
+import Git.Queue
 import Types.Backend
 import Types.Remote
 import Types.Crypto
diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 58e77a6e85..4c35adfb87 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -16,7 +16,7 @@ import Control.Monad (when, unless)
 
 import Annex
 import Messages
-import qualified GitQueue
+import qualified Git.Queue
 import Utility
 
 {- Adds a git command to the queue, possibly running previously queued
@@ -24,24 +24,24 @@ import Utility
 add :: String -> [CommandParam] -> FilePath -> Annex ()
 add command params file = do
 	q <- getState repoqueue
-	store $ GitQueue.add q command params file
+	store $ Git.Queue.add q command params file
 
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
 flushWhenFull = do
 	q <- getState repoqueue
-	when (GitQueue.full q) $ flush False
+	when (Git.Queue.full q) $ flush False
 
 {- Runs (and empties) the queue. -}
 flush :: Bool -> Annex ()
 flush silent = do
 	q <- getState repoqueue
-	unless (0 == GitQueue.size q) $ do
+	unless (0 == Git.Queue.size q) $ do
 		unless silent $
 			showSideAction "Recording state in git..."
 		g <- gitRepo
-		q' <- liftIO $ GitQueue.flush g q
+		q' <- liftIO $ Git.Queue.flush g q
 		store q'
 
-store :: GitQueue.Queue -> Annex ()
+store :: Git.Queue.Queue -> Annex ()
 store q = changeState $ \s -> s { repoqueue = q }
diff --git a/GitQueue.hs b/Git/Queue.hs
similarity index 93%
rename from GitQueue.hs
rename to Git/Queue.hs
index 5da3ba1d6e..e1ec0cd313 100644
--- a/GitQueue.hs
+++ b/Git/Queue.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module GitQueue (
+module Git.Queue (
 	Queue,
 	empty,
 	add,
@@ -21,7 +21,7 @@ import Data.String.Utils
 import Control.Monad (unless, forM_)
 import Utility
 
-import qualified Git
+import Git
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
@@ -72,7 +72,7 @@ full :: Queue -> Bool
 full (Queue n _) = n > maxSize
 
 {- Runs a queue on a git repository. -}
-flush :: Git.Repo -> Queue -> IO Queue
+flush :: Repo -> Queue -> IO Queue
 flush repo (Queue _ m) = do
 	forM_ (M.toList m) $ uncurry $ runAction repo
 	return empty
@@ -80,10 +80,10 @@ flush repo (Queue _ m) = do
 {- Runs an Action on a list of files in a git repository.
  -
  - Complicated by commandline length limits. -}
-runAction :: Git.Repo -> Action -> [FilePath] -> IO ()
+runAction :: Repo -> Action -> [FilePath] -> IO ()
 runAction repo action files = unless (null files) runxargs
 	where
 		runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
-		params = toCommand $ Git.gitCommandLine repo
+		params = toCommand $ gitCommandLine repo
 			(Param (getSubcommand action):getParams action)
 		feedxargs h = hPutStr h $ join "\0" files

From 896726cde425f6c74273b35cde30c1909551ff66 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:32:47 -0400
Subject: [PATCH 1967/2835] rename GitUnionMerge to Git.UnionMerge

Also, moved commit function into Git proper, it's not union merge specific.
---
 Branch.hs                             | 18 +++++------
 Git.hs                                | 14 +++++++++
 GitUnionMerge.hs => Git/UnionMerge.hs | 44 +++++++++------------------
 git-union-merge.hs                    |  6 ++--
 4 files changed, 40 insertions(+), 42 deletions(-)
 rename GitUnionMerge.hs => Git/UnionMerge.hs (67%)

diff --git a/Branch.hs b/Branch.hs
index 7650fe8619..59f2d3134f 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -31,7 +31,7 @@ import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
 import qualified Git
-import qualified GitUnionMerge
+import qualified Git.UnionMerge
 import qualified Annex
 import Utility
 import Types
@@ -72,7 +72,7 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = GitUnionMerge.ls_tree g fullname >>= GitUnionMerge.update_index g
+genIndex g = Git.UnionMerge.ls_tree g fullname >>= Git.UnionMerge.update_index g
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
@@ -128,14 +128,13 @@ create = unlessM (refExists fullname) $ do
 	if e
 		then liftIO $ Git.run g "branch" [Param name, Param originname]
 		else withIndex' True $
-			liftIO $ GitUnionMerge.commit g "branch created" fullname []
+			liftIO $ Git.commit g "branch created" fullname []
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM stageJournalFiles $ do
 	g <- Annex.gitRepo
-	withIndex $ liftIO $
-		GitUnionMerge.commit g message fullname [fullname]
+	withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
@@ -158,8 +157,7 @@ update = do
 		let refs = map (last . words) (lines r)
 		updated <- catMaybes `liftM` mapM updateRef refs
 		unless (null updated && not staged) $ liftIO $
-			GitUnionMerge.commit g "update" fullname
-				(fullname:updated)
+			Git.commit g "update" fullname (fullname:updated)
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
@@ -200,7 +198,7 @@ updateRef ref
 				-- index will be removed. So, documentation
 				-- advises users not to directly modify the
 				-- branch.
-				liftIO $ GitUnionMerge.merge g [ref]
+				liftIO $ Git.UnionMerge.merge g [ref]
 				return $ Just ref
 
 {- Records changed content of a file into the journal. -}
@@ -335,12 +333,12 @@ stageJournalFiles = do
 			(h, s) <- Git.pipeWriteRead g [Param "hash-object",
 				Param "-w", Param "--stdin-paths"] $ unlines paths
 			-- update the index, also in just one command
-			GitUnionMerge.update_index g $
+			Git.UnionMerge.update_index g $
 				index_lines (lines s) $ map fileJournal fs
 			forceSuccess h
 			mapM_ removeFile paths
 		index_lines shas fs = map genline $ zip shas fs
-		genline (sha, file) = GitUnionMerge.update_index_line sha file
+		genline (sha, file) = Git.UnionMerge.update_index_line sha file
 
 {- Produces a filename to use in the journal for a file on the branch.
  -
diff --git a/Git.hs b/Git.hs
index 2b0cdd06d5..7005f24f55 100644
--- a/Git.hs
+++ b/Git.hs
@@ -56,6 +56,7 @@ module Git (
 	hashObject,
 	getSha,
 	shaSize,
+	commit,
 
 	prop_idempotent_deencode
 ) where
@@ -425,6 +426,19 @@ getSha subcommand a = do
 shaSize :: Int
 shaSize = 40
 
+{- Commits the index into the specified branch, 
+ - with the specified parent refs. -}
+commit :: Repo -> String -> String -> [String] -> IO ()
+commit g message newref parentrefs = do
+	tree <- getSha "write-tree" $
+		pipeRead g [Param "write-tree"]
+	sha <- getSha "commit-tree" $ ignorehandle $
+		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
+	run g "update-ref" [Param newref, Param sha]
+	where
+		ignorehandle a = return . snd =<< a
+		ps = concatMap (\r -> ["-p", r]) parentrefs
+
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it into a list of files/lines/whatever. -}
 pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
diff --git a/GitUnionMerge.hs b/Git/UnionMerge.hs
similarity index 67%
rename from GitUnionMerge.hs
rename to Git/UnionMerge.hs
index 84c7b44eae..4e0361b858 100644
--- a/GitUnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -5,9 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module GitUnionMerge (
+module Git.UnionMerge (
 	merge,
-	commit,
 	update_index,
 	update_index_line,
 	ls_tree
@@ -18,7 +17,7 @@ import Data.List
 import Data.Maybe
 import Data.String.Utils
 
-import qualified Git
+import Git
 import Utility
 
 {- Performs a union merge between two branches, staging it in the index.
@@ -29,7 +28,7 @@ import Utility
  -
  - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: Git.Repo -> [String] -> IO ()
+merge :: Repo -> [String] -> IO ()
 merge g (x:y:[]) = do
 	a <- ls_tree g x
 	b <- merge_trees g x y
@@ -40,10 +39,10 @@ merge _ _ = error "wrong number of branches to merge"
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
-update_index :: Git.Repo -> [String] -> IO ()
+update_index :: Repo -> [String] -> IO ()
 update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
-		togit ps content = Git.pipeWrite g (map Param ps) content
+		togit ps content = pipeWrite g (map Param ps) content
 			>>= forceSuccess
 
 {- Generates a line suitable to be fed into update-index, to add
@@ -52,16 +51,16 @@ update_index_line :: String -> FilePath -> String
 update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 
 {- Gets the contents of a tree in a format suitable for update_index. -}
-ls_tree :: Git.Repo -> String -> IO [String]
-ls_tree g x = Git.pipeNullSplit g $ 
+ls_tree :: Repo -> String -> IO [String]
+ls_tree g x = pipeNullSplit g $ 
 	map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: Git.Repo -> String -> String -> IO [String]
+merge_trees :: Repo -> String -> String -> IO [String]
 merge_trees g x y = calc_merge g $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: Git.Repo -> String -> IO [String]
+merge_tree_index :: Repo -> String -> IO [String]
 merge_tree_index g x = calc_merge g $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
@@ -69,9 +68,9 @@ diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: Git.Repo -> [String] -> IO [String]
+calc_merge :: Repo -> [String] -> IO [String]
 calc_merge g differ = do
-	diff <- Git.pipeNullSplit g $ map Param differ
+	diff <- pipeNullSplit g $ map Param differ
 	l <- mapM (mergeFile g) (pairs diff)
 	return $ catMaybes l
 	where
@@ -82,28 +81,15 @@ calc_merge g differ = do
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
-mergeFile :: Git.Repo -> (String, FilePath) -> IO (Maybe String)
+mergeFile :: Repo -> (String, FilePath) -> IO (Maybe String)
 mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- Git.pipeRead g $ map Param ("show":shas)
-		sha <- Git.hashObject g $ unionmerge content
+		content <- pipeRead g $ map Param ("show":shas)
+		sha <- hashObject g $ unionmerge content
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
-		nullsha = take Git.shaSize $ repeat '0'
+		nullsha = take shaSize $ repeat '0'
 		unionmerge = unlines . nub . lines
-
-{- Commits the index into the specified branch, 
- - with the specified parent refs. -}
-commit :: Git.Repo -> String -> String -> [String] -> IO ()
-commit g message newref parentrefs = do
-	tree <- Git.getSha "write-tree" $
-		Git.pipeRead g [Param "write-tree"]
-	sha <- Git.getSha "commit-tree" $ ignorehandle $
-		Git.pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
-	Git.run g "update-ref" [Param newref, Param sha]
-	where
-		ignorehandle a = return . snd =<< a
-		ps = concatMap (\r -> ["-p", r]) parentrefs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 4939fb57d1..38df0df6a2 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -10,7 +10,7 @@ import System.FilePath
 import System.Directory
 import Control.Monad (when)
 
-import qualified GitUnionMerge
+import qualified Git.UnionMerge
 import qualified Git
 
 header :: String
@@ -44,6 +44,6 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
-	GitUnionMerge.merge g [aref, bref]
-	GitUnionMerge.commit g "union merge" newref [aref, bref]
+	Git.UnionMerge.merge g [aref, bref]
+	Git.commit g "union merge" newref [aref, bref]
 	cleanup g

From 56aeeb4565dd419c315d370f6e648abfe009a7d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 14:55:03 -0400
Subject: [PATCH 1968/2835] cabal can now be used to build git-annex.

This is substantially slower than using make, does not build or install
documentation, does not run the test suite, and is not particularly
recommended, but could be useful to some.
---
 .gitignore                                    |  1 +
 Makefile                                      |  6 ++-
 Setup.hs                                      | 17 +++++++
 configure.hs                                  | 14 +++++-
 debian/changelog                              |  4 ++
 doc/install.mdwn                              |  7 +++
 ..._d9f7b851567445c7aa7ebbb440781819._comment |  8 ---
 ..._cf0f829536744098d6846500db998b6a._comment | 17 -------
 git-annex.cabal                               | 49 +++++++++++++++++++
 9 files changed, 96 insertions(+), 27 deletions(-)
 create mode 100644 Setup.hs
 delete mode 100644 doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
 delete mode 100644 doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
 create mode 100644 git-annex.cabal

diff --git a/.gitignore b/.gitignore
index 9a4bc80de3..d5bf54c813 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ html
 Touch.hs
 StatFS.hs
 Remote/S3.hs
+dist
diff --git a/Makefile b/Makefile
index 915b0bf0b2..2f72cdbf9d 100644
--- a/Makefile
+++ b/Makefile
@@ -11,6 +11,8 @@ mans=git-annex.1 git-annex-shell.1 git-union-merge.1
 
 all: $(bins) $(mans) docs
 
+sources: SysConfig.hs StatFS.hs Touch.hs Remote/S3.hs
+
 SysConfig.hs: configure.hs TestConfig.hs
 	$(GHCMAKE) configure
 	./configure
@@ -19,8 +21,10 @@ SysConfig.hs: configure.hs TestConfig.hs
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 
-Remote/S3.o:
+Remote/S3.hs:
 	@ln -sf S3real.hs Remote/S3.hs
+
+Remote/S3.o: Remote/S3.hs
 	@if ! $(GHCMAKE) Remote/S3.hs; then \
 		ln -sf S3stub.hs Remote/S3.hs; \
 		echo "** building without S3 support"; \
diff --git a/Setup.hs b/Setup.hs
new file mode 100644
index 0000000000..547d6a156e
--- /dev/null
+++ b/Setup.hs
@@ -0,0 +1,17 @@
+{- cabal setup file -}
+
+import Distribution.Simple
+import System.Cmd
+
+main = defaultMainWithHooks simpleUserHooks {
+	preConf = makeSources,
+	postClean = makeClean
+}
+
+makeSources _ _ = do
+	system "make sources"
+	return (Nothing, [])
+
+makeClean _ _ _ _ = do
+	system "make clean"
+	return ()
diff --git a/configure.hs b/configure.hs
index c81aa17e69..2e39feb166 100644
--- a/configure.hs
+++ b/configure.hs
@@ -56,12 +56,24 @@ unicodeFilePath = do
 {- Pulls package version out of the changelog. -}
 getVersion :: Test
 getVersion = do
-	changelog <- readFile "debian/changelog"
+	changelog <- readFile "CHANGELOG"
 	let verline = head $ lines changelog
 	let version = middle (words verline !! 1)
+
+	-- Replace Version field in cabal file, so I don't have to maintain
+	-- the version there too.
+	cabal <- readFile cabalfile
+	writeFile tmpcabalfile $ unlines $ map (setversion version) $ lines cabal
+	renameFile tmpcabalfile cabalfile
+
 	return $ Config "packageversion" (StringConfig version)
 	where
 		middle s = drop 1 $ take (length s - 1) s
+		cabalfile = "git-annex.cabal"
+		tmpcabalfile = cabalfile++".tmp"
+		setversion version s
+			| "Version:" `isPrefixOf` s = "Version: " ++ version
+			| otherwise = s
 
 setup :: IO ()
 setup = do
diff --git a/debian/changelog b/debian/changelog
index 8848cf8611..277b542590 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,10 @@ git-annex (0.20110611) UNRELEASED; urgency=low
   * git-union-merge: New git subcommand, that does a generic union merge
     operation, and operates efficiently without touching the working tree.
   * --force will cause add, etc, to operate on ignored files.
+  * cabal can now be used to build git-annex. This is substantially
+    slower than using make, does not build or install documentation,
+    does not run the test suite, and is not particularly recommended,
+    but could be useful to some.
 
  -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 7818aaf152..8a3edcb643 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -32,3 +32,10 @@ To build and use git-annex, you will need:
   * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`
+
+## Using cabal
+
+As a haskell package, git-annex can be built using cabal. For example:
+
+	cabal configure
+	cabal install --bindir=$HOME/bin
diff --git a/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment b/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
deleted file mode 100644
index 616b3c4dd5..0000000000
--- a/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="84.189.1.247"
- subject="Why isn't this package built with Cabal?"
- date="2011-03-23T11:31:06Z"
- content="""
-It would be a lot easier to compile this package, if it had a Cabal file to describe the build; especially the build-time dependencies. Why isn't Cabal used?
-"""]]
diff --git a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
deleted file mode 100644
index 81d5a2c629..0000000000
--- a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
+++ /dev/null
@@ -1,17 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 2"
- date="2011-03-23T15:18:29Z"
- content="""
-Because I haven't learned Cabal yet.
-
-But also because I've had bad experiences with both a) tying a particular program to a particular language's pet build system and then having to add ugliness when I later need to do something in the build that has nothing to do with that language and b) as a user, needing to deal with the pet build systems of languages when I just need to make some small change to the build process that is trivial in a Makefile.
-
-With that said, I do have a configure program written in Haskell, so at least it doesn't use autotools. :)
-
-Update: I did try using cabal, but git-annex includes 3 programs, and they
-all link to a lot of git-annex modules, and cabal wanted to build nearly
-every module 3 times, which was too slow for me and I could not find a way
-around.
-"""]]
diff --git a/git-annex.cabal b/git-annex.cabal
new file mode 100644
index 0000000000..a3b04b60e4
--- /dev/null
+++ b/git-annex.cabal
@@ -0,0 +1,49 @@
+Name: git-annex
+Version: 0.20110611
+Cabal-Version: >= 1.2
+License: GPL
+Maintainer: Joey Hess 
+Author: Joey Hess
+Stability: Stable
+Copyright: 2010-2011 Joey Hess
+License-File: GPL
+Extra-Source-Files: 
+Homepage: http://git-annex.branchable.com/
+Build-type: Custom
+Category: Utility
+Synopsis: manage files with git, without checking their contents into git
+Description:
+ git-annex allows managing files with git, without checking the file
+ contents into git. While that may seem paradoxical, it is useful when
+ dealing with files larger than git can currently easily handle, whether due
+ to limitations in memory, checksumming time, or disk space.
+ .
+ Even without file content tracking, being able to manage files with git,
+ move files around and delete files with versioned directory trees, and use
+ branches and distributed clones, are all very handy reasons to use git. And
+ annexed files can co-exist in the same git repository with regularly
+ versioned files, which is convenient for maintaining documents, Makefiles,
+ etc that are associated with annexed files but that benefit from full
+ revision control.
+
+Executable git-annex
+  Main-Is: git-annex.hs
+  GHC-Options: -O2
+  Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
+   unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
+
+Executable git-annex-shell
+  Main-Is: git-annex-shell.hs
+  GHC-Options: -O2
+  Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
+   unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
+
+Executable git-union-merge
+  Main-Is: git-union-merge.hs
+  GHC-Options: -O2
+
+source-repository head
+  type: git
+  location: git://git-annex.branchable.com/

From 20565027cc2f78679a9c54c8350fb7b34e8ff94f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 15:07:10 -0400
Subject: [PATCH 1969/2835] cabal tweaks

---
 git-annex.cabal | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index f602143b5b..294a3baced 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,6 +1,6 @@
 Name: git-annex
 Version: 3.20110625
-Cabal-Version: >= 1.2
+Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
 Author: Joey Hess
@@ -28,21 +28,18 @@ Description:
 
 Executable git-annex
   Main-Is: git-annex.hs
-  GHC-Options: -O2
   Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
-  GHC-Options: -O2
   Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs
-  GHC-Options: -O2
 
 source-repository head
   type: git

From b3ab44f8bbc6a1e253bb81dfe9d6939adcdcba56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 15:37:35 -0400
Subject: [PATCH 1970/2835] add a filelist for cabal sdist

I hate hard-coded 40 kilobyte lone file lists, and just once would like to
see a build system that does not assume it's a good idea to have a file
list, or a hardcoded file list, or a file list that can only be generated
with a crippled form of globs. But not today, thank you cabal.
---
 Makefile        |  2 +-
 configure.hs    | 37 ++++++++++++++++++++++++++-----------
 git-annex.cabal |  2 +-
 3 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/Makefile b/Makefile
index da80228dd3..0bc02dc352 100644
--- a/Makefile
+++ b/Makefile
@@ -84,7 +84,7 @@ docs: $(mans)
 clean:
 	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \
 		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs
-	rm -rf doc/.ikiwiki html
+	rm -rf doc/.ikiwiki html dist
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
 .PHONY: $(bins) test install
diff --git a/configure.hs b/configure.hs
index 2e39feb166..5f0ff5a1e3 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,6 +2,8 @@
 
 import System.Directory
 import Data.List
+import Data.String.Utils
+import System.Cmd.Utils
 
 import TestConfig
 
@@ -56,24 +58,36 @@ unicodeFilePath = do
 {- Pulls package version out of the changelog. -}
 getVersion :: Test
 getVersion = do
+	version <- getVersionString
+	return $ Config "packageversion" (StringConfig version)
+	
+getVersionString :: IO String
+getVersionString = do
 	changelog <- readFile "CHANGELOG"
 	let verline = head $ lines changelog
-	let version = middle (words verline !! 1)
-
-	-- Replace Version field in cabal file, so I don't have to maintain
-	-- the version there too.
-	cabal <- readFile cabalfile
-	writeFile tmpcabalfile $ unlines $ map (setversion version) $ lines cabal
-	renameFile tmpcabalfile cabalfile
-
-	return $ Config "packageversion" (StringConfig version)
+	return $ middle (words verline !! 1)
 	where
 		middle s = drop 1 $ take (length s - 1) s
+
+{- Set up cabal file with version. -}
+cabalSetup :: IO ()
+cabalSetup = do
+	version <- getVersionString
+	(_, filelist) <- pipeLinesFrom "find" (words ". -name .git -prune -o -name dist -prune -o -not -name *.hi -not -name *.o -not -name configure -not -name *.tmp -type f -print")
+	cabal <- readFile cabalfile
+	writeFile tmpcabalfile $ unlines $ 
+		map (setfield "Version" version) $
+		map (setfield "Extra-Source-Files" $ join ", " $ sort filelist) $
+		lines cabal
+	renameFile tmpcabalfile cabalfile
+	where
 		cabalfile = "git-annex.cabal"
 		tmpcabalfile = cabalfile++".tmp"
-		setversion version s
-			| "Version:" `isPrefixOf` s = "Version: " ++ version
+		setfield field value s
+			| fullfield `isPrefixOf` s = fullfield ++ value
 			| otherwise = s
+			where
+				fullfield = field ++ ": "
 
 setup :: IO ()
 setup = do
@@ -90,3 +104,4 @@ main = do
 	config <- runTests tests
 	writeSysConfig config
 	cleanup
+	cabalSetup
diff --git a/git-annex.cabal b/git-annex.cabal
index 294a3baced..2560b90a94 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -7,7 +7,6 @@ Author: Joey Hess
 Stability: Stable
 Copyright: 2010-2011 Joey Hess
 License-File: GPL
-Extra-Source-Files: 
 Homepage: http://git-annex.branchable.com/
 Build-type: Custom
 Category: Utility
@@ -25,6 +24,7 @@ Description:
  versioned files, which is convenient for maintaining documents, Makefiles,
  etc that are associated with annexed files but that benefit from full
  revision control.
+Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs, ./Backend.hs, ./Backend/File.hs, ./Backend/SHA.hs, ./Backend/URL.hs, ./Backend/WORM.hs, ./BackendList.hs, ./Base64.hs, ./Branch.hs, ./CmdLine.hs, ./Command.hs, ./Command/Add.hs, ./Command/ConfigList.hs, ./Command/Copy.hs, ./Command/Describe.hs, ./Command/Drop.hs, ./Command/DropKey.hs, ./Command/DropUnused.hs, ./Command/Find.hs, ./Command/Fix.hs, ./Command/FromKey.hs, ./Command/Fsck.hs, ./Command/Get.hs, ./Command/InAnnex.hs, ./Command/Init.hs, ./Command/InitRemote.hs, ./Command/Lock.hs, ./Command/Map.hs, ./Command/Merge.hs, ./Command/Migrate.hs, ./Command/Move.hs, ./Command/PreCommit.hs, ./Command/RecvKey.hs, ./Command/Semitrust.hs, ./Command/SendKey.hs, ./Command/SetKey.hs, ./Command/Status.hs, ./Command/Trust.hs, ./Command/Unannex.hs, ./Command/Uninit.hs, ./Command/Unlock.hs, ./Command/Untrust.hs, ./Command/Unused.hs, ./Command/Upgrade.hs, ./Command/Version.hs, ./Command/Whereis.hs, ./Config.hs, ./Content.hs, ./CopyFile.hs, ./Crypto.hs, ./DataUnits.hs, ./Dot.hs, ./Git.hs, ./Git/LsFiles.hs, ./Git/Queue.hs, ./Git/UnionMerge.hs, ./GitAnnex.hs, ./LocationLog.hs, ./Locations.hs, ./Makefile, ./Messages.hs, ./Options.hs, ./README, ./Remote.hs, ./Remote/Bup.hs, ./Remote/Directory.hs, ./Remote/Encryptable.hs, ./Remote/Git.hs, ./Remote/Hook.hs, ./Remote/Rsync.hs, ./Remote/S3real.hs, ./Remote/S3stub.hs, ./Remote/Special.hs, ./RsyncFile.hs, ./Setup.hs, ./Ssh.hs, ./StatFS.hsc, ./SysConfig.hs, ./TestConfig.hs, ./Touch.hsc, ./Trust.hs, ./Types.hs, ./Types/Backend.hs, ./Types/BranchState.hs, ./Types/Crypto.hs, ./Types/Key.hs, ./Types/Remote.hs, ./Types/TrustLevel.hs, ./Types/UUID.hs, ./UUID.hs, ./Upgrade.hs, ./Upgrade/V0.hs, ./Upgrade/V1.hs, ./Upgrade/V2.hs, ./Utility.hs, ./Version.hs, ./configure.hs, ./debian/NEWS, ./debian/changelog, ./debian/compat, ./debian/control, ./debian/copyright, ./debian/doc-base, ./debian/manpages, ./debian/rules, ./doc/GPL, ./doc/backends.mdwn, ./doc/bare_repositories.mdwn, ./doc/bugs.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment, ./doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/bugs/Makefile_is_missing_dependancies.mdwn, ./doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment, ./doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment, ./doc/bugs/No_version_information_from_cli.mdwn, ./doc/bugs/Problems_running_make_on_osx.mdwn, ./doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment, ./doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment, ./doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment, ./doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment, ./doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment, ./doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment, ./doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment, ./doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment, ./doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment, ./doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment, ./doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment, ./doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment, ./doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment, ./doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment, ./doc/bugs/S3_memory_leaks.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment, ./doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn, ./doc/bugs/annex_add_in_annex.mdwn, ./doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn, ./doc/bugs/bare_git_repos.mdwn, ./doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn, ./doc/bugs/building_on_lenny.mdwn, ./doc/bugs/check_for_curl_in_configure.hs.mdwn, ./doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn, ./doc/bugs/conflicting_haskell_packages.mdwn, ./doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment, ./doc/bugs/done.mdwn, ./doc/bugs/dotdot_problem.mdwn, ./doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn, ./doc/bugs/encrypted_S3_stalls.mdwn, ./doc/bugs/error_propigation.mdwn, ./doc/bugs/error_with_file_names_starting_with_dash.mdwn, ./doc/bugs/fat_support.mdwn, ./doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment, ./doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment, ./doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment, ./doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment, ./doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment, ./doc/bugs/free_space_checking.mdwn, ./doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment, ./doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment, ./doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment, ./doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn, ./doc/bugs/fsck_output.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment, ./doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn, ./doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn, ./doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn, ./doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment, ./doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn, ./doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn, ./doc/bugs/git_annex_initremote_walks_.git-annex.mdwn, ./doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn, ./doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn, ./doc/bugs/git_annex_unlock_is_not_atomic.mdwn, ./doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn, ./doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn, ./doc/bugs/git_rename_detection_on_file_move.mdwn, ./doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment, ./doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn, ./doc/bugs/ordering.mdwn, ./doc/bugs/problem_commit_normal_links.mdwn, ./doc/bugs/problems_with_utf8_names.mdwn, ./doc/bugs/scp_interrupt_to_background.mdwn, ./doc/bugs/softlink_mtime.mdwn, ./doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn, ./doc/bugs/tmp_file_handling.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment, ./doc/bugs/unannex_vs_unlock_hook_confusion.mdwn, ./doc/bugs/unhappy_without_UTF8_locale.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment, ./doc/bugs/weird_local_clone_confuses.mdwn, ./doc/cheatsheet.mdwn, ./doc/comments.mdwn, ./doc/contact.mdwn, ./doc/copies.mdwn, ./doc/design.mdwn, ./doc/design/encryption.mdwn, ./doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment, ./doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment, ./doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment, ./doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment, ./doc/distributed_version_control.mdwn, ./doc/download.mdwn, ./doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment, ./doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment, ./doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment, ./doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment, ./doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment, ./doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment, ./doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment, ./doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment, ./doc/encryption.mdwn, ./doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment, ./doc/feeds.mdwn, ./doc/forum.mdwn, ./doc/forum/Behaviour_of_fsck.mdwn, ./doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment, ./doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment, ./doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment, ./doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment, ./doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment, ./doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn, ./doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn, ./doc/forum/Problems_with_large_numbers_of_files.mdwn, ./doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment, ./doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull.mdwn, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment, ./doc/forum/batch_check_on_remote_when_using_copy.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment, ./doc/forum/git-annex_communication_channels.mdwn, ./doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment, ./doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment, ./doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment, ./doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment, ./doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment, ./doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment, ./doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment, ./doc/forum/git-annex_on_OSX.mdwn, ./doc/forum/hashing_objects_directories.mdwn, ./doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment, ./doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment, ./doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment, ./doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment, ./doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment, ./doc/forum/incompatible_versions__63__.mdwn, ./doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex.mdwn, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment, ./doc/forum/migration_to_git-annex_and_rsync.mdwn, ./doc/forum/new_microfeatures.mdwn, ./doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment, ./doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment, ./doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment, ./doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment, ./doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment, ./doc/forum/relying_on_git_for_numcopies.mdwn, ./doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment, ./doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment, ./doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment, ./doc/forum/rsync_over_ssh__63__.mdwn, ./doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment, ./doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment, ./doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn, ./doc/forum/sparse_git_checkouts_with_annex.mdwn, ./doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment, ./doc/forum/unannex_alternatives.mdwn, ./doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment, ./doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment, ./doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment, ./doc/forum/wishlist:_command_options_changes.mdwn, ./doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment, ./doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment, ./doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment, ./doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn, ./doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment, ./doc/forum/wishlist:_git-annex_replicate.mdwn, ./doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment, ./doc/forum/wishlist:_git_annex_status.mdwn, ./doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment, ./doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment, ./doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment, ./doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment, ./doc/forum/wishlist:_git_backend_for_git-annex.mdwn, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment, ./doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment, ./doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn, ./doc/forum/wishlist:alias_system.mdwn, ./doc/forum/working_without_git-annex_commits.mdwn, ./doc/future_proofing.mdwn, ./doc/git-annex-shell.mdwn, ./doc/git-annex.mdwn, ./doc/git-union-merge.mdwn, ./doc/index.mdwn, ./doc/install.mdwn, ./doc/install/Debian.mdwn, ./doc/install/Fedora.mdwn, ./doc/install/FreeBSD.mdwn, ./doc/install/OSX.mdwn, ./doc/install/Ubuntu.mdwn, ./doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment, ./doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment, ./doc/internals.mdwn, ./doc/location_tracking.mdwn, ./doc/logo.png, ./doc/logo_small.png, ./doc/news.mdwn, ./doc/news/LWN_article.mdwn, ./doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn, ./doc/news/version_0.20110521.mdwn, ./doc/news/version_0.20110522.mdwn, ./doc/news/version_0.20110601.mdwn, ./doc/news/version_0.20110610.mdwn, ./doc/news/version_3.20110624.mdwn, ./doc/not.mdwn, ./doc/repomap.png, ./doc/special_remotes.mdwn, ./doc/special_remotes/S3.mdwn, ./doc/special_remotes/bup.mdwn, ./doc/special_remotes/directory.mdwn, ./doc/special_remotes/hook.mdwn, ./doc/special_remotes/rsync.mdwn, ./doc/summary.mdwn, ./doc/templates/bare.tmpl, ./doc/templates/walkthrough.tmpl, ./doc/todo.mdwn, ./doc/todo/S3.mdwn, ./doc/todo/add_--exclude_option_to_git_annex_find.mdwn, ./doc/todo/add_a_git_backend.mdwn, ./doc/todo/auto_remotes.mdwn, ./doc/todo/auto_remotes/discussion.mdwn, ./doc/todo/backendSHA1.mdwn, ./doc/todo/branching.mdwn, ./doc/todo/cache_key_info.mdwn, ./doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment, ./doc/todo/checkout.mdwn, ./doc/todo/done.mdwn, ./doc/todo/file_copy_progress_bar.mdwn, ./doc/todo/fsck.mdwn, ./doc/todo/git-annex-shell.mdwn, ./doc/todo/git-annex_unused_eats_memory.mdwn, ./doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn, ./doc/todo/gitrm.mdwn, ./doc/todo/hidden_files.mdwn, ./doc/todo/immutable_annexed_files.mdwn, ./doc/todo/network_remotes.mdwn, ./doc/todo/object_dir_reorg_v2.mdwn, ./doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment, ./doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment, ./doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment, ./doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment, ./doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment, ./doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment, ./doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment, ./doc/todo/parallel_possibilities.mdwn, ./doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment, ./doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment, ./doc/todo/pushpull.mdwn, ./doc/todo/rsync.mdwn, ./doc/todo/smudge.mdwn, ./doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment, ./doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment, ./doc/todo/speed_up_fsck.mdwn, ./doc/todo/support-non-utf8-locales.mdwn, ./doc/todo/support_S3_multipart_uploads.mdwn, ./doc/todo/symlink_farming_commit_hook.mdwn, ./doc/todo/tahoe_lfs_for_reals.mdwn, ./doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment, ./doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment, ./doc/todo/union_mounting.mdwn, ./doc/todo/use_cp_reflink.mdwn, ./doc/todo/using_url_backend.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment, ./doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/todo/wishlist:_swift_backend.mdwn, ./doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment, ./doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment, ./doc/transferring_data.mdwn, ./doc/trust.mdwn, ./doc/upgrades.mdwn, ./doc/upgrades/SHA_size.mdwn, ./doc/use_case/Alice.mdwn, ./doc/use_case/Bob.mdwn, ./doc/users.mdwn, ./doc/users/chrysn.mdwn, ./doc/users/fmarier.mdwn, ./doc/users/joey.mdwn, ./doc/walkthrough.mdwn, ./doc/walkthrough/Internet_Archive_via_S3.mdwn, ./doc/walkthrough/adding_a_remote.mdwn, ./doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment, ./doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment, ./doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment, ./doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment, ./doc/walkthrough/adding_files.mdwn, ./doc/walkthrough/backups.mdwn, ./doc/walkthrough/creating_a_repository.mdwn, ./doc/walkthrough/fsck:_verifying_your_data.mdwn, ./doc/walkthrough/fsck:_when_things_go_wrong.mdwn, ./doc/walkthrough/getting_file_content.mdwn, ./doc/walkthrough/migrating_data_to_a_new_backend.mdwn, ./doc/walkthrough/modifying_annexed_files.mdwn, ./doc/walkthrough/more.mdwn, ./doc/walkthrough/moving_file_content_between_repositories.mdwn, ./doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment, ./doc/walkthrough/recover_data_from_lost+found.mdwn, ./doc/walkthrough/removing_files.mdwn, ./doc/walkthrough/removing_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/renaming_files.mdwn, ./doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/untrusted_repositories.mdwn, ./doc/walkthrough/unused_data.mdwn, ./doc/walkthrough/using_Amazon_S3.mdwn, ./doc/walkthrough/using_bup.mdwn, ./doc/walkthrough/using_ssh_remotes.mdwn, ./doc/walkthrough/using_the_SHA1_backend.mdwn, ./doc/walkthrough/using_the_URL_backend.mdwn, ./doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn, ./git-annex-shell.hs, ./git-annex.cabal, ./git-annex.hs, ./git-union-merge.hs, ./mdwn2man, ./test.hs, ./testdata/unicode-test-ö
 
 Executable git-annex
   Main-Is: git-annex.hs

From e27e20c6d64200ae0804dce62e31eaf55d5581fa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 16:16:52 -0400
Subject: [PATCH 1971/2835] add

---
 doc/upgrades.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 337c65767a..d62e9dcc78 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -22,6 +22,9 @@ conflicts first before upgrading git-annex.
 
 Involved moving the .git-annex/ directory into a separate git-annex branch.
 
+After this upgrade, you should make sure you include the git-annex branch
+when git pushing and pulling.
+
 ### tips for this upgrade
 
 This upgrade is easier (and faster!) than the previous upgrades.

From b9721fffac0815fcb9c69cca6b9ef2c7982438e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 19:25:21 -0400
Subject: [PATCH 1972/2835] cabal check

---
 git-annex.cabal | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 2560b90a94..5a68e4dea4 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -28,15 +28,17 @@ Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs,
 
 Executable git-annex
   Main-Is: git-annex.hs
-  Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
+  Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3,
+   base < 5
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
-  Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
+  Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3,
+   base < 5
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs

From ceb887d82669b3ec694f31a899b59eefe0f5f352 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 01:11:41 -0400
Subject: [PATCH 1973/2835] tweaks

---
 Content.hs | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/Content.hs b/Content.hs
index d24fc9ac70..937ebd5426 100644
--- a/Content.hs
+++ b/Content.hs
@@ -235,10 +235,11 @@ moveBad key = do
 	g <- Annex.gitRepo
 	let src = gitAnnexLocation g key
 	let dest = gitAnnexBadDir g  takeFileName src
-	liftIO $ createDirectoryIfMissing True (parentDir dest)
-	liftIO $ allowWrite (parentDir src)
-	liftIO $ renameFile src dest
-	liftIO $ removeDirectory (parentDir src)
+	liftIO $ do
+		createDirectoryIfMissing True (parentDir dest)
+		allowWrite (parentDir src)
+		renameFile src dest
+		removeDirectory (parentDir src)
 	logStatus key ValueMissing
 	return dest
 
@@ -252,12 +253,12 @@ getKeysPresent' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
 	if (not exists)
 		then return []
-		else do
+		else liftIO $ do
 			-- 2 levels of hashing
-			levela <- liftIO $ dirContents dir
-			levelb <- liftIO $ mapM dirContents levela
-			contents <- liftIO $ mapM dirContents (concat levelb)
-			files <- liftIO $ filterM present (concat contents)
+			levela <- dirContents dir
+			levelb <- mapM dirContents levela
+			contents <- mapM dirContents (concat levelb)
+			files <- filterM present (concat contents)
 			return $ catMaybes $ map (fileKey . takeFileName) files
 	where
 		present d = do

From cdbcd6f495580ee927a85af0581661b486c8ef77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 15:24:07 -0400
Subject: [PATCH 1974/2835] add web special remote

Generalized LocationLog to PresenceLog, and use a presence log to record
urls for the web special remote.
---
 Backend/File.hs    |   6 +--
 Command/Add.hs     |   2 +-
 Command/Drop.hs    |   2 +-
 Command/DropKey.hs |   2 +-
 Command/Fsck.hs    |   4 +-
 Command/Move.hs    |   2 +-
 Command/SetKey.hs  |   2 +-
 Command/Unannex.hs |   2 +-
 Command/Whereis.hs |   2 +-
 Content.hs         |  11 ++--
 LocationLog.hs     | 103 +------------------------------------
 PresenceLog.hs     | 123 +++++++++++++++++++++++++++++++++++++++++++++
 Remote.hs          |  23 ++++++++-
 Remote/Web.hs      | 107 +++++++++++++++++++++++++++++++++++++++
 UUID.hs            |  16 ------
 debian/changelog   |   2 +
 git-annex.cabal    |   2 +-
 17 files changed, 272 insertions(+), 139 deletions(-)
 create mode 100644 PresenceLog.hs
 create mode 100644 Remote/Web.hs

diff --git a/Backend/File.hs b/Backend/File.hs
index b8d3bb65ad..174da4e6dc 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -135,8 +135,8 @@ showLocations key exclude = do
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
-	ppuuidswanted <- prettyPrintUUIDs uuidswanted
-	ppuuidsskipped <- prettyPrintUUIDs uuidsskipped
+	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
+	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
 	showLongNote $ message ppuuidswanted ppuuidsskipped
 	where
 		filteruuids list x = filter (`notElem` x) list
@@ -195,7 +195,7 @@ checkKeyNumCopies key file numcopies = do
 	let present = length safelocations
 	if present < needed
 		then do
-			ppuuids <- prettyPrintUUIDs untrustedlocations
+			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
 			warning $ missingNote (filename file key) present needed ppuuids
 			return False
 		else return True
diff --git a/Command/Add.hs b/Command/Add.hs
index 5133ee1fd5..6a1ffb5da6 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -51,7 +51,7 @@ perform (file, backend) = do
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	logStatus key ValuePresent
+	logStatus key InfoPresent
 
 	link <- calcGitLink file key
 	liftIO $ createSymbolicLink link file
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 07cec1a677..bd47407413 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -45,5 +45,5 @@ perform key backend numcopies = do
 cleanup :: Key -> CommandCleanup
 cleanup key = do
 	whenM (inAnnex key) $ removeAnnex key
-	logStatus key ValueMissing
+	logStatus key InfoMissing
 	return True
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 780fe0adf0..16a3e35d64 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -40,5 +40,5 @@ perform key = do
 
 cleanup :: Key -> CommandCleanup
 cleanup key = do
-	logStatus key ValueMissing
+	logStatus key InfoMissing
 	return True
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index cb062342d7..988cfd28d1 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -64,11 +64,11 @@ verifyLocationLog key file = do
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
-				fix g u ValuePresent
+				fix g u InfoPresent
 				-- There is no data loss, so do not fail.
 				return True
 		(False, True) -> do
-				fix g u ValueMissing
+				fix g u InfoMissing
 				warning $
 					"** Based on the location log, " ++ file
 					++ "\n** was expected to be present, " ++
diff --git a/Command/Move.hs b/Command/Move.hs
index 03b605ce57..6bf6e05827 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -58,7 +58,7 @@ remoteHasKey remote key present	= do
 	g <- Annex.gitRepo
 	logChange g key remoteuuid status
 	where
-		status = if present then ValuePresent else ValueMissing
+		status = if present then InfoPresent else InfoMissing
 
 {- Moves (or copies) the content of an annexed file to a remote.
  -
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index dbad148b25..546bd8e2d0 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -46,5 +46,5 @@ perform file = do
 cleanup :: CommandCleanup
 cleanup = do
 	key <- cmdlineKey
-	logStatus key ValuePresent
+	logStatus key InfoPresent
 	return True
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index fe413b25b4..d30f8d20f7 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -65,7 +65,7 @@ cleanup file key = do
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 
 	fromAnnex key file
-	logStatus key ValueMissing
+	logStatus key InfoMissing
 	
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 3a7213217a..0e4858f8b7 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -10,7 +10,7 @@ module Command.Whereis where
 import LocationLog
 import Command
 import Messages
-import UUID
+import Remote
 import Types
 
 command :: [Command]
diff --git a/Content.hs b/Content.hs
index 937ebd5426..a2f38ddc96 100644
--- a/Content.hs
+++ b/Content.hs
@@ -65,12 +65,7 @@ calcGitLink file key = do
 		whoops = error $ "unable to normalize " ++ file
 
 {- Updates the LocationLog when a key's presence changes in the current
- - repository.
- -
- - Note that the LocationLog is not updated in bare repositories.
- - Operations that change a bare repository should be done from
- - a non-bare repository, and the LocationLog in that repository be
- - updated instead. -}
+ - repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
 	g <- Annex.gitRepo
@@ -112,7 +107,7 @@ getViaTmpUnchecked key action = do
 	if success
 		then do
 			moveAnnex key tmp
-			logStatus key ValuePresent
+			logStatus key InfoPresent
 			return True
 		else do
 			-- the tmp file is left behind, in case caller wants
@@ -240,7 +235,7 @@ moveBad key = do
 		allowWrite (parentDir src)
 		renameFile src dest
 		removeDirectory (parentDir src)
-	logStatus key ValueMissing
+	logStatus key InfoMissing
 	return dest
 
 {- List of keys whose content exists in .git/annex/objects/ -}
diff --git a/LocationLog.hs b/LocationLog.hs
index b7deb3ed9a..a5db7d1218 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -5,11 +5,6 @@
  -
  - Repositories record their UUID and the date when they --get or --drop
  - a value.
- -
- - A line of the log will look like: "date N UUID"
- - Where N=1 when the repo has the file, and 0 otherwise.
- - (After the UUID can optionally come a white space and other data, 
- - for future expansion.)
  - 
  - Copyright 2010-2011 Joey Hess 
  -
@@ -25,61 +20,16 @@ module LocationLog (
 	loggedKeys
 ) where
 
-import Data.Time.Clock.POSIX
-import Data.Time
-import System.Locale
 import System.FilePath
-import qualified Data.Map as Map
 import Control.Monad (when)
 import Data.Maybe
-import Control.Monad.State (liftIO)
 
 import qualified Git
 import qualified Branch
 import UUID
 import Types
 import Locations
-
-data LogLine = LogLine {
-	date :: POSIXTime,
-	status :: LogStatus,
-	uuid :: UUID
-} deriving (Eq)
-
-data LogStatus = ValuePresent | ValueMissing | Undefined
-	deriving (Eq)
-
-instance Show LogStatus where
-	show ValuePresent = "1"
-	show ValueMissing = "0"
-	show Undefined = "undefined"
-
-instance Read LogStatus where
-	readsPrec _ "1" = [(ValuePresent, "")]
-	readsPrec _ "0" = [(ValueMissing, "")]
-	readsPrec _ _   = [(Undefined, "")]
-
-instance Show LogLine where
-	show (LogLine d s u) = unwords [show d, show s, u]
-
-instance Read LogLine where
-	-- This parser is robust in that even unparsable log lines are
-	-- read without an exception being thrown.
-	-- Such lines have a status of Undefined.
-	readsPrec _ string = 
-		if length w >= 3
-			then maybe bad good pdate
-			else bad
-		where
-			w = words string
-			s = read $ w !! 1
-			u = w !! 2
-			pdate :: Maybe UTCTime
-			pdate = parseTime defaultTimeLocale "%s%Qs" $ head w
-
-			good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s u
-			bad = ret $ LogLine 0 Undefined ""
-			ret v = [(v, "")]
+import PresenceLog
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
@@ -92,59 +42,10 @@ logChange repo key u s = do
 	ls <- readLog f
 	writeLog f (compactLog $ line:ls)
 
-{- Reads a log file.
- - Note that the LogLines returned may be in any order. -}
-readLog :: FilePath -> Annex [LogLine]
-readLog file = return . parseLog =<< Branch.get file
-
-parseLog :: String -> [LogLine]
-parseLog s = filter parsable $ map read $ lines s
-	where
-		-- some lines may be unparseable, avoid them
-		parsable l = status l /= Undefined
-
-{- Stores a set of lines in a log file -}
-writeLog :: FilePath -> [LogLine] -> Annex ()
-writeLog file ls = Branch.change file (unlines $ map show ls)
-
-{- Generates a new LogLine with the current date. -}
-logNow :: LogStatus -> UUID -> Annex LogLine
-logNow s u = do
-	now <- liftIO $ getPOSIXTime
-	return $ LogLine now s u
-
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations key = do
-	ls <- readLog $ logFile key
-	return $ map uuid $ filterPresent ls
-
-{- Filters the list of LogLines to find ones where the value
- - is (or should still be) present. -}
-filterPresent :: [LogLine] -> [LogLine]
-filterPresent ls = filter (\l -> ValuePresent == status l) $ compactLog ls
-
-type LogMap = Map.Map UUID LogLine
-
-{- Compacts a set of logs, returning a subset that contains the current
- - status. -}
-compactLog :: [LogLine] -> [LogLine]
-compactLog ls = compactLog' Map.empty ls
-compactLog' :: LogMap -> [LogLine] -> [LogLine]
-compactLog' m [] = Map.elems m
-compactLog' m (l:ls) = compactLog' (mapLog m l) ls
-
-{- Inserts a log into a map of logs, if the log has better (ie, newer)
- - information about a repo than the other logs in the map -}
-mapLog :: LogMap -> LogLine -> LogMap
-mapLog m l = 
-	if better
-		then Map.insert u l m
-		else m
-	where
-		better = maybe True (\l' -> date l' <= date l) $ Map.lookup u m
-		u = uuid l
+keyLocations key = currentLog $ logFile key
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/PresenceLog.hs b/PresenceLog.hs
new file mode 100644
index 0000000000..71d78f1eda
--- /dev/null
+++ b/PresenceLog.hs
@@ -0,0 +1,123 @@
+{- git-annex presence log
+ -
+ - This is used to store presence information in the git-annex branch in
+ - a way that can be union merged.
+ -
+ - A line of the log will look like: "date N INFO"
+ - Where N=1 when the INFO is present, and 0 otherwise.
+ - 
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module PresenceLog (
+	LogStatus(..),
+	readLog,
+	writeLog,
+	logNow,
+	compactLog,
+	currentLog
+) where
+
+import Data.Time.Clock.POSIX
+import Data.Time
+import System.Locale
+import qualified Data.Map as Map
+import Control.Monad.State (liftIO)
+
+import qualified Branch
+import Types
+
+data LogLine = LogLine {
+	date :: POSIXTime,
+	status :: LogStatus,
+	info :: String
+} deriving (Eq)
+
+data LogStatus = InfoPresent | InfoMissing | Undefined
+	deriving (Eq)
+
+instance Show LogStatus where
+	show InfoPresent = "1"
+	show InfoMissing = "0"
+	show Undefined = "undefined"
+
+instance Read LogStatus where
+	readsPrec _ "1" = [(InfoPresent, "")]
+	readsPrec _ "0" = [(InfoMissing, "")]
+	readsPrec _ _   = [(Undefined, "")]
+
+instance Show LogLine where
+	show (LogLine d s i) = unwords [show d, show s, i]
+
+instance Read LogLine where
+	-- This parser is robust in that even unparsable log lines are
+	-- read without an exception being thrown.
+	-- Such lines have a status of Undefined.
+	readsPrec _ string = 
+		if length w >= 3
+			then maybe bad good pdate
+			else bad
+		where
+			w = words string
+			s = read $ w !! 1
+			i = w !! 2
+			pdate :: Maybe UTCTime
+			pdate = parseTime defaultTimeLocale "%s%Qs" $ head w
+
+			good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s i
+			bad = ret $ LogLine 0 Undefined ""
+			ret v = [(v, "")]
+
+{- Reads a log file.
+ - Note that the LogLines returned may be in any order. -}
+readLog :: FilePath -> Annex [LogLine]
+readLog file = return . parseLog =<< Branch.get file
+
+parseLog :: String -> [LogLine]
+parseLog s = filter parsable $ map read $ lines s
+	where
+		-- some lines may be unparseable, avoid them
+		parsable l = status l /= Undefined
+
+{- Stores a set of lines in a log file -}
+writeLog :: FilePath -> [LogLine] -> Annex ()
+writeLog file ls = Branch.change file (unlines $ map show ls)
+
+{- Generates a new LogLine with the current date. -}
+logNow :: LogStatus -> String -> Annex LogLine
+logNow s i = do
+	now <- liftIO $ getPOSIXTime
+	return $ LogLine now s i
+
+{- Reads a log and returns only the info that is still in effect. -}
+currentLog :: FilePath -> Annex [String]
+currentLog file = do
+	ls <- readLog file
+	return $ map info $ filterPresent ls
+
+{- Returns the info from LogLines that are in effect. -}
+filterPresent :: [LogLine] -> [LogLine]
+filterPresent ls = filter (\l -> InfoPresent == status l) $ compactLog ls
+
+type LogMap = Map.Map String LogLine
+
+{- Compacts a set of logs, returning a subset that contains the current
+ - status. -}
+compactLog :: [LogLine] -> [LogLine]
+compactLog ls = compactLog' Map.empty ls
+compactLog' :: LogMap -> [LogLine] -> [LogLine]
+compactLog' m [] = Map.elems m
+compactLog' m (l:ls) = compactLog' (mapLog m l) ls
+
+{- Inserts a log into a map of logs, if the log has better (ie, newer)
+ - information than the other logs in the map -}
+mapLog :: LogMap -> LogLine -> LogMap
+mapLog m l = 
+	if better
+		then Map.insert i l m
+		else m
+	where
+		better = maybe True (\l' -> date l' <= date l) $ Map.lookup i m
+		i = info l
diff --git a/Remote.hs b/Remote.hs
index 1accabf6d9..28c2e39cdd 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -24,6 +24,7 @@ module Remote (
 	nameToUUID,
 	remotesWithUUID,
 	remotesWithoutUUID,
+	prettyPrintUUIDs,
 
 	remoteLog,
 	readRemoteLog,
@@ -34,7 +35,7 @@ module Remote (
 	prop_idempotent_configEscape
 ) where
 
-import Control.Monad (filterM)
+import Control.Monad (filterM, liftM2)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
@@ -54,6 +55,7 @@ import qualified Remote.S3
 import qualified Remote.Bup
 import qualified Remote.Directory
 import qualified Remote.Rsync
+import qualified Remote.Web
 import qualified Remote.Hook
 
 remoteTypes :: [RemoteType Annex]
@@ -63,6 +65,7 @@ remoteTypes =
 	, Remote.Bup.remote
 	, Remote.Directory.remote
 	, Remote.Rsync.remote
+	, Remote.Web.remote
 	, Remote.Hook.remote
 	]
 
@@ -120,6 +123,24 @@ nameToUUID n = do
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
+{- Pretty-prints a list of UUIDs of remotes. -}
+prettyPrintUUIDs :: [UUID] -> Annex String
+prettyPrintUUIDs uuids = do
+	here <- getUUID =<< Annex.gitRepo
+	-- Show descriptions from the uuid log, falling back to remote names,
+	-- as some remotes may not be in the uuid log.
+	m <- liftM2 M.union uuidMap $
+		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList
+	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
+	where
+		prettify m u here = base ++ ishere
+			where
+				base = if not $ null $ findlog m u
+					then u ++ "  -- " ++ findlog m u
+					else u
+				ishere = if here == u then " <-- here" else ""
+		findlog m u = M.findWithDefault "" u m
+
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
diff --git a/Remote/Web.hs b/Remote/Web.hs
new file mode 100644
index 0000000000..201f923cf7
--- /dev/null
+++ b/Remote/Web.hs
@@ -0,0 +1,107 @@
+{- Web remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Web (
+	remote
+) where
+
+import Control.Monad.State (liftIO)
+import Control.Exception
+import System.FilePath
+import Network.Curl.Easy
+import Network.Curl.Opts
+import Network.Curl.Types
+import Network.Curl.Code
+
+import Types
+import Types.Remote
+import qualified Git
+import Messages
+import Utility
+import UUID
+import Config
+import PresenceLog
+
+remote :: RemoteType Annex
+remote = RemoteType {
+	typename = "web",
+	enumerate = list,
+	generate = gen,
+	setup = error "not supported"
+}
+
+-- There is only one web remote, and it always exists.
+-- (If the web should cease to exist, remove this module and redistribute
+-- a new release to the survivors by carrier pigeon.)
+list :: Annex [Git.Repo]
+list = return [Git.repoRemoteNameSet Git.repoFromUnknown "remote.web.dummy"]
+
+-- Dummy uuid for the whole web. Do not alter.
+webUUID :: UUID
+webUUID = "00000000-0000-0000-0000-000000000001"
+
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen r _ _ = 
+	return $ Remote {
+		uuid = webUUID,
+		cost = expensiveRemoteCost,
+		name = Git.repoDescribe r,
+		storeKey = upload,
+		retrieveKeyFile = download,
+		removeKey = remove,
+		hasKey = check,
+		hasKeyCheap = False,
+		config = Nothing
+	}
+
+{- The urls for a key are stored in remote/web/key.log in the git-annex branch. -}
+urlLog :: Key -> FilePath
+urlLog key = "remote/web"  show key ++ ".log"
+
+urls :: Key -> Annex [URLString]
+urls key = currentLog (urlLog key)
+
+download :: Key -> FilePath -> Annex Bool
+download key file = download' file =<< urls key
+download' :: FilePath -> [URLString] -> Annex Bool
+download' _ [] = return False
+download' file (url:us) = do
+	showProgress -- make way for curl progress bar
+	ok <- liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
+	if ok then return ok else download' file us
+
+upload :: Key -> Annex Bool
+upload _ = do
+	warning "upload to web not supported"
+	return False
+
+remove :: Key -> Annex Bool
+remove _ = do
+	warning "removal from web not supported"
+	return False
+
+check :: Key -> Annex (Either IOException Bool)
+check key = do
+	us <- urls key
+	if null us
+		then return $ Right False
+		else return . Right =<< check' us
+check' :: [URLString] -> Annex Bool
+check' [] = return False
+check' (u:us) = do
+	showNote ("checking " ++ u)
+	e <- liftIO $ urlexists u
+	if e then return e else check' us
+
+urlexists :: URLString -> IO Bool
+urlexists url = do
+	curl <- initialize
+	_ <- setopt curl (CurlURL url)
+	_ <- setopt curl (CurlNoBody True)
+	_ <- setopt curl (CurlFailOnError True)
+	res <- perform curl
+	return $ res == CurlOK
diff --git a/UUID.hs b/UUID.hs
index 0723abeca4..3ccc729069 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -17,7 +17,6 @@ module UUID (
 	getUncachedUUID,
 	prepUUID,
 	genUUID,
-	prettyPrintUUIDs,
 	describeUUID,
 	uuidMap,
 	uuidLog
@@ -86,21 +85,6 @@ prepUUID = do
 		uuid <- liftIO $ genUUID
 		setConfig configkey uuid
 
-{- Pretty-prints a list of UUIDs -}
-prettyPrintUUIDs :: [UUID] -> Annex String
-prettyPrintUUIDs uuids = do
-	here <- getUUID =<< Annex.gitRepo
-	m <- uuidMap
-	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
-	where
-		prettify m u here = base ++ ishere
-			where
-				base = if not $ null $ findlog m u
-					then u ++ "  -- " ++ findlog m u
-					else u
-				ishere = if here == u then " <-- here" else ""
-		findlog m u = M.findWithDefault "" u m
-
 {- Records a description for a uuid in the uuidLog. -}
 describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
diff --git a/debian/changelog b/debian/changelog
index f3cbbef9ce..53fc99e8c3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,7 @@
 git-annex (3.20110625) UNRELEASED; urgency=low
 
+  * Now the web can be used as a special remote. This feature
+    replaces the old URL backend.
   * Sped back up fsck, copy --from, and other commands that often
     have to read a lot of information from the git-annex branch. Such
     commands are now faster than they were before introduction of the
diff --git a/git-annex.cabal b/git-annex.cabal
index 5a68e4dea4..aa11ba381f 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -24,7 +24,7 @@ Description:
  versioned files, which is convenient for maintaining documents, Makefiles,
  etc that are associated with annexed files but that benefit from full
  revision control.
-Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs, ./Backend.hs, ./Backend/File.hs, ./Backend/SHA.hs, ./Backend/URL.hs, ./Backend/WORM.hs, ./BackendList.hs, ./Base64.hs, ./Branch.hs, ./CmdLine.hs, ./Command.hs, ./Command/Add.hs, ./Command/ConfigList.hs, ./Command/Copy.hs, ./Command/Describe.hs, ./Command/Drop.hs, ./Command/DropKey.hs, ./Command/DropUnused.hs, ./Command/Find.hs, ./Command/Fix.hs, ./Command/FromKey.hs, ./Command/Fsck.hs, ./Command/Get.hs, ./Command/InAnnex.hs, ./Command/Init.hs, ./Command/InitRemote.hs, ./Command/Lock.hs, ./Command/Map.hs, ./Command/Merge.hs, ./Command/Migrate.hs, ./Command/Move.hs, ./Command/PreCommit.hs, ./Command/RecvKey.hs, ./Command/Semitrust.hs, ./Command/SendKey.hs, ./Command/SetKey.hs, ./Command/Status.hs, ./Command/Trust.hs, ./Command/Unannex.hs, ./Command/Uninit.hs, ./Command/Unlock.hs, ./Command/Untrust.hs, ./Command/Unused.hs, ./Command/Upgrade.hs, ./Command/Version.hs, ./Command/Whereis.hs, ./Config.hs, ./Content.hs, ./CopyFile.hs, ./Crypto.hs, ./DataUnits.hs, ./Dot.hs, ./Git.hs, ./Git/LsFiles.hs, ./Git/Queue.hs, ./Git/UnionMerge.hs, ./GitAnnex.hs, ./LocationLog.hs, ./Locations.hs, ./Makefile, ./Messages.hs, ./Options.hs, ./README, ./Remote.hs, ./Remote/Bup.hs, ./Remote/Directory.hs, ./Remote/Encryptable.hs, ./Remote/Git.hs, ./Remote/Hook.hs, ./Remote/Rsync.hs, ./Remote/S3real.hs, ./Remote/S3stub.hs, ./Remote/Special.hs, ./RsyncFile.hs, ./Setup.hs, ./Ssh.hs, ./StatFS.hsc, ./SysConfig.hs, ./TestConfig.hs, ./Touch.hsc, ./Trust.hs, ./Types.hs, ./Types/Backend.hs, ./Types/BranchState.hs, ./Types/Crypto.hs, ./Types/Key.hs, ./Types/Remote.hs, ./Types/TrustLevel.hs, ./Types/UUID.hs, ./UUID.hs, ./Upgrade.hs, ./Upgrade/V0.hs, ./Upgrade/V1.hs, ./Upgrade/V2.hs, ./Utility.hs, ./Version.hs, ./configure.hs, ./debian/NEWS, ./debian/changelog, ./debian/compat, ./debian/control, ./debian/copyright, ./debian/doc-base, ./debian/manpages, ./debian/rules, ./doc/GPL, ./doc/backends.mdwn, ./doc/bare_repositories.mdwn, ./doc/bugs.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment, ./doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/bugs/Makefile_is_missing_dependancies.mdwn, ./doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment, ./doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment, ./doc/bugs/No_version_information_from_cli.mdwn, ./doc/bugs/Problems_running_make_on_osx.mdwn, ./doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment, ./doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment, ./doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment, ./doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment, ./doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment, ./doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment, ./doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment, ./doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment, ./doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment, ./doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment, ./doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment, ./doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment, ./doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment, ./doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment, ./doc/bugs/S3_memory_leaks.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment, ./doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn, ./doc/bugs/annex_add_in_annex.mdwn, ./doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn, ./doc/bugs/bare_git_repos.mdwn, ./doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn, ./doc/bugs/building_on_lenny.mdwn, ./doc/bugs/check_for_curl_in_configure.hs.mdwn, ./doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn, ./doc/bugs/conflicting_haskell_packages.mdwn, ./doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment, ./doc/bugs/done.mdwn, ./doc/bugs/dotdot_problem.mdwn, ./doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn, ./doc/bugs/encrypted_S3_stalls.mdwn, ./doc/bugs/error_propigation.mdwn, ./doc/bugs/error_with_file_names_starting_with_dash.mdwn, ./doc/bugs/fat_support.mdwn, ./doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment, ./doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment, ./doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment, ./doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment, ./doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment, ./doc/bugs/free_space_checking.mdwn, ./doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment, ./doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment, ./doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment, ./doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn, ./doc/bugs/fsck_output.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment, ./doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn, ./doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn, ./doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn, ./doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment, ./doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn, ./doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn, ./doc/bugs/git_annex_initremote_walks_.git-annex.mdwn, ./doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn, ./doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn, ./doc/bugs/git_annex_unlock_is_not_atomic.mdwn, ./doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn, ./doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn, ./doc/bugs/git_rename_detection_on_file_move.mdwn, ./doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment, ./doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn, ./doc/bugs/ordering.mdwn, ./doc/bugs/problem_commit_normal_links.mdwn, ./doc/bugs/problems_with_utf8_names.mdwn, ./doc/bugs/scp_interrupt_to_background.mdwn, ./doc/bugs/softlink_mtime.mdwn, ./doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn, ./doc/bugs/tmp_file_handling.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment, ./doc/bugs/unannex_vs_unlock_hook_confusion.mdwn, ./doc/bugs/unhappy_without_UTF8_locale.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment, ./doc/bugs/weird_local_clone_confuses.mdwn, ./doc/cheatsheet.mdwn, ./doc/comments.mdwn, ./doc/contact.mdwn, ./doc/copies.mdwn, ./doc/design.mdwn, ./doc/design/encryption.mdwn, ./doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment, ./doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment, ./doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment, ./doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment, ./doc/distributed_version_control.mdwn, ./doc/download.mdwn, ./doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment, ./doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment, ./doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment, ./doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment, ./doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment, ./doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment, ./doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment, ./doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment, ./doc/encryption.mdwn, ./doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment, ./doc/feeds.mdwn, ./doc/forum.mdwn, ./doc/forum/Behaviour_of_fsck.mdwn, ./doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment, ./doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment, ./doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment, ./doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment, ./doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment, ./doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn, ./doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn, ./doc/forum/Problems_with_large_numbers_of_files.mdwn, ./doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment, ./doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull.mdwn, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment, ./doc/forum/batch_check_on_remote_when_using_copy.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment, ./doc/forum/git-annex_communication_channels.mdwn, ./doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment, ./doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment, ./doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment, ./doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment, ./doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment, ./doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment, ./doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment, ./doc/forum/git-annex_on_OSX.mdwn, ./doc/forum/hashing_objects_directories.mdwn, ./doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment, ./doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment, ./doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment, ./doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment, ./doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment, ./doc/forum/incompatible_versions__63__.mdwn, ./doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex.mdwn, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment, ./doc/forum/migration_to_git-annex_and_rsync.mdwn, ./doc/forum/new_microfeatures.mdwn, ./doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment, ./doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment, ./doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment, ./doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment, ./doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment, ./doc/forum/relying_on_git_for_numcopies.mdwn, ./doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment, ./doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment, ./doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment, ./doc/forum/rsync_over_ssh__63__.mdwn, ./doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment, ./doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment, ./doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn, ./doc/forum/sparse_git_checkouts_with_annex.mdwn, ./doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment, ./doc/forum/unannex_alternatives.mdwn, ./doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment, ./doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment, ./doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment, ./doc/forum/wishlist:_command_options_changes.mdwn, ./doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment, ./doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment, ./doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment, ./doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn, ./doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment, ./doc/forum/wishlist:_git-annex_replicate.mdwn, ./doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment, ./doc/forum/wishlist:_git_annex_status.mdwn, ./doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment, ./doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment, ./doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment, ./doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment, ./doc/forum/wishlist:_git_backend_for_git-annex.mdwn, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment, ./doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment, ./doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn, ./doc/forum/wishlist:alias_system.mdwn, ./doc/forum/working_without_git-annex_commits.mdwn, ./doc/future_proofing.mdwn, ./doc/git-annex-shell.mdwn, ./doc/git-annex.mdwn, ./doc/git-union-merge.mdwn, ./doc/index.mdwn, ./doc/install.mdwn, ./doc/install/Debian.mdwn, ./doc/install/Fedora.mdwn, ./doc/install/FreeBSD.mdwn, ./doc/install/OSX.mdwn, ./doc/install/Ubuntu.mdwn, ./doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment, ./doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment, ./doc/internals.mdwn, ./doc/location_tracking.mdwn, ./doc/logo.png, ./doc/logo_small.png, ./doc/news.mdwn, ./doc/news/LWN_article.mdwn, ./doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn, ./doc/news/version_0.20110521.mdwn, ./doc/news/version_0.20110522.mdwn, ./doc/news/version_0.20110601.mdwn, ./doc/news/version_0.20110610.mdwn, ./doc/news/version_3.20110624.mdwn, ./doc/not.mdwn, ./doc/repomap.png, ./doc/special_remotes.mdwn, ./doc/special_remotes/S3.mdwn, ./doc/special_remotes/bup.mdwn, ./doc/special_remotes/directory.mdwn, ./doc/special_remotes/hook.mdwn, ./doc/special_remotes/rsync.mdwn, ./doc/summary.mdwn, ./doc/templates/bare.tmpl, ./doc/templates/walkthrough.tmpl, ./doc/todo.mdwn, ./doc/todo/S3.mdwn, ./doc/todo/add_--exclude_option_to_git_annex_find.mdwn, ./doc/todo/add_a_git_backend.mdwn, ./doc/todo/auto_remotes.mdwn, ./doc/todo/auto_remotes/discussion.mdwn, ./doc/todo/backendSHA1.mdwn, ./doc/todo/branching.mdwn, ./doc/todo/cache_key_info.mdwn, ./doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment, ./doc/todo/checkout.mdwn, ./doc/todo/done.mdwn, ./doc/todo/file_copy_progress_bar.mdwn, ./doc/todo/fsck.mdwn, ./doc/todo/git-annex-shell.mdwn, ./doc/todo/git-annex_unused_eats_memory.mdwn, ./doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn, ./doc/todo/gitrm.mdwn, ./doc/todo/hidden_files.mdwn, ./doc/todo/immutable_annexed_files.mdwn, ./doc/todo/network_remotes.mdwn, ./doc/todo/object_dir_reorg_v2.mdwn, ./doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment, ./doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment, ./doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment, ./doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment, ./doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment, ./doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment, ./doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment, ./doc/todo/parallel_possibilities.mdwn, ./doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment, ./doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment, ./doc/todo/pushpull.mdwn, ./doc/todo/rsync.mdwn, ./doc/todo/smudge.mdwn, ./doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment, ./doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment, ./doc/todo/speed_up_fsck.mdwn, ./doc/todo/support-non-utf8-locales.mdwn, ./doc/todo/support_S3_multipart_uploads.mdwn, ./doc/todo/symlink_farming_commit_hook.mdwn, ./doc/todo/tahoe_lfs_for_reals.mdwn, ./doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment, ./doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment, ./doc/todo/union_mounting.mdwn, ./doc/todo/use_cp_reflink.mdwn, ./doc/todo/using_url_backend.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment, ./doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/todo/wishlist:_swift_backend.mdwn, ./doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment, ./doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment, ./doc/transferring_data.mdwn, ./doc/trust.mdwn, ./doc/upgrades.mdwn, ./doc/upgrades/SHA_size.mdwn, ./doc/use_case/Alice.mdwn, ./doc/use_case/Bob.mdwn, ./doc/users.mdwn, ./doc/users/chrysn.mdwn, ./doc/users/fmarier.mdwn, ./doc/users/joey.mdwn, ./doc/walkthrough.mdwn, ./doc/walkthrough/Internet_Archive_via_S3.mdwn, ./doc/walkthrough/adding_a_remote.mdwn, ./doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment, ./doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment, ./doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment, ./doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment, ./doc/walkthrough/adding_files.mdwn, ./doc/walkthrough/backups.mdwn, ./doc/walkthrough/creating_a_repository.mdwn, ./doc/walkthrough/fsck:_verifying_your_data.mdwn, ./doc/walkthrough/fsck:_when_things_go_wrong.mdwn, ./doc/walkthrough/getting_file_content.mdwn, ./doc/walkthrough/migrating_data_to_a_new_backend.mdwn, ./doc/walkthrough/modifying_annexed_files.mdwn, ./doc/walkthrough/more.mdwn, ./doc/walkthrough/moving_file_content_between_repositories.mdwn, ./doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment, ./doc/walkthrough/recover_data_from_lost+found.mdwn, ./doc/walkthrough/removing_files.mdwn, ./doc/walkthrough/removing_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/renaming_files.mdwn, ./doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/untrusted_repositories.mdwn, ./doc/walkthrough/unused_data.mdwn, ./doc/walkthrough/using_Amazon_S3.mdwn, ./doc/walkthrough/using_bup.mdwn, ./doc/walkthrough/using_ssh_remotes.mdwn, ./doc/walkthrough/using_the_SHA1_backend.mdwn, ./doc/walkthrough/using_the_URL_backend.mdwn, ./doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn, ./git-annex-shell.hs, ./git-annex.cabal, ./git-annex.hs, ./git-union-merge.hs, ./mdwn2man, ./test.hs, ./testdata/unicode-test-ö
+Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs, ./Backend.hs, ./Backend/File.hs, ./Backend/SHA.hs, ./Backend/URL.hs, ./Backend/WORM.hs, ./BackendList.hs, ./Base64.hs, ./Branch.hs, ./CmdLine.hs, ./Command.hs, ./Command/Add.hs, ./Command/ConfigList.hs, ./Command/Copy.hs, ./Command/Describe.hs, ./Command/Drop.hs, ./Command/DropKey.hs, ./Command/DropUnused.hs, ./Command/Find.hs, ./Command/Fix.hs, ./Command/FromKey.hs, ./Command/Fsck.hs, ./Command/Get.hs, ./Command/InAnnex.hs, ./Command/Init.hs, ./Command/InitRemote.hs, ./Command/Lock.hs, ./Command/Map.hs, ./Command/Merge.hs, ./Command/Migrate.hs, ./Command/Move.hs, ./Command/PreCommit.hs, ./Command/RecvKey.hs, ./Command/Semitrust.hs, ./Command/SendKey.hs, ./Command/SetKey.hs, ./Command/Status.hs, ./Command/Trust.hs, ./Command/Unannex.hs, ./Command/Uninit.hs, ./Command/Unlock.hs, ./Command/Untrust.hs, ./Command/Unused.hs, ./Command/Upgrade.hs, ./Command/Version.hs, ./Command/Whereis.hs, ./Config.hs, ./Content.hs, ./CopyFile.hs, ./Crypto.hs, ./DataUnits.hs, ./Dot.hs, ./Git.hs, ./Git/LsFiles.hs, ./Git/Queue.hs, ./Git/UnionMerge.hs, ./GitAnnex.hs, ./LocationLog.hs, ./Locations.hs, ./Makefile, ./Messages.hs, ./Options.hs, ./PresenceLog.hs, ./README, ./Remote.hs, ./Remote/Bup.hs, ./Remote/Directory.hs, ./Remote/Encryptable.hs, ./Remote/Git.hs, ./Remote/Hook.hs, ./Remote/Rsync.hs, ./Remote/S3real.hs, ./Remote/S3stub.hs, ./Remote/Special.hs, ./Remote/Web.hs, ./RsyncFile.hs, ./Setup.hs, ./Ssh.hs, ./StatFS.hs, ./StatFS.hsc, ./SysConfig.hs, ./TestConfig.hs, ./Touch.hs, ./Touch.hsc, ./Trust.hs, ./Types.hs, ./Types/Backend.hs, ./Types/BranchState.hs, ./Types/Crypto.hs, ./Types/Key.hs, ./Types/Remote.hs, ./Types/TrustLevel.hs, ./Types/UUID.hs, ./UUID.hs, ./Upgrade.hs, ./Upgrade/V0.hs, ./Upgrade/V1.hs, ./Upgrade/V2.hs, ./Utility.hs, ./Version.hs, ./configure.hs, ./debian/NEWS, ./debian/changelog, ./debian/compat, ./debian/control, ./debian/copyright, ./debian/doc-base, ./debian/manpages, ./debian/rules, ./doc/.ikiwiki/indexdb, ./doc/.ikiwiki/lockfile, ./doc/GPL, ./doc/backends.mdwn, ./doc/bare_repositories.mdwn, ./doc/bugs.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment, ./doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/bugs/Makefile_is_missing_dependancies.mdwn, ./doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment, ./doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment, ./doc/bugs/No_version_information_from_cli.mdwn, ./doc/bugs/Problems_running_make_on_osx.mdwn, ./doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment, ./doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment, ./doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment, ./doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment, ./doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment, ./doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment, ./doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment, ./doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment, ./doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment, ./doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment, ./doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment, ./doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment, ./doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment, ./doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment, ./doc/bugs/S3_memory_leaks.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment, ./doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn, ./doc/bugs/annex_add_in_annex.mdwn, ./doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn, ./doc/bugs/bare_git_repos.mdwn, ./doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn, ./doc/bugs/building_on_lenny.mdwn, ./doc/bugs/check_for_curl_in_configure.hs.mdwn, ./doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn, ./doc/bugs/conflicting_haskell_packages.mdwn, ./doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment, ./doc/bugs/done.mdwn, ./doc/bugs/dotdot_problem.mdwn, ./doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn, ./doc/bugs/encrypted_S3_stalls.mdwn, ./doc/bugs/error_propigation.mdwn, ./doc/bugs/error_with_file_names_starting_with_dash.mdwn, ./doc/bugs/fat_support.mdwn, ./doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment, ./doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment, ./doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment, ./doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment, ./doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment, ./doc/bugs/free_space_checking.mdwn, ./doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment, ./doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment, ./doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment, ./doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn, ./doc/bugs/fsck_output.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment, ./doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn, ./doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn, ./doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn, ./doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment, ./doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn, ./doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn, ./doc/bugs/git_annex_initremote_walks_.git-annex.mdwn, ./doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn, ./doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn, ./doc/bugs/git_annex_unlock_is_not_atomic.mdwn, ./doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn, ./doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn, ./doc/bugs/git_rename_detection_on_file_move.mdwn, ./doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment, ./doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn, ./doc/bugs/ordering.mdwn, ./doc/bugs/problem_commit_normal_links.mdwn, ./doc/bugs/problems_with_utf8_names.mdwn, ./doc/bugs/scp_interrupt_to_background.mdwn, ./doc/bugs/softlink_mtime.mdwn, ./doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn, ./doc/bugs/tmp_file_handling.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment, ./doc/bugs/unannex_vs_unlock_hook_confusion.mdwn, ./doc/bugs/unhappy_without_UTF8_locale.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment, ./doc/bugs/weird_local_clone_confuses.mdwn, ./doc/cheatsheet.mdwn, ./doc/comments.mdwn, ./doc/contact.mdwn, ./doc/copies.mdwn, ./doc/design.mdwn, ./doc/design/encryption.mdwn, ./doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment, ./doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment, ./doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment, ./doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment, ./doc/distributed_version_control.mdwn, ./doc/download.mdwn, ./doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment, ./doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment, ./doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment, ./doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment, ./doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment, ./doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment, ./doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment, ./doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment, ./doc/encryption.mdwn, ./doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment, ./doc/feeds.mdwn, ./doc/forum.mdwn, ./doc/forum/Behaviour_of_fsck.mdwn, ./doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment, ./doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment, ./doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment, ./doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment, ./doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment, ./doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn, ./doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn, ./doc/forum/Problems_with_large_numbers_of_files.mdwn, ./doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment, ./doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull.mdwn, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment, ./doc/forum/batch_check_on_remote_when_using_copy.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment, ./doc/forum/git-annex_communication_channels.mdwn, ./doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment, ./doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment, ./doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment, ./doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment, ./doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment, ./doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment, ./doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment, ./doc/forum/git-annex_on_OSX.mdwn, ./doc/forum/hashing_objects_directories.mdwn, ./doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment, ./doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment, ./doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment, ./doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment, ./doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment, ./doc/forum/incompatible_versions__63__.mdwn, ./doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex.mdwn, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment, ./doc/forum/migration_to_git-annex_and_rsync.mdwn, ./doc/forum/new_microfeatures.mdwn, ./doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment, ./doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment, ./doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment, ./doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment, ./doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment, ./doc/forum/relying_on_git_for_numcopies.mdwn, ./doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment, ./doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment, ./doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment, ./doc/forum/rsync_over_ssh__63__.mdwn, ./doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment, ./doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment, ./doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn, ./doc/forum/sparse_git_checkouts_with_annex.mdwn, ./doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment, ./doc/forum/unannex_alternatives.mdwn, ./doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment, ./doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment, ./doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment, ./doc/forum/wishlist:_command_options_changes.mdwn, ./doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment, ./doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment, ./doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment, ./doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn, ./doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment, ./doc/forum/wishlist:_git-annex_replicate.mdwn, ./doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment, ./doc/forum/wishlist:_git_annex_status.mdwn, ./doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment, ./doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment, ./doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment, ./doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment, ./doc/forum/wishlist:_git_backend_for_git-annex.mdwn, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment, ./doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment, ./doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn, ./doc/forum/wishlist:alias_system.mdwn, ./doc/forum/working_without_git-annex_commits.mdwn, ./doc/future_proofing.mdwn, ./doc/git-annex-shell.mdwn, ./doc/git-annex.mdwn, ./doc/git-union-merge.mdwn, ./doc/index.mdwn, ./doc/install.mdwn, ./doc/install/Debian.mdwn, ./doc/install/Fedora.mdwn, ./doc/install/FreeBSD.mdwn, ./doc/install/OSX.mdwn, ./doc/install/Ubuntu.mdwn, ./doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment, ./doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment, ./doc/internals.mdwn, ./doc/location_tracking.mdwn, ./doc/logo.png, ./doc/logo_small.png, ./doc/news.mdwn, ./doc/news/LWN_article.mdwn, ./doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn, ./doc/news/version_0.20110521.mdwn, ./doc/news/version_0.20110522.mdwn, ./doc/news/version_0.20110601.mdwn, ./doc/news/version_0.20110610.mdwn, ./doc/news/version_3.20110624.mdwn, ./doc/not.mdwn, ./doc/repomap.png, ./doc/special_remotes.mdwn, ./doc/special_remotes/S3.mdwn, ./doc/special_remotes/bup.mdwn, ./doc/special_remotes/directory.mdwn, ./doc/special_remotes/hook.mdwn, ./doc/special_remotes/rsync.mdwn, ./doc/summary.mdwn, ./doc/templates/bare.tmpl, ./doc/templates/walkthrough.tmpl, ./doc/todo.mdwn, ./doc/todo/S3.mdwn, ./doc/todo/add_--exclude_option_to_git_annex_find.mdwn, ./doc/todo/add_a_git_backend.mdwn, ./doc/todo/auto_remotes.mdwn, ./doc/todo/auto_remotes/discussion.mdwn, ./doc/todo/backendSHA1.mdwn, ./doc/todo/branching.mdwn, ./doc/todo/cache_key_info.mdwn, ./doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment, ./doc/todo/checkout.mdwn, ./doc/todo/done.mdwn, ./doc/todo/file_copy_progress_bar.mdwn, ./doc/todo/fsck.mdwn, ./doc/todo/git-annex-shell.mdwn, ./doc/todo/git-annex_unused_eats_memory.mdwn, ./doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn, ./doc/todo/gitrm.mdwn, ./doc/todo/hidden_files.mdwn, ./doc/todo/immutable_annexed_files.mdwn, ./doc/todo/network_remotes.mdwn, ./doc/todo/object_dir_reorg_v2.mdwn, ./doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment, ./doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment, ./doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment, ./doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment, ./doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment, ./doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment, ./doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment, ./doc/todo/parallel_possibilities.mdwn, ./doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment, ./doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment, ./doc/todo/pushpull.mdwn, ./doc/todo/rsync.mdwn, ./doc/todo/smudge.mdwn, ./doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment, ./doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment, ./doc/todo/speed_up_fsck.mdwn, ./doc/todo/support-non-utf8-locales.mdwn, ./doc/todo/support_S3_multipart_uploads.mdwn, ./doc/todo/symlink_farming_commit_hook.mdwn, ./doc/todo/tahoe_lfs_for_reals.mdwn, ./doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment, ./doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment, ./doc/todo/union_mounting.mdwn, ./doc/todo/use_cp_reflink.mdwn, ./doc/todo/using_url_backend.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment, ./doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/todo/wishlist:_swift_backend.mdwn, ./doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment, ./doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment, ./doc/transferring_data.mdwn, ./doc/trust.mdwn, ./doc/upgrades.mdwn, ./doc/upgrades/SHA_size.mdwn, ./doc/use_case/Alice.mdwn, ./doc/use_case/Bob.mdwn, ./doc/users.mdwn, ./doc/users/chrysn.mdwn, ./doc/users/fmarier.mdwn, ./doc/users/joey.mdwn, ./doc/walkthrough.mdwn, ./doc/walkthrough/Internet_Archive_via_S3.mdwn, ./doc/walkthrough/adding_a_remote.mdwn, ./doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment, ./doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment, ./doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment, ./doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment, ./doc/walkthrough/adding_files.mdwn, ./doc/walkthrough/backups.mdwn, ./doc/walkthrough/creating_a_repository.mdwn, ./doc/walkthrough/fsck:_verifying_your_data.mdwn, ./doc/walkthrough/fsck:_when_things_go_wrong.mdwn, ./doc/walkthrough/getting_file_content.mdwn, ./doc/walkthrough/migrating_data_to_a_new_backend.mdwn, ./doc/walkthrough/modifying_annexed_files.mdwn, ./doc/walkthrough/more.mdwn, ./doc/walkthrough/moving_file_content_between_repositories.mdwn, ./doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment, ./doc/walkthrough/recover_data_from_lost+found.mdwn, ./doc/walkthrough/removing_files.mdwn, ./doc/walkthrough/removing_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/renaming_files.mdwn, ./doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/untrusted_repositories.mdwn, ./doc/walkthrough/unused_data.mdwn, ./doc/walkthrough/using_Amazon_S3.mdwn, ./doc/walkthrough/using_bup.mdwn, ./doc/walkthrough/using_ssh_remotes.mdwn, ./doc/walkthrough/using_the_SHA1_backend.mdwn, ./doc/walkthrough/using_the_URL_backend.mdwn, ./doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn, ./git-annex, ./git-annex-shell, ./git-annex-shell.1, ./git-annex-shell.hs, ./git-annex.1, ./git-annex.cabal, ./git-annex.hs, ./git-union-merge, ./git-union-merge.1, ./git-union-merge.hs, ./html/GPL, ./html/backends.html, ./html/bare_repositories.html, ./html/bugs.html, ./html/bugs/Displayed_copy_speed_is_wrong.html, ./html/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.html, ./html/bugs/Makefile_is_missing_dependancies.html, ./html/bugs/Name_scheme_does_not_follow_git__39__s_rules.html, ./html/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.html, ./html/bugs/No_version_information_from_cli.html, ./html/bugs/Problems_running_make_on_osx.html, ./html/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.html, ./html/bugs/S3_memory_leaks.html, ./html/bugs/Unfortunate_interaction_with_Calibre.html, ./html/bugs/WORM:_Handle_long_filenames_correctly.html, ./html/bugs/add_range_argument_to___34__git_annex_dropunused__34___.html, ./html/bugs/annex_add_in_annex.html, ./html/bugs/backend_version_upgrade_leaves_repo_unusable.html, ./html/bugs/bare_git_repos.html, ./html/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.html, ./html/bugs/building_on_lenny.html, ./html/bugs/check_for_curl_in_configure.hs.html, ./html/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.html, ./html/bugs/conflicting_haskell_packages.html, ./html/bugs/copy_fast_confusing_with_broken_locationlog.html, ./html/bugs/done.html, ./html/bugs/dotdot_problem.html, ./html/bugs/dropping_files_with_a_URL_backend_fails.html, ./html/bugs/encrypted_S3_stalls.html, ./html/bugs/error_propigation.html, ./html/bugs/error_with_file_names_starting_with_dash.html, ./html/bugs/fat_support.html, ./html/bugs/free_space_checking.html, ./html/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.html, ./html/bugs/fsck_output.html, ./html/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.html, ./html/bugs/git-annex_directory_hashing_problems_on_osx.html, ./html/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.html, ./html/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.html, ./html/bugs/git_annex_copy_--fast_does_not_copy_files.html, ./html/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.html, ./html/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.html, ./html/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.html, ./html/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.html, ./html/bugs/git_annex_initremote_walks_.git-annex.html, ./html/bugs/git_annex_migrate_leaves_old_backend_versions_around.html, ./html/bugs/git_annex_should_use___39__git_add_-f__39___internally.html, ./html/bugs/git_annex_unlock_is_not_atomic.html, ./html/bugs/git_annex_unused_failes_on_empty_repository.html, ./html/bugs/git_annex_unused_seems_to_check_for_current_path.html, ./html/bugs/git_rename_detection_on_file_move.html, ./html/bugs/minor_bug:_errors_are_not_verbose_enough.html, ./html/bugs/ordering.html, ./html/bugs/problem_commit_normal_links.html, ./html/bugs/problems_with_utf8_names.html, ./html/bugs/scp_interrupt_to_background.html, ./html/bugs/softlink_mtime.html, ./html/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.html, ./html/bugs/tmp_file_handling.html, ./html/bugs/touch.hsc_has_problems_on_non-linux_based_systems.html, ./html/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.html, ./html/bugs/unannex_vs_unlock_hook_confusion.html, ./html/bugs/unhappy_without_UTF8_locale.html, ./html/bugs/upgrade_left_untracked_.git-annex__47____42___directories.html, ./html/bugs/weird_local_clone_confuses.html, ./html/cheatsheet.html, ./html/comments.html, ./html/contact.html, ./html/copies.html, ./html/design.html, ./html/design/encryption.html, ./html/distributed_version_control.html, ./html/download.html, ./html/encryption.html, ./html/feeds.html, ./html/forum.html, ./html/forum/Behaviour_of_fsck.html, ./html/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.html, ./html/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.html, ./html/forum/Need_new_build_instructions_for_Debian_stable.html, ./html/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.html, ./html/forum/OSX__39__s_haskell-platform_statically_links_things.html, ./html/forum/Problems_with_large_numbers_of_files.html, ./html/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.html, ./html/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.html, ./html/forum/__34__git_annex_lock__34___very_slow_for_big_repo.html, ./html/forum/bainstorming:_git_annex_push___38___pull.html, ./html/forum/batch_check_on_remote_when_using_copy.html, ./html/forum/can_git-annex_replace_ddm__63__.html, ./html/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.html, ./html/forum/git-annex_communication_channels.html, ./html/forum/git-annex_on_OSX.html, ./html/forum/hashing_objects_directories.html, ./html/forum/incompatible_versions__63__.html, ./html/forum/migrate_existing_git_repository_to_git-annex.html, ./html/forum/migration_to_git-annex_and_rsync.html, ./html/forum/new_microfeatures.html, ./html/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.html, ./html/forum/relying_on_git_for_numcopies.html, ./html/forum/rsync_over_ssh__63__.html, ./html/forum/seems_to_build_fine_on_haskell_platform_2011.html, ./html/forum/sparse_git_checkouts_with_annex.html, ./html/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.html, ./html/forum/unannex_alternatives.html, ./html/forum/wishlist:_command_options_changes.html, ./html/forum/wishlist:_define_remotes_that_must_have_all_files.html, ./html/forum/wishlist:_do_round_robin_downloading_of_data.html, ./html/forum/wishlist:_git-annex_replicate.html, ./html/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.html, ./html/forum/wishlist:_git_annex_status.html, ./html/forum/wishlist:_git_backend_for_git-annex.html, ./html/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.html, ./html/forum/wishlist:_special_remote_for_sftp_or_rsync.html, ./html/forum/wishlist:_support_for_more_ssh_urls_.html, ./html/forum/wishlist:_traffic_accounting_for_git-annex.html, ./html/forum/wishlist:alias_system.html, ./html/forum/working_without_git-annex_commits.html, ./html/future_proofing.html, ./html/git-annex-shell.html, ./html/git-annex.html, ./html/git-union-merge.html, ./html/ikiwiki/ikiwiki.js, ./html/ikiwiki/relativedate.js, ./html/ikiwiki/toggle.js, ./html/index.html, ./html/install.html, ./html/install/Debian.html, ./html/install/Fedora.html, ./html/install/FreeBSD.html, ./html/install/OSX.html, ./html/install/Ubuntu.html, ./html/internals.html, ./html/location_tracking.html, ./html/logo.png, ./html/logo_small.png, ./html/news.html, ./html/not.html, ./html/repomap.png, ./html/special_remotes.html, ./html/special_remotes/S3.html, ./html/special_remotes/bup.html, ./html/special_remotes/directory.html, ./html/special_remotes/hook.html, ./html/special_remotes/rsync.html, ./html/summary.html, ./html/templates/bare.tmpl, ./html/templates/walkthrough.tmpl, ./html/todo.html, ./html/todo/S3.html, ./html/todo/add_--exclude_option_to_git_annex_find.html, ./html/todo/add_a_git_backend.html, ./html/todo/auto_remotes.html, ./html/todo/auto_remotes/discussion.html, ./html/todo/backendSHA1.html, ./html/todo/branching.html, ./html/todo/cache_key_info.html, ./html/todo/checkout.html, ./html/todo/done.html, ./html/todo/file_copy_progress_bar.html, ./html/todo/fsck.html, ./html/todo/git-annex-shell.html, ./html/todo/git-annex_unused_eats_memory.html, ./html/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.html, ./html/todo/gitrm.html, ./html/todo/hidden_files.html, ./html/todo/immutable_annexed_files.html, ./html/todo/network_remotes.html, ./html/todo/object_dir_reorg_v2.html, ./html/todo/parallel_possibilities.html, ./html/todo/pushpull.html, ./html/todo/rsync.html, ./html/todo/smudge.html, ./html/todo/speed_up_fsck.html, ./html/todo/support-non-utf8-locales.html, ./html/todo/support_S3_multipart_uploads.html, ./html/todo/symlink_farming_commit_hook.html, ./html/todo/tahoe_lfs_for_reals.html, ./html/todo/union_mounting.html, ./html/todo/use_cp_reflink.html, ./html/todo/using_url_backend.html, ./html/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.html, ./html/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.html, ./html/todo/wishlist:___34__git_annex_add__34___multiple_processes.html, ./html/todo/wishlist:_support_for_more_ssh_urls_.html, ./html/todo/wishlist:_swift_backend.html, ./html/transferring_data.html, ./html/trust.html, ./html/upgrades.html, ./html/upgrades/SHA_size.html, ./html/use_case/Alice.html, ./html/use_case/Bob.html, ./html/users.html, ./html/users/chrysn.html, ./html/users/fmarier.html, ./html/users/joey.html, ./html/walkthrough.html, ./html/walkthrough/Internet_Archive_via_S3.html, ./html/walkthrough/adding_a_remote.html, ./html/walkthrough/adding_files.html, ./html/walkthrough/backups.html, ./html/walkthrough/creating_a_repository.html, ./html/walkthrough/fsck:_verifying_your_data.html, ./html/walkthrough/fsck:_when_things_go_wrong.html, ./html/walkthrough/getting_file_content.html, ./html/walkthrough/migrating_data_to_a_new_backend.html, ./html/walkthrough/modifying_annexed_files.html, ./html/walkthrough/more.html, ./html/walkthrough/moving_file_content_between_repositories.html, ./html/walkthrough/recover_data_from_lost+found.html, ./html/walkthrough/removing_files.html, ./html/walkthrough/removing_files:_When_things_go_wrong.html, ./html/walkthrough/renaming_files.html, ./html/walkthrough/transferring_files:_When_things_go_wrong.html, ./html/walkthrough/untrusted_repositories.html, ./html/walkthrough/unused_data.html, ./html/walkthrough/using_Amazon_S3.html, ./html/walkthrough/using_bup.html, ./html/walkthrough/using_ssh_remotes.html, ./html/walkthrough/using_the_SHA1_backend.html, ./html/walkthrough/using_the_URL_backend.html, ./html/walkthrough/what_to_do_when_you_lose_a_repository.html, ./mdwn2man, ./test.hs, ./testdata/unicode-test-ö
 
 Executable git-annex
   Main-Is: git-annex.hs

From 3efba481b50ad8de6efe5869db90ac69f5c022a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 15:31:36 -0400
Subject: [PATCH 1975/2835] remove Extra-Source-Files nonsense

will need to find a better way to make sdist work, this is not livable
---
 configure.hs    | 4 ----
 git-annex.cabal | 1 -
 2 files changed, 5 deletions(-)

diff --git a/configure.hs b/configure.hs
index 5f0ff5a1e3..d47dddf1df 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,8 +2,6 @@
 
 import System.Directory
 import Data.List
-import Data.String.Utils
-import System.Cmd.Utils
 
 import TestConfig
 
@@ -73,11 +71,9 @@ getVersionString = do
 cabalSetup :: IO ()
 cabalSetup = do
 	version <- getVersionString
-	(_, filelist) <- pipeLinesFrom "find" (words ". -name .git -prune -o -name dist -prune -o -not -name *.hi -not -name *.o -not -name configure -not -name *.tmp -type f -print")
 	cabal <- readFile cabalfile
 	writeFile tmpcabalfile $ unlines $ 
 		map (setfield "Version" version) $
-		map (setfield "Extra-Source-Files" $ join ", " $ sort filelist) $
 		lines cabal
 	renameFile tmpcabalfile cabalfile
 	where
diff --git a/git-annex.cabal b/git-annex.cabal
index aa11ba381f..a4b8aef993 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -24,7 +24,6 @@ Description:
  versioned files, which is convenient for maintaining documents, Makefiles,
  etc that are associated with annexed files but that benefit from full
  revision control.
-Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs, ./Backend.hs, ./Backend/File.hs, ./Backend/SHA.hs, ./Backend/URL.hs, ./Backend/WORM.hs, ./BackendList.hs, ./Base64.hs, ./Branch.hs, ./CmdLine.hs, ./Command.hs, ./Command/Add.hs, ./Command/ConfigList.hs, ./Command/Copy.hs, ./Command/Describe.hs, ./Command/Drop.hs, ./Command/DropKey.hs, ./Command/DropUnused.hs, ./Command/Find.hs, ./Command/Fix.hs, ./Command/FromKey.hs, ./Command/Fsck.hs, ./Command/Get.hs, ./Command/InAnnex.hs, ./Command/Init.hs, ./Command/InitRemote.hs, ./Command/Lock.hs, ./Command/Map.hs, ./Command/Merge.hs, ./Command/Migrate.hs, ./Command/Move.hs, ./Command/PreCommit.hs, ./Command/RecvKey.hs, ./Command/Semitrust.hs, ./Command/SendKey.hs, ./Command/SetKey.hs, ./Command/Status.hs, ./Command/Trust.hs, ./Command/Unannex.hs, ./Command/Uninit.hs, ./Command/Unlock.hs, ./Command/Untrust.hs, ./Command/Unused.hs, ./Command/Upgrade.hs, ./Command/Version.hs, ./Command/Whereis.hs, ./Config.hs, ./Content.hs, ./CopyFile.hs, ./Crypto.hs, ./DataUnits.hs, ./Dot.hs, ./Git.hs, ./Git/LsFiles.hs, ./Git/Queue.hs, ./Git/UnionMerge.hs, ./GitAnnex.hs, ./LocationLog.hs, ./Locations.hs, ./Makefile, ./Messages.hs, ./Options.hs, ./PresenceLog.hs, ./README, ./Remote.hs, ./Remote/Bup.hs, ./Remote/Directory.hs, ./Remote/Encryptable.hs, ./Remote/Git.hs, ./Remote/Hook.hs, ./Remote/Rsync.hs, ./Remote/S3real.hs, ./Remote/S3stub.hs, ./Remote/Special.hs, ./Remote/Web.hs, ./RsyncFile.hs, ./Setup.hs, ./Ssh.hs, ./StatFS.hs, ./StatFS.hsc, ./SysConfig.hs, ./TestConfig.hs, ./Touch.hs, ./Touch.hsc, ./Trust.hs, ./Types.hs, ./Types/Backend.hs, ./Types/BranchState.hs, ./Types/Crypto.hs, ./Types/Key.hs, ./Types/Remote.hs, ./Types/TrustLevel.hs, ./Types/UUID.hs, ./UUID.hs, ./Upgrade.hs, ./Upgrade/V0.hs, ./Upgrade/V1.hs, ./Upgrade/V2.hs, ./Utility.hs, ./Version.hs, ./configure.hs, ./debian/NEWS, ./debian/changelog, ./debian/compat, ./debian/control, ./debian/copyright, ./debian/doc-base, ./debian/manpages, ./debian/rules, ./doc/.ikiwiki/indexdb, ./doc/.ikiwiki/lockfile, ./doc/GPL, ./doc/backends.mdwn, ./doc/bare_repositories.mdwn, ./doc/bugs.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment, ./doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/bugs/Makefile_is_missing_dependancies.mdwn, ./doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment, ./doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment, ./doc/bugs/No_version_information_from_cli.mdwn, ./doc/bugs/Problems_running_make_on_osx.mdwn, ./doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment, ./doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment, ./doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment, ./doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment, ./doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment, ./doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment, ./doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment, ./doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment, ./doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment, ./doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment, ./doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment, ./doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment, ./doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment, ./doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment, ./doc/bugs/S3_memory_leaks.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment, ./doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn, ./doc/bugs/annex_add_in_annex.mdwn, ./doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn, ./doc/bugs/bare_git_repos.mdwn, ./doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn, ./doc/bugs/building_on_lenny.mdwn, ./doc/bugs/check_for_curl_in_configure.hs.mdwn, ./doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn, ./doc/bugs/conflicting_haskell_packages.mdwn, ./doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment, ./doc/bugs/done.mdwn, ./doc/bugs/dotdot_problem.mdwn, ./doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn, ./doc/bugs/encrypted_S3_stalls.mdwn, ./doc/bugs/error_propigation.mdwn, ./doc/bugs/error_with_file_names_starting_with_dash.mdwn, ./doc/bugs/fat_support.mdwn, ./doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment, ./doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment, ./doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment, ./doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment, ./doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment, ./doc/bugs/free_space_checking.mdwn, ./doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment, ./doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment, ./doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment, ./doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn, ./doc/bugs/fsck_output.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment, ./doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn, ./doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn, ./doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn, ./doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment, ./doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn, ./doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn, ./doc/bugs/git_annex_initremote_walks_.git-annex.mdwn, ./doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn, ./doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn, ./doc/bugs/git_annex_unlock_is_not_atomic.mdwn, ./doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn, ./doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn, ./doc/bugs/git_rename_detection_on_file_move.mdwn, ./doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment, ./doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn, ./doc/bugs/ordering.mdwn, ./doc/bugs/problem_commit_normal_links.mdwn, ./doc/bugs/problems_with_utf8_names.mdwn, ./doc/bugs/scp_interrupt_to_background.mdwn, ./doc/bugs/softlink_mtime.mdwn, ./doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn, ./doc/bugs/tmp_file_handling.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment, ./doc/bugs/unannex_vs_unlock_hook_confusion.mdwn, ./doc/bugs/unhappy_without_UTF8_locale.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment, ./doc/bugs/weird_local_clone_confuses.mdwn, ./doc/cheatsheet.mdwn, ./doc/comments.mdwn, ./doc/contact.mdwn, ./doc/copies.mdwn, ./doc/design.mdwn, ./doc/design/encryption.mdwn, ./doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment, ./doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment, ./doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment, ./doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment, ./doc/distributed_version_control.mdwn, ./doc/download.mdwn, ./doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment, ./doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment, ./doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment, ./doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment, ./doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment, ./doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment, ./doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment, ./doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment, ./doc/encryption.mdwn, ./doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment, ./doc/feeds.mdwn, ./doc/forum.mdwn, ./doc/forum/Behaviour_of_fsck.mdwn, ./doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment, ./doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment, ./doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment, ./doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment, ./doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment, ./doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn, ./doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn, ./doc/forum/Problems_with_large_numbers_of_files.mdwn, ./doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment, ./doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull.mdwn, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment, ./doc/forum/batch_check_on_remote_when_using_copy.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment, ./doc/forum/git-annex_communication_channels.mdwn, ./doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment, ./doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment, ./doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment, ./doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment, ./doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment, ./doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment, ./doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment, ./doc/forum/git-annex_on_OSX.mdwn, ./doc/forum/hashing_objects_directories.mdwn, ./doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment, ./doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment, ./doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment, ./doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment, ./doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment, ./doc/forum/incompatible_versions__63__.mdwn, ./doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex.mdwn, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment, ./doc/forum/migration_to_git-annex_and_rsync.mdwn, ./doc/forum/new_microfeatures.mdwn, ./doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment, ./doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment, ./doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment, ./doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment, ./doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment, ./doc/forum/relying_on_git_for_numcopies.mdwn, ./doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment, ./doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment, ./doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment, ./doc/forum/rsync_over_ssh__63__.mdwn, ./doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment, ./doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment, ./doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn, ./doc/forum/sparse_git_checkouts_with_annex.mdwn, ./doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment, ./doc/forum/unannex_alternatives.mdwn, ./doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment, ./doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment, ./doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment, ./doc/forum/wishlist:_command_options_changes.mdwn, ./doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment, ./doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment, ./doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment, ./doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn, ./doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment, ./doc/forum/wishlist:_git-annex_replicate.mdwn, ./doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment, ./doc/forum/wishlist:_git_annex_status.mdwn, ./doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment, ./doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment, ./doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment, ./doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment, ./doc/forum/wishlist:_git_backend_for_git-annex.mdwn, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment, ./doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment, ./doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn, ./doc/forum/wishlist:alias_system.mdwn, ./doc/forum/working_without_git-annex_commits.mdwn, ./doc/future_proofing.mdwn, ./doc/git-annex-shell.mdwn, ./doc/git-annex.mdwn, ./doc/git-union-merge.mdwn, ./doc/index.mdwn, ./doc/install.mdwn, ./doc/install/Debian.mdwn, ./doc/install/Fedora.mdwn, ./doc/install/FreeBSD.mdwn, ./doc/install/OSX.mdwn, ./doc/install/Ubuntu.mdwn, ./doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment, ./doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment, ./doc/internals.mdwn, ./doc/location_tracking.mdwn, ./doc/logo.png, ./doc/logo_small.png, ./doc/news.mdwn, ./doc/news/LWN_article.mdwn, ./doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn, ./doc/news/version_0.20110521.mdwn, ./doc/news/version_0.20110522.mdwn, ./doc/news/version_0.20110601.mdwn, ./doc/news/version_0.20110610.mdwn, ./doc/news/version_3.20110624.mdwn, ./doc/not.mdwn, ./doc/repomap.png, ./doc/special_remotes.mdwn, ./doc/special_remotes/S3.mdwn, ./doc/special_remotes/bup.mdwn, ./doc/special_remotes/directory.mdwn, ./doc/special_remotes/hook.mdwn, ./doc/special_remotes/rsync.mdwn, ./doc/summary.mdwn, ./doc/templates/bare.tmpl, ./doc/templates/walkthrough.tmpl, ./doc/todo.mdwn, ./doc/todo/S3.mdwn, ./doc/todo/add_--exclude_option_to_git_annex_find.mdwn, ./doc/todo/add_a_git_backend.mdwn, ./doc/todo/auto_remotes.mdwn, ./doc/todo/auto_remotes/discussion.mdwn, ./doc/todo/backendSHA1.mdwn, ./doc/todo/branching.mdwn, ./doc/todo/cache_key_info.mdwn, ./doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment, ./doc/todo/checkout.mdwn, ./doc/todo/done.mdwn, ./doc/todo/file_copy_progress_bar.mdwn, ./doc/todo/fsck.mdwn, ./doc/todo/git-annex-shell.mdwn, ./doc/todo/git-annex_unused_eats_memory.mdwn, ./doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn, ./doc/todo/gitrm.mdwn, ./doc/todo/hidden_files.mdwn, ./doc/todo/immutable_annexed_files.mdwn, ./doc/todo/network_remotes.mdwn, ./doc/todo/object_dir_reorg_v2.mdwn, ./doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment, ./doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment, ./doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment, ./doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment, ./doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment, ./doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment, ./doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment, ./doc/todo/parallel_possibilities.mdwn, ./doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment, ./doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment, ./doc/todo/pushpull.mdwn, ./doc/todo/rsync.mdwn, ./doc/todo/smudge.mdwn, ./doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment, ./doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment, ./doc/todo/speed_up_fsck.mdwn, ./doc/todo/support-non-utf8-locales.mdwn, ./doc/todo/support_S3_multipart_uploads.mdwn, ./doc/todo/symlink_farming_commit_hook.mdwn, ./doc/todo/tahoe_lfs_for_reals.mdwn, ./doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment, ./doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment, ./doc/todo/union_mounting.mdwn, ./doc/todo/use_cp_reflink.mdwn, ./doc/todo/using_url_backend.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment, ./doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/todo/wishlist:_swift_backend.mdwn, ./doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment, ./doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment, ./doc/transferring_data.mdwn, ./doc/trust.mdwn, ./doc/upgrades.mdwn, ./doc/upgrades/SHA_size.mdwn, ./doc/use_case/Alice.mdwn, ./doc/use_case/Bob.mdwn, ./doc/users.mdwn, ./doc/users/chrysn.mdwn, ./doc/users/fmarier.mdwn, ./doc/users/joey.mdwn, ./doc/walkthrough.mdwn, ./doc/walkthrough/Internet_Archive_via_S3.mdwn, ./doc/walkthrough/adding_a_remote.mdwn, ./doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment, ./doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment, ./doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment, ./doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment, ./doc/walkthrough/adding_files.mdwn, ./doc/walkthrough/backups.mdwn, ./doc/walkthrough/creating_a_repository.mdwn, ./doc/walkthrough/fsck:_verifying_your_data.mdwn, ./doc/walkthrough/fsck:_when_things_go_wrong.mdwn, ./doc/walkthrough/getting_file_content.mdwn, ./doc/walkthrough/migrating_data_to_a_new_backend.mdwn, ./doc/walkthrough/modifying_annexed_files.mdwn, ./doc/walkthrough/more.mdwn, ./doc/walkthrough/moving_file_content_between_repositories.mdwn, ./doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment, ./doc/walkthrough/recover_data_from_lost+found.mdwn, ./doc/walkthrough/removing_files.mdwn, ./doc/walkthrough/removing_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/renaming_files.mdwn, ./doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/untrusted_repositories.mdwn, ./doc/walkthrough/unused_data.mdwn, ./doc/walkthrough/using_Amazon_S3.mdwn, ./doc/walkthrough/using_bup.mdwn, ./doc/walkthrough/using_ssh_remotes.mdwn, ./doc/walkthrough/using_the_SHA1_backend.mdwn, ./doc/walkthrough/using_the_URL_backend.mdwn, ./doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn, ./git-annex, ./git-annex-shell, ./git-annex-shell.1, ./git-annex-shell.hs, ./git-annex.1, ./git-annex.cabal, ./git-annex.hs, ./git-union-merge, ./git-union-merge.1, ./git-union-merge.hs, ./html/GPL, ./html/backends.html, ./html/bare_repositories.html, ./html/bugs.html, ./html/bugs/Displayed_copy_speed_is_wrong.html, ./html/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.html, ./html/bugs/Makefile_is_missing_dependancies.html, ./html/bugs/Name_scheme_does_not_follow_git__39__s_rules.html, ./html/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.html, ./html/bugs/No_version_information_from_cli.html, ./html/bugs/Problems_running_make_on_osx.html, ./html/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.html, ./html/bugs/S3_memory_leaks.html, ./html/bugs/Unfortunate_interaction_with_Calibre.html, ./html/bugs/WORM:_Handle_long_filenames_correctly.html, ./html/bugs/add_range_argument_to___34__git_annex_dropunused__34___.html, ./html/bugs/annex_add_in_annex.html, ./html/bugs/backend_version_upgrade_leaves_repo_unusable.html, ./html/bugs/bare_git_repos.html, ./html/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.html, ./html/bugs/building_on_lenny.html, ./html/bugs/check_for_curl_in_configure.hs.html, ./html/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.html, ./html/bugs/conflicting_haskell_packages.html, ./html/bugs/copy_fast_confusing_with_broken_locationlog.html, ./html/bugs/done.html, ./html/bugs/dotdot_problem.html, ./html/bugs/dropping_files_with_a_URL_backend_fails.html, ./html/bugs/encrypted_S3_stalls.html, ./html/bugs/error_propigation.html, ./html/bugs/error_with_file_names_starting_with_dash.html, ./html/bugs/fat_support.html, ./html/bugs/free_space_checking.html, ./html/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.html, ./html/bugs/fsck_output.html, ./html/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.html, ./html/bugs/git-annex_directory_hashing_problems_on_osx.html, ./html/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.html, ./html/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.html, ./html/bugs/git_annex_copy_--fast_does_not_copy_files.html, ./html/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.html, ./html/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.html, ./html/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.html, ./html/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.html, ./html/bugs/git_annex_initremote_walks_.git-annex.html, ./html/bugs/git_annex_migrate_leaves_old_backend_versions_around.html, ./html/bugs/git_annex_should_use___39__git_add_-f__39___internally.html, ./html/bugs/git_annex_unlock_is_not_atomic.html, ./html/bugs/git_annex_unused_failes_on_empty_repository.html, ./html/bugs/git_annex_unused_seems_to_check_for_current_path.html, ./html/bugs/git_rename_detection_on_file_move.html, ./html/bugs/minor_bug:_errors_are_not_verbose_enough.html, ./html/bugs/ordering.html, ./html/bugs/problem_commit_normal_links.html, ./html/bugs/problems_with_utf8_names.html, ./html/bugs/scp_interrupt_to_background.html, ./html/bugs/softlink_mtime.html, ./html/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.html, ./html/bugs/tmp_file_handling.html, ./html/bugs/touch.hsc_has_problems_on_non-linux_based_systems.html, ./html/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.html, ./html/bugs/unannex_vs_unlock_hook_confusion.html, ./html/bugs/unhappy_without_UTF8_locale.html, ./html/bugs/upgrade_left_untracked_.git-annex__47____42___directories.html, ./html/bugs/weird_local_clone_confuses.html, ./html/cheatsheet.html, ./html/comments.html, ./html/contact.html, ./html/copies.html, ./html/design.html, ./html/design/encryption.html, ./html/distributed_version_control.html, ./html/download.html, ./html/encryption.html, ./html/feeds.html, ./html/forum.html, ./html/forum/Behaviour_of_fsck.html, ./html/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.html, ./html/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.html, ./html/forum/Need_new_build_instructions_for_Debian_stable.html, ./html/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.html, ./html/forum/OSX__39__s_haskell-platform_statically_links_things.html, ./html/forum/Problems_with_large_numbers_of_files.html, ./html/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.html, ./html/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.html, ./html/forum/__34__git_annex_lock__34___very_slow_for_big_repo.html, ./html/forum/bainstorming:_git_annex_push___38___pull.html, ./html/forum/batch_check_on_remote_when_using_copy.html, ./html/forum/can_git-annex_replace_ddm__63__.html, ./html/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.html, ./html/forum/git-annex_communication_channels.html, ./html/forum/git-annex_on_OSX.html, ./html/forum/hashing_objects_directories.html, ./html/forum/incompatible_versions__63__.html, ./html/forum/migrate_existing_git_repository_to_git-annex.html, ./html/forum/migration_to_git-annex_and_rsync.html, ./html/forum/new_microfeatures.html, ./html/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.html, ./html/forum/relying_on_git_for_numcopies.html, ./html/forum/rsync_over_ssh__63__.html, ./html/forum/seems_to_build_fine_on_haskell_platform_2011.html, ./html/forum/sparse_git_checkouts_with_annex.html, ./html/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.html, ./html/forum/unannex_alternatives.html, ./html/forum/wishlist:_command_options_changes.html, ./html/forum/wishlist:_define_remotes_that_must_have_all_files.html, ./html/forum/wishlist:_do_round_robin_downloading_of_data.html, ./html/forum/wishlist:_git-annex_replicate.html, ./html/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.html, ./html/forum/wishlist:_git_annex_status.html, ./html/forum/wishlist:_git_backend_for_git-annex.html, ./html/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.html, ./html/forum/wishlist:_special_remote_for_sftp_or_rsync.html, ./html/forum/wishlist:_support_for_more_ssh_urls_.html, ./html/forum/wishlist:_traffic_accounting_for_git-annex.html, ./html/forum/wishlist:alias_system.html, ./html/forum/working_without_git-annex_commits.html, ./html/future_proofing.html, ./html/git-annex-shell.html, ./html/git-annex.html, ./html/git-union-merge.html, ./html/ikiwiki/ikiwiki.js, ./html/ikiwiki/relativedate.js, ./html/ikiwiki/toggle.js, ./html/index.html, ./html/install.html, ./html/install/Debian.html, ./html/install/Fedora.html, ./html/install/FreeBSD.html, ./html/install/OSX.html, ./html/install/Ubuntu.html, ./html/internals.html, ./html/location_tracking.html, ./html/logo.png, ./html/logo_small.png, ./html/news.html, ./html/not.html, ./html/repomap.png, ./html/special_remotes.html, ./html/special_remotes/S3.html, ./html/special_remotes/bup.html, ./html/special_remotes/directory.html, ./html/special_remotes/hook.html, ./html/special_remotes/rsync.html, ./html/summary.html, ./html/templates/bare.tmpl, ./html/templates/walkthrough.tmpl, ./html/todo.html, ./html/todo/S3.html, ./html/todo/add_--exclude_option_to_git_annex_find.html, ./html/todo/add_a_git_backend.html, ./html/todo/auto_remotes.html, ./html/todo/auto_remotes/discussion.html, ./html/todo/backendSHA1.html, ./html/todo/branching.html, ./html/todo/cache_key_info.html, ./html/todo/checkout.html, ./html/todo/done.html, ./html/todo/file_copy_progress_bar.html, ./html/todo/fsck.html, ./html/todo/git-annex-shell.html, ./html/todo/git-annex_unused_eats_memory.html, ./html/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.html, ./html/todo/gitrm.html, ./html/todo/hidden_files.html, ./html/todo/immutable_annexed_files.html, ./html/todo/network_remotes.html, ./html/todo/object_dir_reorg_v2.html, ./html/todo/parallel_possibilities.html, ./html/todo/pushpull.html, ./html/todo/rsync.html, ./html/todo/smudge.html, ./html/todo/speed_up_fsck.html, ./html/todo/support-non-utf8-locales.html, ./html/todo/support_S3_multipart_uploads.html, ./html/todo/symlink_farming_commit_hook.html, ./html/todo/tahoe_lfs_for_reals.html, ./html/todo/union_mounting.html, ./html/todo/use_cp_reflink.html, ./html/todo/using_url_backend.html, ./html/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.html, ./html/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.html, ./html/todo/wishlist:___34__git_annex_add__34___multiple_processes.html, ./html/todo/wishlist:_support_for_more_ssh_urls_.html, ./html/todo/wishlist:_swift_backend.html, ./html/transferring_data.html, ./html/trust.html, ./html/upgrades.html, ./html/upgrades/SHA_size.html, ./html/use_case/Alice.html, ./html/use_case/Bob.html, ./html/users.html, ./html/users/chrysn.html, ./html/users/fmarier.html, ./html/users/joey.html, ./html/walkthrough.html, ./html/walkthrough/Internet_Archive_via_S3.html, ./html/walkthrough/adding_a_remote.html, ./html/walkthrough/adding_files.html, ./html/walkthrough/backups.html, ./html/walkthrough/creating_a_repository.html, ./html/walkthrough/fsck:_verifying_your_data.html, ./html/walkthrough/fsck:_when_things_go_wrong.html, ./html/walkthrough/getting_file_content.html, ./html/walkthrough/migrating_data_to_a_new_backend.html, ./html/walkthrough/modifying_annexed_files.html, ./html/walkthrough/more.html, ./html/walkthrough/moving_file_content_between_repositories.html, ./html/walkthrough/recover_data_from_lost+found.html, ./html/walkthrough/removing_files.html, ./html/walkthrough/removing_files:_When_things_go_wrong.html, ./html/walkthrough/renaming_files.html, ./html/walkthrough/transferring_files:_When_things_go_wrong.html, ./html/walkthrough/untrusted_repositories.html, ./html/walkthrough/unused_data.html, ./html/walkthrough/using_Amazon_S3.html, ./html/walkthrough/using_bup.html, ./html/walkthrough/using_ssh_remotes.html, ./html/walkthrough/using_the_SHA1_backend.html, ./html/walkthrough/using_the_URL_backend.html, ./html/walkthrough/what_to_do_when_you_lose_a_repository.html, ./mdwn2man, ./test.hs, ./testdata/unicode-test-ö
 
 Executable git-annex
   Main-Is: git-annex.hs

From 6ba866ca738d6a63858916f84979cfd346bcb403 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 15:39:30 -0400
Subject: [PATCH 1976/2835] updates for web remote and removing URL backend

---
 debian/control                              |  1 +
 doc/backends.mdwn                           |  4 ----
 doc/cheatsheet.mdwn                         |  1 -
 doc/git-annex.mdwn                          | 14 ++++--------
 doc/install.mdwn                            |  1 +
 doc/walkthrough/using_the_SHA1_backend.mdwn |  2 +-
 doc/walkthrough/using_the_URL_backend.mdwn  | 24 ---------------------
 git-annex.cabal                             |  4 ++--
 8 files changed, 9 insertions(+), 42 deletions(-)
 delete mode 100644 doc/walkthrough/using_the_URL_backend.mdwn

diff --git a/debian/control b/debian/control
index 8c533af114..07a9e4bbb2 100644
--- a/debian/control
+++ b/debian/control
@@ -10,6 +10,7 @@ Build-Depends:
 	libghc-sha-dev,
 	libghc-dataenc-dev,
 	libghc-utf8-string-dev,
+	libghc-curl-dev,
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
 	ikiwiki,
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 4290da33ba..03502eaa18 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -27,10 +27,6 @@ these backends.
   part of the key. Useful for archival tasks where the filename extension
   contains metadata that should be preserved.
 
-These backends store file contents in other key/value stores.
-
-* `URL` -- This backend downloads the file's content from an external URL.
-
 The `annex.backends` git-config setting can be used to list the backends
 git-annex should use. The first one listed will be used by default when
 new files are added.
diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 3f5960972d..e88efc0ab0 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -5,7 +5,6 @@ A suppliment to the [[walkthrough]].
 [[!inline feeds=no show=0 template=walkthrough pagenames="""
 	walkthrough/using_Amazon_S3
 	walkthrough/using_bup
-	walkthrough/using_the_URL_backend
 	walkthrough/using_the_SHA1_backend
 	walkthrough/migrating_data_to_a_new_backend
 	walkthrough/untrusted_repositories
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a5ddf3172a..92fd824543 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -277,13 +277,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * fromkey file
 
-  This can be used to manually set up a file to link to a specified key
-  in the key-value backend. How you determine an existing key in the backend
-  varies. For the URL backend, the key is based on an URL to the content.
-
-  Example:
-
-	git annex fromkey --key=URL--http://www.archive.org/somefile somefile
+  This plumbing-level command can be used to manually set up a file
+  to link to a specified key in the key-value backend.
 
 * dropkey [key ...]
 
@@ -299,8 +294,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * setkey file
 
-  This plumbing-level command sets the annexed data for a key to the content of
-  the specified file, and then removes the file.
+  This plumbing-level command sets the annexed data for a key to the
+  content of the specified file, and then removes the file.
 
   Example:
 
@@ -402,7 +397,6 @@ Here are all the supported configuration settings.
 
   Space-separated list of names of the key-value backends to use. 
   The first listed is used to store new files by default. 
-  (default: "WORM SHA1 URL")
 
 * `remote..annex-cost`
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 1a83c2fd63..8a5e8d5ecd 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -19,6 +19,7 @@ To build and use git-annex, you will need:
   * [dataenc](http://hackage.haskell.org/package/dataenc)
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
+  * [curl](http://hackage.haskell.org/package/curl)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
 * Shell commands
   * [git](http://git-scm.com/)
diff --git a/doc/walkthrough/using_the_SHA1_backend.mdwn b/doc/walkthrough/using_the_SHA1_backend.mdwn
index c04729e2cd..70dc2ef759 100644
--- a/doc/walkthrough/using_the_SHA1_backend.mdwn
+++ b/doc/walkthrough/using_the_SHA1_backend.mdwn
@@ -1,4 +1,4 @@
-Another handy alternative to the default [[backend|backends]] is the
+A handy alternative to the default [[backend|backends]] is the
 SHA1 backend. This backend provides more git-style assurance that your data
 has not been damaged. And the checksum means that when you add the same
 content to the annex twice, only one copy need be stored in the backend.
diff --git a/doc/walkthrough/using_the_URL_backend.mdwn b/doc/walkthrough/using_the_URL_backend.mdwn
deleted file mode 100644
index 50da4dad89..0000000000
--- a/doc/walkthrough/using_the_URL_backend.mdwn
+++ /dev/null
@@ -1,24 +0,0 @@
-git-annex has multiple key-value [[backends]]. So far you have been using
-the default, WORM (Write Once, Read Many) backend.
-
-Another handy backend is the URL backend, which can fetch file's content
-from remote URLs. Here's how to set up some files in your repository
-that use this backend:
-
-	# git annex fromkey --key=URL--http://www.archive.org/somefile somefile
-	fromkey somefile ok
-	# git commit -m "added a file from the Internet Archive"
-
-Now you if you ask git-annex to get that file, it will download it, 
-and cache it locally.
-
-	# git annex get somefile
-	get somefile (downloading)
-	#########################################################################100.0%
-	ok
-
-You can always drop files downloaded by the URL backend. It is assumed
-that the URL is stable; no local backup is kept.
-
-	# git annex drop somefile
-	drop somefile (ok)
diff --git a/git-annex.cabal b/git-annex.cabal
index a4b8aef993..fab2c71b96 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -29,14 +29,14 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, curl,
    base < 5
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, curl,
    base < 5
 
 Executable git-union-merge

From 2cdacfbae6519eceed2d5dcbea052de244a0b8ec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 16:00:44 -0400
Subject: [PATCH 1977/2835] remove URL backend

---
 Backend/URL.hs | 61 --------------------------------------------------
 BackendList.hs |  2 --
 debian/NEWS    |  6 +++++
 3 files changed, 6 insertions(+), 63 deletions(-)
 delete mode 100644 Backend/URL.hs

diff --git a/Backend/URL.hs b/Backend/URL.hs
deleted file mode 100644
index e41004dd46..0000000000
--- a/Backend/URL.hs
+++ /dev/null
@@ -1,61 +0,0 @@
-{- git-annex "URL" backend
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.URL (backends) where
-
-import Control.Monad.State (liftIO)
-
-import Types
-import Types.Backend
-import Utility
-import Messages
-import Types.Key
-
-backends :: [Backend Annex]
-backends = [backend]
-
-backend :: Backend Annex
-backend = Backend {
-	name = "URL",
-	getKey = keyValue,
-	storeFileKey = dummyStore,
-	retrieveKeyFile = downloadUrl,
-	-- allow keys to be removed; presumably they can always be
-	-- downloaded again
-	removeKey = dummyRemove,
-	-- similarly, keys are always assumed to be out there on the web
-	hasKey = dummyOk,
-	-- and nothing needed to fsck
-	fsckKey = dummyFsck,
-	-- and key upgrade not needed
-	upgradableKey = \_ -> return False
-}
-
--- cannot generate url from filename
-keyValue :: FilePath -> Annex (Maybe Key)
-keyValue _ = return Nothing
-
--- cannot change url contents
-dummyStore :: FilePath -> Key -> Annex Bool
-dummyStore _ _ = return False
-
-dummyRemove :: Key -> Maybe a -> Annex Bool
-dummyRemove _ _ = return True
-
-dummyFsck :: Key -> Maybe FilePath -> Maybe a -> Annex Bool
-dummyFsck _ _ _ = return True
-
-dummyOk :: Key -> Annex Bool
-dummyOk _ = return True
-
-downloadUrl :: Key -> FilePath -> Annex Bool
-downloadUrl key file = do
-	showNote $ "downloading"
-	showProgress -- make way for curl progress bar
-	liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
-	where
-		url = keyName key
diff --git a/BackendList.hs b/BackendList.hs
index bc3fd83142..e4e1d76fe2 100644
--- a/BackendList.hs
+++ b/BackendList.hs
@@ -10,12 +10,10 @@ module BackendList (allBackends) where
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
 import qualified Backend.SHA
-import qualified Backend.URL
 import Types
 
 allBackends :: [Backend Annex]
 allBackends = concat 
 	[ Backend.WORM.backends
 	, Backend.SHA.backends
-	, Backend.URL.backends
 	]
diff --git a/debian/NEWS b/debian/NEWS
index 22835ace84..ad4e946e6c 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,3 +1,9 @@
+git-annex (3.20110702) unstable; urgency=low
+
+  The URL backend has been removed. Instead the new web remote can be used.
+
+ -- Joey Hess   Fri, 01 Jul 2011 15:40:51 -0400
+
 git-annex (3.20110624) exerimental; urgency=low
 
   There has been another change to the git-annex data store.

From a140f7148f3ea0bef2d8c060c7847b3d1be4d25e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 16:01:26 -0400
Subject: [PATCH 1978/2835] documentation for using the web

---
 debian/changelog                   |  3 ++-
 doc/cheatsheet.mdwn                |  1 +
 doc/git-annex.mdwn                 |  4 ++++
 doc/special_remotes.mdwn           |  1 +
 doc/special_remotes/web.mdwn       |  7 +++++++
 doc/walkthrough/using_the_web.mdwn | 32 ++++++++++++++++++++++++++++++
 6 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 doc/special_remotes/web.mdwn
 create mode 100644 doc/walkthrough/using_the_web.mdwn

diff --git a/debian/changelog b/debian/changelog
index 53fc99e8c3..b4a77de7dc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,8 @@
-git-annex (3.20110625) UNRELEASED; urgency=low
+git-annex (3.20110702) UNRELEASED; urgency=low
 
   * Now the web can be used as a special remote. This feature
     replaces the old URL backend.
+  * addurl: New command to download an url and store it in the annex.
   * Sped back up fsck, copy --from, and other commands that often
     have to read a lot of information from the git-annex branch. Such
     commands are now faster than they were before introduction of the
diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index e88efc0ab0..9f7c146c82 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -5,6 +5,7 @@ A suppliment to the [[walkthrough]].
 [[!inline feeds=no show=0 template=walkthrough pagenames="""
 	walkthrough/using_Amazon_S3
 	walkthrough/using_bup
+	walkthrough/using_the_web
 	walkthrough/using_the_SHA1_backend
 	walkthrough/migrating_data_to_a_new_backend
 	walkthrough/untrusted_repositories
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 92fd824543..c450887488 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -275,6 +275,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Returns a repository to the default semi trusted state.
 
+* addurl [url ...]
+
+  Downloads each url to a file, which is added to the annex.
+
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 12e0aedb1b..afc6a2cf23 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -10,6 +10,7 @@ They cannot be used by other git commands though.
 * [[bup]]
 * [[directory]]
 * [[rsync]]
+* [[web]]
 * [[hook]]
 * [[tahoe-lafs|forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] - limited testing
 
diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
new file mode 100644
index 0000000000..68df31ef4d
--- /dev/null
+++ b/doc/special_remotes/web.mdwn
@@ -0,0 +1,7 @@
+git-annex can use the WWW as a special remote, downloading urls to files.
+See [[walkthrough/using_web_web]] for usage examples.
+
+## notes
+
+Currently git-annex only supports downloading content from the web; 
+it cannot upload to it or remove content.
diff --git a/doc/walkthrough/using_the_web.mdwn b/doc/walkthrough/using_the_web.mdwn
new file mode 100644
index 0000000000..9d5525758a
--- /dev/null
+++ b/doc/walkthrough/using_the_web.mdwn
@@ -0,0 +1,32 @@
+The web can be used as a [[special_remote|special_remotes]] too.
+
+	# git annex addurl http://example.com/video.mpeg
+	addurl video.mpeg (downloading http://example.com/video.mpeg)
+	########################################################## 100.0%
+	ok
+
+Now the file is downloaded, and has been added to the annex like any other
+file. So it can be copied to other repositories, and so on.
+
+Note that git-annex assumes that, if the web site does not 404, the file is
+still present on the web, and this counts as one [[copy|copies]] of the
+file. So it will let you remove your last copy, trusting it can be
+downloaded again:
+
+	# git annex drop video.mpeg
+	drop video.mpeg (checking http://example.com/video.mpeg) ok
+
+If you don't [[trust]] the web to this degree, just let git-annex know:
+
+	# git annex untrust web
+	untrust web ok
+
+With the result that it will hang onto files:
+
+	# git annex drop video.mpeg
+	drop video.mpeg (unsafe) 
+	  Could only verify the existence of 0 out of 1 necessary copies
+	  Also these untrusted repositories may contain the file:
+	  	00000000-0000-0000-0000-000000000001  -- web
+	  (Use --force to override this check, or adjust annex.numcopies.)
+	failed

From 6bddebdb79ca8ed168e143d533a6101c7d469628 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:15:46 -0400
Subject: [PATCH 1979/2835] add the addurl command

---
 Command/AddUrl.hs | 74 +++++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs       |  2 ++
 LocationLog.hs    |  5 +---
 PresenceLog.hs    |  6 ++++
 Remote/Web.hs     | 69 ++++++++++++++++++++++++++-----------------
 5 files changed, 125 insertions(+), 31 deletions(-)
 create mode 100644 Command/AddUrl.hs

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
new file mode 100644
index 0000000000..713a486a5c
--- /dev/null
+++ b/Command/AddUrl.hs
@@ -0,0 +1,74 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.AddUrl where
+
+import Control.Monad.State (liftIO, when)
+import Network.URI
+import Data.String.Utils
+import System.Directory
+
+import Command
+import qualified Backend
+import qualified Remote.Web
+import qualified Command.Add
+import Messages
+import Content
+import PresenceLog
+
+command :: [Command]
+command = [repoCommand "addurl" paramPath seek "add urls to annex"]
+
+seek :: [CommandSeek]
+seek = [withStrings start]
+
+start :: CommandStartString
+start s = do
+	let u = parseURI s
+	case u of
+		Nothing -> error $ "bad url " ++ s
+		Just url -> do
+			file <- liftIO $ url2file url
+			showStart "addurl" file
+			next $ perform s file
+			
+perform :: String -> FilePath -> CommandPerform
+perform url file = do
+	[(_, backend)] <- Backend.chooseBackends [file]
+	showNote $ "downloading " ++ url
+	ok <- Remote.Web.download file [url]
+	if ok
+		then do
+			stored <- Backend.storeFileKey file backend
+			case stored of
+				Nothing -> stop
+				Just (key, _) -> do
+					moveAnnex key file
+					Remote.Web.setUrl key url InfoPresent
+					next $ Command.Add.cleanup file key
+		else stop
+
+url2file :: URI -> IO FilePath
+url2file url = do
+	let parts = filter safe $ split "/" $ uriPath url
+	if null parts
+		then fallback
+		else do
+			let file = last parts
+			e <- doesFileExist file
+			if e then fallback else return file
+	where
+		fallback = do
+			let file = replace "/" "_" $ show url
+			e <- doesFileExist file
+			when e $ error "already have this url"
+			return file
+		safe s
+			| null s = False
+			| s == "." = False
+			| s == ".." = False
+			| otherwise = True
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 58b512f718..85eb2bf26e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -46,6 +46,7 @@ import qualified Command.Uninit
 import qualified Command.Trust
 import qualified Command.Untrust
 import qualified Command.Semitrust
+import qualified Command.AddUrl
 import qualified Command.Map
 import qualified Command.Upgrade
 import qualified Command.Version
@@ -68,6 +69,7 @@ cmds = concat
 	, Command.Trust.command
 	, Command.Untrust.command
 	, Command.Semitrust.command
+	, Command.AddUrl.command
 	, Command.FromKey.command
 	, Command.DropKey.command
 	, Command.SetKey.command
diff --git a/LocationLog.hs b/LocationLog.hs
index a5db7d1218..19a8eb83a1 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -37,10 +37,7 @@ logChange repo key u s = do
 	when (null u) $
 		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
 			" (have you run git annex init there?)"
-	line <- logNow s u
-	let f = logFile key
-	ls <- readLog f
-	writeLog f (compactLog $ line:ls)
+	addLog (logFile key) =<< logNow s u
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 71d78f1eda..0777db209b 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -13,6 +13,7 @@
 
 module PresenceLog (
 	LogStatus(..),
+	addLog,
 	readLog,
 	writeLog,
 	logNow,
@@ -70,6 +71,11 @@ instance Read LogLine where
 			bad = ret $ LogLine 0 Undefined ""
 			ret v = [(v, "")]
 
+addLog :: FilePath -> LogLine -> Annex ()
+addLog file line = do
+	ls <- readLog file
+	writeLog file (compactLog $ line:ls)
+
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 201f923cf7..342acef918 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -6,7 +6,9 @@
  -}
 
 module Remote.Web (
-	remote
+	remote,
+	setUrl,
+	download
 ) where
 
 import Control.Monad.State (liftIO)
@@ -20,11 +22,13 @@ import Network.Curl.Code
 import Types
 import Types.Remote
 import qualified Git
+import qualified Annex
 import Messages
 import Utility
 import UUID
 import Config
 import PresenceLog
+import LocationLog
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -50,10 +54,10 @@ gen r _ _ =
 		uuid = webUUID,
 		cost = expensiveRemoteCost,
 		name = Git.repoDescribe r,
-		storeKey = upload,
-		retrieveKeyFile = download,
-		removeKey = remove,
-		hasKey = check,
+		storeKey = uploadKey,
+		retrieveKeyFile = downloadKey,
+		removeKey = dropKey,
+		hasKey = checkKey,
 		hasKeyCheap = False,
 		config = Nothing
 	}
@@ -62,40 +66,44 @@ gen r _ _ =
 urlLog :: Key -> FilePath
 urlLog key = "remote/web"  show key ++ ".log"
 
-urls :: Key -> Annex [URLString]
-urls key = currentLog (urlLog key)
+getUrls :: Key -> Annex [URLString]
+getUrls key = currentLog (urlLog key)
 
-download :: Key -> FilePath -> Annex Bool
-download key file = download' file =<< urls key
-download' :: FilePath -> [URLString] -> Annex Bool
-download' _ [] = return False
-download' file (url:us) = do
-	showProgress -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
-	if ok then return ok else download' file us
+{- Records a change in an url for a key. -}
+setUrl :: Key -> URLString -> LogStatus -> Annex ()
+setUrl key url status = do
+	g <- Annex.gitRepo
+	addLog (urlLog key) =<< logNow status url
 
-upload :: Key -> Annex Bool
-upload _ = do
+	-- update location log to indicate that the web has the key, or not
+	us <- getUrls key
+	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
+
+downloadKey :: Key -> FilePath -> Annex Bool
+downloadKey key file = download file =<< getUrls key
+
+uploadKey :: Key -> Annex Bool
+uploadKey _ = do
 	warning "upload to web not supported"
 	return False
 
-remove :: Key -> Annex Bool
-remove _ = do
+dropKey :: Key -> Annex Bool
+dropKey _ = do
 	warning "removal from web not supported"
 	return False
 
-check :: Key -> Annex (Either IOException Bool)
-check key = do
-	us <- urls key
+checkKey :: Key -> Annex (Either IOException Bool)
+checkKey key = do
+	us <- getUrls key
 	if null us
 		then return $ Right False
-		else return . Right =<< check' us
-check' :: [URLString] -> Annex Bool
-check' [] = return False
-check' (u:us) = do
+		else return . Right =<< checkKey' us
+checkKey' :: [URLString] -> Annex Bool
+checkKey' [] = return False
+checkKey' (u:us) = do
 	showNote ("checking " ++ u)
 	e <- liftIO $ urlexists u
-	if e then return e else check' us
+	if e then return e else checkKey' us
 
 urlexists :: URLString -> IO Bool
 urlexists url = do
@@ -105,3 +113,10 @@ urlexists url = do
 	_ <- setopt curl (CurlFailOnError True)
 	res <- perform curl
 	return $ res == CurlOK
+
+download :: FilePath -> [URLString] -> Annex Bool
+download _ [] = return False
+download file (url:us) = do
+	showProgress -- make way for curl progress bar
+	ok <- liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
+	if ok then return ok else download file us

From fb58d1a560f7c4c94826ec63de16e0276d1f17f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:17:02 -0400
Subject: [PATCH 1980/2835] wording

---
 Backend/SHA.hs    | 2 +-
 doc/backends.mdwn | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 8ed00b7073..8930e4b938 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -1,4 +1,4 @@
-{- git-annex SHA abstract backend
+{- git-annex SHA backend
  -
  - Copyright 2011 Joey Hess 
  -
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 03502eaa18..9e698032d8 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -9,7 +9,7 @@ to retrieve the file's content (its value).
 Multiple pluggable backends are supported, and a single repository
 can use different backends for different files.
 
-These backends can transfer file contents in configured git remotes.
+These backends can transfer file contents between configured git remotes.
 It's also possible to use [[special_remotes]], such as Amazon S3 with
 these backends.
 

From 79016c197ca87182dfae9f6dfb994ff5079fc952 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:23:01 -0400
Subject: [PATCH 1981/2835] add hashing to web log files

---
 LocationLog.hs | 17 ++++++++++++++++-
 Locations.hs   | 14 --------------
 Remote/Web.hs  |  6 ++++--
 Upgrade/V2.hs  |  2 +-
 4 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index 19a8eb83a1..eb48b7916a 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -17,7 +17,9 @@ module LocationLog (
 	readLog,
 	writeLog,
 	keyLocations,
-	loggedKeys
+	loggedKeys,
+	logFile,
+	logFileKey	
 ) where
 
 import System.FilePath
@@ -28,6 +30,7 @@ import qualified Git
 import qualified Branch
 import UUID
 import Types
+import Types.Key
 import Locations
 import PresenceLog
 
@@ -49,3 +52,15 @@ keyLocations key = currentLog $ logFile key
 loggedKeys :: Annex [Key]
 loggedKeys =
 	return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files
+
+{- The filename of the log file for a given key. -}
+logFile :: Key -> String
+logFile key = hashDirLower key ++ keyFile key ++ ".log"
+
+{- Converts a log filename into a key. -}
+logFileKey :: FilePath -> Maybe Key
+logFileKey file
+	| end == ".log" = readKey beginning
+	| otherwise = Nothing
+	where
+		(beginning, end) = splitAt (length file - 4) file
diff --git a/Locations.hs b/Locations.hs
index 6142b2393a..347b08ce14 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -19,8 +19,6 @@ module Locations (
 	gitAnnexUnusedLog,
 	gitAnnexJournalDir,
 	isLinkToAnnex,
-	logFile,
-	logFileKey,
 	hashDirMixed,
 	hashDirLower,
 
@@ -115,18 +113,6 @@ gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r  "journal"
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
-{- The filename of the log file for a given key. -}
-logFile :: Key -> String
-logFile key = hashDirLower key ++ keyFile key ++ ".log"
-
-{- Converts a log filename into a key. -}
-logFileKey :: FilePath -> Maybe Key
-logFileKey file
-	| end == ".log" = readKey beginning
-	| otherwise = Nothing
-	where
-		(beginning, end) = splitAt (length file - 4) file
-
 {- Converts a key into a filename fragment.
  -
  - Escape "/" in the key name, to keep a flat tree of files and avoid
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 342acef918..7425dec87e 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -29,6 +29,7 @@ import UUID
 import Config
 import PresenceLog
 import LocationLog
+import Locations
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -62,9 +63,10 @@ gen r _ _ =
 		config = Nothing
 	}
 
-{- The urls for a key are stored in remote/web/key.log in the git-annex branch. -}
+{- The urls for a key are stored in remote/web/hash/key.log 
+ - in the git-annex branch. -}
 urlLog :: Key -> FilePath
-urlLog key = "remote/web"  show key ++ ".log"
+urlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
 
 getUrls :: Key -> Annex [URLString]
 getUrls key = currentLog (urlLog key)
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index ce0424b302..14e328edb4 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -20,7 +20,7 @@ import qualified Git
 import qualified Branch
 import Messages
 import Utility
-import Locations
+import LocationLog
 import Content
 
 olddir :: Git.Repo -> FilePath

From 457d28c676f48d61c523df74d665040e24132ed7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:24:11 -0400
Subject: [PATCH 1982/2835] wording

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index b4a77de7dc..4be7e674a6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,7 @@
 git-annex (3.20110702) UNRELEASED; urgency=low
 
-  * Now the web can be used as a special remote. This feature
-    replaces the old URL backend.
+  * Now the web can be used as a special remote. 
+    This feature replaces the old URL backend.
   * addurl: New command to download an url and store it in the annex.
   * Sped back up fsck, copy --from, and other commands that often
     have to read a lot of information from the git-annex branch. Such

From 5d154b84365770ff27c09137ad039f930505ebfd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:28:31 -0400
Subject: [PATCH 1983/2835] document web special remote log files

---
 doc/internals.mdwn | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 1096845e5e..a4ec5c417b 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -72,7 +72,7 @@ Example:
 
 ## `aaa/bbb/*.log`
 
-The remainder of the log files record [[location_tracking]] information
+These log files record [[location_tracking]] information
 for file contents. Again these are placed in two levels of subdirectories
 for hashing. The name of the key is the filename, and the content
 consists of a timestamp, either 1 (present) or 0 (not present), and
@@ -85,3 +85,9 @@ Example:
 
 These files are designed to be auto-merged using git's union merge driver.
 The timestamps allow the most recent information to be identified.
+
+## `remote/web/aaa/bbb/*.log`
+
+These log files record urls used by the
+[[web_special_remote|special_remotes/web]]. Their format is similar
+to the location tracking files, but with urls rather than UUIDs.

From ace9de37e8db9c99db4b121392ca63091e48bfac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 18:46:07 -0400
Subject: [PATCH 1984/2835] download urls via tmp file, and support resuming

---
 Command/AddUrl.hs                                | 16 ++++++++++++----
 Remote/Web.hs                                    | 14 ++++++++------
 ...t_2_80b9e848edfdc7be21baab7d0cef0e3a._comment |  3 +++
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 713a486a5c..ebf0810bae 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -16,9 +16,13 @@ import Command
 import qualified Backend
 import qualified Remote.Web
 import qualified Command.Add
+import qualified Annex
 import Messages
 import Content
 import PresenceLog
+import Types.Key
+import Locations
+import Utility
 
 command :: [Command]
 command = [repoCommand "addurl" paramPath seek "add urls to annex"]
@@ -38,16 +42,20 @@ start s = do
 			
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
-	[(_, backend)] <- Backend.chooseBackends [file]
+	g <- Annex.gitRepo
 	showNote $ "downloading " ++ url
-	ok <- Remote.Web.download file [url]
+	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }
+	let tmp = gitAnnexTmpLocation g dummykey
+	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	ok <- Remote.Web.download [url] tmp
 	if ok
 		then do
-			stored <- Backend.storeFileKey file backend
+			[(_, backend)] <- Backend.chooseBackends [file]
+			stored <- Backend.storeFileKey tmp backend
 			case stored of
 				Nothing -> stop
 				Just (key, _) -> do
-					moveAnnex key file
+					moveAnnex key tmp
 					Remote.Web.setUrl key url InfoPresent
 					next $ Command.Add.cleanup file key
 		else stop
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 7425dec87e..304f191d31 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -82,7 +82,9 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = download file =<< getUrls key
+downloadKey key file = do
+	us <- getUrls key
+	download us file
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
@@ -116,9 +118,9 @@ urlexists url = do
 	res <- perform curl
 	return $ res == CurlOK
 
-download :: FilePath -> [URLString] -> Annex Bool
-download _ [] = return False
-download file (url:us) = do
+download :: [URLString] -> FilePath -> Annex Bool
+download [] _ = return False
+download (url:us) file = do
 	showProgress -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
-	if ok then return ok else download file us
+	ok <- liftIO $ boolSystem "curl" [Params "-C - -# -o", File file, File url]
+	if ok then return ok else download us file
diff --git a/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
index a32461615f..6dba86c47c 100644
--- a/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
+++ b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
@@ -7,4 +7,7 @@
 Whoops! You'd only told me O(N) twice before..
 
 So this is not too high priority. I think I would like to get the per-remote storage sorted out anyway, since probably it will be the thing needed to convert the URL backend into a special remote, which would then allow ripping out the otherwise unused pluggable backend infrastructure.
+
+Update: Per-remote storage is now sorted out, so this could be implemented
+if it actually made sense to do so.
 """]]

From e6b9539a655f15c402aad3f5fa3690c8e4fb06aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 21:52:27 -0400
Subject: [PATCH 1985/2835] make curl follow redirs

---
 Remote/Web.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 304f191d31..71591b7aa0 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -115,6 +115,7 @@ urlexists url = do
 	_ <- setopt curl (CurlURL url)
 	_ <- setopt curl (CurlNoBody True)
 	_ <- setopt curl (CurlFailOnError True)
+	_ <- setopt curl (CurlFollowLocation True)
 	res <- perform curl
 	return $ res == CurlOK
 
@@ -122,5 +123,5 @@ download :: [URLString] -> FilePath -> Annex Bool
 download [] _ = return False
 download (url:us) file = do
 	showProgress -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-C - -# -o", File file, File url]
+	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
 	if ok then return ok else download us file

From 80459918307730f9abc9c6b681d4888758b6d522 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 14:40:57 -0400
Subject: [PATCH 1986/2835] better cabal command

(which will work once I upload to hackage)
---
 doc/install.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 8a5e8d5ecd..25eb8bedfb 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -39,5 +39,4 @@ Then just [[download]] git-annex and run: `make; make install`
 
 As a haskell package, git-annex can be built using cabal. For example:
 
-	cabal configure
-	cabal install --bindir=$HOME/bin
+	cabal install git-annex --bindir=$HOME/bin

From 648827861024783ef6fa9c03e8ca5210c68fc7cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 14:53:55 -0400
Subject: [PATCH 1987/2835] cabal sdist: plan C

---
 Makefile                |  8 ++++++++
 configure.hs            | 14 --------------
 git-annex.cabal         |  1 +
 testdata/unicode-test-ö |  1 -
 4 files changed, 9 insertions(+), 15 deletions(-)
 delete mode 100644 testdata/unicode-test-ö

diff --git a/Makefile b/Makefile
index 0bc02dc352..afd2de99e0 100644
--- a/Makefile
+++ b/Makefile
@@ -87,4 +87,12 @@ clean:
 	rm -rf doc/.ikiwiki html dist
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
+# Workaround for cabal sdist not running Setup hooks, so I cannot
+# generate a file list there.
+sdist: clean
+	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi
+	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -type f -print)!i" < git-annex.cabal.orig > git-annex.cabal
+	@cabal sdist
+	@mv git-annex.cabal.orig git-annex.cabal
+
 .PHONY: $(bins) test install
diff --git a/configure.hs b/configure.hs
index d47dddf1df..8639af44b1 100644
--- a/configure.hs
+++ b/configure.hs
@@ -17,7 +17,6 @@ tests =
 	, TestCase "curl" $ testCmd "curl" "curl --version >/dev/null"
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
-	, TestCase "unicode FilePath support" $ unicodeFilePath
 	] ++ shaTestCases [1, 256, 512, 224, 384]
 
 shaTestCases :: [Int] -> [TestCase]
@@ -40,19 +39,6 @@ testCp k option = TestCase cmd $ testCmd k run
 		cmd = "cp " ++ option
 		run = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new"
 
-{- Checks if FilePaths contain decoded unicode, or not. The testdata
- - directory contains a "unicode-test-ü" file; try to find the file,
- - and see if the "ü" is encoded correctly.
- -
- - Note that the file is shipped with git-annex, rather than created,
- - to avoid other potential unicode issues.
- -}
-unicodeFilePath :: Test
-unicodeFilePath = do
-	fs <- getDirectoryContents "testdata"
-	let file = head $ filter (isInfixOf "unicode-test") fs
-	return $ Config "unicodefilepath" (BoolConfig $ isInfixOf "ü" file)
-
 {- Pulls package version out of the changelog. -}
 getVersion :: Test
 getVersion = do
diff --git a/git-annex.cabal b/git-annex.cabal
index fab2c71b96..59220f47b3 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -7,6 +7,7 @@ Author: Joey Hess
 Stability: Stable
 Copyright: 2010-2011 Joey Hess
 License-File: GPL
+Extra-Source-Files: use-make-sdist-instead
 Homepage: http://git-annex.branchable.com/
 Build-type: Custom
 Category: Utility
diff --git a/testdata/unicode-test-ö b/testdata/unicode-test-ö
deleted file mode 100644
index 45b983be36..0000000000
--- a/testdata/unicode-test-ö
+++ /dev/null
@@ -1 +0,0 @@
-hi

From 48db40857cfc84454ae0121b8d014e3c5b17e753 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 15:08:05 -0400
Subject: [PATCH 1988/2835] releasing version 3.20110702

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4be7e674a6..266a747ee0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110702) UNRELEASED; urgency=low
+git-annex (3.20110702) unstable; urgency=low
 
   * Now the web can be used as a special remote. 
     This feature replaces the old URL backend.
@@ -17,7 +17,7 @@ git-annex (3.20110702) UNRELEASED; urgency=low
     does not run the test suite, and is not particularly recommended,
     but could be useful to some.
 
- -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
+ -- Joey Hess   Sat, 02 Jul 2011 15:00:18 -0400
 
 git-annex (3.20110624) experimental; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 59220f47b3..54a526160e 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110625
+Version: 3.20110702
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 791dfaac65434cf6b6c9423e4ec3ff3252a71b13 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 15:08:49 -0400
Subject: [PATCH 1989/2835] add news item for git-annex 3.20110702

---
 doc/news/version_0.20110521.mdwn |  5 -----
 doc/news/version_3.20110702.mdwn | 22 ++++++++++++++++++++++
 2 files changed, 22 insertions(+), 5 deletions(-)
 delete mode 100644 doc/news/version_0.20110521.mdwn
 create mode 100644 doc/news/version_3.20110702.mdwn

diff --git a/doc/news/version_0.20110521.mdwn b/doc/news/version_0.20110521.mdwn
deleted file mode 100644
index f64392a760..0000000000
--- a/doc/news/version_0.20110521.mdwn
+++ /dev/null
@@ -1,5 +0,0 @@
-git-annex 0.20110521 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * status: New subcommand to show info about an annex, including its size.
-   * --backend now overrides any backend configured in .gitattributes files.
-   * Add --debug option. Closes: #[627499](http://bugs.debian.org/627499)"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110702.mdwn b/doc/news/version_3.20110702.mdwn
new file mode 100644
index 0000000000..0bb3adf2d8
--- /dev/null
+++ b/doc/news/version_3.20110702.mdwn
@@ -0,0 +1,22 @@
+News for git-annex 3.20110702:
+
+   The URL backend has been removed. Instead the new web remote can be used.
+
+git-annex 3.20110702 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Now the web can be used as a special remote.
+     This feature replaces the old URL backend.
+   * addurl: New command to download an url and store it in the annex.
+   * Sped back up fsck, copy --from, and other commands that often
+     have to read a lot of information from the git-annex branch. Such
+     commands are now faster than they were before introduction of the
+     git-annex branch.
+   * Always ensure git-annex branch exists.
+   * Modify location log parser to allow future expansion.
+   * --force will cause add, etc, to operate on ignored files.
+   * Avoid mangling encoding when storing the description of repository
+     and other content.
+   * cabal can now be used to build git-annex. This is substantially
+     slower than using make, does not build or install documentation,
+     does not run the test suite, and is not particularly recommended,
+     but could be useful to some."""]]
\ No newline at end of file

From 17a2b13e648c70db768a778d3b29ab933e8a39c2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 15:11:20 -0400
Subject: [PATCH 1990/2835] formatting

---
 doc/news/version_3.20110624.mdwn | 23 ++++++++++++-----------
 doc/news/version_3.20110702.mdwn |  4 ++--
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/doc/news/version_3.20110624.mdwn b/doc/news/version_3.20110624.mdwn
index 5a91fa4295..6204673bd2 100644
--- a/doc/news/version_3.20110624.mdwn
+++ b/doc/news/version_3.20110624.mdwn
@@ -1,15 +1,16 @@
 News for git-annex 3.20110624:
 
-   There has been another change to the git-annex data store.
-   Use `git annex upgrade` to migrate your repositories to the new
-   layout. See <http://git-annex.branchable.com/upgrades/> or
-   /usr/share/doc/git-annex/html/upgrades.html
-   The significant change this time is that the .git-annex/ directory
-   is gone; instead there is a git-annex branch that is automatically
-   maintained by git-annex, and encapsulates all its state nicely out
-   of your way.
-   You should make sure you include the git-annex branch when
-   git pushing and pulling.
+There has been another change to the git-annex data store.
+Use `git annex upgrade` to migrate your repositories to the new
+layout. See [[upgrades]].
+
+The significant change this time is that the .git-annex/ directory
+is gone; instead there is a git-annex branch that is automatically
+maintained by git-annex, and encapsulates all its state nicely out
+of your way.
+
+You should make sure you include the git-annex branch when
+git pushing and pulling.
 
 git-annex 3.20110624 released with [[!toggle text="these changes"]]
 [[!toggleable text="""
@@ -29,4 +30,4 @@ git-annex 3.20110624 released with [[!toggle text="these changes"]]
      trusting repositories that are not configured remotes.
    * unlock: Made atomic.
    * git-union-merge: New git subcommand, that does a generic union merge
-     operation, and operates efficiently without touching the working tree."""]]
\ No newline at end of file
+     operation, and operates efficiently without touching the working tree."""]]
diff --git a/doc/news/version_3.20110702.mdwn b/doc/news/version_3.20110702.mdwn
index 0bb3adf2d8..a5bb925552 100644
--- a/doc/news/version_3.20110702.mdwn
+++ b/doc/news/version_3.20110702.mdwn
@@ -1,6 +1,6 @@
 News for git-annex 3.20110702:
 
-   The URL backend has been removed. Instead the new web remote can be used.
+The URL backend has been removed. Instead the new web remote can be used.
 
 git-annex 3.20110702 released with [[!toggle text="these changes"]]
 [[!toggleable text="""
@@ -19,4 +19,4 @@ git-annex 3.20110702 released with [[!toggle text="these changes"]]
    * cabal can now be used to build git-annex. This is substantially
      slower than using make, does not build or install documentation,
      does not run the test suite, and is not particularly recommended,
-     but could be useful to some."""]]
\ No newline at end of file
+     but could be useful to some."""]]

From 686d08718b84a70d4b36039fdf595cc4c0fa176a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 17:10:25 -0400
Subject: [PATCH 1991/2835] link to hackage

---
 doc/download.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index c0d2d261d3..ddb2ee82a2 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -10,5 +10,8 @@ Other mirrors of the git repository:
 To download a tarball of a particular release, use an url like
 
 
+From time to time, releases of git-annex are uploaded
+[to hackage](http://hackage.haskell.org/package/git-annex).
+
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.

From 84a9fee6f2afaa260054ef42da3b08ca0dddb130 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 17:10:43 -0400
Subject: [PATCH 1992/2835] fix sdist to include symlinks, without which it
 doesn't build

oops.. will upload a new minor version to hackage
---
 Makefile        | 2 +-
 git-annex.cabal | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index afd2de99e0..786fd919e4 100644
--- a/Makefile
+++ b/Makefile
@@ -91,7 +91,7 @@ clean:
 # generate a file list there.
 sdist: clean
 	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi
-	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -type f -print)!i" < git-annex.cabal.orig > git-annex.cabal
+	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -not -type d -print)!i" < git-annex.cabal.orig > git-annex.cabal
 	@cabal sdist
 	@mv git-annex.cabal.orig git-annex.cabal
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 54a526160e..d5a066cdff 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110702
+Version: 3.20110702.2
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From e3ffa4330c0c5855c6e3542aac83a2c54fc8c4de Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 17:14:27 -0400
Subject: [PATCH 1993/2835] update

---
 doc/download.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index ddb2ee82a2..e1257d2618 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -8,7 +8,7 @@ Other mirrors of the git repository:
 * [at github](https://github.com/joeyh/git-annex)
 
 To download a tarball of a particular release, use an url like
-
+
 
 From time to time, releases of git-annex are uploaded
 [to hackage](http://hackage.haskell.org/package/git-annex).

From bd5884d8dd873d6bc7a5e938afb2d835b7bd968f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 17:29:20 -0400
Subject: [PATCH 1994/2835] reorder

---
 doc/install.mdwn | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 25eb8bedfb..23a0da902d 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -6,7 +6,13 @@
 * [[Fedora]]
 * [[FreeBSD]]
 
-## Generic instructions
+## Using cabal
+
+As a haskell package, git-annex can be built using cabal. For example:
+
+	cabal install git-annex --bindir=$HOME/bin
+
+## Installation by hand
 
 To build and use git-annex, you will need:
 
@@ -34,9 +40,3 @@ To build and use git-annex, you will need:
   * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`
-
-## Using cabal
-
-As a haskell package, git-annex can be built using cabal. For example:
-
-	cabal install git-annex --bindir=$HOME/bin

From f626512b07f3675f184101c4c945861381862339 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 19:16:28 -0400
Subject: [PATCH 1995/2835] further improved git cat-file error handling

---
 Branch.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 59f2d3134f..d9cbe61a72 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -241,8 +241,7 @@ catFile file = do
 				[Param "cat-file", Param "--batch"]
 			let gitcmd = join " " ("git" : toCommand cmd)
 			(_, from, to) <- liftIO $ hPipeBoth "sh"
-				-- want stderr on stdin to see error messages
-				["-c", "exec " ++ gitcmd ++ " 2>&1"]
+				["-c", "exec " ++ gitcmd ++ " 2>/dev/null"]
 			setState state { catFileHandles = Just (from, to) }
 			ask (from, to)
 		ask (from, to) = liftIO $ do
@@ -255,7 +254,9 @@ catFile file = do
 					| length sha == Git.shaSize &&
 					  blob == "blob" -> handle from size
 					| otherwise -> empty
-				_ -> empty
+				_
+					| header == want ++ " missing" -> empty
+					| otherwise -> error $ "unknown response from git cat-file " ++ header
 		handle from size = case reads size of
 			[(bytes, "")] -> readcontent from bytes
 			_ -> empty

From 591e293f43f7e4450b570f0e6e95c1a33cd50f44 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 19:22:11 -0400
Subject: [PATCH 1996/2835] simplify git cat-file startup

---
 Branch.hs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index d9cbe61a72..c0c96611bf 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -237,11 +237,9 @@ catFile file = do
 	where
 		startup state = do
 			g <- Annex.gitRepo
-			let cmd = Git.gitCommandLine g
-				[Param "cat-file", Param "--batch"]
-			let gitcmd = join " " ("git" : toCommand cmd)
-			(_, from, to) <- liftIO $ hPipeBoth "sh"
-				["-c", "exec " ++ gitcmd ++ " 2>/dev/null"]
+			(_, from, to) <- liftIO $ hPipeBoth "git" $
+				toCommand $ Git.gitCommandLine g
+					[Param "cat-file", Param "--batch"]
 			setState state { catFileHandles = Just (from, to) }
 			ask (from, to)
 		ask (from, to) = liftIO $ do

From 3904d2e4b90a7a36b3e4a8ed58caebbd7fc0ae22 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 3 Jul 2011 11:32:17 +0000
Subject: [PATCH 1997/2835]

---
 .../annex_unannex__47__uninit_should_handle_copies.mdwn   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn

diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
new file mode 100644
index 0000000000..6d1799cd72
--- /dev/null
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
@@ -0,0 +1,8 @@
+Just starting using v3, even more awesome, thanks!
+
+With git-annex, I take the habit to do copies of files without restriction, as they end up into (cheap) symlink copies.
+However, if 2 copies are unannexed, only one is restored, the other becomes a broken symlink, so I kind of loose some information 
+(my use case: I have a repo on which I recently started using annex, but most of the files, which i would want to be annexed, are only in git,
+so my plan is to unninit this repo, delete the .git dir, and then annex everything, as I don't mind the history).
+
+Rafaël

From d97cc2fd52fab7a18d5c06a008c4d532615babef Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 3 Jul 2011 11:56:45 +0000
Subject: [PATCH 1998/2835] Added a comment

---
 .../comment_1_7683bf02cf9e97830fb4690314501568._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment

diff --git a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment
new file mode 100644
index 0000000000..c556fbd771
--- /dev/null
+++ b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 1"
+ date="2011-07-03T11:56:45Z"
+ content="""
+And what about emitting a warning, as git does, that some files were not annex-added (when not using --force)?
+"""]]

From e6ca68250e1be5f092de1bc5f2c40f5838785aea Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 3 Jul 2011 14:39:42 +0000
Subject: [PATCH 1999/2835] Added a comment: git annex fetch

---
 .../comment_6_31ea08c008500560c0b96c6601bc6362._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment

diff --git a/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment b/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment
new file mode 100644
index 0000000000..868e9677c8
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex fetch"
+ date="2011-07-03T14:39:41Z"
+ content="""
+I'm not sure it is worth adding a command for such a small feature, but I would certainly use it: having something like \"git annex fetch remote\" do \"git fetch remote && git annex copy --from=remote\", and \"git annex push remote\" do \"git push remote && git annex copy --to=remote\". And maybe the same for a pull operation?
+"""]]

From de408626b72b0bdb98cf1534d7406910318516c2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 3 Jul 2011 17:57:00 +0000
Subject: [PATCH 2000/2835] Added a comment: git annex fetch

---
 ...comment_7_94045b9078b1fff877933b012d1b49e2._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment

diff --git a/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment b/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment
new file mode 100644
index 0000000000..e39e162322
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex fetch"
+ date="2011-07-03T17:57:00Z"
+ content="""
+My last comment is a bit confused. The \"git fetch\" command allows to get all the information from a remote, and it is then possible to merge while being offline (without access to the remote). I would like a \"git annex fetch remote\" command to be able to get all annexed files from remote, so that if I later merge with remote, all annexed files are already here. And \"git annex fetch\" could (optionally) call \"git fetch\" before getting the files.
+
+It seems also that in my last post, I should have written \"git annex get --from=remote\" instead of \"git annex copy --from=remote\", because \"annex copy --from\" copies all files, even if the local repo already have them (is this the case? if yes, when is it useful?)
+"""]]

From bd54dadb0b92945db9fc004d03d1fb32a453225c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 12:27:47 -0400
Subject: [PATCH 2001/2835] response

---
 ...annex_unannex__47__uninit_should_handle_copies.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
index 6d1799cd72..751e1afa9b 100644
--- a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
@@ -6,3 +6,13 @@ However, if 2 copies are unannexed, only one is restored, the other becomes a br
 so my plan is to unninit this repo, delete the .git dir, and then annex everything, as I don't mind the history).
 
 Rafaël
+
+> The only way for git-annex to support this in its current state would be
+> for the unannex command to copy the file content from the annex, rather
+> than moving it out. Then multiple links to the same content could be
+> unannexed.
+> 
+> But, this would be slower, and would depend on a later `unused` and
+> `dropunused` to actually remove the content. While doable, by use case
+> for unannex is more to quickly undo a mistaken add, and it's unlikely there
+> are multiple symlinks to the same content in this situation. --[[Joey]] 

From 109814d8bb4ea3fe61f8f02181e26f9cdad6ac5a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 4 Jul 2011 16:29:40 +0000
Subject: [PATCH 2002/2835] Comment moderation

---
 ...nt_2_273a45e6977d40d39e0d9ab924a83240._comment |  9 +++++++++
 ...nt_1_818f38aa988177d3a9415055e084f0fb._comment | 15 +++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment
 create mode 100644 doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment
new file mode 100644
index 0000000000..b01590a7a7
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="git annex fsck --from remote"
+ date="2011-06-25T16:20:44Z"
+ content="""
+Currently fsck silently ignores --to/--from.
+It should at least complain if it is not supported.
+"""]]
diff --git a/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment
new file mode 100644
index 0000000000..11b44b8094
--- /dev/null
+++ b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="filtering based on git-commits"
+ date="2011-06-23T13:56:35Z"
+ content="""
+additional filter criteria could come from the git history:
+
+* `git annex get --touched-in HEAD~5..` to fetch what has recently been worked on
+* `git annex get --touched-by chrysn --touched-in version-1.0..HEAD` to fetch what i've been workin on recently (based on regexp or substring match in author; git experts could probably craft much more meaningful expressions)
+
+these options could also apply to `git annex find` -- actually, looking at the normal file system tools for such tasks, that might even be sufficient (think `git annex find --numcopies-gt 3 --present-on lanserver1 --drop` like `find -iname '*foo*' -delete`
+
+(i was about to open a new forum discussion for commit-based getting, but this is close enough to be usefully joint in a discussion)
+"""]]

From 18d82459c62f5a27c5fd8d7279cc760486ae82fa Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Mon, 4 Jul 2011 16:57:26 +0000
Subject: [PATCH 2003/2835] Added a comment

---
 ...comment_1_c896ff6589f62178b60e606771e4f2bf._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment

diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment
new file mode 100644
index 0000000000..839992477a
--- /dev/null
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 1"
+ date="2011-07-04T16:57:25Z"
+ content="""
+You convince me for unannex, but isn't the goal of uninit to revert all annex operations? In the current state, a clean revert is not possible (because of the broken symlinks after uninit). Instead of copying, using hard links is out of question?
+
+For my needs, is the command \"git annex unlock .\" (from the root of the repo) a correct workaround?
+"""]]

From 5c63b409d4b046f2179ae3c542bfd47d86c8c90c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 15:50:30 -0400
Subject: [PATCH 2004/2835] uninit: Delete the git-annex branch.

---
 Command/Uninit.hs | 17 ++++++++++++-----
 debian/changelog  |  6 ++++++
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index c47ac0c3af..22006f7dcd 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -9,6 +9,7 @@ module Command.Uninit where
 
 import Control.Monad.State (liftIO)
 import System.Directory
+import System.Exit
 
 import Command
 import Messages
@@ -18,6 +19,8 @@ import qualified Git
 import qualified Annex
 import qualified Command.Unannex
 import qualified Command.Init
+import qualified Branch
+import Content
 
 command :: [Command]
 command = [repoCommand "uninit" paramPath seek 
@@ -27,15 +30,19 @@ seek :: [CommandSeek]
 seek = [withFilesInGit Command.Unannex.start, withNothing start]
 
 start :: CommandStartNothing
-start = do
-	showStart "uninit" ""
-	next perform
+start = next perform
 
 perform :: CommandPerform
-perform = do
+perform = next cleanup
+
+cleanup :: CommandCleanup
+cleanup = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
-	next $ return True
+	saveState
+	liftIO $ Git.run g "branch" [Param "-D", Param Branch.name]
+	-- bypass normal shutdown, which writes to the deleted branch
+	liftIO exitSuccess
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/debian/changelog b/debian/changelog
index 266a747ee0..fb5b80ac0e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110703) UNRELEASED; urgency=low
+
+  * uninit: Delete the git-annex branch.
+
+ -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
+
 git-annex (3.20110702) unstable; urgency=low
 
   * Now the web can be used as a special remote. 

From 5beb6bc76fb3edbc28c238eb9596fc828aa49bfc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 15:55:03 -0400
Subject: [PATCH 2005/2835] uninit: delete .git/annex/

---
 Command/Uninit.hs | 9 ++++++---
 debian/changelog  | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 22006f7dcd..5cb66e83a1 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -21,6 +21,7 @@ import qualified Command.Unannex
 import qualified Command.Init
 import qualified Branch
 import Content
+import Locations
 
 command :: [Command]
 command = [repoCommand "uninit" paramPath seek 
@@ -40,9 +41,11 @@ cleanup = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
 	saveState
-	liftIO $ Git.run g "branch" [Param "-D", Param Branch.name]
-	-- bypass normal shutdown, which writes to the deleted branch
-	liftIO exitSuccess
+	liftIO $ do
+		Git.run g "branch" [Param "-D", Param Branch.name]
+		removeDirectoryRecursive (gitAnnexDir g)
+		-- avoid normal shutdown
+		exitSuccess
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/debian/changelog b/debian/changelog
index fb5b80ac0e..3564acd17b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,6 @@
 git-annex (3.20110703) UNRELEASED; urgency=low
 
-  * uninit: Delete the git-annex branch.
+  * uninit: Delete the git-annex branch and .git/annex/
 
  -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
 

From 22a4f5b348c72a07fac99786613f6efc2eeb3b17 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 16:06:28 -0400
Subject: [PATCH 2006/2835] unannex: In --fast mode, file content is left in
 the annex, and a hard link made to it.

---
 Command/Unannex.hs | 13 +++++++++++--
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  3 +++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index d30f8d20f7..f0c1b27c6c 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -10,6 +10,7 @@ module Command.Unannex where
 import Control.Monad.State (liftIO)
 import Control.Monad (unless)
 import System.Directory
+import System.Posix.Files
 
 import Command
 import qualified Annex
@@ -22,6 +23,7 @@ import Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Messages
+import Locations
 
 command :: [Command]
 command = [repoCommand "unannex" paramPath seek "undo accidential add command"]
@@ -64,8 +66,15 @@ cleanup file key = do
 	-- git rm deletes empty directories; put them back
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 
-	fromAnnex key file
-	logStatus key InfoMissing
+	fast <- Annex.getState Annex.fast
+	if fast
+		then liftIO $ do
+			-- fast mode: hard link to content in annex
+			createLink (gitAnnexLocation g key) file
+			allowWrite file
+		else do
+			fromAnnex key file
+			logStatus key InfoMissing
 	
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
diff --git a/debian/changelog b/debian/changelog
index 3564acd17b..844c6c3add 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (3.20110703) UNRELEASED; urgency=low
 
   * uninit: Delete the git-annex branch and .git/annex/
+  * unannex: In --fast mode, file content is left in the annex, and a
+    hard link made to it.
 
  -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index c450887488..dfb23f5f21 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -239,6 +239,9 @@ Many git-annex commands will stage changes for later `git commit` by you.
   file and don't want its contents any more. In that case you should use
   `git annex drop` instead, and you can also `git rm` the file.
 
+  In --fast mode, this command leaves content in the annex, simply making
+  a hard link to it.
+
 * uninit
 
   Use this to stop using git annex. It will unannex every file in the

From d7ce51af5abd5c8582a31c39c3968597f5cae34e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 16:13:44 -0400
Subject: [PATCH 2007/2835] fix usage for setkey

---
 Command/SetKey.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 546bd8e2d0..b000a4e8bf 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -16,7 +16,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [repoCommand "setkey" (paramRepeating paramKey) seek
+command = [repoCommand "setkey" (paramPath) seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]

From 71c783bf24f2aa4ab911d8279081bcad08951ece Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 16:19:04 -0400
Subject: [PATCH 2008/2835] uninit: Use unannex in --fast mode, to support
 unannexing multiple files that link to the same content.

---
 Command/Uninit.hs                             | 20 +++++++++++++------
 debian/changelog                              |  2 ++
 ...nnex__47__uninit_should_handle_copies.mdwn |  2 ++
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 5cb66e83a1..1497bbfd19 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -28,7 +28,15 @@ command = [repoCommand "uninit" paramPath seek
         "de-initialize git-annex and clean out repository"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit Command.Unannex.start, withNothing start]
+seek = [withFilesInGit startUnannex, withNothing start]
+
+startUnannex :: CommandStartString
+startUnannex file = do
+	-- Force fast mode before running unannex. This way, if multiple
+	-- files link to a key, it will be left in the annex and hardlinked
+	-- to by each.
+	Annex.changeState $ \s -> s { Annex.fast = True }
+	Command.Unannex.start file
 
 start :: CommandStartNothing
 start = next perform
@@ -40,12 +48,12 @@ cleanup :: CommandCleanup
 cleanup = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
+	mapM_ removeAnnex =<< getKeysPresent
+	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
+	-- avoid normal shutdown
 	saveState
-	liftIO $ do
-		Git.run g "branch" [Param "-D", Param Branch.name]
-		removeDirectoryRecursive (gitAnnexDir g)
-		-- avoid normal shutdown
-		exitSuccess
+	liftIO $ Git.run g "branch" [Param "-D", Param Branch.name]
+	liftIO $ exitSuccess
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/debian/changelog b/debian/changelog
index 844c6c3add..8f559bb028 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20110703) UNRELEASED; urgency=low
   * uninit: Delete the git-annex branch and .git/annex/
   * unannex: In --fast mode, file content is left in the annex, and a
     hard link made to it.
+  * uninit: Use unannex in --fast mode, to support unannexing multiple
+    files that link to the same content.
 
  -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
 
diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
index 751e1afa9b..e830f11564 100644
--- a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
@@ -16,3 +16,5 @@ Rafaël
 > `dropunused` to actually remove the content. While doable, by use case
 > for unannex is more to quickly undo a mistaken add, and it's unlikely there
 > are multiple symlinks to the same content in this situation. --[[Joey]] 
+
+[[!tag done]]

From 02f2c744bd7335155a40f069b4d7bb91413765e9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY"
 
Date: Mon, 4 Jul 2011 20:25:39 +0000
Subject: [PATCH 2009/2835] Added a comment

---
 .../comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment

diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment
new file mode 100644
index 0000000000..21c0c449b0
--- /dev/null
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY"
+ nickname="Joey"
+ subject="comment 2"
+ date="2011-07-04T20:25:38Z"
+ content="""
+Indeed, uninit needed to be improved. I've done so. Also, unannex --fast can be used to make hard links to content left in the annex.
+"""]]

From 9869ebb260de96a04e464e8348e6de749f2b9a11 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Mon, 4 Jul 2011 22:54:15 +0000
Subject: [PATCH 2010/2835]

---
 doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
new file mode 100644
index 0000000000..10804e45dd
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
@@ -0,0 +1 @@
+On several of my repos, the upgrade to v3 seemed to take forever. A Crl-C followed by another "git annex upgrade" "solved" the problem in some cases. Sometimes, I had to also delete the .git/annex/journal dir to have the upgrade. I didn't notice anything special about the non-working repos to help diagnose the problem.

From 0ef0f277c17cc5ebedd47213494f2b54c5deec53 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 4 Jul 2011 22:58:46 +0000
Subject: [PATCH 2011/2835] Added a comment

---
 .../comment_1_5f60006c9bb095167d817f234a14d20b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment
new file mode 100644
index 0000000000..0cf0ad4618
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-04T22:58:46Z"
+ content="""
+Well if it happens again why don't you use `ps` or `strace` to see what it's doing.
+"""]]

From 5c69ac14eb47e284ab4f4dec44ed6ab3581d416f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 19:31:45 -0400
Subject: [PATCH 2012/2835] Drop the dependency on the haskell curl bindings,
 use regular haskell HTTP.

---
 Remote/Web.hs           | 29 ++++++++++++++++++-----------
 debian/changelog        |  1 +
 debian/control          |  2 +-
 doc/install.mdwn        |  2 +-
 doc/install/Fedora.mdwn |  1 +
 doc/install/OSX.mdwn    |  1 +
 git-annex.cabal         |  4 ++--
 7 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 71591b7aa0..d3d140d73b 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -14,10 +14,9 @@ module Remote.Web (
 import Control.Monad.State (liftIO)
 import Control.Exception
 import System.FilePath
-import Network.Curl.Easy
-import Network.Curl.Opts
-import Network.Curl.Types
-import Network.Curl.Code
+import Network.Browser
+import Network.HTTP
+import Network.URI
 
 import Types
 import Types.Remote
@@ -31,6 +30,8 @@ import PresenceLog
 import LocationLog
 import Locations
 
+type URLString = String
+
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "web",
@@ -111,13 +112,19 @@ checkKey' (u:us) = do
 
 urlexists :: URLString -> IO Bool
 urlexists url = do
-	curl <- initialize
-	_ <- setopt curl (CurlURL url)
-	_ <- setopt curl (CurlNoBody True)
-	_ <- setopt curl (CurlFailOnError True)
-	_ <- setopt curl (CurlFollowLocation True)
-	res <- perform curl
-	return $ res == CurlOK
+	case parseURI url of
+		Nothing -> return False
+		Just u -> do
+			(_, r) <- Network.Browser.browse $ do
+				setErrHandler ignore
+				setOutHandler ignore
+				setAllowRedirects True
+				request (mkRequest HEAD u :: Request_String)
+			case rspCode r of
+				(2,_,_) -> return True
+				_ -> return False
+	where
+		ignore = const $ return ()
 
 download :: [URLString] -> FilePath -> Annex Bool
 download [] _ = return False
diff --git a/debian/changelog b/debian/changelog
index 8f559bb028..37d03ceb20 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20110703) UNRELEASED; urgency=low
     hard link made to it.
   * uninit: Use unannex in --fast mode, to support unannexing multiple
     files that link to the same content.
+  * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
 
  -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
 
diff --git a/debian/control b/debian/control
index 07a9e4bbb2..4347233841 100644
--- a/debian/control
+++ b/debian/control
@@ -9,8 +9,8 @@ Build-Depends:
 	libghc-pcre-light-dev,
 	libghc-sha-dev,
 	libghc-dataenc-dev,
+	libghc-http-dev,
 	libghc-utf8-string-dev,
-	libghc-curl-dev,
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
 	ikiwiki,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 23a0da902d..38963695b8 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -25,7 +25,7 @@ To build and use git-annex, you will need:
   * [dataenc](http://hackage.haskell.org/package/dataenc)
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
-  * [curl](http://hackage.haskell.org/package/curl)
+  * [HTTP](http://hackage.haskell.org/package/HTTP)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
 * Shell commands
   * [git](http://git-scm.com/)
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 0050295e86..7814eec940 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -9,6 +9,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install HTTP
 sudo cabal install hS3
 
 git clone git://git-annex.branchable.com/
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 81ffe1d035..ade4fa30e8 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,6 +7,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck  
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install HTTP
 sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
diff --git a/git-annex.cabal b/git-annex.cabal
index d5a066cdff..f7a2bde858 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -30,14 +30,14 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, curl,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP
    base < 5
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, curl,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP
    base < 5
 
 Executable git-union-merge

From c9a81fa841e3502f0426627de2d0eed439680b6d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 19:36:36 -0400
Subject: [PATCH 2013/2835] update build deps

---
 git-annex.cabal | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index f7a2bde858..3779780ca9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -30,15 +30,11 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
    base < 5
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
-  Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
-   unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP
-   base < 5
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs

From e5ee6508ddab17487e6b6aa7f4d3ac7665d8f40c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 19:54:46 -0400
Subject: [PATCH 2014/2835] use a more specific type for executeFile

Apparently the generic -> IO a type fails with some version of GHC.
Possibly due to System.Posix.Process.executeFile having a more specific
type.
---
 Utility.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility.hs b/Utility.hs
index 7831a4ab4b..63fa6eda35 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -108,7 +108,7 @@ boolSystemEnv command params env = do
 			executeFile command True (toCommand params) env
 
 {- executeFile with debug logging -}
-executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO a
+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
 executeFile c path p e = do
 	debugM "Utility.executeFile" $
 		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e

From bddbb66ea4caf22e799e1bd8261e01a789697cc1 Mon Sep 17 00:00:00 2001
From: "https://lithitux.org/openidserver/users/pavel" 
Date: Tue, 5 Jul 2011 15:54:21 +0000
Subject: [PATCH 2015/2835] Added a comment: "Me too"

---
 ..._cd0123392b16d89db41b45464165c247._comment | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment
new file mode 100644
index 0000000000..4bef5f6454
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://lithitux.org/openidserver/users/pavel"
+ nickname="pavel"
+ subject=""Me too""
+ date="2011-07-05T15:54:19Z"
+ content="""
+I've also seen this apparent hang during upgrade to v3. A few more details:
+
+The annex in question has just under 18k files (and hence that many log files), which can slow down directory operations when they're all in the same place (like, for example, .git/annex/journal).
+
+git-annex uses virtually no CPU time and disk IO when it's hanging like this; the first time it happened, 'ps' showed three defunct git processes, with two \"git-annex\" processes and three \"git\" procs:
+
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex cat-file --batch
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex hash-object -w --stdin-paths
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex update-index -z --index-info
+
+I Ctrl+C'd that and tried again, but it hung again -- this time without the defunct gits.
+
+An strace of the process and its children at the time of hang can be found at http://pastebin.com/4kNh4zEJ . It showed somewhat weird behaviour: When I attached with strace, it would scroll through a whole bunch of syscalls making up the open-fstat-read-close-write loop on .git/annex/journal files, but then would block on a write (sorry, don't have that in my scrollback any more so can't give more details) until I Ctrl+C'd strace; when attaching again, it would again scroll through the syscalls for a second or so and then hang with no output.
+
+Ultimately I detached/reattached with strace about two dozen times and that caused it (?) to finish the upgrade; not really sure how to explain it, but it seems like too much of a timing coincidence.
+
+"""]]

From 44e973dd0948afb25c47f176db83c34f90f49932 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 13:26:59 -0400
Subject: [PATCH 2016/2835] fork a process to feed git hash-object

This is another workaround for bug #624389. I hope it will fix
http://git-annex.branchable.com/bugs/problem_with_upgrade_v2_-__62___v3/
---
 Branch.hs | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index c0c96611bf..d091b6edab 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -27,6 +27,8 @@ import Data.Maybe
 import Data.List
 import System.IO
 import System.IO.Binary
+import System.Posix.Process
+import System.Exit
 import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
@@ -329,12 +331,19 @@ stageJournalFiles = do
 			let paths = map (dir ) fs
 			-- inject all the journal files directly into git
 			-- in one quick command
-			(h, s) <- Git.pipeWriteRead g [Param "hash-object",
-				Param "-w", Param "--stdin-paths"] $ unlines paths
+			(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $
+				Git.gitCommandLine g [Param "hash-object", Param "-w", Param "--stdin-paths"]
+			_ <- forkProcess $ do
+				hPutStr toh $ unlines paths
+				hClose toh
+				exitSuccess
+			hClose toh
+			s <- hGetContents fromh
 			-- update the index, also in just one command
 			Git.UnionMerge.update_index g $
 				index_lines (lines s) $ map fileJournal fs
-			forceSuccess h
+			hClose fromh
+			forceSuccess pid
 			mapM_ removeFile paths
 		index_lines shas fs = map genline $ zip shas fs
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file

From 5070340ca7477b746be8a2f4f1877c3115e1af2e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 5 Jul 2011 17:31:22 +0000
Subject: [PATCH 2017/2835] Added a comment

---
 ...t_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment
new file mode 100644
index 0000000000..e314e73fa0
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-07-05T17:31:22Z"
+ content="""
+I've seen this kind of piping stall that is unblocked by strace before. It can vary with versions of GHC, so it would be good to know what version built git-annex (and on what OS version). I filed a bug report upstream before at .
+
+I really need a full strace -f from the top, or at least a complete `strace -o log` of git-annex from one hang through to another hang. The strace you pastebinned does not seem complete. If I can work out which specific git command is being written to when it hangs I can lift the writing out into a separate thread or process to fix it.
+
+@pavel, you mentioned three defunct git processes, and then showed ps output for 3 git processes. Were there 6 git processes in total? And then when you ran it again you said there were no defunct gits -- where the other 3 git processes running once again?
+
+As best I can make out from the (apparently) running git processes, it seems like the journal files for the upgrade had all been written, and the hang occurred when staging them all into the index in preparation for a commit. I have committed a change that lifts the code that does that write out into a new process, which, if I am guessing right on the limited info I have, will avoid the hang.
+
+However, since I can't reproduce it, even when I put 200 thousand files in the journal and have git-annex process them, I can't be sure.
+"""]]

From 82eb082ab9a33713ed1ec3674be2b95f9b81d861 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 14:36:21 -0400
Subject: [PATCH 2018/2835] my fix is confirmed to have worked

Also audited for other uses of pipeBoth and hPipeBoth and they mostly seem
safe.
---
 doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
index 10804e45dd..7f37668ad1 100644
--- a/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
@@ -1 +1,3 @@
 On several of my repos, the upgrade to v3 seemed to take forever. A Crl-C followed by another "git annex upgrade" "solved" the problem in some cases. Sometimes, I had to also delete the .git/annex/journal dir to have the upgrade. I didn't notice anything special about the non-working repos to help diagnose the problem.
+
+[[!tag done]]

From 502bac1c719eaec7eda38fdcb314b4e6c9ba65f1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 5 Jul 2011 18:37:21 +0000
Subject: [PATCH 2019/2835] Added a comment

---
 .../comment_4_91439d4dbbf1461e281b276eb0003691._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment
new file mode 100644
index 0000000000..7bc32c259f
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-07-05T18:37:21Z"
+ content="""
+I've managed to reproduce this and confirmed my fix works.
+"""]]

From d31b84c777b6ba7158be8947fc2236b2a15e29bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 14:58:33 -0400
Subject: [PATCH 2020/2835] better display of thrown errors

---
 CmdLine.hs     | 4 +++-
 Command.hs     | 9 +++++++--
 Command/Map.hs | 1 +
 Messages.hs    | 6 ++++--
 4 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 85423e5e89..46b980fbcb 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -87,7 +87,9 @@ tryRun' state errnum (a:as) = do
 		a
 	case result of
 		Left err -> do
-			Annex.eval state $ showErr err
+			Annex.eval state $ do
+				showEndFail
+				showErr err
 			tryRun' state (errnum + 1) as
 		Right (True,state') -> tryRun' state' errnum as
 		Right (False,state') -> tryRun' state' (errnum + 1) as
diff --git a/Command.hs b/Command.hs
index d36f675d24..c666ddbd29 100644
--- a/Command.hs
+++ b/Command.hs
@@ -96,10 +96,15 @@ prepCommand Command { cmdseek = seek } params = do
 doCommand :: CommandStart -> CommandCleanup
 doCommand = start
 	where
-		start   = stage $ maybe (return True) perform
-		perform = stage $ maybe (showEndFail >> return False) cleanup
+		start   = stage $ maybe success perform
+		perform = stage $ maybe failure cleanup
 		cleanup = stage $ \r -> showEndResult r >> return r
 		stage a b = b >>= a
+		success = return True
+		failure = do
+			showProgress
+			showEndFail
+			return False
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
diff --git a/Command/Map.hs b/Command/Map.hs
index 7bb435ff81..940db54c89 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -175,6 +175,7 @@ scan r = do
 			showEndOk
 			return r'
 		Nothing -> do
+			showProgress
 			showEndFail
 			return r
 
diff --git a/Messages.hs b/Messages.hs
index c44e44eea1..038e4c0bc7 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -43,14 +43,16 @@ showEndOk :: Annex ()
 showEndOk = verbose $ liftIO $ putStrLn "ok"
 
 showEndFail :: Annex ()
-showEndFail = verbose $ liftIO $ putStrLn "\nfailed"
+showEndFail = verbose $ liftIO $ putStrLn "failed"
 
 showEndResult :: Bool -> Annex ()
 showEndResult True = showEndOk
 showEndResult False = showEndFail
 
 showErr :: (Show a) => a -> Annex ()
-showErr e = warning $ "git-annex: " ++ show e
+showErr e = do
+	liftIO $ hFlush stdout
+	liftIO $ hPutStrLn stderr $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
 warning w = do

From d360ca3ed9ebfc4abc34694138ff5628838fed5f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 15:06:20 -0400
Subject: [PATCH 2021/2835] make upgrade more robust

don't remove .git-annex until state has been succeffully saved
---
 Upgrade/V2.hs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 14e328edb4..4824f4bbaf 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -54,11 +54,13 @@ upgrade = do
 	when e $ do
 		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
 		mapM_ (\f -> inject f f) =<< logFiles (olddir g)
-		liftIO $ do
-			Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
-			unless bare $ gitAttributesUnWrite g
 
 	saveState
+
+	when e $ liftIO $ do
+		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
+		unless bare $ gitAttributesUnWrite g
+
 	unless bare $ push
 
 	return True

From fb433a5ba2e69485b4dd2e84919a8da23fc039de Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 5 Jul 2011 19:06:48 +0000
Subject: [PATCH 2022/2835] Added a comment

---
 .../comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment
new file mode 100644
index 0000000000..8649dc77a8
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-07-05T19:06:48Z"
+ content="""
+By the way, the original bug reporter mentioned deleting .git/annex/journal. This is not recommended, and doing it during an upgrade can result in git-annex losing location tracking information. You should probably run `git annex fsck` or reset to the old git tree (and `git config annex.version 2`) and upgrade again.
+"""]]

From d583e04d23005d16d9f25388b84ce27eadecac69 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 15:21:38 -0400
Subject: [PATCH 2023/2835] releasing version 3.20110705

---
 debian/changelog | 5 +++--
 git-annex.cabal  | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 37d03ceb20..773c91b7eb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110703) UNRELEASED; urgency=low
+git-annex (3.20110705) unstable; urgency=low
 
   * uninit: Delete the git-annex branch and .git/annex/
   * unannex: In --fast mode, file content is left in the annex, and a
@@ -6,8 +6,9 @@ git-annex (3.20110703) UNRELEASED; urgency=low
   * uninit: Use unannex in --fast mode, to support unannexing multiple
     files that link to the same content.
   * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
+  * Fix a pipeline stall when upgrading (caused by #624389).
 
- -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
+ -- Joey Hess   Tue, 05 Jul 2011 14:37:39 -0400
 
 git-annex (3.20110702) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 3779780ca9..3ea7e5fb39 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110702.2
+Version: 3.20110705
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 674768abac3efb2646479c6afba76d9ff27fd802 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 15:22:12 -0400
Subject: [PATCH 2024/2835] add news item for git-annex 3.20110705

---
 doc/news/version_0.20110522.mdwn | 5 -----
 doc/news/version_3.20110705.mdwn | 9 +++++++++
 2 files changed, 9 insertions(+), 5 deletions(-)
 delete mode 100644 doc/news/version_0.20110522.mdwn
 create mode 100644 doc/news/version_3.20110705.mdwn

diff --git a/doc/news/version_0.20110522.mdwn b/doc/news/version_0.20110522.mdwn
deleted file mode 100644
index 5dccb993e2..0000000000
--- a/doc/news/version_0.20110522.mdwn
+++ /dev/null
@@ -1,5 +0,0 @@
-git-annex 0.20110522 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Closer emulation of git's behavior when told to use "foo/.git" as a
-     git repository instead of just "foo". Closes: #[627563](http://bugs.debian.org/627563)
-   * Fix bug in --exclude introduced in 0.20110516."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110705.mdwn b/doc/news/version_3.20110705.mdwn
new file mode 100644
index 0000000000..bb4665c047
--- /dev/null
+++ b/doc/news/version_3.20110705.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110705 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * uninit: Delete the git-annex branch and .git/annex/
+   * unannex: In --fast mode, file content is left in the annex, and a
+     hard link made to it.
+   * uninit: Use unannex in --fast mode, to support unannexing multiple
+     files that link to the same content.
+   * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
+   * Fix a pipeline stall when upgrading (caused by #624389)."""]]
\ No newline at end of file

From 9f1577f74684d8d627e75d3021eb1ff50ef7492f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 18:31:46 -0400
Subject: [PATCH 2025/2835] remove unused backend machinery

The only remaining vestiage of backends is different types of keys. These
are still called "backends", mostly to avoid needing to change user interface
and configuration. But everything to do with storing keys in different
backends was gone; instead different types of remotes are used.

In the refactoring, lots of code was moved out of odd corners like
Backend.File, to closer to where it's used, like Command.Drop and
Command.Fsck. Quite a lot of dead code was removed. Several data structures
became simpler, which may result in better runtime efficiency. There should
be no user-visible changes.
---
 Annex.hs              |  11 +--
 Backend.hs            | 168 ++++++++++----------------------
 Backend/File.hs       | 220 ------------------------------------------
 Backend/SHA.hs        |   5 +-
 Backend/WORM.hs       |   6 +-
 BackendList.hs        |  19 ----
 CmdLine.hs            |   3 +-
 Command/Add.hs        |   4 +-
 Command/AddUrl.hs     |   4 +-
 Command/Drop.hs       |  60 ++++++++++--
 Command/DropUnused.hs |   5 +-
 Command/FromKey.hs    |   3 +-
 Command/Fsck.hs       |  77 ++++++++++++++-
 Command/Get.hs        |  49 ++++++++--
 Command/Migrate.hs    |  17 ++--
 Command/Status.hs     |   6 +-
 Command/Unannex.hs    |  13 ++-
 Command/Unlock.hs     |   3 +-
 Config.hs             |  13 +++
 LocationLog.hs        |   2 +-
 Remote.hs             |  33 ++++++-
 Remote/Git.hs         |   4 +-
 Types/Backend.hs      |  16 +--
 Upgrade/V1.hs         |   7 +-
 test.hs               |   5 +-
 25 files changed, 308 insertions(+), 445 deletions(-)
 delete mode 100644 Backend/File.hs
 delete mode 100644 BackendList.hs

diff --git a/Annex.hs b/Annex.hs
index c21cfb37ca..f7e3e29f82 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -34,7 +34,6 @@ type Annex = StateT AnnexState IO
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [Backend Annex]
-	, supportedBackends :: [Backend Annex]
 	, remotes :: [Remote Annex]
 	, repoqueue :: Queue
 	, quiet :: Bool
@@ -52,12 +51,11 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-newState :: [Backend Annex] -> Git.Repo -> AnnexState
-newState allbackends gitrepo = AnnexState
+newState :: Git.Repo -> AnnexState
+newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
-	, supportedBackends = allbackends
 	, repoqueue = empty
 	, quiet = False
 	, force = False
@@ -75,9 +73,8 @@ newState allbackends gitrepo = AnnexState
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
-new :: Git.Repo -> [Backend Annex] -> IO AnnexState
-new gitrepo allbackends =
-	newState allbackends `liftM` (liftIO . Git.configRead) gitrepo
+new :: Git.Repo -> IO AnnexState
+new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
diff --git a/Backend.hs b/Backend.hs
index b1cd4c8f0e..cf976d2b8a 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -1,16 +1,4 @@
-{- git-annex key-value storage backends
- -
- - git-annex uses a key-value abstraction layer to allow files contents to be
- - stored in different ways. In theory, any key-value storage system could be
- - used to store the file contents, and git-annex would then retrieve them
- - as needed and put them in `.git/annex/`.
- - 
- - When a file is annexed, a key is generated from its content and/or metadata.
- - This key can later be used to retrieve the file's content (its value). This
- - key generation must be stable for a given file content, name, and size.
- - 
- - Multiple pluggable backends are supported, and more than one can be used
- - to store different files' contents in a given repository.
+{- git-annex key/value backends
  -
  - Copyright 2010 Joey Hess 
  -
@@ -19,15 +7,10 @@
 
 module Backend (
 	list,
-	storeFileKey,
-	retrieveKeyFile,
-	removeKey,
-	hasKey,
-	fsckKey,
-	upgradableKey,
+	orderedList,
+	genKey,
 	lookupFile,
 	chooseBackends,
-	keyBackend,
 	lookupBackendName,
 	maybeLookupBackendName
 ) where
@@ -36,7 +19,6 @@ import Control.Monad.State (liftIO, when)
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
-import System.Directory
 
 import Locations
 import qualified Git
@@ -45,12 +27,20 @@ import Types
 import Types.Key
 import qualified Types.Backend as B
 import Messages
-import Content
-import DataUnits
+
+-- When adding a new backend, import it here and add it to the list.
+import qualified Backend.WORM
+import qualified Backend.SHA
+
+list :: [Backend Annex]
+list = concat 
+	[ Backend.WORM.backends
+	, Backend.SHA.backends
+	]
 
 {- List of backends in the order to try them when storing a new key. -}
-list :: Annex [Backend Annex]
-list = do
+orderedList :: Annex [Backend Annex]
+orderedList = do
 	l <- Annex.getState Annex.backends -- list is cached here
 	if not $ null l
 		then return l
@@ -59,92 +49,49 @@ list = do
 			d <- Annex.getState Annex.forcebackend
 			handle d s
 	where
-		parseBackendList l [] = l
-		parseBackendList bs s = map (lookupBackendName bs) $ words s
+		parseBackendList [] = list
+		parseBackendList s = map lookupBackendName $ words s
 		handle Nothing s = return s
 		handle (Just "") s = return s
 		handle (Just name) s = do
-			bs <- Annex.getState Annex.supportedBackends
-			let l' = (lookupBackendName bs name):s
+			let l' = (lookupBackendName name):s
 			Annex.changeState $ \state -> state { Annex.backends = l' }
 			return l'
 		getstandard = do
-			bs <- Annex.getState Annex.supportedBackends
 			g <- Annex.gitRepo
-			return $ parseBackendList bs $
+			return $ parseBackendList $
 				Git.configGet g "annex.backends" ""
 
-{- Looks up a backend in a list. May fail if unknown. -}
-lookupBackendName :: [Backend Annex] -> String -> Backend Annex
-lookupBackendName bs s = maybe unknown id $ maybeLookupBackendName bs s
-	where
-		unknown = error $ "unknown backend " ++ s
-maybeLookupBackendName :: [Backend Annex] -> String -> Maybe (Backend Annex)
-maybeLookupBackendName bs s =
-	if 1 /= length matches
-		then Nothing
-		else Just $ head matches
-	where matches = filter (\b -> s == B.name b) bs
-
-{- Attempts to store a file in one of the backends. -}
-storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
-storeFileKey file trybackend = do
-	bs <- list
+{- Generates a key for a file, trying each backend in turn until one
+ - accepts it. -}
+genKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
+genKey file trybackend = do
+	bs <- orderedList
 	let bs' = maybe bs (:bs) trybackend
-	storeFileKey' bs' file
-storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
-storeFileKey' [] _ = return Nothing
-storeFileKey' (b:bs) file = maybe nextbackend store =<< (B.getKey b) file
-	where
-		nextbackend = storeFileKey' bs file
-		store key = do
-			stored <- (B.storeFileKey b) file key
-			if (not stored)
-				then nextbackend
-				else return $ Just (key, b)
-
-{- Attempts to retrieve an key from one of the backends, saving it to
- - a specified location. -}
-retrieveKeyFile :: Backend Annex -> Key -> FilePath -> Annex Bool
-retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest
-
-{- Removes a key from a backend. -}
-removeKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool
-removeKey backend key numcopies = (B.removeKey backend) key numcopies
-
-{- Checks if a key is present in its backend. -}
-hasKey :: Key -> Annex Bool
-hasKey key = do
-	backend <- keyBackend key
-	(B.hasKey backend) key
-
-{- Checks a key for problems. -}
-fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-fsckKey backend key file numcopies = do
-	size_ok <- checkKeySize key
-	backend_ok <-(B.fsckKey backend) key file numcopies
-	return $ size_ok && backend_ok
-
-{- Checks if a key is upgradable to a newer representation. -}
-upgradableKey :: Backend Annex -> Key -> Annex Bool
-upgradableKey backend key = (B.upgradableKey backend) key
+	genKey' bs' file
+genKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
+genKey' [] _ = return Nothing
+genKey' (b:bs) file = do
+	r <- (B.getKey b) file
+	case r of
+		Nothing -> genKey' bs file
+		Just k -> return $ Just (k, b)
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile file = do
-	bs <- Annex.getState Annex.supportedBackends
 	tl <- liftIO $ try getsymlink
 	case tl of
 		Left _ -> return Nothing
-		Right l -> makekey bs l
+		Right l -> makekey l
 	where
 		getsymlink = do
 			l <- readSymbolicLink file
 			return $ takeFileName l
-		makekey bs l = maybe (return Nothing) (makeret bs l) (fileKey l)
-		makeret bs l k =
-			case maybeLookupBackendName bs bname of
+		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
+		makeret l k =
+			case maybeLookupBackendName bname of
 					Just backend -> return $ Just (k, backend)
 					Nothing -> do
 						when (isLinkToAnnex l) $
@@ -164,37 +111,20 @@ chooseBackends fs = do
 	forced <- Annex.getState Annex.forcebackend
 	if forced /= Nothing
 		then do
-			l <- list
+			l <- orderedList
 			return $ map (\f -> (f, Just $ head l)) fs
 		else do
-			bs <- Annex.getState Annex.supportedBackends
 			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
-			return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs
+			return $ map (\(f,b) -> (f, maybeLookupBackendName b)) pairs
 
-{- Returns the backend to use for a key. -}
-keyBackend :: Key -> Annex (Backend Annex)
-keyBackend key = do
-	bs <- Annex.getState Annex.supportedBackends
-	return $ lookupBackendName bs $ keyBackendName key
-
-{- The size of the data for a key is checked against the size encoded in
- - the key's metadata, if available. -}
-checkKeySize :: Key -> Annex Bool
-checkKeySize key = do
-	g <- Annex.gitRepo
-	let file = gitAnnexLocation g key
-	present <- liftIO $ doesFileExist file
-	case (present, keySize key) of
-		(_, Nothing) -> return True
-		(False, _) -> return True
-		(True, Just size) -> do
-			stat <- liftIO $ getFileStatus file
-			let size' = fromIntegral (fileSize stat)
-			if size == size'
-				then return True
-				else do
-					dest <- moveBad key
-					warning $ "Bad file size (" ++
-						compareSizes storageUnits True size size' ++ 
-						"); moved to " ++ dest
-					return False
+{- Looks up a backend by name. May fail if unknown. -}
+lookupBackendName :: String -> Backend Annex
+lookupBackendName s = maybe unknown id $ maybeLookupBackendName s
+	where
+		unknown = error $ "unknown backend " ++ s
+maybeLookupBackendName :: String -> Maybe (Backend Annex)
+maybeLookupBackendName s =
+	if 1 /= length matches
+		then Nothing
+		else Just $ head matches
+	where matches = filter (\b -> s == B.name b) list
diff --git a/Backend/File.hs b/Backend/File.hs
deleted file mode 100644
index 174da4e6dc..0000000000
--- a/Backend/File.hs
+++ /dev/null
@@ -1,220 +0,0 @@
-{- git-annex pseudo-backend
- -
- - This backend does not really do any independant data storage,
- - it relies on the file contents in .git/annex/ in this repo,
- - and other accessible repos.
- -
- - This is an abstract backend; name, getKey and fsckKey have to be implemented
- - to complete it.
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.File (backend, checkKey) where
-
-import Data.List
-import Data.String.Utils
-
-import Types.Backend
-import LocationLog
-import qualified Remote
-import qualified Git
-import Content
-import qualified Annex
-import Types
-import UUID
-import Messages
-import Trust
-import Types.Key
-
-backend :: Backend Annex
-backend = Backend {
-	name = mustProvide,
-	getKey = mustProvide,
-	storeFileKey = dummyStore,
-	retrieveKeyFile = copyKeyFile,
-	removeKey = checkRemoveKey,
-	hasKey = inAnnex,
-	fsckKey = checkKeyOnly,
-	upgradableKey = checkUpgradableKey
-}
-
-mustProvide :: a
-mustProvide = error "must provide this field"
-
-{- Storing a key is a no-op. -}
-dummyStore :: FilePath -> Key -> Annex Bool
-dummyStore _ _ = return True
-
-{- Try to find a copy of the file in one of the remotes,
- - and copy it to here. -}
-copyKeyFile :: Key -> FilePath -> Annex Bool
-copyKeyFile key file = do
-	remotes <- Remote.keyPossibilities key
-	if null remotes
-		then do
-			showNote "not available"
-			showLocations key []
-			return False
-		else trycopy remotes remotes
-	where
-		trycopy full [] = do
-			showTriedRemotes full
-			showLocations key []
-			return False
-		trycopy full (r:rs) = do
-			probablythere <- probablyPresent r
-			if probablythere
-				then docopy r (trycopy full rs)
-				else trycopy full rs
-		-- This check is to avoid an ugly message if a remote is a
-		-- drive that is not mounted.
-		probablyPresent r =
-			if Remote.hasKeyCheap r
-				then do
-					res <- Remote.hasKey r key
-					case res of
-						Right b -> return b
-						Left _ -> return False
-				else return True
-		docopy r continue = do
-			showNote $ "from " ++ Remote.name r ++ "..."
-			copied <- Remote.retrieveKeyFile r key file
-			if copied
-				then return True
-				else continue
-
-{- Checks remotes to verify that enough copies of a key exist to allow
- - for a key to be safely removed (with no data loss), and fails with an
- - error if not. -}
-checkRemoveKey :: Key -> Maybe Int -> Annex Bool
-checkRemoveKey key numcopiesM = do
-	force <- Annex.getState Annex.force
-	if force || numcopiesM == Just 0
-		then return True
-		else do
-			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
-			untrusteduuids <- trustGet UnTrusted
-			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
-			numcopies <- getNumCopies numcopiesM
-			findcopies numcopies trusteduuids tocheck []
-	where
-		findcopies need have [] bad
-			| length have >= need = return True
-			| otherwise = notEnoughCopies need have bad
-		findcopies need have (r:rs) bad
-			| length have >= need = return True
-			| otherwise = do
-				let u = Remote.uuid r
-				let dup = u `elem` have
-				haskey <- Remote.hasKey r key
-				case (dup, haskey) of
-					(False, Right True)	-> findcopies need (u:have) rs bad
-					(False, Left _)		-> findcopies need have rs (r:bad)
-					_			-> findcopies need have rs bad
-		notEnoughCopies need have bad = do
-			unsafe
-			showLongNote $
-				"Could only verify the existence of " ++
-				show (length have) ++ " out of " ++ show need ++ 
-				" necessary copies"
-			showTriedRemotes bad
-			showLocations key have
-			hint
-			return False
-		unsafe = showNote "unsafe"
-		hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)"
-
-showLocations :: Key -> [UUID] -> Annex ()
-showLocations key exclude = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-	uuids <- keyLocations key
-	untrusteduuids <- trustGet UnTrusted
-	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
-	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
-	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
-	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
-	showLongNote $ message ppuuidswanted ppuuidsskipped
-	where
-		filteruuids list x = filter (`notElem` x) list
-		message [] [] = "No other repository is known to contain the file."
-		message rs [] = "Try making some of these repositories available:\n" ++ rs
-		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
-		message rs us = message rs [] ++ message [] us
-
-showTriedRemotes :: [Remote.Remote Annex] -> Annex ()
-showTriedRemotes [] = return ()	
-showTriedRemotes remotes =
-	showLongNote $ "Unable to access these remotes: " ++
-		(join ", " $ map Remote.name remotes)
-
-{- If a value is specified, it is used; otherwise the default is looked up
- - in git config. forcenumcopies overrides everything. -}
-getNumCopies :: Maybe Int -> Annex Int
-getNumCopies v = 
-	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
-	where
-		use (Just n) = return n
-		use Nothing = do
-			g <- Annex.gitRepo
-			return $ read $ Git.configGet g config "1"
-		config = "annex.numcopies"
-
-{- Ideally, all keys have file size metadata. Old keys may not. -}
-checkUpgradableKey :: Key -> Annex Bool
-checkUpgradableKey key
-	| keySize key == Nothing = return True
-	| otherwise = return False
-
-{- This is used to check that numcopies is satisfied for the key on fsck.
- - This trusts data in the the location log, and so can check all keys, even
- - those with data not present in the current annex.
- -
- - The passed action is first run to allow backends deriving this one
- - to do their own checks.
- -}
-checkKey :: (Key -> Annex Bool) -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-checkKey a key file numcopies = do
-	a_ok <- a key
-	copies_ok <- checkKeyNumCopies key file numcopies
-	return $ a_ok && copies_ok
-
-checkKeyOnly :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-checkKeyOnly = checkKey (\_ -> return True)
-
-checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-checkKeyNumCopies key file numcopies = do
-	needed <- getNumCopies numcopies
-	locations <- keyLocations key
-	untrusted <- trustGet UnTrusted
-	let untrustedlocations = intersect untrusted locations
-	let safelocations = filter (`notElem` untrusted) locations
-	let present = length safelocations
-	if present < needed
-		then do
-			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
-			warning $ missingNote (filename file key) present needed ppuuids
-			return False
-		else return True
-	where
-		filename Nothing k = show k
-		filename (Just f) _ = f
-
-missingNote :: String -> Int -> Int -> String -> String
-missingNote file 0 _ [] = 
-		"** No known copies exist of " ++ file
-missingNote file 0 _ untrusted =
-		"Only these untrusted locations may have copies of " ++ file ++
-		"\n" ++ untrusted ++
-		"Back it up to trusted locations with git-annex copy."
-missingNote file present needed [] =
-		"Only " ++ show present ++ " of " ++ show needed ++ 
-		" trustworthy copies exist of " ++ file ++
-		"\nBack it up with git-annex copy."
-missingNote file present needed untrusted = 
-		missingNote file present needed [] ++
-		"\nThe following untrusted locations may also have copies: " ++
-		"\n" ++ untrusted
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 8930e4b938..bd6e411a05 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -16,7 +16,6 @@ import Data.Maybe
 import System.Posix.Files
 import System.FilePath
 
-import qualified Backend.File
 import Messages
 import qualified Annex
 import Locations
@@ -42,10 +41,10 @@ genBackend size
 	| shaCommand size == Nothing = Nothing
 	| otherwise = Just b
 	where
-		b = Backend.File.backend 
+		b = Types.Backend.Backend
 			{ name = shaName size
 			, getKey = keyValue size
-			, fsckKey = Backend.File.checkKey $ checkKeyChecksum size
+			, fsckKey = checkKeyChecksum size
 			}
 
 genBackendE :: SHASize -> Maybe (Backend Annex)
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index dc2e48adce..036d0564cb 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -11,7 +11,6 @@ import Control.Monad.State
 import System.FilePath
 import System.Posix.Files
 
-import qualified Backend.File
 import Types.Backend
 import Types
 import Types.Key
@@ -20,9 +19,10 @@ backends :: [Backend Annex]
 backends = [backend]
 
 backend :: Backend Annex
-backend = Backend.File.backend {
+backend = Types.Backend.Backend {
 	name = "WORM",
-	getKey = keyValue
+	getKey = keyValue,
+	fsckKey = const (return True)
 }
 
 {- The key includes the file size, modification time, and the
diff --git a/BackendList.hs b/BackendList.hs
deleted file mode 100644
index e4e1d76fe2..0000000000
--- a/BackendList.hs
+++ /dev/null
@@ -1,19 +0,0 @@
-{- git-annex backend list
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module BackendList (allBackends) where
-
--- When adding a new backend, import it here and add it to the list.
-import qualified Backend.WORM
-import qualified Backend.SHA
-import Types
-
-allBackends :: [Backend Annex]
-allBackends = concat 
-	[ Backend.WORM.backends
-	, Backend.SHA.backends
-	]
diff --git a/CmdLine.hs b/CmdLine.hs
index 46b980fbcb..b807046df3 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,7 +22,6 @@ import qualified Git
 import Content
 import Types
 import Command
-import BackendList
 import Version
 import Options
 import Messages
@@ -32,7 +31,7 @@ import UUID
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
 dispatch args cmds options header gitrepo = do
 	setupConsole
-	state <- Annex.new gitrepo allBackends
+	state <- Annex.new gitrepo
 	(actions, state') <- Annex.run state $ parseCmd args header cmds options
 	tryRun state' $ [startup] ++ actions ++ [shutdown]
 
diff --git a/Command/Add.hs b/Command/Add.hs
index 6a1ffb5da6..2831e1b35f 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -42,8 +42,8 @@ start pair@(file, _) = notAnnexed file $ do
 
 perform :: BackendFile -> CommandPerform
 perform (file, backend) = do
-	stored <- Backend.storeFileKey file backend
-	case stored of
+	k <- Backend.genKey file backend
+	case k of
 		Nothing -> stop
 		Just (key, _) -> do
 			moveAnnex key file
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index ebf0810bae..e80fe9621b 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -51,8 +51,8 @@ perform url file = do
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
-			stored <- Backend.storeFileKey tmp backend
-			case stored of
+			k <- Backend.genKey tmp backend
+			case k of
 				Nothing -> stop
 				Just (key, _) -> do
 					moveAnnex key tmp
diff --git a/Command/Drop.hs b/Command/Drop.hs
index bd47407413..14f098349e 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -8,12 +8,15 @@
 module Command.Drop where
 
 import Command
-import qualified Backend
+import qualified Remote
+import qualified Annex
 import LocationLog
 import Types
 import Content
 import Messages
 import Utility
+import Trust
+import Config
 
 command :: [Command]
 command = [repoCommand "drop" paramPath seek
@@ -25,19 +28,19 @@ seek = [withAttrFilesInGit "annex.numcopies" start]
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}
 start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, backend) -> do
-	inbackend <- Backend.hasKey key
-	if inbackend
+start (file, attr) = isAnnexed file $ \(key, _) -> do
+	present <- inAnnex key
+	if present
 		then do
 			showStart "drop" file
-			next $ perform key backend numcopies
+			next $ perform key numcopies
 		else stop
 	where
 		numcopies = readMaybe attr :: Maybe Int
 
-perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform
-perform key backend numcopies = do
-	success <- Backend.removeKey backend key numcopies
+perform :: Key -> Maybe Int -> CommandPerform
+perform key numcopies = do
+	success <- dropKey key numcopies
 	if success
 		then next $ cleanup key
 		else stop
@@ -47,3 +50,44 @@ cleanup key = do
 	whenM (inAnnex key) $ removeAnnex key
 	logStatus key InfoMissing
 	return True
+
+{- Checks remotes to verify that enough copies of a key exist to allow
+ - for a key to be safely removed (with no data loss), and fails with an
+ - error if not. -}
+dropKey :: Key -> Maybe Int -> Annex Bool
+dropKey key numcopiesM = do
+	force <- Annex.getState Annex.force
+	if force || numcopiesM == Just 0
+		then return True
+		else do
+			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
+			untrusteduuids <- trustGet UnTrusted
+			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
+			numcopies <- getNumCopies numcopiesM
+			findcopies numcopies trusteduuids tocheck []
+	where
+		findcopies need have [] bad
+			| length have >= need = return True
+			| otherwise = notEnoughCopies need have bad
+		findcopies need have (r:rs) bad
+			| length have >= need = return True
+			| otherwise = do
+				let u = Remote.uuid r
+				let dup = u `elem` have
+				haskey <- Remote.hasKey r key
+				case (dup, haskey) of
+					(False, Right True)	-> findcopies need (u:have) rs bad
+					(False, Left _)		-> findcopies need have rs (r:bad)
+					_			-> findcopies need have rs bad
+		notEnoughCopies need have bad = do
+			unsafe
+			showLongNote $
+				"Could only verify the existence of " ++
+				show (length have) ++ " out of " ++ show need ++ 
+				" necessary copies"
+			Remote.showTriedRemotes bad
+			Remote.showLocations key have
+			hint
+			return False
+		unsafe = showNote "unsafe"
+		hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)"
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 2125abdc3c..55007c1f73 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -21,7 +21,6 @@ import qualified Command.Drop
 import qualified Command.Move
 import qualified Remote
 import qualified Git
-import Backend
 import Types.Key
 import Utility
 
@@ -64,9 +63,7 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 			r <- Remote.byName name
 			showNote $ "from " ++ Remote.name r ++ "..."
 			next $ Command.Move.fromCleanup r True key
-		droplocal = do
-			backend <- keyBackend key
-			Command.Drop.perform key backend (Just 0) -- force drop
+		droplocal = Command.Drop.perform key (Just 0) -- force drop
 
 performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 34816d6574..fb9ab0775a 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -15,7 +15,6 @@ import Control.Monad (unless)
 import Command
 import qualified AnnexQueue
 import Utility
-import qualified Backend
 import Content
 import Messages
 import Types.Key
@@ -30,7 +29,7 @@ seek = [withFilesMissing start]
 start :: CommandStartString
 start file = notBareRepo $ do
 	key <- cmdlineKey
-	inbackend <- Backend.hasKey key
+	inbackend <- inAnnex key
 	unless inbackend $ error $
 		"key ("++keyName key++") is not present in backend"
 	showStart "fromkey" file
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 988cfd28d1..446d25a449 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -9,10 +9,15 @@ module Command.Fsck where
 
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
+import System.Directory
+import Data.List
+import System.Posix.Files
 
 import Command
-import qualified Backend
 import qualified Annex
+import qualified Remote
+import qualified Types.Backend
+import qualified Types.Key
 import UUID
 import Types
 import Messages
@@ -20,6 +25,9 @@ import Utility
 import Content
 import LocationLog
 import Locations
+import Trust
+import DataUnits
+import Config
 
 command :: [Command]
 command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
@@ -40,7 +48,7 @@ perform key file backend numcopies = do
 	-- the location log is checked first, so that if it has bad data
 	-- that gets corrected
 	locationlogok <- verifyLocationLog key file
-	backendok <- Backend.fsckKey backend key (Just file) numcopies
+	backendok <- fsckKey backend key (Just file) numcopies
 	if locationlogok && backendok
 		then next $ return True
 		else stop
@@ -80,3 +88,68 @@ verifyLocationLog key file = do
 		fix g u s = do
 			showNote "fixing location log"
 			logChange g key u s
+
+{- Checks a key for problems. -}
+fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+fsckKey backend key file numcopies = do
+	size_ok <- checkKeySize key
+	copies_ok <- checkKeyNumCopies key file numcopies
+	backend_ok <-(Types.Backend.fsckKey backend) key
+	return $ size_ok && copies_ok && backend_ok
+
+{- The size of the data for a key is checked against the size encoded in
+ - the key's metadata, if available. -}
+checkKeySize :: Key -> Annex Bool
+checkKeySize key = do
+	g <- Annex.gitRepo
+	let file = gitAnnexLocation g key
+	present <- liftIO $ doesFileExist file
+	case (present, Types.Key.keySize key) of
+		(_, Nothing) -> return True
+		(False, _) -> return True
+		(True, Just size) -> do
+			stat <- liftIO $ getFileStatus file
+			let size' = fromIntegral (fileSize stat)
+			if size == size'
+				then return True
+				else do
+					dest <- moveBad key
+					warning $ "Bad file size (" ++
+						compareSizes storageUnits True size size' ++ 
+						"); moved to " ++ dest
+					return False
+
+
+checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKeyNumCopies key file numcopies = do
+	needed <- getNumCopies numcopies
+	locations <- keyLocations key
+	untrusted <- trustGet UnTrusted
+	let untrustedlocations = intersect untrusted locations
+	let safelocations = filter (`notElem` untrusted) locations
+	let present = length safelocations
+	if present < needed
+		then do
+			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
+			warning $ missingNote (filename file key) present needed ppuuids
+			return False
+		else return True
+	where
+		filename Nothing k = show k
+		filename (Just f) _ = f
+
+missingNote :: String -> Int -> Int -> String -> String
+missingNote file 0 _ [] = 
+		"** No known copies exist of " ++ file
+missingNote file 0 _ untrusted =
+		"Only these untrusted locations may have copies of " ++ file ++
+		"\n" ++ untrusted ++
+		"Back it up to trusted locations with git-annex copy."
+missingNote file present needed [] =
+		"Only " ++ show present ++ " of " ++ show needed ++ 
+		" trustworthy copies exist of " ++ file ++
+		"\nBack it up with git-annex copy."
+missingNote file present needed untrusted = 
+		missingNote file present needed [] ++
+		"\nThe following untrusted locations may also have copies: " ++
+		"\n" ++ untrusted
diff --git a/Command/Get.hs b/Command/Get.hs
index 50dc009feb..cc780cb6a3 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -8,7 +8,6 @@
 module Command.Get where
 
 import Command
-import qualified Backend
 import qualified Annex
 import qualified Remote
 import Types
@@ -24,7 +23,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
 start :: CommandStartString
-start file = isAnnexed file $ \(key, backend) -> do
+start file = isAnnexed file $ \(key, _) -> do
 	inannex <- inAnnex key
 	if inannex
 		then stop
@@ -32,14 +31,52 @@ start file = isAnnexed file $ \(key, backend) -> do
 			showStart "get" file
 			from <- Annex.getState Annex.fromremote
 			case from of
-				Nothing -> next $ perform key backend
+				Nothing -> next $ perform key
 				Just name -> do
 					src <- Remote.byName name
 					next $ Command.Move.fromPerform src False key
 
-perform :: Key -> Backend Annex -> CommandPerform
-perform key backend = do
-	ok <- getViaTmp key (Backend.retrieveKeyFile backend key)
+perform :: Key -> CommandPerform
+perform key = do
+	ok <- getViaTmp key (getKeyFile key)
 	if ok
 		then next $ return True -- no cleanup needed
 		else stop
+
+{- Try to find a copy of the file in one of the remotes,
+ - and copy it to here. -}
+getKeyFile :: Key -> FilePath -> Annex Bool
+getKeyFile key file = do
+	remotes <- Remote.keyPossibilities key
+	if null remotes
+		then do
+			showNote "not available"
+			Remote.showLocations key []
+			return False
+		else trycopy remotes remotes
+	where
+		trycopy full [] = do
+			Remote.showTriedRemotes full
+			Remote.showLocations key []
+			return False
+		trycopy full (r:rs) = do
+			probablythere <- probablyPresent r
+			if probablythere
+				then docopy r (trycopy full rs)
+				else trycopy full rs
+		-- This check is to avoid an ugly message if a remote is a
+		-- drive that is not mounted.
+		probablyPresent r =
+			if Remote.hasKeyCheap r
+				then do
+					res <- Remote.hasKey r key
+					case res of
+						Right b -> return b
+						Left _ -> return False
+				else return True
+		docopy r continue = do
+			showNote $ "from " ++ Remote.name r ++ "..."
+			copied <- Remote.retrieveKeyFile r key file
+			if copied
+				then return True
+				else continue
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 09ff6df7da..495bf9fb63 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -15,6 +15,7 @@ import System.FilePath
 import Command
 import qualified Annex
 import qualified Backend
+import qualified Types.Key
 import Locations
 import Types
 import Content
@@ -32,18 +33,20 @@ start :: CommandStartBackendFile
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
-	upgradable <- Backend.upgradableKey oldbackend key
-	if (newbackend /= oldbackend || upgradable) && exists
+	if (newbackend /= oldbackend || upgradableKey key) && exists
 		then do
 			showStart "migrate" file
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = do
-			backends <- Backend.list
-			return $ head backends
+		choosebackend Nothing = return . head =<< Backend.orderedList
 		choosebackend (Just backend) = return backend
 
+{- Checks if a key is upgradable to a newer representation. -}
+{- Ideally, all keys have file size metadata. Old keys may not. -}
+upgradableKey :: Key -> Bool
+upgradableKey key = Types.Key.keySize key == Nothing
+
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
 	g <- Annex.gitRepo
@@ -55,9 +58,9 @@ perform file oldkey newbackend = do
 	let src = gitAnnexLocation g oldkey
 	let tmpfile = gitAnnexTmpDir g  takeFileName file
 	liftIO $ createLink src tmpfile
-	stored <- Backend.storeFileKey tmpfile $ Just newbackend
+	k <- Backend.genKey tmpfile $ Just newbackend
 	liftIO $ cleantmp tmpfile
-	case stored of
+	case k of
 		Nothing -> stop
 		Just (newkey, _) -> do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
diff --git a/Command/Status.hs b/Command/Status.hs
index 53589030b3..2448f65a40 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -25,6 +25,7 @@ import DataUnits
 import Content
 import Types.Key
 import Locations
+import Backend
 
 -- a named computation that produces a statistic
 type Stat = StatState (Maybe (String, StatState String))
@@ -95,9 +96,8 @@ showStat s = calc =<< s
 		calc Nothing = return ()
 
 supported_backends :: Stat
-supported_backends = stat "supported backends" $
-	lift (Annex.getState Annex.supportedBackends) >>=
-		return . unwords . (map B.name)
+supported_backends = stat "supported backends" $ 
+	return $ unwords $ map B.name Backend.list
 
 supported_remote_types :: Stat
 supported_remote_types = stat "supported remote types" $
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index f0c1b27c6c..f22503ee06 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -13,10 +13,10 @@ import System.Directory
 import System.Posix.Files
 
 import Command
+import qualified Command.Drop
 import qualified Annex
 import qualified AnnexQueue
 import Utility
-import qualified Backend
 import LocationLog
 import Types
 import Content
@@ -33,7 +33,7 @@ seek = [withFilesInGit start]
 
 {- The unannex subcommand undoes an add. -}
 start :: CommandStartString
-start file = isAnnexed file $ \(key, backend) -> do
+start file = isAnnexed file $ \(key, _) -> do
 	ishere <- inAnnex key
 	if ishere
 		then do
@@ -46,13 +46,12 @@ start file = isAnnexed file $ \(key, backend) -> do
 				Annex.changeState $ \s -> s { Annex.force = True }
 
 			showStart "unannex" file
-			next $ perform file key backend
+			next $ perform file key
 		else stop
 
-perform :: FilePath -> Key -> Backend Annex -> CommandPerform
-perform file key backend = do
-	-- force backend to always remove
-	ok <- Backend.removeKey backend key (Just 0)
+perform :: FilePath -> Key -> CommandPerform
+perform file key = do
+	ok <- Command.Drop.dropKey key (Just 0) -- always remove
 	if ok
 		then next $ cleanup file key
 		else stop
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index ca8b62502a..8a897c3657 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -12,7 +12,6 @@ import System.Directory hiding (copyFile)
 
 import Command
 import qualified Annex
-import qualified Backend
 import Types
 import Messages
 import Locations
@@ -38,7 +37,7 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: FilePath -> Key -> CommandPerform
 perform dest key = do
-	unlessM (Backend.hasKey key) $ error "content not present"
+	unlessM (inAnnex key) $ error "content not present"
 	
 	checkDiskSpace key
 
diff --git a/Config.hs b/Config.hs
index 1016845e75..9cbf2d52fd 100644
--- a/Config.hs
+++ b/Config.hs
@@ -86,3 +86,16 @@ remoteNotIgnored r = do
 		match a = do
 			n <- Annex.getState a
 			return $ n == Git.repoRemoteName r
+
+{- If a value is specified, it is used; otherwise the default is looked up
+ - in git config. forcenumcopies overrides everything. -}
+getNumCopies :: Maybe Int -> Annex Int
+getNumCopies v = 
+	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
+	where
+		use (Just n) = return n
+		use Nothing = do
+			g <- Annex.gitRepo
+			return $ read $ Git.configGet g config "1"
+		config = "annex.numcopies"
+
diff --git a/LocationLog.hs b/LocationLog.hs
index eb48b7916a..28b423e2fa 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -19,7 +19,7 @@ module LocationLog (
 	keyLocations,
 	loggedKeys,
 	logFile,
-	logFileKey	
+	logFileKey
 ) where
 
 import System.FilePath
diff --git a/Remote.hs b/Remote.hs
index 28c2e39cdd..623b85733c 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -14,10 +14,10 @@ module Remote (
 	removeKey,
 	hasKey,
 	hasKeyCheap,
+
 	keyPossibilities,
 	keyPossibilitiesTrusted,
 	forceTrust,
-
 	remoteTypes,
 	genList,
 	byName,
@@ -25,6 +25,8 @@ module Remote (
 	remotesWithUUID,
 	remotesWithoutUUID,
 	prettyPrintUUIDs,
+	showTriedRemotes,
+	showLocations,
 
 	remoteLog,
 	readRemoteLog,
@@ -40,6 +42,7 @@ import Data.List
 import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
+import Data.String.Utils
 
 import qualified Branch
 import Types
@@ -49,6 +52,7 @@ import qualified Annex
 import Config
 import Trust
 import LocationLog
+import Messages
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -181,9 +185,34 @@ keyPossibilities' withtrusted key = do
 
 	return (sort validremotes, validtrusteduuids)
 
+{- Displays known locations of a key. -}
+showLocations :: Key -> [UUID] -> Annex ()
+showLocations key exclude = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+	uuids <- keyLocations key
+	untrusteduuids <- trustGet UnTrusted
+	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
+	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
+	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
+	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
+	showLongNote $ message ppuuidswanted ppuuidsskipped
+	where
+		filteruuids l x = filter (`notElem` x) l
+		message [] [] = "No other repository is known to contain the file."
+		message rs [] = "Try making some of these repositories available:\n" ++ rs
+		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
+		message rs us = message rs [] ++ message [] us
+
+showTriedRemotes :: [Remote Annex] -> Annex ()
+showTriedRemotes [] = return ()	
+showTriedRemotes remotes =
+	showLongNote $ "Unable to access these remotes: " ++
+		(join ", " $ map name remotes)
+
 forceTrust :: TrustLevel -> String -> Annex ()
 forceTrust level remotename = do
-	r <- Remote.nameToUUID remotename
+	r <- nameToUUID remotename
 	Annex.changeState $ \s ->
 		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 471417e345..b4006d7fd1 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -112,7 +112,7 @@ inAnnex r key = if Git.repoIsUrl r
 		checklocal = do
 			-- run a local check inexpensively,
 			-- by making an Annex monad using the remote
-			a <- Annex.new r []
+			a <- Annex.new r
 			Annex.eval a (Content.inAnnex key)
 		checkremote = do
 			showNote ("checking " ++ Git.repoDescribe r ++ "...")
@@ -142,7 +142,7 @@ copyToRemote r key
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
 		liftIO $ do
-			a <- Annex.new r []
+			a <- Annex.new r
 			Annex.eval a $ do
 				ok <- Content.getViaTmp key $
 					rsyncOrCopyFile r keysrc
diff --git a/Types/Backend.hs b/Types/Backend.hs
index 8100eaf285..f86d0845cd 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -16,22 +16,8 @@ data Backend a = Backend {
 	name :: String,
 	-- converts a filename to a key
 	getKey :: FilePath -> a (Maybe Key),
-	-- stores a file's contents to a key
-	storeFileKey :: FilePath -> Key -> a Bool,
-	-- retrieves a key's contents to a file
-	retrieveKeyFile :: Key -> FilePath -> a Bool,
-	-- removes a key, optionally checking that enough copies are stored
-	-- elsewhere
-	removeKey :: Key -> Maybe Int -> a Bool,
-	-- checks if a backend is storing the content of a key
-	hasKey :: Key -> a Bool,
 	-- called during fsck to check a key
-	-- (second parameter may be the filename associated with it)
-	-- (third parameter may be the number of copies that there should
-	-- be of the key)
-	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool,
-	-- Is a newer repesentation possible for a key?
-	upgradableKey :: Key -> a Bool
+	fsckKey :: Key -> a Bool
 }
 
 instance Show (Backend a) where
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 39b8e47c5f..c0bbeebaf4 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -191,17 +191,16 @@ logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
-	bs <- Annex.getState Annex.supportedBackends
 	tl <- liftIO $ try getsymlink
 	case tl of
 		Left _ -> return Nothing
-		Right l -> makekey bs l
+		Right l -> makekey l
 	where
 		getsymlink = do
 			l <- readSymbolicLink file
 			return $ takeFileName l
-		makekey bs l = do
-			case maybeLookupBackendName bs bname of
+		makekey l = do
+			case maybeLookupBackendName bname of
 				Nothing -> do
 					unless (null kname || null bname ||
 					        not (isLinkToAnnex l)) $
diff --git a/test.hs b/test.hs
index 44a792f14c..76ffe40479 100644
--- a/test.hs
+++ b/test.hs
@@ -25,7 +25,6 @@ import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
 
 import qualified Annex
-import qualified BackendList
 import qualified Backend
 import qualified Git
 import qualified Locations
@@ -483,7 +482,7 @@ annexeval :: Types.Annex a -> IO a
 annexeval a = do
 	g <- Git.repoFromCwd
 	g' <- Git.configRead g
-	s <- Annex.new g' BackendList.allBackends
+	s <- Annex.new g'
 	Annex.eval s a
 
 innewrepo :: Assertion -> Assertion
@@ -684,4 +683,4 @@ backendWORM :: Types.Backend Types.Annex
 backendWORM = backend_ "WORM"
 
 backend_ :: String -> Types.Backend Types.Annex
-backend_ name = Backend.lookupBackendName BackendList.allBackends name
+backend_ name = Backend.lookupBackendName name

From 6040d8aed17de582f5d5c179040e29c599315e31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 20:16:57 -0400
Subject: [PATCH 2026/2835] factor out RemoteLog

---
 Command/InitRemote.hs |  9 ++--
 Remote.hs             | 85 +------------------------------------
 RemoteLog.hs          | 97 +++++++++++++++++++++++++++++++++++++++++++
 test.hs               |  3 +-
 4 files changed, 106 insertions(+), 88 deletions(-)
 create mode 100644 RemoteLog.hs

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index a3054630c3..15962ad991 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -15,6 +15,7 @@ import Data.String.Utils
 
 import Command
 import qualified Remote
+import qualified RemoteLog
 import qualified Types.Remote as R
 import Types
 import UUID
@@ -42,7 +43,7 @@ start ws = do
 
 	where
 		name = head ws
-		config = Remote.keyValToConfig $ tail ws
+		config = RemoteLog.keyValToConfig $ tail ws
 		needname = do
 			let err s = error $ "Specify a name for the remote. " ++ s
 			names <- remoteNames
@@ -58,13 +59,13 @@ perform t u c = do
 
 cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
 cleanup u c = do
-	Remote.configSet u c
+	RemoteLog.configSet u c
         return True
 
 {- Look up existing remote's UUID and config by name, or generate a new one -}
 findByName :: String -> Annex (UUID, R.RemoteConfig)
 findByName name = do
-	m <- Remote.readRemoteLog
+	m <- RemoteLog.readRemoteLog
 	maybe generate return $ findByName' name m
 	where
 		generate = do
@@ -83,7 +84,7 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 
 remoteNames :: Annex [String]
 remoteNames = do
-	m <- Remote.readRemoteLog
+	m <- RemoteLog.readRemoteLog
 	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m
 
 {- find the specified remote type -}
diff --git a/Remote.hs b/Remote.hs
index 623b85733c..a86e1022cf 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -17,7 +17,6 @@ module Remote (
 
 	keyPossibilities,
 	keyPossibilitiesTrusted,
-	forceTrust,
 	remoteTypes,
 	genList,
 	byName,
@@ -27,24 +26,14 @@ module Remote (
 	prettyPrintUUIDs,
 	showTriedRemotes,
 	showLocations,
-
-	remoteLog,
-	readRemoteLog,
-	configSet,
-	keyValToConfig,
-	configToKeyVal,
-	
-	prop_idempotent_configEscape
+	forceTrust
 ) where
 
 import Control.Monad (filterM, liftM2)
 import Data.List
 import qualified Data.Map as M
-import Data.Maybe
-import Data.Char
 import Data.String.Utils
 
-import qualified Branch
 import Types
 import Types.Remote
 import UUID
@@ -53,6 +42,7 @@ import Config
 import Trust
 import LocationLog
 import Messages
+import RemoteLog
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -215,74 +205,3 @@ forceTrust level remotename = do
 	r <- nameToUUID remotename
 	Annex.changeState $ \s ->
 		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
-
-{- Filename of remote.log. -}
-remoteLog :: FilePath
-remoteLog = "remote.log"
-
-{- Adds or updates a remote's config in the log. -}
-configSet :: UUID -> RemoteConfig -> Annex ()
-configSet u c = do
-	m <- readRemoteLog
-	Branch.change remoteLog $ unlines $ sort $
-		map toline $ M.toList $ M.insert u c m
-	where
-		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')
-
-{- Map of remotes by uuid containing key/value config maps. -}
-readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
-
-remoteLogParse :: String -> M.Map UUID RemoteConfig
-remoteLogParse s =
-	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s
-	where
-		parseline l
-			| length w > 2 = Just (u, c)
-			| otherwise = Nothing
-			where
-				w = words l
-				u = w !! 0
-				c = keyValToConfig $ tail w
-
-{- Given Strings like "key=value", generates a RemoteConfig. -}
-keyValToConfig :: [String] -> RemoteConfig
-keyValToConfig ws = M.fromList $ map (/=/) ws
-	where
-		(/=/) s = (k, v)
-			where
-				k = takeWhile (/= '=') s
-				v = configUnEscape $ drop (1 + length k) s
-
-configToKeyVal :: M.Map String String -> [String]
-configToKeyVal m = map toword $ sort $ M.toList m
-	where
-		toword (k, v) = k ++ "=" ++ configEscape v
-
-configEscape :: String -> String
-configEscape = (>>= escape)
-	where
-		escape c
-			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"
-			| otherwise = [c]
-
-configUnEscape :: String -> String
-configUnEscape = unescape
-	where
-		unescape [] = []
-		unescape (c:rest)
-			| c == '&' = entity rest
-			| otherwise = c : unescape rest
-		entity s = if ok
-				then chr (read num) : unescape rest
-				else '&' : unescape s
-			where
-				num = takeWhile isNumber s
-				r = drop (length num) s
-				rest = drop 1 r
-				ok = not (null num) && 
-					not (null r) && r !! 0 == ';'
-
-{- for quickcheck -}
-prop_idempotent_configEscape :: String -> Bool
-prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
diff --git a/RemoteLog.hs b/RemoteLog.hs
new file mode 100644
index 0000000000..c2065db9da
--- /dev/null
+++ b/RemoteLog.hs
@@ -0,0 +1,97 @@
+{- git-annex remote log
+ - 
+ - Copyright 2011 Joey Hess 
+ - 
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module RemoteLog (
+	remoteLog,
+	readRemoteLog,
+	configSet,
+	keyValToConfig,
+	configToKeyVal,
+
+	prop_idempotent_configEscape
+) where
+
+import Data.List
+import qualified Data.Map as M
+import Data.Maybe
+import Data.Char
+
+import qualified Branch
+import Types
+import Types.Remote
+import UUID
+
+{- Filename of remote.log. -}
+remoteLog :: FilePath
+remoteLog = "remote.log"
+
+{- Adds or updates a remote's config in the log. -}
+configSet :: UUID -> RemoteConfig -> Annex ()
+configSet u c = do
+	m <- readRemoteLog
+	Branch.change remoteLog $ unlines $ sort $
+		map toline $ M.toList $ M.insert u c m
+	where
+		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')
+
+{- Map of remotes by uuid containing key/value config maps. -}
+readRemoteLog :: Annex (M.Map UUID RemoteConfig)
+readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
+
+remoteLogParse :: String -> M.Map UUID RemoteConfig
+remoteLogParse s =
+	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s
+	where
+		parseline l
+			| length w > 2 = Just (u, c)
+			| otherwise = Nothing
+			where
+				w = words l
+				u = w !! 0
+				c = keyValToConfig $ tail w
+
+{- Given Strings like "key=value", generates a RemoteConfig. -}
+keyValToConfig :: [String] -> RemoteConfig
+keyValToConfig ws = M.fromList $ map (/=/) ws
+	where
+		(/=/) s = (k, v)
+			where
+				k = takeWhile (/= '=') s
+				v = configUnEscape $ drop (1 + length k) s
+
+configToKeyVal :: M.Map String String -> [String]
+configToKeyVal m = map toword $ sort $ M.toList m
+	where
+		toword (k, v) = k ++ "=" ++ configEscape v
+
+configEscape :: String -> String
+configEscape = (>>= escape)
+	where
+		escape c
+			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"
+			| otherwise = [c]
+
+configUnEscape :: String -> String
+configUnEscape = unescape
+	where
+		unescape [] = []
+		unescape (c:rest)
+			| c == '&' = entity rest
+			| otherwise = c : unescape rest
+		entity s = if ok
+				then chr (read num) : unescape rest
+				else '&' : unescape s
+			where
+				num = takeWhile isNumber s
+				r = drop (length num) s
+				rest = drop 1 r
+				ok = not (null num) && 
+					not (null r) && r !! 0 == ';'
+
+{- for quickcheck -}
+prop_idempotent_configEscape :: String -> Bool
+prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
diff --git a/test.hs b/test.hs
index 76ffe40479..9dad37e0c0 100644
--- a/test.hs
+++ b/test.hs
@@ -36,6 +36,7 @@ import qualified LocationLog
 import qualified UUID
 import qualified Trust
 import qualified Remote
+import qualified RemoteLog
 import qualified Content
 import qualified Command.DropUnused
 import qualified Types.Key
@@ -73,7 +74,7 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
 	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
-	, qctest "prop_idempotent_configEscape" Remote.prop_idempotent_configEscape
+	, qctest "prop_idempotent_configEscape" RemoteLog.prop_idempotent_configEscape
 	, qctest "prop_parentDir_basics" Utility.prop_parentDir_basics
 	, qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane

From c98b5cf36e785cdf2c971eaf9b0329db06b68ef8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 20:24:10 -0400
Subject: [PATCH 2027/2835] rename

---
 Command/RecvKey.hs                   | 2 +-
 Command/SendKey.hs                   | 2 +-
 Command/Unlock.hs                    | 2 +-
 Crypto.hs                            | 2 +-
 Remote/Directory.hs                  | 2 +-
 Remote/Git.hs                        | 4 ++--
 Remote/Rsync.hs                      | 2 +-
 Remote/S3real.hs                     | 2 +-
 Base64.hs => Utility/Base64.hs       | 2 +-
 CopyFile.hs => Utility/CopyFile.hs   | 2 +-
 RsyncFile.hs => Utility/RsyncFile.hs | 2 +-
 11 files changed, 12 insertions(+), 12 deletions(-)
 rename Base64.hs => Utility/Base64.hs (88%)
 rename CopyFile.hs => Utility/CopyFile.hs (94%)
 rename RsyncFile.hs => Utility/RsyncFile.hs (97%)

diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index b49116de45..e2f7c74abb 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -14,7 +14,7 @@ import Command
 import CmdLine
 import Content
 import Utility
-import RsyncFile
+import Utility.RsyncFile
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index c2f793f8fe..02fedb349e 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -15,7 +15,7 @@ import qualified Annex
 import Command
 import Content
 import Utility
-import RsyncFile
+import Utility.RsyncFile
 import Messages
 
 command :: [Command]
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 8a897c3657..d189545f5d 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -16,7 +16,7 @@ import Types
 import Messages
 import Locations
 import Content
-import CopyFile
+import Utility.CopyFile
 import Utility
 
 command :: [Command]
diff --git a/Crypto.hs b/Crypto.hs
index e84e397f2e..485fb6e931 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -46,7 +46,7 @@ import Types
 import Types.Key
 import Types.Remote
 import Utility
-import Base64
+import Utility.Base64
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 991ccbe481..05d42136f1 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -22,7 +22,7 @@ import qualified Git
 import qualified Annex
 import UUID
 import Locations
-import CopyFile
+import Utility.CopyFile
 import Config
 import Content
 import Utility
diff --git a/Remote/Git.hs b/Remote/Git.hs
index b4006d7fd1..4a8f8ee928 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -22,8 +22,8 @@ import UUID
 import Utility
 import qualified Content
 import Messages
-import CopyFile
-import RsyncFile
+import Utility.CopyFile
+import Utility.RsyncFile
 import Ssh
 import Config
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index aa2507fff4..80e194fed1 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -29,7 +29,7 @@ import Remote.Special
 import Remote.Encryptable
 import Crypto
 import Messages
-import RsyncFile
+import Utility.RsyncFile
 
 type RsyncUrl = String
 
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 829c58ad06..52d1ed1be1 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -37,7 +37,7 @@ import Remote.Special
 import Remote.Encryptable
 import Crypto
 import Content
-import Base64
+import Utility.Base64
 
 remote :: RemoteType Annex
 remote = RemoteType {
diff --git a/Base64.hs b/Utility/Base64.hs
similarity index 88%
rename from Base64.hs
rename to Utility/Base64.hs
index 153049751d..dd739fd4fb 100644
--- a/Base64.hs
+++ b/Utility/Base64.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Base64 (toB64, fromB64) where
+module Utility.Base64 (toB64, fromB64) where
 
 import Codec.Binary.Base64
 import Data.Bits.Utils
diff --git a/CopyFile.hs b/Utility/CopyFile.hs
similarity index 94%
rename from CopyFile.hs
rename to Utility/CopyFile.hs
index b08ede3c88..5ee4a91df7 100644
--- a/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module CopyFile (copyFile) where
+module Utility.CopyFile (copyFile) where
 
 import System.Directory (doesFileExist, removeFile)
 
diff --git a/RsyncFile.hs b/Utility/RsyncFile.hs
similarity index 97%
rename from RsyncFile.hs
rename to Utility/RsyncFile.hs
index 48d927fcf2..c68909d2dc 100644
--- a/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module RsyncFile where
+module Utility.RsyncFile where
 
 import Data.String.Utils
 

From cab4ac247ca990a03537f7611b299efca8edaffe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 20:36:43 -0400
Subject: [PATCH 2028/2835] rename

---
 Command/Fsck.hs                      | 2 +-
 Command/Map.hs                       | 4 ++--
 Command/Status.hs                    | 2 +-
 Content.hs                           | 2 +-
 Remote/Bup.hs                        | 2 +-
 Remote/Git.hs                        | 2 +-
 Ssh.hs => Remote/Ssh.hs              | 4 ++--
 DataUnits.hs => Utility/DataUnits.hs | 2 +-
 Dot.hs => Utility/Dot.hs             | 2 +-
 9 files changed, 11 insertions(+), 11 deletions(-)
 rename Ssh.hs => Remote/Ssh.hs (96%)
 rename DataUnits.hs => Utility/DataUnits.hs (99%)
 rename Dot.hs => Utility/Dot.hs (97%)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 446d25a449..ec3f1d8e7d 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -26,7 +26,7 @@ import Content
 import LocationLog
 import Locations
 import Trust
-import DataUnits
+import Utility.DataUnits
 import Config
 
 command :: [Command]
diff --git a/Command/Map.hs b/Command/Map.hs
index 940db54c89..0391ab8e8f 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -21,8 +21,8 @@ import Types
 import Utility
 import UUID
 import Trust
-import Ssh
-import qualified Dot
+import Remote.Ssh
+import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
 data Link = Link Git.Repo Git.Repo
diff --git a/Command/Status.hs b/Command/Status.hs
index 2448f65a40..1ec4782362 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -21,7 +21,7 @@ import qualified Command.Unused
 import qualified Git
 import Command
 import Types
-import DataUnits
+import Utility.DataUnits
 import Content
 import Types.Key
 import Locations
diff --git a/Content.hs b/Content.hs
index a2f38ddc96..94f8b8c2ac 100644
--- a/Content.hs
+++ b/Content.hs
@@ -43,7 +43,7 @@ import qualified Branch
 import Utility
 import StatFS
 import Types.Key
-import DataUnits
+import Utility.DataUnits
 import Config
 
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 11c0ec4daf..5a44397f0d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -28,7 +28,7 @@ import Locations
 import Config
 import Utility
 import Messages
-import Ssh
+import Remote.Ssh
 import Remote.Special
 import Remote.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 4a8f8ee928..fb85123825 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -24,7 +24,7 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Ssh
+import Remote.Ssh
 import Config
 
 remote :: RemoteType Annex
diff --git a/Ssh.hs b/Remote/Ssh.hs
similarity index 96%
rename from Ssh.hs
rename to Remote/Ssh.hs
index 21e72c0839..0d4842a1ab 100644
--- a/Ssh.hs
+++ b/Remote/Ssh.hs
@@ -1,11 +1,11 @@
-{- git-annex repository access with ssh
+{- git-annex remote access with ssh
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Ssh where
+module Remote.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/DataUnits.hs b/Utility/DataUnits.hs
similarity index 99%
rename from DataUnits.hs
rename to Utility/DataUnits.hs
index c81c6e42e5..7af2eadafb 100644
--- a/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module DataUnits (
+module Utility.DataUnits (
 	dataUnits,
 	storageUnits,
 	memoryUnits,
diff --git a/Dot.hs b/Utility/Dot.hs
similarity index 97%
rename from Dot.hs
rename to Utility/Dot.hs
index deba10201e..8696849963 100644
--- a/Dot.hs
+++ b/Utility/Dot.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Dot where -- import qualified
+module Utility.Dot where -- import qualified
 
 {- generates a graph description from a list of lines -}
 graph :: [String] -> String

From 497b1e60926d822f8acdeb6f3df80f597e81086e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 20:53:58 -0400
Subject: [PATCH 2029/2835] Fix sign bug in disk free space checking.

Giulio Eulisse reported that on OSX, bad free space numbers were being
shown. It thought he had negative free space.

While the documentation is not clear, especially across OS's, it seems
likely that statfs uses unsigned long. It doesn't make sense for any
numbers to be negative.
---
 StatFS.hsc       | 2 +-
 debian/changelog | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index 0828f1378c..feb82aa9af 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -96,7 +96,7 @@ foreign import ccall unsafe "sys/vfs.h statfs64"
   c_statfs :: CString -> Ptr CStatfs -> IO CInt
 #endif
 
-toI :: CLong -> Integer
+toI :: CULong -> Integer
 toI = toInteger
 
 getFileSystemStats :: String -> IO (Maybe FileSystemStats)
diff --git a/debian/changelog b/debian/changelog
index 773c91b7eb..8d1a396cb5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110706) UNRELEASED; urgency=low
+
+  * Fix sign bug in disk free space checking.
+
+ -- Joey Hess   Tue, 05 Jul 2011 20:52:11 -0400
+
 git-annex (3.20110705) unstable; urgency=low
 
   * uninit: Delete the git-annex branch and .git/annex/

From 10e72acb0100820e30dab4398ba90b6232b56fcd Mon Sep 17 00:00:00 2001
From: "https://lithitux.org/openidserver/users/pavel" 
Date: Wed, 6 Jul 2011 08:14:26 +0000
Subject: [PATCH 2030/2835] Added a comment

---
 ...comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment
new file mode 100644
index 0000000000..0852db0795
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://lithitux.org/openidserver/users/pavel"
+ nickname="pavel"
+ subject="comment 6"
+ date="2011-07-06T08:14:26Z"
+ content="""
+Ah, great, thanks very much for the quick fix!
+
+Yes, when I mentioned three defunct git processes, there were three processes shown as \"git [defunct]\", plus the three git processes I listed, plus two \"git-annex\" processes. Upon cancel/resume, there were no defunct git processes when I checked, but by the time I found the bug report on the forum and commented I'd already successfully upgraded by annex (by repeatedly attaching strace) and couldn't really easily get at either additional 'ps' info or a fuller strace than what I posted (that was just the log from one of the attach/detach cycles), so it's a relief you managed to pinpoint the problem.
+"""]]

From b5733069db61dc9e03bc68a9e3aa2a8946cc3dec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jul 2011 16:06:10 -0400
Subject: [PATCH 2031/2835] tweak

---
 Remote.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index a86e1022cf..de1c7f38db 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,15 +15,15 @@ module Remote (
 	hasKey,
 	hasKeyCheap,
 
-	keyPossibilities,
-	keyPossibilitiesTrusted,
 	remoteTypes,
 	genList,
 	byName,
-	nameToUUID,
+	prettyPrintUUIDs,
 	remotesWithUUID,
 	remotesWithoutUUID,
-	prettyPrintUUIDs,
+	keyPossibilities,
+	keyPossibilitiesTrusted,
+	nameToUUID,
 	showTriedRemotes,
 	showLocations,
 	forceTrust

From a6c2bea91f17b42899b49a05ae6c2ce71c944cbc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Wed, 6 Jul 2011 20:39:11 +0000
Subject: [PATCH 2032/2835]

---
 ...unlock__34___files_without_copying_the_file_data__63__.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
new file mode 100644
index 0000000000..21c80819a4
--- /dev/null
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
@@ -0,0 +1,3 @@
+I have a DLink Boxee media player and it can not play content from symbolic links, it needs to access regular media files. Unfortunately unlocking/locking is quite slow for such a large amount of data due to the required data copying, but it should not even be needed since I do not need write access to any file to watch the movie or to play the song.
+
+Is it currently possible or would it be possible to add a commands like "unlock" which would not copy the file data but simply move files out from the data store into the tree while still keeping the files read only? A corresponding "lock" command would also be needed to restore the normal symbolic link tree structure.

From c5296dee1fa0dd5252b496b7974808538edd4564 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Wed, 6 Jul 2011 21:29:37 +0000
Subject: [PATCH 2033/2835]

---
 ...unlock__34___files_without_copying_the_file_data__63__.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
index 21c80819a4..1a7930fec4 100644
--- a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
@@ -1,3 +1,6 @@
 I have a DLink Boxee media player and it can not play content from symbolic links, it needs to access regular media files. Unfortunately unlocking/locking is quite slow for such a large amount of data due to the required data copying, but it should not even be needed since I do not need write access to any file to watch the movie or to play the song.
 
 Is it currently possible or would it be possible to add a commands like "unlock" which would not copy the file data but simply move files out from the data store into the tree while still keeping the files read only? A corresponding "lock" command would also be needed to restore the normal symbolic link tree structure.
+
+Update:
+I tried the rsync special remote http://git-annex.branchable.com/special_remotes/rsync/ and it works but the file structure created reflects the data store not the view given by the symbolic links.

From 8da1dd033684b854b595d39f0b2e86fbfef4d265 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Thu, 7 Jul 2011 11:08:35 +0000
Subject: [PATCH 2034/2835]

---
 doc/bugs/making_annex-merge_try_a_fast-forward.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/making_annex-merge_try_a_fast-forward.mdwn

diff --git a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
new file mode 100644
index 0000000000..532e771bd1
--- /dev/null
+++ b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
@@ -0,0 +1,3 @@
+While merging the git-annex branch, annex-merge does not end up in a fast-forward even when it would be possible.
+But as sometimes annex-merge takes time, it would probably be worth it
+(but maybe I miss something with my workflow...).

From f854d5ae7a17bc2ee07f9581ea6e659cd1afc341 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 7 Jul 2011 15:27:28 +0000
Subject: [PATCH 2035/2835] Added a comment

---
 ...comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment
new file mode 100644
index 0000000000..3ab518714d
--- /dev/null
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-07T15:27:28Z"
+ content="""
+The rsync or directory special remotes would work if the media player uses metadata in the files, rather than directory locations.
+
+Beyond that there is the [[todo/smudge]] idea, which is hoped to be supported sometime.
+"""]]

From 944cc2fde8c1ad37827adfdbd5e83eb18fe4f9b0 Mon Sep 17 00:00:00 2001
From: ssqq 
Date: Thu, 7 Jul 2011 18:49:49 +0000
Subject: [PATCH 2036/2835]

---
 ...annex_add_eats_files_when_filename_is_too_long.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
new file mode 100644
index 0000000000..6baf5ce815
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
@@ -0,0 +1,10 @@
+Recently I ran into the following situation under Ubuntu with an encrypted home directory (which shortens the length that filenames can be):
+
+$ git annex add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+git-annex: /home/lhuhn/annex/.git/annex/tmp/155_518_WORM-s426663-m1310064100--687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+git-annex: 1 failed
+
+The file seems to be completely gone.  It no longer exists in the current directory, or under .git/annex.
+
+I don't mind horribly that git-annex failed due to the name length limit, but it shouldn't have deleted my file in the process (fortunately the file wasn't very important, or hard to recover).

From 1ffe7f777037e08abc4b068a4547e2baa0e714ee Mon Sep 17 00:00:00 2001
From: ssqq 
Date: Thu, 7 Jul 2011 19:44:40 +0000
Subject: [PATCH 2037/2835]

---
 ...acter___40__and_probably_others__41__.mdwn | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn

diff --git a/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn
new file mode 100644
index 0000000000..28d11d6ee6
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn
@@ -0,0 +1,32 @@
+"git annex upgrade" has lost track of some of my files.  Most of them have "&" characters.  The others contain "%" characters (I haven't tried the testcase below with "%" however).
+
+Testcase:
+
+    # (With git annex v2)
+    mkdir ~/testannex1
+    cd ~/testannex1
+    git init
+    git annex init "testannex1"
+    touch '02 - Afternoons & Coffeespoons.mp3'
+    touch 'no ampersand.mp3'
+    git annex add '02 - Afternoons & Coffeespoons.mp3'
+    git annex add 'no ampersand.mp3'
+    git commit -m added
+    git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    git annex whereis 'no ampersand.mp3'
+    # (Upgrade git-annex binary to v3 and then...)
+    git annex upgrade
+    git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    git annex whereis 'no ampersand.mp3'
+
+This produces:
+
+    12:38:40 ~/testannex1 (master)$ git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    whereis 02 - Afternoons & Coffeespoons.mp3 (0 copies) 
+    failed
+    git-annex: 1 failed
+    12:38:40 ~/testannex1 (master)$ git annex whereis 'no ampersand.mp3'
+    whereis no ampersand.mp3 (1 copy) 
+      	a7b680fc-a8d0-11e0-b0fe-4f94e86d1fb7  -- testannex1 <-- here
+    ok
+

From 33ebaf3f9ff77526b0fae2ee39d049117ffd188a Mon Sep 17 00:00:00 2001
From: ssqq 
Date: Thu, 7 Jul 2011 19:45:31 +0000
Subject: [PATCH 2038/2835]

---
 ...it_annex_add_eats_files_when_filename_is_too_long.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
index 6baf5ce815..af807b65d2 100644
--- a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
@@ -1,9 +1,9 @@
 Recently I ran into the following situation under Ubuntu with an encrypted home directory (which shortens the length that filenames can be):
 
-$ git annex add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
-add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
-git-annex: /home/lhuhn/annex/.git/annex/tmp/155_518_WORM-s426663-m1310064100--687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
-git-annex: 1 failed
+    $ git annex add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+    add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+    git-annex: /home/lhuhn/annex/.git/annex/tmp/155_518_WORM-s426663-m1310064100--687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+    git-annex: 1 failed
 
 The file seems to be completely gone.  It no longer exists in the current directory, or under .git/annex.
 

From 0c52362359687b47b5c70ca9985e4ee6fc9409dd Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 7 Jul 2011 20:27:33 +0000
Subject: [PATCH 2039/2835] Added a comment

---
 ..._9650284913bec2a00cf551b90ab5d8ff._comment | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment
new file mode 100644
index 0000000000..1df159181d
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-07T20:27:33Z"
+ content="""
+When I reproduce this, the file is not gone, it's been moved under .git/annex/objects. There is no way an add can delete a file, since all it does is rename it. It would be good for it to error unwind and move the file back though.
+
+
+joey@gnu:~/tmp/a>touch 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+joey@gnu:~/tmp/a>git annex add *.gif
+add 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+git-annex: /home/joey/tmp/a/.git/annex/tmp/8e2_6a4_WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+joey@gnu:~/tmp/a>touch 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+joey@gnu:~/tmp/a>git annex add *.gif
+add 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+git-annex: /home/joey/tmp/a/.git/annex/tmp/8e2_6a4_WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+joey@gnu:~/tmp/a>find .git/annex/objects -type f
+.git/annex/objects/Mk/92/WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif/WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+

+"""]]

From 2fb771f135ad0a5adec0349a6270cadc518e04f6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 17:04:21 -0400
Subject: [PATCH 2040/2835] Bugfix: Forgot to de-escape keys when upgrading.

Could result in bad location log data for keys that contain [&:%] in their
names. (A workaround for this problem is to run git annex fsck.)

`git annex unused --from remote` could also run into the broken code.
---
 LocationLog.hs   | 2 +-
 debian/changelog | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index 28b423e2fa..aab817f3fc 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -60,7 +60,7 @@ logFile key = hashDirLower key ++ keyFile key ++ ".log"
 {- Converts a log filename into a key. -}
 logFileKey :: FilePath -> Maybe Key
 logFileKey file
-	| end == ".log" = readKey beginning
+	| end == ".log" = fileKey beginning
 	| otherwise = Nothing
 	where
 		(beginning, end) = splitAt (length file - 4) file
diff --git a/debian/changelog b/debian/changelog
index 8d1a396cb5..ef0ec77191 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,9 @@
 git-annex (3.20110706) UNRELEASED; urgency=low
 
   * Fix sign bug in disk free space checking.
+  * Bugfix: Forgot to de-escape keys when upgrading. Could result in
+    bad location log data for keys that contain [&:%] in their names.
+    (A workaround for this problem is to run git annex fsck.)
 
  -- Joey Hess   Tue, 05 Jul 2011 20:52:11 -0400
 

From dac158c7b206b2f4d82ae6b993189e8148ad5f09 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 7 Jul 2011 21:04:23 +0000
Subject: [PATCH 2041/2835] Added a comment

---
 ...mment_1_861506e40e0d04d2be98bbfe9188be89._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment

diff --git a/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment
new file mode 100644
index 0000000000..194b36ac10
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-07T21:04:23Z"
+ content="""
+What an evil little bug. In retrospect, this probably bit my own test upgrades, but I ran `git annex fsck` everywhere and so avoided the location log breakage.
+
+I've fixed the bug, which also involved files with other punctuation in their names [&:%] when using the WORM backend. 
+
+The only way I have to recover repos that have already been upgraded is to run `git annex fsck --fast` in each clone of such a repo, which will let it rebuild the location log information. I think that is the best way to recover; ie I can't think of a way to recover that doesn't need to do everything fsck does anyway.
+"""]]

From 67dcc1f171f0bbe2b57d20fbafce9f6c9b8f781e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:29:36 -0400
Subject: [PATCH 2042/2835] add: Avoid a failure mode that resulted in the file
 seemingly being deleted (content put in the annex but no symlink present).

---
 Command/Add.hs                                                | 4 ++--
 debian/changelog                                              | 2 ++
 .../git_annex_add_eats_files_when_filename_is_too_long.mdwn   | 4 ++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 2831e1b35f..e7d16b6c07 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -51,11 +51,11 @@ perform (file, backend) = do
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	logStatus key InfoPresent
-
 	link <- calcGitLink file key
 	liftIO $ createSymbolicLink link file
 
+	logStatus key InfoPresent
+
 	-- touch the symlink to have the same mtime as the file it points to
 	s <- liftIO $ getFileStatus file
 	let mtime = modificationTime s
diff --git a/debian/changelog b/debian/changelog
index ef0ec77191..eb9037771b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20110706) UNRELEASED; urgency=low
   * Bugfix: Forgot to de-escape keys when upgrading. Could result in
     bad location log data for keys that contain [&:%] in their names.
     (A workaround for this problem is to run git annex fsck.)
+  * add: Avoid a failure mode that resulted in the file seemingly being
+    deleted (content put in the annex but no symlink present).
 
  -- Joey Hess   Tue, 05 Jul 2011 20:52:11 -0400
 
diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
index af807b65d2..d17e569f17 100644
--- a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
@@ -8,3 +8,7 @@ Recently I ran into the following situation under Ubuntu with an encrypted home
 The file seems to be completely gone.  It no longer exists in the current directory, or under .git/annex.
 
 I don't mind horribly that git-annex failed due to the name length limit, but it shouldn't have deleted my file in the process (fortunately the file wasn't very important, or hard to recover).
+
+> [[done]], as noted it did not delete content and now it makes the symlink
+> before trying to write to the location log, avoiding that gotcha.
+> --[[Joey]] 

From 4d4f297c9620348c8c86ddd2155cf49bf38424ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:37:49 -0400
Subject: [PATCH 2043/2835] releasing version 3.20110707

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index eb9037771b..626e388371 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110706) UNRELEASED; urgency=low
+git-annex (3.20110707) unstable; urgency=low
 
   * Fix sign bug in disk free space checking.
   * Bugfix: Forgot to de-escape keys when upgrading. Could result in
@@ -7,7 +7,7 @@ git-annex (3.20110706) UNRELEASED; urgency=low
   * add: Avoid a failure mode that resulted in the file seemingly being
     deleted (content put in the annex but no symlink present).
 
- -- Joey Hess   Tue, 05 Jul 2011 20:52:11 -0400
+ -- Joey Hess   Thu, 07 Jul 2011 19:29:39 -0400
 
 git-annex (3.20110705) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 3ea7e5fb39..eb02ae0cad 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110705
+Version: 3.20110707
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From f7be0d5077ef95fa76332c8ede5aec3cb0acf408 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:38:05 -0400
Subject: [PATCH 2044/2835] add news item for git-annex 3.20110707

---
 doc/news/version_0.20110601.mdwn | 9 ---------
 doc/news/version_3.20110707.mdwn | 8 ++++++++
 2 files changed, 8 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_0.20110601.mdwn
 create mode 100644 doc/news/version_3.20110707.mdwn

diff --git a/doc/news/version_0.20110601.mdwn b/doc/news/version_0.20110601.mdwn
deleted file mode 100644
index 59079088da..0000000000
--- a/doc/news/version_0.20110601.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 0.20110601 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Minor bugfixes and error message improvements.
-   * Massively sped up `git annex lock` by avoiding use of the uber-slow
-     `git reset`, and only running `git checkout` once, even when many files
-     are being locked.
-   * Fix locking of files with staged changes.
-   * Somewhat sped up `git commit` of modifications to unlocked files.
-   * Build fix for older ghc."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110707.mdwn b/doc/news/version_3.20110707.mdwn
new file mode 100644
index 0000000000..3f489ae5d6
--- /dev/null
+++ b/doc/news/version_3.20110707.mdwn
@@ -0,0 +1,8 @@
+git-annex 3.20110707 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix sign bug in disk free space checking.
+   * Bugfix: Forgot to de-escape keys when upgrading. Could result in
+     bad location log data for keys that contain [&:%] in their names.
+     (A workaround for this problem is to run git annex fsck.)
+   * add: Avoid a failure mode that resulted in the file seemingly being
+     deleted (content put in the annex but no symlink present)."""]]
\ No newline at end of file

From 8c1fa1ab5f15c096e76fd2727f41b3bc042a2f3a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:49:24 -0400
Subject: [PATCH 2045/2835] add a nasty workaround for a nasty cabal limitation

It croaks on long filenames.. probably >= 100 chars
100 characters was a (historial) limit on filenames in tarballs.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 786fd919e4..ccf98f6252 100644
--- a/Makefile
+++ b/Makefile
@@ -91,7 +91,7 @@ clean:
 # generate a file list there.
 sdist: clean
 	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi
-	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -not -type d -print)!i" < git-annex.cabal.orig > git-annex.cabal
+	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -not -type d -print | perl -ne 'print unless length >= 100')!i" < git-annex.cabal.orig > git-annex.cabal
 	@cabal sdist
 	@mv git-annex.cabal.orig git-annex.cabal
 

From 9633deab573922c8178e714c72ea9709dfd23104 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Fri, 8 Jul 2011 00:21:32 +0000
Subject: [PATCH 2046/2835] Added a comment: this happens also when the user
 has not the permission to set the file mode

---
 .../comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment
new file mode 100644
index 0000000000..bd53627bbc
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="this happens also when the user has not the permission to set the file mode"
+ date="2011-07-08T00:21:31Z"
+ content="""
+For example if the file is owned by root, I guess git-annex fails when it tries to remove write permissions (I retested with the last version of today (whose \"version\" subcommand still outputs 3.20110702)).By the way, it would be nice to have a log file created containing the list of all failures, to avoid having to scan manually all the output of a long git-annex operation.
+"""]]

From 4857f4e86156c5f119374110aff6c62a884a5d96 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Fri, 8 Jul 2011 00:45:30 +0000
Subject: [PATCH 2047/2835] Added a comment

---
 .../comment_3_5776864d78d56849001dd12e3adb9cbe._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment
new file mode 100644
index 0000000000..f9d1b5d682
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 3"
+ date="2011-07-08T00:45:30Z"
+ content="""
+comment on the output of 'git-annex version' (from my last comment): now I get the right version 3.20110707. But I checked in my console that the three commands \"git checkout 3.20110707\", \"make\" and \"./git-annex version\" gave me before 3.20110702, I don't know why...
+"""]]

From 2a108982ad95f9bba9bb1ae1c6152ef9b10be53c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:10:42 -0400
Subject: [PATCH 2048/2835] add monad-control to build depends

Will use this to handle exceptions in the Annex monad, yay.
---
 debian/control          | 1 +
 doc/install.mdwn        | 1 +
 doc/install/Fedora.mdwn | 1 +
 doc/install/OSX.mdwn    | 1 +
 git-annex.cabal         | 2 +-
 5 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index 4347233841..d519c42b71 100644
--- a/debian/control
+++ b/debian/control
@@ -13,6 +13,7 @@ Build-Depends:
 	libghc-utf8-string-dev,
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
+	libghc-monad-control-dev,
 	ikiwiki,
 	perlmagick,
 	git | git-core,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 38963695b8..49ddd913f0 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -23,6 +23,7 @@ To build and use git-annex, you will need:
   * [utf8-string](http://hackage.haskell.org/package/utf8-string)
   * [SHA](http://hackage.haskell.org/package/SHA)
   * [dataenc](http://hackage.haskell.org/package/dataenc)
+  * [monad-control](http://hackage.haskell.org/package/monad-control)
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [HTTP](http://hackage.haskell.org/package/HTTP)
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 7814eec940..0c1da2e6a5 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -9,6 +9,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install monad-control
 sudo cabal install HTTP
 sudo cabal install hS3
 
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index ade4fa30e8..23cb1b62e2 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,6 +7,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck  
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install monad-control
 sudo cabal install HTTP
 sudo cabal install hS3 # optional
 
diff --git a/git-annex.cabal b/git-annex.cabal
index eb02ae0cad..29ad80a58a 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5
+   base < 5, monad-control
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From 2640ee820f4269ccc1b5f4cd184aaf895fcf405d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 20:59:05 -0400
Subject: [PATCH 2049/2835] cleanup

---
 LocationLog.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index aab817f3fc..fe09482b9f 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -30,7 +30,6 @@ import qualified Git
 import qualified Branch
 import UUID
 import Types
-import Types.Key
 import Locations
 import PresenceLog
 

From 40c6ba99f51875db28f3e1e8b309812c66594e32 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 21:29:31 -0400
Subject: [PATCH 2050/2835] add: Be even more robust to avoid ever leaving the
 file seemingly deleted.

A failure at any point after the file is annexed will result in an undo
that puts the original file back into place and wipes the location log.
---
 Command/Add.hs   | 44 ++++++++++++++++++++++++++++++++++----------
 debian/changelog |  6 ++++++
 2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index e7d16b6c07..5c7cad044e 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -9,6 +9,10 @@ module Command.Add where
 
 import Control.Monad.State (liftIO)
 import System.Posix.Files
+import System.Directory
+import Control.Exception.Control (handle)
+import Control.Exception.Base (throwIO)
+import Control.Exception.Extensible (IOException)
 
 import Command
 import qualified Annex
@@ -20,6 +24,7 @@ import Content
 import Messages
 import Utility
 import Touch
+import Locations
 
 command :: [Command]
 command = [repoCommand "add" paramPath seek "add files to annex"]
@@ -46,20 +51,39 @@ perform (file, backend) = do
 	case k of
 		Nothing -> stop
 		Just (key, _) -> do
-			moveAnnex key file
+			handle (undo file key) $ moveAnnex key file
 			next $ cleanup file key
 
+{- On error, put the file back so it doesn't seem to have vanished.
+ - This can be called before or after the symlink is in place. -}
+undo :: FilePath -> Key -> IOException -> Annex a
+undo file key e = do
+	unlessM (inAnnex key) $ rethrow -- no cleanup to do
+	liftIO $ whenM (doesFileExist file) $ do removeFile file
+	handle tryharder $ fromAnnex key file
+	logStatus key InfoMissing
+	rethrow
+	where
+		rethrow = liftIO $ throwIO e
+
+		-- fromAnnex could fail if the file ownership is weird
+		tryharder :: IOException -> Annex ()
+		tryharder _ = do
+			g <- Annex.gitRepo
+			liftIO $ renameFile (gitAnnexLocation g key) file
+
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	link <- calcGitLink file key
-	liftIO $ createSymbolicLink link file
-
-	logStatus key InfoPresent
-
-	-- touch the symlink to have the same mtime as the file it points to
-	s <- liftIO $ getFileStatus file
-	let mtime = modificationTime s
-	liftIO $ touch file (TimeSpec mtime) False
+	handle (undo file key) $ do
+		link <- calcGitLink file key
+		liftIO $ createSymbolicLink link file
+		logStatus key InfoPresent
+	
+		-- touch the symlink to have the same mtime as the
+		-- file it points to
+		s <- liftIO $ getFileStatus file
+		let mtime = modificationTime s
+		liftIO $ touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/debian/changelog b/debian/changelog
index 626e388371..80fe84256b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110708) UNRELEASED; urgency=low
+
+  * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+
+ -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
+
 git-annex (3.20110707) unstable; urgency=low
 
   * Fix sign bug in disk free space checking.

From 151b1d85c7f5cc4d52f29ac64499622ce6c3a5c8 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 8 Jul 2011 01:32:30 +0000
Subject: [PATCH 2051/2835] Added a comment

---
 .../comment_4_371ec7b4ae73280ede31edfe90b42a95._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment
new file mode 100644
index 0000000000..1ba57c1992
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-07-08T01:32:30Z"
+ content="""
+Indeed, I've made it even more robust now, handling the case where the file has weird permissions too, and undoing the failed add so the file is always back at the start state. Had to add a dependency on another haskell module to allow this, so it took some time to figure out how to do it..
+
+"""]]

From a8fe35f645234d91b8259a12106ed7b0bc8b55dd Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 8 Jul 2011 03:39:22 +0000
Subject: [PATCH 2052/2835]

---
 ...e_of_massively_disconnected_operation.mdwn | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 doc/forum/example_of_massively_disconnected_operation.mdwn

diff --git a/doc/forum/example_of_massively_disconnected_operation.mdwn b/doc/forum/example_of_massively_disconnected_operation.mdwn
new file mode 100644
index 0000000000..00a5d8d6c5
--- /dev/null
+++ b/doc/forum/example_of_massively_disconnected_operation.mdwn
@@ -0,0 +1,33 @@
+I found this archival drive that had been offline since October 26th 2010. Since I released git-annex 0.02 on October 27th, this must have been made using the very first release of git-annex, ever.
+
+So, I synced it back up! :) --[[Joey]]
+
+
+commit 4151f4595fe6205d4aed653617ab23eb3335130a
+Author: Joey Hess 
+Date:   Tue Oct 26 02:18:03 2010 -0400
+
+joey> git pull
+remote: Counting objects: 428782, done.
+remote: Compressing objects: 100% (280714/280714), done.
+remote: Total 416692 (delta 150923), reused 389593 (delta 125143)
+Receiving objects: 100% (416692/416692), 44.71 MiB | 495 KiB/s, done.
+Resolving deltas: 100% (150923/150923), completed with 818 local objects.
+ * [new branch]      git-annex  -> origin/git-annex
+   1893f9c..9ebcc0e  master     -> origin/master
+Updating 1893f9c..9ebcc0e
+Checking out files: 100% (76884/76884), done.
+joey> git annex version
+git-annex version: 3.20110611
+local repository version: unknown
+default repository version: 3
+supported repository versions: 3
+upgrade supported from repository versions: 0 1 2
+joey> git config annex.version 0
+joey> git annex upgrade
+upgrade . (v0 to v1...) (v1 to v2) (moving content...) (updating symlinks...)  (moving location logs...) (v2 to v3) (merging origin/git-annex into git-annex...)
+
+  git-annex branch created
+  Be sure to push this branch when pushing to remotes.
+ok
+


From 7da059e5570652de5f882149ff981672d3424355 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 8 Jul 2011 01:26:18 -0400
Subject: [PATCH 2053/2835] update

---
 doc/upgrades.mdwn | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index d62e9dcc78..c4aac53482 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -88,5 +88,6 @@ Involved a reorganisation of the layout of .git/annex/. Symlinks changed.
 Handled more or less transparently, although git-annex was just 2 weeks
 old at the time, and had few users other than Joey.
 
-This upgrade is believed to still be supported, but has not been tested
-lately.
+Before doing this upgrade, set annex.version:
+
+	git config annex.version 0

From 085eeaa6529df0b14f7b6b1c9ca576caa2b18c0b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sat, 9 Jul 2011 01:49:09 +0000
Subject: [PATCH 2054/2835]

---
 ..._39__git_add__39___for_parent_relative_path.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn

diff --git a/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
new file mode 100644
index 0000000000..1243bbfb20
--- /dev/null
+++ b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
@@ -0,0 +1,13 @@
+The following commands show the failure:
+
+$ mkdir d && touch d/f
+
+$ mkdir g && cd g && git annex add ../d/f 
+
+add ... ok
+
+error: Invalid path '.git/annex/objects/Jx/...
+
+...
+
+Then it seems it is enough to 'git add ../d/f' to complete the operation.

From 7919de73af81d1788867ecbcbcc17e25b348ad1d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jul 2011 13:52:53 -0400
Subject: [PATCH 2055/2835] Bugfix: Make add ../ work.

The complication of check-attr returning absolute paths that have to be
converted back to relative paths..
---
 Git.hs                                              | 13 ++++++++-----
 debian/changelog                                    |  1 +
 ..._39__git_add__39___for_parent_relative_path.mdwn |  2 ++
 test.hs                                             | 10 +++++++++-
 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/Git.hs b/Git.hs
index 7005f24f55..7827f80514 100644
--- a/Git.hs
+++ b/Git.hs
@@ -530,6 +530,7 @@ checkAttr repo attr files = do
 	-- directory. Convert to absolute, and then convert the filenames
 	-- in its output back to relative.
 	cwd <- getCurrentDirectory
+	let top = workTree repo
 	let absfiles = map (absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
         _ <- forkProcess $ do
@@ -539,19 +540,21 @@ checkAttr repo attr files = do
                 exitSuccess
         hClose toh
 	s <- hGetContents fromh
-	return $ map (topair $ cwd++"/") $ lines s
+	return $ map (topair cwd top) $ lines s
 	where
 		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]
-		topair cwd l = (relfile, value)
+		topair cwd top l = (relfile, value)
 			where 
-				relfile 
-					| startswith cwd file = drop (length cwd) file
-					| otherwise = file
+				relfile
+					| startswith cwd' file = drop (length cwd') file
+					| otherwise = relPathDirToFile top' file
 				file = decodeGitFile $ join sep $ take end bits
 				value = bits !! end
 				end = length bits - 1
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
+				cwd' = cwd ++ "/"
+				top' = top ++ "/"
 
 {- Some git commands output encoded filenames. Decode that (annoyingly
  - complex) encoding. -}
diff --git a/debian/changelog b/debian/changelog
index 80fe84256b..ec3176d12a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110708) UNRELEASED; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+  * Bugfix: Make add ../ work.
 
  -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
 
diff --git a/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
index 1243bbfb20..f129abf623 100644
--- a/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
+++ b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
@@ -11,3 +11,5 @@ error: Invalid path '.git/annex/objects/Jx/...
 ...
 
 Then it seems it is enough to 'git add ../d/f' to complete the operation.
+
+> Thanks for reporting, [[fixed|done]] --[[Joey]] 
diff --git a/test.hs b/test.hs
index 9dad37e0c0..51ccc600e6 100644
--- a/test.hs
+++ b/test.hs
@@ -108,7 +108,7 @@ test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 		reponame = "test repo"
 
 test_add :: Test
-test_add = "git-annex add" ~: TestList [basic, sha1dup]
+test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 	where
 		-- this test case runs in the main repo, to set up a basic
 		-- annexed file that later tests will use
@@ -129,6 +129,14 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup]
 			git_annex "add" ["-q", sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
 			annexed_present sha1annexedfiledup
 			annexed_present sha1annexedfile
+		subdirs = TestCase $ intmpclonerepo $ do
+			createDirectory "dir"
+			writeFile "dir/foo" $ content annexedfile
+			git_annex "add" ["-q", "dir"] @? "add of subdir failed"
+			createDirectory "dir2"
+			writeFile "dir2/foo" $ content annexedfile
+			changeWorkingDirectory "dir"
+			git_annex "add" ["-q", "../dir2"] @? "add of ../subdir failed"
 
 test_setkey :: Test
 test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do

From c38dd9adc844688fb7ebdc8f41db16cf52de0939 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jul 2011 15:37:29 -0400
Subject: [PATCH 2056/2835] analysis

---
 ...making_annex-merge_try_a_fast-forward.mdwn | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
index 532e771bd1..a2bd8c7479 100644
--- a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
+++ b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
@@ -1,3 +1,29 @@
 While merging the git-annex branch, annex-merge does not end up in a fast-forward even when it would be possible.
 But as sometimes annex-merge takes time, it would probably be worth it
 (but maybe I miss something with my workflow...).
+
+> I don't think a fast-forward will make things much faster.
+> 
+> git-annex needs its index file to be updated to reflect the merge.
+> With the union merge it does now, this can be accomplished by using
+> `git-diff-index` to efficiently get a list of files that have changed,
+> and only merge those changes into the index with `git-update-index`.
+> Then the index gets committed, generating the merge.
+> 
+> To fast-forward, it would just reset the git-annex branch to the new
+> head of the remote it's merging to. But then the index needs to be
+> updated to reflect this new head too. To do that needs the same method
+> described above, essentially (with the difference that it can replace
+> files in the index with the version from the git-annex branch, rather
+> than merging in the changes... but only if the index is known to be
+> already committed and have no other changes, which would require both
+> an attempt to commit it first, and
+> locking). 
+> 
+> So will take basically the same amount of time, except
+> it would not need to commit the index at the end of the merge. The
+> most expensive work is the `git-diff-index` and `git-update-index`,
+> which are not avoided.
+> 
+> Although, perhaps fast-forward merge would use slightly
+> less space. --[[Joey]]

From 0fee31a164aaaa951a8b8bd66acce65b1e94ca43 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 13 Jul 2011 10:35:18 +0000
Subject: [PATCH 2057/2835]

---
 ..._normal_files_in_the_git-annex_git_repository__63__.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn

diff --git a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
new file mode 100644
index 0000000000..1105cd0271
--- /dev/null
+++ b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
@@ -0,0 +1,6 @@
+Is it possible to story ordinary files in the git repository, or is this going to confuse git-annex? In other words, can I safely run
+
+  git add .gitattributes
+  git commit -m 'remember attributes' .gitattributes
+
+..., or do I have to use `git-annex add` all time?

From 709f75f1878961d70e913a0f1020a134a7e16bc2 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 13 Jul 2011 10:35:47 +0000
Subject: [PATCH 2058/2835]

---
 ...re_normal_files_in_the_git-annex_git_repository__63__.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
index 1105cd0271..6d1083dd57 100644
--- a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
+++ b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
@@ -1,6 +1,6 @@
 Is it possible to story ordinary files in the git repository, or is this going to confuse git-annex? In other words, can I safely run
 
-  git add .gitattributes
-  git commit -m 'remember attributes' .gitattributes
+    git add .gitattributes
+    git commit -m 'remember attributes' .gitattributes
 
 ..., or do I have to use `git-annex add` all time?

From a4f2dd2fc60da291f5aa37ba814affeae134ff90 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 13 Jul 2011 16:21:26 +0000
Subject: [PATCH 2059/2835] Added a comment: Solved

---
 .../comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment

diff --git a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment
new file mode 100644
index 0000000000..8873edcde8
--- /dev/null
+++ b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.12.196.1"
+ subject="Solved"
+ date="2011-07-13T16:21:25Z"
+ content="""
+I got my answer on #vcs-home: Yes, git-annex and git get along fine.
+"""]]

From 020787bb974da8eafb5002b9427d44e21b3c802e Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Thu, 14 Jul 2011 12:15:42 +0000
Subject: [PATCH 2060/2835]

---
 ..._command_has_tons_of_redundant_-a_paramters.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn

diff --git a/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
new file mode 100644
index 0000000000..80bf562ec7
--- /dev/null
+++ b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
@@ -0,0 +1,13 @@
+This doesn't look right:
+
+    simons   11148  0.0  0.0  15572  1268 pts/1    SN+  04:00   0:00              |               \_ git annex unannex stuff
+    simons   11150  0.5  0.0 130504 11212 pts/1    SN+  04:00   3:40              |               |   \_ git-annex unannex stuff
+    simons   11152  0.0  0.1  39536 23932 pts/1    SN+  04:00   0:00              |               |       \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex ls-files --cached -z -- stuff
+    simons   11288  0.0  0.0      0     0 pts/1    ZN+  04:01   0:00              |               |       \_ [git] 
+    simons   11339  0.0  0.0      0     0 pts/1    ZN+  04:02   0:00              |               |       \_ [git-annex] 
+    simons   11442  0.0  0.0      0     0 pts/1    ZN+  04:06   0:00              |               |       \_ [git] 
+    simons   11443  0.0  0.0      0     0 pts/1    ZN+  04:06   0:05              |               |       \_ [git] 
+    simons   16541  0.0  0.0      0     0 pts/1    ZN+  04:14   0:00              |               |       \_ [git] 
+    simons   16543  0.3  0.0  15644  1744 pts/1    SN+  04:14   2:13              |               |       \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex cat-file --batch
+    simons   14224  0.0  0.0 100744   796 pts/1    SN+  14:10   0:00              |               |       \_ xargs -0 git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex
+    simons   14225  0.4  0.1  32684 18652 pts/1    DN+  14:10   0:00              |               |           \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a

From dddbc09ff0199967602ad5d4112b2b4e04780177 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jul 2011 16:41:17 -0400
Subject: [PATCH 2061/2835] allow configStore to be run incrementally to
 override configs

---
 Git.hs | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/Git.hs b/Git.hs
index 7827f80514..2d03163bbf 100644
--- a/Git.hs
+++ b/Git.hs
@@ -466,13 +466,24 @@ hConfigRead repo h = do
 	val <- hGetContentsStrict h
 	configStore repo val
 
-{- Parses a git config and returns a version of the repo using it. -}
+{- Stores a git config into a repo, returning the new version of the repo.
+ - The git config may be multiple lines, or a single line. Config settings
+ - can be updated inrementally. -}
 configStore :: Repo -> String -> IO Repo
 configStore repo s = do
-	rs <- configRemotes r
-	return $ r { remotes = rs }
+	let repo' = repo { config = Map.union (configParse s) (config repo) }
+	rs <- configRemotes repo'
+	return $ repo' { remotes = rs }
+
+{- Parses git config --list output into a config map. -}
+configParse :: String -> Map.Map String String
+configParse s = Map.fromList $ map pair $ lines s
 	where
-		r = repo { config = configParse s }
+		pair l = (key l, val l)
+		key l = head $ keyval l
+		val l = join sep $ drop 1 $ keyval l
+		keyval l = split sep l :: [String]
+		sep = "="
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
 configRemotes :: Repo -> IO [Repo]
@@ -503,16 +514,6 @@ configRemotes repo = mapM construct remotepairs
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 
-{- Parses git config --list output into a config map. -}
-configParse :: String -> Map.Map String String
-configParse s = Map.fromList $ map pair $ lines s
-	where
-		pair l = (key l, val l)
-		key l = head $ keyval l
-		val l = join sep $ drop 1 $ keyval l
-		keyval l = split sep l :: [String]
-		sep = "="
-
 {- Returns a single git config setting, or a default value if not set. -}
 configGet :: Repo -> String -> String -> String
 configGet repo key defaultValue = 

From 0c46cbab09af8cc8761668885e58944d397b856d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jul 2011 16:44:23 -0400
Subject: [PATCH 2062/2835] Support the standard git -c name=value

This allows eg, `git-annex -c annex.rsync-options=-6 get file`

The overridden git configs are not passed on to git plumbing commands
that are run. Perhaps someone will find a need to do that, but I don't yet
and it would require storing more state to know what config settings
have been overridden and need to be passed on.
---
 GitAnnex.hs        | 11 ++++++++++-
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  4 ++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index 85eb2bf26e..6f4e5d4921 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -8,12 +8,14 @@
 module GitAnnex where
 
 import System.Console.GetOpt
+import Control.Monad.State (liftIO)
 
 import qualified Git
 import CmdLine
 import Command
 import Options
 import Utility
+import Types
 import Types.TrustLevel
 import qualified Annex
 import qualified Remote
@@ -102,9 +104,11 @@ options = commonOptions ++
 	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)
 		"override trust setting"
 	, Option [] ["semitrust"] (ReqArg (Remote.forceTrust SemiTrusted) paramRemote)
-		"override trust setting back to default value"
+		"override trust setting back to default"
 	, Option [] ["untrust"] (ReqArg (Remote.forceTrust UnTrusted) paramRemote)
 		"override trust setting to untrusted"
+	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
+		"override git configuration setting"
 	]
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
@@ -112,6 +116,11 @@ options = commonOptions ++
 		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
+		setgitconfig :: String -> Annex ()
+		setgitconfig v = do
+			g <- Annex.gitRepo
+			g' <- liftIO $ Git.configStore g v
+			Annex.changeState $ \s -> s { Annex.repo = g' }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/debian/changelog b/debian/changelog
index ec3176d12a..673084d2d8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110708) UNRELEASED; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
   * Bugfix: Make add ../ work.
+  * Support the standard git -c name=value
 
  -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index dfb23f5f21..11f617f1ba 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -387,6 +387,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Specifies a key to operate on.
 
+* -c name=value
+
+  Used to override git configuration settings. May be specified multiple times.
+
 # CONFIGURATION
 
 Like other git commands, git-annex is configured via `.git/config`.

From ded259112449f592bc42207e89c82268f3795f12 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jul 2011 16:56:06 -0400
Subject: [PATCH 2063/2835] unannex: Clean up use of git commit -a.

This was more complex than would be expected. unannex has to use git commit -a
since it's removing files from git; git commit filelist won't do.

Allow commands to be added to the Git queue that have no associated files,
and run such commands once.
---
 AnnexQueue.hs                                   |  6 +++---
 Command/Add.hs                                  |  4 ++--
 Command/Fix.hs                                  |  2 +-
 Command/FromKey.hs                              |  2 +-
 Command/Lock.hs                                 |  2 +-
 Command/Unannex.hs                              |  2 +-
 Git/Queue.hs                                    | 17 ++++++++++-------
 Upgrade/V1.hs                                   |  8 ++++----
 debian/changelog                                |  1 +
 ...mand_has_tons_of_redundant_-a_paramters.mdwn |  2 ++
 10 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 4c35adfb87..b1678df07f 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -21,10 +21,10 @@ import Utility
 
 {- Adds a git command to the queue, possibly running previously queued
  - actions if enough have accumulated. -}
-add :: String -> [CommandParam] -> FilePath -> Annex ()
-add command params file = do
+add :: String -> [CommandParam] -> [FilePath] -> Annex ()
+add command params files = do
 	q <- getState repoqueue
-	store $ Git.Queue.add q command params file
+	store $ Git.Queue.add q command params files
 
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
diff --git a/Command/Add.hs b/Command/Add.hs
index 5c7cad044e..51b95b9b5b 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -87,6 +87,6 @@ cleanup file key = do
 
 	force <- Annex.getState Annex.force
 	if force
-		then AnnexQueue.add "add" [Param "-f", Param "--"] file
-		else AnnexQueue.add "add" [Param "--"] file
+		then AnnexQueue.add "add" [Param "-f", Param "--"] [file]
+		else AnnexQueue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 60627e9df0..47b0c4c9a0 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -44,5 +44,5 @@ perform file link = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	AnnexQueue.add "add" [Param "--"] file
+	AnnexQueue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index fb9ab0775a..d59f1de397 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -45,5 +45,5 @@ perform file = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	AnnexQueue.add "add" [Param "--"] file
+	AnnexQueue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/Lock.hs b/Command/Lock.hs
index e55cd9e79a..d39df5f335 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -33,5 +33,5 @@ perform file = do
 	-- Checkout from HEAD to get rid of any changes that might be 
 	-- staged in the index, and get back to the previous symlink to
 	-- the content.
-	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] file
+	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] [file]
 	next $ return True -- no cleanup needed
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index f22503ee06..d3623ed997 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -78,6 +78,6 @@ cleanup file key = do
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
-	AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] "-a"
+	AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] []
 	
 	return True
diff --git a/Git/Queue.hs b/Git/Queue.hs
index e1ec0cd313..e080476b79 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -53,15 +53,15 @@ empty :: Queue
 empty = Queue 0 M.empty
 
 {- Adds an action to a queue. -}
-add :: Queue -> String -> [CommandParam] -> FilePath -> Queue
-add (Queue n m) subcommand params file = Queue (n + 1) m'
+add :: Queue -> String -> [CommandParam] -> [FilePath] -> Queue
+add (Queue n m) subcommand params files = Queue (n + 1) m'
 	where
 		action = Action subcommand params
 		-- There are probably few items in the map, but there
 		-- can be a lot of files per item. So, optimise adding
 		-- files.
-		m' = M.insertWith' const action files m
-		files = file:(M.findWithDefault [] action m)
+		m' = M.insertWith' const action fs m
+		fs = files ++ (M.findWithDefault [] action m)
 
 {- Number of items in a queue. -}
 size :: Queue -> Int
@@ -79,11 +79,14 @@ flush repo (Queue _ m) = do
 
 {- Runs an Action on a list of files in a git repository.
  -
- - Complicated by commandline length limits. -}
+ - Complicated by commandline length limits.
+ -
+ - Intentionally runs the command even if the list of files is empty;
+ - this allows queueing commands that do not need a list of files. -}
 runAction :: Repo -> Action -> [FilePath] -> IO ()
-runAction repo action files = unless (null files) runxargs
+runAction repo action files =
+	pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 	where
-		runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 		params = toCommand $ gitCommandLine repo
 			(Param (getSubcommand action):getParams action)
 		feedxargs h = hPutStr h $ join "\0" files
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c0bbeebaf4..165a482628 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -104,7 +104,7 @@ updateSymlinks = do
 					link <- calcGitLink f k
 					liftIO $ removeFile f
 					liftIO $ createSymbolicLink link f
-					AnnexQueue.add "add" [Param "--"] f
+					AnnexQueue.add "add" [Param "--"] [f]
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
@@ -134,9 +134,9 @@ moveLocationLogs = do
 				old <- readLog f
 				new <- readLog dest
 				writeLog dest (old++new)
-				AnnexQueue.add "add" [Param "--"] dest
-				AnnexQueue.add "add" [Param "--"] f
-				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] f
+				AnnexQueue.add "add" [Param "--"] [dest]
+				AnnexQueue.add "add" [Param "--"] [f]
+				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
 		
 oldlog2key :: FilePath -> Maybe (FilePath, Key)
 oldlog2key l = 
diff --git a/debian/changelog b/debian/changelog
index 673084d2d8..9a7c3e7b0f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20110708) UNRELEASED; urgency=low
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
   * Bugfix: Make add ../ work.
   * Support the standard git -c name=value
+  * unannex: Clean up use of git commit -a.
 
  -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
 
diff --git a/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
index 80bf562ec7..181b02b5c1 100644
--- a/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
+++ b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
@@ -11,3 +11,5 @@ This doesn't look right:
     simons   16543  0.3  0.0  15644  1744 pts/1    SN+  04:14   2:13              |               |       \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex cat-file --batch
     simons   14224  0.0  0.0 100744   796 pts/1    SN+  14:10   0:00              |               |       \_ xargs -0 git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex
     simons   14225  0.4  0.1  32684 18652 pts/1    DN+  14:10   0:00              |               |           \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a
+
+> [[Fixed|done]] --[[Joey]] 

From 9bb797c0eae3c9d2f119a734762a6d5fa7321a80 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jul 2011 17:18:53 -0400
Subject: [PATCH 2064/2835] unannex: only commit, no -a

-a is actually not needed; only commit staged changes
---
 Command/Unannex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index d3623ed997..960f99722f 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -78,6 +78,6 @@ cleanup file key = do
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
-	AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] []
+	AnnexQueue.add "commit" [Param "-m", Param "content removed from git annex"] []
 	
 	return True

From e78475737636a5d1e0d0a36b475c300cc7bb56cc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 03:12:05 -0400
Subject: [PATCH 2065/2835] hlint tweaks

Did all sources except Remotes/* and Command/*
---
 Backend.hs         | 10 +++----
 Backend/SHA.hs     |  2 +-
 Backend/WORM.hs    |  2 +-
 Branch.hs          |  6 ++--
 CmdLine.hs         | 22 +++++++-------
 Command.hs         |  6 ++--
 Content.hs         | 71 +++++++++++++++++++++++-----------------------
 Crypto.hs          | 29 ++++++++++---------
 Git.hs             | 51 ++++++++++++++++-----------------
 Git/LsFiles.hs     |  8 +++---
 Git/Queue.hs       |  4 +--
 Git/UnionMerge.hs  |  2 +-
 LocationLog.hs     |  3 +-
 Locations.hs       |  2 +-
 Messages.hs        |  2 +-
 PresenceLog.hs     |  4 +--
 Remote.hs          |  7 +++--
 RemoteLog.hs       | 10 +++----
 TestConfig.hs      | 23 ++++++++-------
 Types/Key.hs       |  4 +--
 Types/Remote.hs    |  3 +-
 UUID.hs            |  4 +--
 Upgrade/V0.hs      |  2 +-
 Upgrade/V1.hs      | 25 +++++++---------
 Upgrade/V2.hs      | 12 ++++----
 Utility.hs         |  4 +--
 Version.hs         |  3 +-
 configure.hs       |  5 ++--
 git-annex-shell.hs |  4 +--
 git-annex.cabal    |  2 +-
 git-union-merge.hs |  5 ++--
 test.hs            | 14 ++++-----
 32 files changed, 172 insertions(+), 179 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index cf976d2b8a..3429e8f42c 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -19,6 +19,7 @@ import Control.Monad.State (liftIO, when)
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
+import Data.Maybe
 
 import Locations
 import qualified Git
@@ -33,10 +34,7 @@ import qualified Backend.WORM
 import qualified Backend.SHA
 
 list :: [Backend Annex]
-list = concat 
-	[ Backend.WORM.backends
-	, Backend.SHA.backends
-	]
+list = Backend.WORM.backends ++ Backend.SHA.backends
 
 {- List of backends in the order to try them when storing a new key. -}
 orderedList :: Annex [Backend Annex]
@@ -54,7 +52,7 @@ orderedList = do
 		handle Nothing s = return s
 		handle (Just "") s = return s
 		handle (Just name) s = do
-			let l' = (lookupBackendName name):s
+			let l' = lookupBackendName name : s
 			Annex.changeState $ \state -> state { Annex.backends = l' }
 			return l'
 		getstandard = do
@@ -119,7 +117,7 @@ chooseBackends fs = do
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend Annex
-lookupBackendName s = maybe unknown id $ maybeLookupBackendName s
+lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: String -> Maybe (Backend Annex)
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index bd6e411a05..dc27b30003 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -114,7 +114,7 @@ checkKeyChecksum size key = do
 	fast <- Annex.getState Annex.fast
 	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
-	if (not present || fast)
+	if not present || fast
 		then return True
 		else do
 			s <- shaN size file
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 036d0564cb..831c9e8ced 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -35,7 +35,7 @@ backend = Types.Backend.Backend {
 keyValue :: FilePath -> Annex (Maybe Key)
 keyValue file = do
 	stat <- liftIO $ getFileStatus file
-	return $ Just $ Key {
+	return $ Just Key {
 		keyName = takeFileName file,
 		keyBackendName = name backend,
 		keySize = Just $ fromIntegral $ fileSize stat,
diff --git a/Branch.hs b/Branch.hs
index d091b6edab..c8e6bc2bb4 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -87,7 +87,7 @@ withIndex' bootstrapping a = do
 
 	e <- liftIO $ doesFileExist f
 	unless e $ do
-		unless bootstrapping $ create
+		unless bootstrapping create
 		liftIO $ createDirectoryIfMissing True $ takeDirectory f
 		liftIO $ unless bootstrapping $ genIndex g
 
@@ -187,7 +187,7 @@ updateRef ref
 			Param (name++".."++ref),
 			Params "--oneline -n1"
 			]
-		if (null diffs)
+		if null diffs
 			then return Nothing
 			else do
 				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name ++ "..."
@@ -305,7 +305,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = getJournalFilesRaw >>= return . map fileJournal
+getJournalFiles = fmap (map fileJournal) getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do
diff --git a/CmdLine.hs b/CmdLine.hs
index b807046df3..c33c497856 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -39,14 +39,13 @@ dispatch args cmds options header gitrepo = do
  - list of actions to be run in the Annex monad. -}
 parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool]
 parseCmd argv header cmds options = do
-	(flags, params) <- liftIO $ getopt
+	(flags, params) <- liftIO getopt
 	when (null params) $ error $ "missing command" ++ usagemsg
 	case lookupCmd (head params) of
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			when (cmdusesrepo command) $
-				checkVersion
+			when (cmdusesrepo command) checkVersion
 			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
@@ -78,9 +77,9 @@ usage header cmds options =
  - (but explicitly thrown errors terminate the whole command).
  -}
 tryRun :: Annex.AnnexState -> [Annex Bool] -> IO ()
-tryRun state actions = tryRun' state 0 actions
-tryRun' :: Annex.AnnexState -> Integer -> [Annex Bool] -> IO ()
-tryRun' state errnum (a:as) = do
+tryRun = tryRun' 0
+tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
+tryRun' errnum state (a:as) = do
 	result <- try $ Annex.run state $ do
 		AnnexQueue.flushWhenFull
 		a
@@ -89,11 +88,10 @@ tryRun' state errnum (a:as) = do
 			Annex.eval state $ do
 				showEndFail
 				showErr err
-			tryRun' state (errnum + 1) as
-		Right (True,state') -> tryRun' state' errnum as
-		Right (False,state') -> tryRun' state' (errnum + 1) as
-tryRun' _ errnum [] = do
-	when (errnum > 0) $ error $ show errnum ++ " failed"
+			tryRun' (errnum + 1) state as
+		Right (True,state') -> tryRun' errnum state' as
+		Right (False,state') -> tryRun' (errnum + 1) state' as
+tryRun' errnum _ [] = when (errnum > 0) $ error $ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
@@ -105,5 +103,5 @@ startup = do
 shutdown :: Annex Bool
 shutdown = do
 	saveState
-	liftIO $ Git.reap
+	liftIO Git.reap
 	return True
diff --git a/Command.hs b/Command.hs
index c666ddbd29..729e442fc4 100644
--- a/Command.hs
+++ b/Command.hs
@@ -115,7 +115,7 @@ isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
 	g <- Annex.gitRepo
-	when (Git.repoIsLocalBare g) $ do
+	when (Git.repoIsLocalBare g) $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
@@ -175,9 +175,9 @@ withFilesUnlocked' typechanged a params = do
 	unlockedfiles' <- filterFiles unlockedfiles
 	backendPairs a unlockedfiles'
 withKeys :: CommandSeekKeys
-withKeys a params = return $ map a $ map parse params
+withKeys a params = return $ map (a . parse) params
 	where
-		parse p = maybe (error "bad key") id $ readKey p
+		parse p = fromMaybe (error "bad key") $ readKey p
 withTempFile :: CommandSeekStrings
 withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
diff --git a/Content.hs b/Content.hs
index 94f8b8c2ac..c63042dfb5 100644
--- a/Content.hs
+++ b/Content.hs
@@ -57,8 +57,8 @@ inAnnex key = do
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
 	g <- Annex.gitRepo
-	cwd <- liftIO $ getCurrentDirectory
-	let absfile = maybe whoops id $ absNormPath cwd file
+	cwd <- liftIO getCurrentDirectory
+	let absfile = fromMaybe whoops $ absNormPath cwd file
 	return $ relPathDirToFile (parentDir absfile) 
 			(Git.workTree g)  ".git"  annexLocation key
 	where
@@ -94,15 +94,19 @@ getViaTmp key action = do
 
 	getViaTmpUnchecked key action
 
+prepTmp :: Key -> Annex FilePath
+prepTmp key = do
+	g <- Annex.gitRepo
+	let tmp = gitAnnexTmpLocation g key
+	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	return tmp
+
 {- Like getViaTmp, but does not check that there is enough disk space
  - for the incoming key. For use when the key content is already on disk
  - and not being copied into place. -}
 getViaTmpUnchecked :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmpUnchecked key action = do
-	g <- Annex.gitRepo
-	let tmp = gitAnnexTmpLocation g key
-
-	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	tmp <- prepTmp key
 	success <- action tmp
 	if success
 		then do
@@ -117,9 +121,7 @@ getViaTmpUnchecked key action = do
 {- Creates a temp file, runs an action on it, and cleans up the temp file. -}
 withTmp :: Key -> (FilePath -> Annex a) -> Annex a
 withTmp key action = do
-	g <- Annex.gitRepo
-	let tmp = gitAnnexTmpLocation g key
-	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	tmp <- prepTmp key
 	res <- action tmp
 	liftIO $ whenM (doesFileExist tmp) $ liftIO $ removeFile tmp
 	return res
@@ -133,23 +135,21 @@ checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- Annex.gitRepo
 	r <- getConfig g "diskreserve" ""
-	let reserve = maybe megabyte id $ readSize dataUnits r
+	let reserve = fromMaybe megabyte $ readSize dataUnits r
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
 		(_, Nothing) -> return ()
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
-			if (need + reserve > have + adjustment)
-				then needmorespace (need + reserve - have - adjustment)
-				else return ()
+			when (need + reserve > have + adjustment) $
+				needmorespace (need + reserve - have - adjustment)
 	where
 		megabyte :: Integer
 		megabyte = 1000000
-		needmorespace n = do
-			unlessM (Annex.getState Annex.force) $
-				error $ "not enough free space, need " ++ 
-					roughSize storageUnits True n ++
-					" more (use --force to override this check or adjust annex.diskreserve)"
+		needmorespace n = unlessM (Annex.getState Annex.force) $
+			error $ "not enough free space, need " ++ 
+				roughSize storageUnits True n ++
+				" more (use --force to override this check or adjust annex.diskreserve)"
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
@@ -200,28 +200,27 @@ moveAnnex key src = do
 			preventWrite dest
 			preventWrite dir
 
-{- Removes a key's file from .git/annex/objects/ -}
-removeAnnex :: Key -> Annex ()
-removeAnnex key = do
+withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
+withObjectLoc key a = do
 	g <- Annex.gitRepo
 	let file = gitAnnexLocation g key
 	let dir = parentDir file
-	liftIO $ do
-		allowWrite dir
-		removeFile file
-		removeDirectory dir
+	a (dir, file)
+
+{- Removes a key's file from .git/annex/objects/ -}
+removeAnnex :: Key -> Annex ()
+removeAnnex key = withObjectLoc key $ \(dir, file) -> liftIO $ do
+	allowWrite dir
+	removeFile file
+	removeDirectory dir
 
 {- Moves a key's file out of .git/annex/objects/ -}
 fromAnnex :: Key -> FilePath -> Annex ()
-fromAnnex key dest = do
-	g <- Annex.gitRepo
-	let file = gitAnnexLocation g key
-	let dir = parentDir file
-	liftIO $ do
-		allowWrite dir
-		allowWrite file
-		renameFile file dest
-		removeDirectory dir
+fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
+	allowWrite dir
+	allowWrite file
+	renameFile file dest
+	removeDirectory dir
 
 {- Moves a key out of .git/annex/objects/ into .git/annex/bad, and
  - returns the file it was moved to. -}
@@ -246,7 +245,7 @@ getKeysPresent = do
 getKeysPresent' :: FilePath -> Annex [Key]
 getKeysPresent' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
-	if (not exists)
+	if not exists
 		then return []
 		else liftIO $ do
 			-- 2 levels of hashing
@@ -254,7 +253,7 @@ getKeysPresent' dir = do
 			levelb <- mapM dirContents levela
 			contents <- mapM dirContents (concat levelb)
 			files <- filterM present (concat contents)
-			return $ catMaybes $ map (fileKey . takeFileName) files
+			return $ mapMaybe (fileKey . takeFileName) files
 	where
 		present d = do
 			result <- try $
diff --git a/Crypto.hs b/Crypto.hs
index 485fb6e931..4fc41ede04 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -33,6 +33,7 @@ import Data.Digest.Pure.SHA
 import System.Cmd.Utils
 import Data.String.Utils
 import Data.List
+import Data.Maybe
 import System.IO
 import System.Posix.IO
 import System.Posix.Types
@@ -125,11 +126,11 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 	return $ EncryptedCipher encipher (KeyIds ks')
 	where
 		encrypt = [ Params "--encrypt" ]
-		recipients l = 
-			-- Force gpg to only encrypt to the specified
-			-- recipients, not configured defaults.
-			[ Params "--no-encrypt-to --no-default-recipient"] ++
-			(concat $ map (\k -> [Param "--recipient", Param k]) l)
+		recipients l = force_recipients :
+			concatMap (\k -> [Param "--recipient", Param k]) l
+		-- Force gpg to only encrypt to the specified
+		-- recipients, not configured defaults.
+		force_recipients = Params "--no-encrypt-to --no-default-recipient"
 
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
 decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
@@ -152,24 +153,24 @@ encryptKey c k =
 
 {- Runs an action, passing it a handle from which it can 
  - stream encrypted content. -}
-withEncryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a
+withEncryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
 withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"]
 
 {- Runs an action, passing it a handle from which it can
  - stream decrypted content. -}
-withDecryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a
+withDecryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
 withDecryptedHandle = gpgCipherHandle [Param "--decrypt"]
 
 {- Streams encrypted content to an action. -}
-withEncryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a
+withEncryptedContent :: Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 withEncryptedContent = pass withEncryptedHandle
 
 {- Streams decrypted content to an action. -}
-withDecryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a
+withDecryptedContent :: Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 withDecryptedContent = pass withDecryptedHandle
 
-pass :: (Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a) 
-      -> Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a
+pass :: (Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a) 
+      -> Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 pass to c i a = to c i $ \h -> a =<< L.hGetContents h
 
 gpgParams :: [CommandParam] -> IO [String]
@@ -202,7 +203,7 @@ gpgPipeStrict params input = do
  -
  - Note that to avoid deadlock with the cleanup stage,
  - the action must fully consume gpg's input before returning. -}
-gpgCipherHandle :: [CommandParam] -> Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a
+gpgCipherHandle :: [CommandParam] -> Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
 gpgCipherHandle params c a b = do
 	-- pipe the passphrase into gpg on a fd
 	(frompipe, topipe) <- createPipe
@@ -235,10 +236,10 @@ configKeyIds c = do
 	where
 		parseWithColons s = map keyIdField $ filter pubKey $ lines s
 		pubKey = isPrefixOf "pub:"
-		keyIdField s = (split ":" s) !! 4
+		keyIdField s = split ":" s !! 4
 
 configGet :: RemoteConfig -> String -> String
-configGet c key = maybe missing id $ M.lookup key c
+configGet c key = fromMaybe missing $ M.lookup key c
 	where missing = error $ "missing " ++ key ++ " in remote config"
 
 hmacWithCipher :: Cipher -> String -> String
diff --git a/Git.hs b/Git.hs
index 2d03163bbf..9b7ac7ea91 100644
--- a/Git.hs
+++ b/Git.hs
@@ -69,11 +69,10 @@ import System.Posix.User
 import System.Posix.Process
 import System.Path
 import System.Cmd.Utils
-import IO (bracket_)
+import IO (bracket_, try)
 import Data.String.Utils
 import System.IO
-import IO (try)
-import qualified Data.Map as Map hiding (map, split)
+import qualified Data.Map as M hiding (map, split)
 import Network.URI
 import Data.Maybe
 import Data.Char
@@ -93,7 +92,7 @@ data RepoLocation = Dir FilePath | Url URI | Unknown
 
 data Repo = Repo {
 	location :: RepoLocation,
-	config :: Map.Map String String,
+	config :: M.Map String String,
 	remotes :: [Repo],
 	-- remoteName holds the name used for this repo in remotes
 	remoteName :: Maybe String 
@@ -103,7 +102,7 @@ newFrom :: RepoLocation -> Repo
 newFrom l = 
 	Repo {
 		location = l,
-		config = Map.empty,
+		config = M.empty,
 		remotes = [],
 		remoteName = Nothing
 	}
@@ -140,7 +139,7 @@ repoFromUrl url
 	| startswith "file://" url = repoFromAbsPath $ uriPath u
 	| otherwise = return $ newFrom $ Url u
 		where
-			u = maybe bad id $ parseURI url
+			u = fromMaybe bad $ parseURI url
 			bad = error $ "bad url " ++ url
 
 {- Creates a repo that has an unknown location. -}
@@ -208,7 +207,7 @@ repoIsSsh Repo { location = Url url }
 repoIsSsh _ = False
 
 configAvail ::Repo -> Bool
-configAvail Repo { config = c } = c /= Map.empty
+configAvail Repo { config = c } = c /= M.empty
 
 repoIsLocalBare :: Repo -> Bool
 repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r
@@ -228,7 +227,7 @@ assertUrl repo action =
 				" not supported"
 
 configBare :: Repo -> Bool
-configBare repo = maybe unknown configTrue $ Map.lookup "core.bare" $ config repo
+configBare repo = maybe unknown configTrue $ M.lookup "core.bare" $ config repo
 	where
 		unknown = error $ "it is not known if git repo " ++
 			repoDescribe repo ++
@@ -272,14 +271,14 @@ workTreeFile repo@(Repo { location = Dir d }) file = do
 	let file' = absfile cwd
 	unless (inrepo file') $
 		error $ file ++ " is not located inside git repository " ++ absrepo
-	if (inrepo $ addTrailingPathSeparator cwd)
+	if inrepo $ addTrailingPathSeparator cwd
 		then return $ relPathDirToFile cwd file'
 		else return $ drop (length absrepo) file'
 	where
 		-- normalize both repo and file, so that repo
 		-- will be substring of file
 		absrepo = maybe bad addTrailingPathSeparator $ absNormPath "/" d
-		absfile c = maybe file id $ secureAbsNormPath c file
+		absfile c = fromMaybe file $ secureAbsNormPath c file
 		inrepo f = absrepo `isPrefixOf` f
 		bad = error $ "bad repo" ++ repoDescribe repo
 workTreeFile repo _ = assertLocal repo $ error "internal"
@@ -303,7 +302,7 @@ uriRegName' a = fixup $ uriRegName a
 			| rest !! len == ']' = take len rest
 			| otherwise = x
 			where
-				len  = (length rest) - 1
+				len  = length rest - 1
 		fixup x = x
 
 {- Hostname of an URL repo. -}
@@ -348,7 +347,7 @@ gitCommandLine repo _ = assertLocal repo $ error "internal"
 {- Runs git in the specified repo. -}
 runBool :: Repo -> String -> [CommandParam] -> IO Bool
 runBool repo subcommand params = assertLocal repo $
-	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
+	boolSystem "git" $ gitCommandLine repo $ Param subcommand : params
 
 {- Runs git in the specified repo, throwing an error if it fails. -}
 run :: Repo -> String -> [CommandParam] -> IO ()
@@ -471,13 +470,13 @@ hConfigRead repo h = do
  - can be updated inrementally. -}
 configStore :: Repo -> String -> IO Repo
 configStore repo s = do
-	let repo' = repo { config = Map.union (configParse s) (config repo) }
+	let repo' = repo { config = configParse s `M.union` config repo }
 	rs <- configRemotes repo'
 	return $ repo' { remotes = rs }
 
 {- Parses git config --list output into a config map. -}
-configParse :: String -> Map.Map String String
-configParse s = Map.fromList $ map pair $ lines s
+configParse :: String -> M.Map String String
+configParse s = M.fromList $ map pair $ lines s
 	where
 		pair l = (key l, val l)
 		key l = head $ keyval l
@@ -489,8 +488,8 @@ configParse s = Map.fromList $ map pair $ lines s
 configRemotes :: Repo -> IO [Repo]
 configRemotes repo = mapM construct remotepairs
 	where
-		remotepairs = Map.toList $ filterremotes $ config repo
-		filterremotes = Map.filterWithKey (\k _ -> isremote k)
+		remotepairs = M.toList $ filterremotes $ config repo
+		filterremotes = M.filterWithKey (\k _ -> isremote k)
 		isremote k = startswith "remote." k && endswith ".url" k
 		construct (k,v) = do
 			r <- gen v
@@ -499,15 +498,15 @@ configRemotes repo = mapM construct remotepairs
 			| isURI v = repoFromUrl v
 			| otherwise = repoFromRemotePath v repo
 		-- git remotes can be written scp style -- [user@]host:dir
-		scpstyle v = ":" `isInfixOf` v && (not $ "//" `isInfixOf` v)
+		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
 			where
 				bits = split ":" v
-				host = bits !! 0
+				host = head bits
 				dir = join ":" $ drop 1 bits
 				slash d	| d == "" = "/~/" ++ dir
-					| d !! 0 == '/' = dir
-					| d !! 0 == '~' = '/':dir
+					| head d == '/' = dir
+					| head d == '~' = '/':dir
 					| otherwise = "/~/" ++ dir
 
 {- Checks if a string from git config is a true value. -}
@@ -517,11 +516,11 @@ configTrue s = map toLower s == "true"
 {- Returns a single git config setting, or a default value if not set. -}
 configGet :: Repo -> String -> String -> String
 configGet repo key defaultValue = 
-	Map.findWithDefault defaultValue key (config repo)
+	M.findWithDefault defaultValue key (config repo)
 
 {- Access to raw config Map -}
-configMap :: Repo -> Map.Map String String
-configMap repo = config repo
+configMap :: Repo -> M.Map String String
+configMap = config
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
@@ -680,8 +679,8 @@ seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
 seekUp want dir = do
 	ok <- want dir
 	if ok
-		then return (Just dir)
-		else case (parentDir dir) of
+		then return $ Just dir
+		else case parentDir dir of
 			"" -> return Nothing
 			d -> seekUp want d
 
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index b88c9144e6..23b383a09d 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -21,7 +21,7 @@ import Utility
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo $
-	[Params "ls-files --cached -z --"] ++ map File l
+	Params "ls-files --cached -z --" : map File l
 
 {- Scans for files at the specified locations that are not checked into
  - git. -}
@@ -44,12 +44,12 @@ staged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
 staged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
 	where
 		start = [Params "diff --cached --name-only -z"]
-		end = [Param "--"] ++ map File l
+		end = Param "--" : map File l
 
 {- Returns a list of files that have unstaged changes. -}
 changedUnstaged :: Repo -> [FilePath] -> IO [FilePath]
 changedUnstaged repo l = pipeNullSplit repo $
-	[Params "diff --name-only -z --"] ++ map File l
+	Params "diff --name-only -z --" : map File l
 
 {- Returns a list of the files in the specified locations that are staged
  - for commit, and whose type has changed. -}
@@ -65,4 +65,4 @@ typeChanged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
 typeChanged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
 	where
 		start = [Params "diff --name-only --diff-filter=T -z"]
-		end = [Param "--"] ++ map File l
+		end = Param "--" : map File l
diff --git a/Git/Queue.hs b/Git/Queue.hs
index e080476b79..0016be4727 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -18,7 +18,7 @@ import qualified Data.Map as M
 import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
-import Control.Monad (unless, forM_)
+import Control.Monad (forM_)
 import Utility
 
 import Git
@@ -61,7 +61,7 @@ add (Queue n m) subcommand params files = Queue (n + 1) m'
 		-- can be a lot of files per item. So, optimise adding
 		-- files.
 		m' = M.insertWith' const action fs m
-		fs = files ++ (M.findWithDefault [] action m)
+		fs = files ++ M.findWithDefault [] action m
 
 {- Number of items in a queue. -}
 size :: Queue -> Int
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 4e0361b858..b0da071703 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -91,5 +91,5 @@ mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
-		nullsha = take shaSize $ repeat '0'
+		nullsha = replicate shaSize '0'
 		unionmerge = unlines . nub . lines
diff --git a/LocationLog.hs b/LocationLog.hs
index fe09482b9f..768483fa1b 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -49,8 +49,7 @@ keyLocations key = currentLog $ logFile key
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Annex [Key]
-loggedKeys =
-	return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files
+loggedKeys = return . mapMaybe (logFileKey . takeFileName) =<< Branch.files
 
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
diff --git a/Locations.hs b/Locations.hs
index 347b08ce14..2dbf2f55ea 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -52,7 +52,7 @@ import qualified Git
 {- The directory git annex uses for local state, relative to the .git
  - directory -}
 annexDir :: FilePath
-annexDir = addTrailingPathSeparator $ "annex"
+annexDir = addTrailingPathSeparator "annex"
 
 {- The directory git annex uses for locally available object content,
  - relative to the .git directory -}
diff --git a/Messages.hs b/Messages.hs
index 038e4c0bc7..5f150aafb4 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -37,7 +37,7 @@ showProgress :: Annex ()
 showProgress = verbose $ liftIO $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indent s
+showLongNote s = verbose $ liftIO $ putStr $ '\n' : indent s
 
 showEndOk :: Annex ()
 showEndOk = verbose $ liftIO $ putStrLn "ok"
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 0777db209b..9c516a8db1 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -94,7 +94,7 @@ writeLog file ls = Branch.change file (unlines $ map show ls)
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> String -> Annex LogLine
 logNow s i = do
-	now <- liftIO $ getPOSIXTime
+	now <- liftIO getPOSIXTime
 	return $ LogLine now s i
 
 {- Reads a log and returns only the info that is still in effect. -}
@@ -112,7 +112,7 @@ type LogMap = Map.Map String LogLine
 {- Compacts a set of logs, returning a subset that contains the current
  - status. -}
 compactLog :: [LogLine] -> [LogLine]
-compactLog ls = compactLog' Map.empty ls
+compactLog = compactLog' Map.empty
 compactLog' :: LogMap -> [LogLine] -> [LogLine]
 compactLog' m [] = Map.elems m
 compactLog' m (l:ls) = compactLog' (mapLog m l) ls
diff --git a/Remote.hs b/Remote.hs
index de1c7f38db..1a5006f6fb 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -33,6 +33,7 @@ import Control.Monad (filterM, liftM2)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
+import Data.Maybe
 
 import Types
 import Types.Remote
@@ -97,7 +98,7 @@ byName' "" = return $ Left "no remote specified"
 byName' n = do
 	allremotes <- genList
 	let match = filter matching allremotes
-	if (null match)
+	if null match
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
 		else return $ Right $ head match
 	where
@@ -110,7 +111,7 @@ nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
-		Left e -> return . (maybe (error e) id) =<< byDescription
+		Left e -> return . fromMaybe (error e) =<< byDescription
 		Right r -> return $ uuid r
 	where
 		byDescription = return . M.lookup n . invertMap =<< uuidMap
@@ -122,7 +123,7 @@ prettyPrintUUIDs :: [UUID] -> Annex String
 prettyPrintUUIDs uuids = do
 	here <- getUUID =<< Annex.gitRepo
 	-- Show descriptions from the uuid log, falling back to remote names,
-	-- as some remotes may not be in the uuid log.
+	-- as some remotes may not be in the uuid log
 	m <- liftM2 M.union uuidMap $
 		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList
 	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
diff --git a/RemoteLog.hs b/RemoteLog.hs
index c2065db9da..69a82f4987 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -36,7 +36,7 @@ configSet u c = do
 	Branch.change remoteLog $ unlines $ sort $
 		map toline $ M.toList $ M.insert u c m
 	where
-		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')
+		toline (u', c') = u' ++ " " ++ unwords (configToKeyVal c')
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
@@ -44,14 +44,14 @@ readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
-	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s
+	M.fromList $ mapMaybe parseline $ filter (not . null) $ lines s
 	where
 		parseline l
 			| length w > 2 = Just (u, c)
 			| otherwise = Nothing
 			where
 				w = words l
-				u = w !! 0
+				u = head w
 				c = keyValToConfig $ tail w
 
 {- Given Strings like "key=value", generates a RemoteConfig. -}
@@ -90,8 +90,8 @@ configUnEscape = unescape
 				r = drop (length num) s
 				rest = drop 1 r
 				ok = not (null num) && 
-					not (null r) && r !! 0 == ';'
+					not (null r) && head r == ';'
 
 {- for quickcheck -}
 prop_idempotent_configEscape :: String -> Bool
-prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
+prop_idempotent_configEscape s = s == (configUnEscape . configEscape) s
diff --git a/TestConfig.hs b/TestConfig.hs
index bab297003a..8cfae7f0c7 100644
--- a/TestConfig.hs
+++ b/TestConfig.hs
@@ -45,7 +45,7 @@ writeSysConfig config = writeFile "SysConfig.hs" body
 
 runTests :: [TestCase] -> IO [Config]
 runTests [] = return []
-runTests ((TestCase tname t):ts) = do
+runTests (TestCase tname t : ts) = do
 	testStart tname
 	c <- t
 	testEnd c
@@ -62,7 +62,7 @@ requireCmd k cmdline = do
 		handle r = do
 			testEnd r
 			error $ "** the " ++ c ++ " command is required"
-		c = (words cmdline) !! 0
+		c = head $ words cmdline
 
 {- Checks if a command is available by running a command line. -}
 testCmd :: ConfigKey -> String -> Test
@@ -74,7 +74,7 @@ testCmd k cmdline = do
  - turn. The Config is set to the first one found. -}
 selectCmd :: ConfigKey -> [String] -> String -> Test
 selectCmd k = searchCmd
-		(\match -> return $ Config k $ StringConfig match)
+		(return . Config k . StringConfig)
 		(\cmds -> do
 			testEnd $ Config k $ BoolConfig False
 			error $ "* need one of these commands, but none are available: " ++ show cmds
@@ -82,7 +82,7 @@ selectCmd k = searchCmd
 
 maybeSelectCmd :: ConfigKey -> [String] -> String -> Test
 maybeSelectCmd k = searchCmd
-		(\match -> return $ Config k $ MaybeStringConfig $ Just match)
+		(return . Config k . MaybeStringConfig . Just)
 		(\_ -> return $ Config k $ MaybeStringConfig Nothing)
 
 searchCmd :: (String -> Test) -> ([String] -> Test) -> [String] -> String -> Test
@@ -91,7 +91,7 @@ searchCmd success failure cmds param = search cmds
 		search [] = failure cmds
 		search (c:cs) = do
 			ret <- system $ quiet c ++ " " ++ param
-			if (ret == ExitSuccess)
+			if ret == ExitSuccess
 				then success c
 				else search cs
 
@@ -104,8 +104,11 @@ testStart s = do
 	hFlush stdout
 
 testEnd :: Config -> IO ()
-testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes"
-testEnd (Config _ (BoolConfig False)) = putStrLn $ " no"
-testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s
-testEnd (Config _ (MaybeStringConfig (Just s))) = putStrLn $ " " ++ s
-testEnd (Config _ (MaybeStringConfig Nothing)) = putStrLn $ " not available"
+testEnd (Config _ (BoolConfig True)) = status "yes"
+testEnd (Config _ (BoolConfig False)) = status "no"
+testEnd (Config _ (StringConfig s)) = status s
+testEnd (Config _ (MaybeStringConfig (Just s))) = status s
+testEnd (Config _ (MaybeStringConfig Nothing)) = status "not available"
+
+status :: String -> IO ()
+status s = putStrLn $ ' ':s
diff --git a/Types/Key.hs b/Types/Key.hs
index 1d9bf8e11c..b26bb89896 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -48,7 +48,7 @@ instance Show Key where
 			"" +++ y = y
 			x +++ "" = x
 			x +++ y = x ++ fieldSep:y
-			c ?: (Just v) = c:(show v)
+			c ?: (Just v) = c : show v
 			_ ?: _ = ""
 
 readKey :: String -> Maybe Key
@@ -73,4 +73,4 @@ readKey s = if key == Just stubKey then Nothing else key
 		addfield _ _ _ = Nothing
 
 prop_idempotent_key_read_show :: Key -> Bool
-prop_idempotent_key_read_show k = Just k == (readKey $ show k)
+prop_idempotent_key_read_show k = Just k == (readKey . show) k
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 1d67ad5cd7..8d9622c519 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -11,6 +11,7 @@ module Types.Remote where
 
 import Control.Exception
 import Data.Map as M
+import Data.Ord
 
 import qualified Git
 import Types.Key
@@ -62,4 +63,4 @@ instance Eq (Remote a) where
 
 -- order remotes by cost
 instance Ord (Remote a) where
-	compare x y = compare (cost x) (cost y)
+	compare = comparing cost
diff --git a/UUID.hs b/UUID.hs
index 3ccc729069..cfa519baa3 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -49,7 +49,7 @@ genUUID :: IO UUID
 genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
 	where
 		command = SysConfig.uuid
-		params = if (command == "uuid")
+		params = if command == "uuid"
 			-- request a random uuid be generated
 			then ["-m"]
 			-- uuidgen generates random uuid by default
@@ -82,7 +82,7 @@ prepUUID :: Annex ()
 prepUUID = do
 	u <- getUUID =<< Annex.gitRepo
 	when ("" == u) $ do
-		uuid <- liftIO $ genUUID
+		uuid <- liftIO genUUID
 		setConfig configkey uuid
 
 {- Records a description for a uuid in the uuidLog. -}
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index eabd030098..071fd12ee1 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -48,7 +48,7 @@ lookupFile0 = Upgrade.V1.lookupFile1
 getKeysPresent0 :: FilePath -> Annex [Key]
 getKeysPresent0 dir = do
 	exists <- liftIO $ doesDirectoryExist dir
-	if (not exists)
+	if not exists
 		then return []
 		else do
 			contents <- liftIO $ getDirectoryContents dir
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 165a482628..8a3d37a642 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -94,7 +94,7 @@ updateSymlinks = do
 	showNote "updating symlinks..."
 	g <- Annex.gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
-	forM_ files $ fixlink
+	forM_ files fixlink
 	where
 		fixlink f = do
 			r <- lookupFile1 f
@@ -119,7 +119,7 @@ moveLocationLogs = do
 				if exists
 					then do
 						contents <- liftIO $ getDirectoryContents dir
-						return $ catMaybes $ map oldlog2key contents
+						return $ mapMaybe oldlog2key contents
 					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
@@ -196,17 +196,14 @@ lookupFile1 file = do
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = do
-			l <- readSymbolicLink file
-			return $ takeFileName l
-		makekey l = do
-			case maybeLookupBackendName bname of
-				Nothing -> do
-					unless (null kname || null bname ||
-					        not (isLinkToAnnex l)) $
-						warning skip
-					return Nothing
-				Just backend -> return $ Just (k, backend)
+		getsymlink = return . takeFileName =<< readSymbolicLink file
+		makekey l = case maybeLookupBackendName bname of
+			Nothing -> do
+				unless (null kname || null bname ||
+				        not (isLinkToAnnex l)) $
+					warning skip
+				return Nothing
+			Just backend -> return $ Just (k, backend)
 			where
 				k = fileKey1 l
 				bname = keyBackendName k
@@ -221,7 +218,7 @@ getKeyFilesPresent1  = do
 getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
 getKeyFilesPresent1' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
-	if (not exists)
+	if not exists
 		then return []
 		else do
 			dirs <- liftIO $ getDirectoryContents dir
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 4824f4bbaf..99c7806d27 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -10,7 +10,7 @@ module Upgrade.V2 where
 import System.Directory
 import System.FilePath
 import Control.Monad.State (unless, when, liftIO)
-import List
+import Data.List
 import Data.Maybe
 
 import Types.Key
@@ -61,7 +61,7 @@ upgrade = do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
 		unless bare $ gitAttributesUnWrite g
 
-	unless bare $ push
+	unless bare push
 
 	return True
 
@@ -70,11 +70,11 @@ locationLogs repo = liftIO $ do
 	levela <- dirContents dir
 	levelb <- mapM tryDirContents levela
 	files <- mapM tryDirContents (concat levelb)
-	return $ catMaybes $ map islogfile (concat files)
+	return $ mapMaybe islogfile (concat files)
 	where
 		tryDirContents d = catch (dirContents d) (return . const [])
 		dir = gitStateDir repo
-		islogfile f = maybe Nothing (\k -> Just $ (k, f)) $
+		islogfile f = maybe Nothing (\k -> Just (k, f)) $
 				logFileKey $ takeFileName f
 
 inject :: FilePath -> FilePath -> Annex ()
@@ -131,10 +131,10 @@ gitAttributesUnWrite repo = do
 	whenM (doesFileExist attributes) $ do
 		c <- readFileStrict attributes
 		liftIO $ viaTmp writeFile attributes $ unlines $
-			filter (\l -> not $ l `elem` attrLines) $ lines c
+			filter (`notElem` attrLines) $ lines c
 		Git.run repo "add" [File attributes]
 
 stateDir :: FilePath
-stateDir = addTrailingPathSeparator $ ".git-annex"
+stateDir = addTrailingPathSeparator ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
 gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
diff --git a/Utility.hs b/Utility.hs
index 63fa6eda35..786c6a1770 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -141,9 +141,9 @@ shellUnEscape s = word : shellUnEscape rest
 
 {- For quickcheck. -}
 prop_idempotent_shellEscape :: String -> Bool
-prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s)
+prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
 prop_idempotent_shellEscape_multiword :: [String] -> Bool
-prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s)
+prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
diff --git a/Version.hs b/Version.hs
index 7e7a4c7ce6..7e6910fbe4 100644
--- a/Version.hs
+++ b/Version.hs
@@ -43,8 +43,7 @@ checkVersion :: Annex ()
 checkVersion = getVersion >>= handle
 	where
 		handle Nothing = error "First run: git-annex init"
-		handle (Just v) = do
-			unless (v `elem` supportedVersions) $ do
+		handle (Just v) = unless (v `elem` supportedVersions) $
 			error $ "Repository version " ++ v ++ 
 				" is not supported. " ++
 				msg v
diff --git a/configure.hs b/configure.hs
index 8639af44b1..bfdfa32dd5 100644
--- a/configure.hs
+++ b/configure.hs
@@ -7,7 +7,7 @@ import TestConfig
 
 tests :: [TestCase]
 tests =
-	[ TestCase "version" $ getVersion
+	[ TestCase "version" getVersion
 	, testCp "cp_a" "-a"
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
@@ -77,8 +77,7 @@ setup = do
 	writeFile testFile "test file contents"
 
 cleanup :: IO ()
-cleanup = do
-	removeDirectoryRecursive tmpDir
+cleanup = removeDirectoryRecursive tmpDir
 
 main :: IO ()
 main = do
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index a64552c721..29ac63aea1 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -58,10 +58,10 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params =
 	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
-		dispatch (cmd:(filterparams params)) cmds commonOptions header
+		dispatch (cmd : filterparams params) cmds commonOptions header
 
 external :: [String] -> IO ()
-external params = do
+external params =
 	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $
 		error "git-shell failed"
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 29ad80a58a..bbbcbf9fb4 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110707
+Version: 3.20110708
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 38df0df6a2..e763376077 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -23,8 +23,7 @@ tmpIndex :: Git.Repo -> FilePath
 tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
-setup g = do
-	cleanup g -- idempotency
+setup g = cleanup g -- idempotency
 
 cleanup :: Git.Repo -> IO ()
 cleanup g = do
@@ -34,7 +33,7 @@ cleanup g = do
 parseArgs :: IO [String]
 parseArgs = do
 	args <- getArgs
-	if (length args /= 3)
+	if length args /= 3
 		then usage
 		else return args
 
diff --git a/test.hs b/test.hs
index 51ccc600e6..2352df36a6 100644
--- a/test.hs
+++ b/test.hs
@@ -19,7 +19,7 @@ import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
 import Control.Exception (throw)
-import Maybe
+import Data.Maybe
 import qualified Data.Map as M
 import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
@@ -48,7 +48,7 @@ instance Arbitrary Types.Key.Key where
 	arbitrary = do
 		n <- arbitrary
 		b <- elements ['A'..'Z']
-		return $ Types.Key.Key {
+		return Types.Key.Key {
 			Types.Key.keyName = n,
 			Types.Key.keyBackendName = [b],
 			Types.Key.keySize = Nothing,
@@ -278,7 +278,7 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do
 	-- write different content, to verify that lock
 	-- throws it away
 	changecontent annexedfile
-	writeFile annexedfile $ (content annexedfile) ++ "foo"
+	writeFile annexedfile $ content annexedfile ++ "foo"
 	git_annex "lock" ["-q", annexedfile] @? "lock failed"
 	annexed_present annexedfile
 	git_annex "unlock" ["-q", annexedfile] @? "unlock failed"		
@@ -287,7 +287,7 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do
 	git_annex "add" ["-q", annexedfile] @? "add of modified file failed"
 	runchecks [checklink, checkunwritable] annexedfile
 	c <- readFile annexedfile
-	assertEqual ("content of modified file") c (changedcontent annexedfile)
+	assertEqual "content of modified file" c (changedcontent annexedfile)
 	r' <- git_annex "drop" ["-q", annexedfile]
 	not r' @? "drop wrongly succeeded with no known copy of modified file"
 
@@ -312,9 +312,9 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 					@? "git commit of edited file failed"
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
-		assertEqual ("content of modified file") c (changedcontent annexedfile)
+		assertEqual "content of modified file" c (changedcontent annexedfile)
 		r <- git_annex "drop" ["-q", annexedfile]
-		(not r) @? "drop wrongly succeeded with no known copy of modified file"
+		not r @? "drop wrongly succeeded with no known copy of modified file"
 
 test_fix :: Test
 test_fix = "git-annex fix" ~: intmpclonerepo $ do
@@ -331,7 +331,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 	git_annex "fix" ["-q", newfile] @? "fix of moved file failed"
 	runchecks [checklink, checkunwritable] newfile
 	c <- readFile newfile
-	assertEqual ("content of moved file") c (content annexedfile)
+	assertEqual "content of moved file" c (content annexedfile)
 	where
 		subdir = "s"
 		newfile = subdir ++ "/" ++ annexedfile

From 185f0b687081f47d059cc0503f4f6b671868f753 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 15 Jul 2011 15:33:35 +0000
Subject: [PATCH 2066/2835]

---
 ...nannex_command_doesn__39__t_all_files.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/unannex_command_doesn__39__t_all_files.mdwn

diff --git a/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn b/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
new file mode 100644
index 0000000000..c3128c0fc2
--- /dev/null
+++ b/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
@@ -0,0 +1,21 @@
+    $ git init ; git annex init test ; dd if=/dev/urandom of=file1 count=128 ; cp file1 file2 ; git annex add --backend=SHA1 file? ; git commit -m init ; git annex unannex ; ls -l
+    Initialized empty Git repository in /tmp/annex/.git/
+    init test ok
+    128+0 records in
+    128+0 records out
+    65536 bytes (66 kB) copied, 0.007173 s, 9.1 MB/s
+    add file1 (checksum...) ok
+    add file2 (checksum...) ok
+    (Recording state in git...)
+    [master (root-commit) 2177b10] init
+     2 files changed, 2 insertions(+), 0 deletions(-)
+     create mode 120000 file1
+     create mode 120000 file2
+    unannex file1 ok
+    (Recording state in git...)
+    [master bef78b1] content removed from git annex
+     1 files changed, 0 insertions(+), 1 deletions(-)
+     delete mode 120000 file1
+    total 72
+    -rw-r--r-- 1 simons users 65536 Jul 15 17:29 file1
+    lrwxrwxrwx 1 simons users   132 Jul 15 17:29 file2 -> .git/annex/objects/jp/Fk/SHA1-s65536--795b58cc4e5190b02e7026fd9e94a10c98c6475f/SHA1-s65536--795b58cc4e5190b02e7026fd9e94a10c98c6475f

From 6c396a256c93464d726c66a95132536941871ee8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 12:47:14 -0400
Subject: [PATCH 2067/2835] finished hlint pass

---
 Command/Add.hs        |  6 +++---
 Command/DropUnused.hs |  9 ++++-----
 Command/Fsck.hs       |  2 +-
 Command/InAnnex.hs    |  2 +-
 Command/InitRemote.hs |  8 ++++----
 Command/Map.hs        | 19 ++++++++++---------
 Command/Move.hs       |  2 +-
 Command/SetKey.hs     |  4 ++--
 Command/Status.hs     | 16 ++++++++--------
 Command/Uninit.hs     |  5 +++--
 Command/Unused.hs     | 16 ++++++++--------
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  4 ++--
 Locations.hs          |  4 ++--
 Remote/Bup.hs         | 20 +++++++++++---------
 Remote/Directory.hs   |  5 +++--
 Remote/Encryptable.hs |  4 ++--
 Remote/Git.hs         |  6 +++---
 Remote/Hook.hs        | 16 +++++++++-------
 Remote/Rsync.hs       | 12 ++++++------
 Remote/S3real.hs      | 24 ++++++++++--------------
 Remote/Special.hs     |  2 +-
 Remote/Ssh.hs         |  2 +-
 Remote/Web.hs         |  4 ++--
 Touch.hsc             |  5 ++---
 Utility/CopyFile.hs   | 12 +++++-------
 Utility/DataUnits.hs  |  6 +++---
 Utility/Dot.hs        | 10 +++++-----
 Utility/RsyncFile.hs  |  2 +-
 29 files changed, 114 insertions(+), 115 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 51b95b9b5b..58c0143dd0 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -39,7 +39,7 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
 start :: CommandStartBackendFile
 start pair@(file, _) = notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
-	if (isSymbolicLink s) || (not $ isRegularFile s)
+	if isSymbolicLink s || not (isRegularFile s)
 		then stop
 		else do
 			showStart "add" file
@@ -58,8 +58,8 @@ perform (file, backend) = do
  - This can be called before or after the symlink is in place. -}
 undo :: FilePath -> Key -> IOException -> Annex a
 undo file key e = do
-	unlessM (inAnnex key) $ rethrow -- no cleanup to do
-	liftIO $ whenM (doesFileExist file) $ do removeFile file
+	unlessM (inAnnex key) rethrow -- no cleanup to do
+	liftIO $ whenM (doesFileExist file) $ removeFile file
 	handle tryharder $ fromAnnex key file
 	logStatus key InfoMissing
 	rethrow
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 55007c1f73..a01e08ab51 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -49,7 +49,7 @@ start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
 	]
 	where
 		search [] = stop
-		search ((m, a):rest) = do
+		search ((m, a):rest) =
 			case M.lookup s m of
 				Nothing -> search rest
 				Just key -> do
@@ -78,10 +78,9 @@ readUnusedLog prefix = do
 	let f = gitAnnexUnusedLog prefix g
 	e <- liftIO $ doesFileExist f
 	if e
-		then do
-			l <- liftIO $ readFile f
-			return $ M.fromList $ map parse $ lines l
-		else return $ M.empty
+		then return . M.fromList . map parse . lines
+			=<< liftIO (readFile f)
+		else return M.empty
 	where
 		parse line = (head ws, fromJust $ readKey $ unwords $ tail ws)
 			where
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index ec3f1d8e7d..0d3ecb58f1 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -94,7 +94,7 @@ fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 fsckKey backend key file numcopies = do
 	size_ok <- checkKeySize key
 	copies_ok <- checkKeyNumCopies key file numcopies
-	backend_ok <-(Types.Backend.fsckKey backend) key
+	backend_ok <- (Types.Backend.fsckKey backend) key
 	return $ size_ok && copies_ok && backend_ok
 
 {- The size of the data for a key is checked against the size encoded in
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index b5b59ccf7d..24f7162ace 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -25,4 +25,4 @@ start key = do
 	present <- inAnnex key
 	if present
 		then stop
-		else liftIO $ exitFailure
+		else liftIO exitFailure
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 15962ad991..9859308e56 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -24,7 +24,7 @@ import Messages
 command :: [Command]
 command = [repoCommand "initremote"
 	(paramPair paramName $
-		paramOptional $ paramRepeating $ paramKeyValue) seek
+		paramOptional $ paramRepeating paramKeyValue) seek
 	"sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
@@ -32,7 +32,7 @@ seek = [withWords start]
 
 start :: CommandStartWords
 start ws = do
-	when (null ws) $ needname
+	when (null ws) needname
 
 	(u, c) <- findByName name
 	let fullconfig = M.union config c	
@@ -69,7 +69,7 @@ findByName name = do
 	maybe generate return $ findByName' name m
 	where
 		generate = do
-			uuid <- liftIO $ genUUID
+			uuid <- liftIO genUUID
 			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
@@ -85,7 +85,7 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 remoteNames :: Annex [String]
 remoteNames = do
 	m <- RemoteLog.readRemoteLog
-	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m
+	return $ mapMaybe (M.lookup nameKey . snd) $ M.toList m
 
 {- find the specified remote type -}
 findType :: R.RemoteConfig -> Annex (R.RemoteType Annex)
diff --git a/Command/Map.hs b/Command/Map.hs
index 0391ab8e8f..557ae27871 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -12,6 +12,7 @@ import Control.Exception.Extensible
 import System.Cmd.Utils
 import qualified Data.Map as M
 import Data.List.Utils
+import Data.Maybe
 
 import Command
 import qualified Annex
@@ -58,7 +59,7 @@ start = do
  - the repositories first, followed by uuids that were not matched
  - to a repository.
  -}
-drawMap :: [Git.Repo] -> (M.Map UUID String) -> [UUID] -> String
+drawMap :: [Git.Repo] -> M.Map UUID String -> [UUID] -> String
 drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 	where
 		repos = map (node umap rs) rs
@@ -78,23 +79,23 @@ basehostname r = head $ split "." $ hostname r
 
 {- A name to display for a repo. Uses the name from uuid.log if available,
  - or the remote name if not. -}
-repoName :: (M.Map UUID String) -> Git.Repo -> String
+repoName :: M.Map UUID String -> Git.Repo -> String
 repoName umap r
 	| null repouuid = fallback
 	| otherwise = M.findWithDefault fallback repouuid umap
 	where
 		repouuid = getUncachedUUID r
-		fallback = maybe "unknown" id $ Git.repoRemoteName r
+		fallback = fromMaybe "unknown" $ Git.repoRemoteName r
 
 {- A unique id for the node for a repo. Uses the annex.uuid if available. -}
 nodeId :: Git.Repo -> String
 nodeId r =
-	case (getUncachedUUID r) of
+	case getUncachedUUID r of
 		"" -> Git.repoLocation r
 		u -> u
 
 {- A node representing a repo. -}
-node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
+node :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> String
 node umap fullinfo r = unlines $ n:edges
 	where
 		n = Dot.subGraph (hostname r) (basehostname r) "lightblue" $
@@ -105,14 +106,14 @@ node umap fullinfo r = unlines $ n:edges
 			| otherwise = reachable
 
 {- An edge between two repos. The second repo is a remote of the first. -}
-edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
+edge :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
 edge umap fullinfo from to =
 	Dot.graphEdge (nodeId from) (nodeId fullto) edgename
 	where
 		-- get the full info for the remote, to get its UUID
 		fullto = findfullinfo to
 		findfullinfo n =
-			case (filter (same n) fullinfo) of
+			case filter (same n) fullinfo of
 				[] -> n
 				(n':_) -> n'
 		{- Only name an edge if the name is different than the name
@@ -120,7 +121,7 @@ edge umap fullinfo from to =
 		 - different from its hostname. (This reduces visual clutter.) -}
 		edgename = maybe Nothing calcname $ Git.repoRemoteName to
 		calcname n
-			| n == repoName umap fullto || n == hostname fullto = Nothing
+			| n `elem` [repoName umap fullto, hostname fullto] = Nothing
 			| otherwise = Just n
 
 unreachable :: String -> String
@@ -188,7 +189,7 @@ tryScan r
 	| otherwise = safely $ Git.configRead r
 	where
 		safely a = do
-			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
+			result <- liftIO (try a :: IO (Either SomeException Git.Repo))
 			case result of
 				Left _ -> return Nothing
 				Right r' -> return $ Just r'
diff --git a/Command/Move.hs b/Command/Move.hs
index 6bf6e05827..a98276e7ec 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -124,7 +124,7 @@ fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
 	remotes <- Remote.keyPossibilities key
-	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
+	if u == Remote.uuid src || not (any (== src) remotes)
 		then stop
 		else do
 			showAction move file
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index b000a4e8bf..f2a5259bac 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -16,7 +16,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [repoCommand "setkey" (paramPath) seek
+command = [repoCommand "setkey" paramPath seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
@@ -34,7 +34,7 @@ perform file = do
 	-- the file might be on a different filesystem, so mv is used
 	-- rather than simply calling moveToObjectDir; disk space is also
 	-- checked this way.
-	ok <- getViaTmp key $ \dest -> do
+	ok <- getViaTmp key $ \dest ->
 		if dest /= file
 			then liftIO $
 				boolSystem "mv" [File file, File dest]
diff --git a/Command/Status.hs b/Command/Status.hs
index 1ec4782362..aef4df2329 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -32,8 +32,8 @@ type Stat = StatState (Maybe (String, StatState String))
 
 -- cached info that multiple Stats may need
 data StatInfo = StatInfo
-	{ keysPresentCache :: (Maybe (SizeList Key))
-	, keysReferencedCache :: (Maybe (SizeList Key))
+	{ keysPresentCache :: Maybe (SizeList Key)
+	, keysReferencedCache :: Maybe (SizeList Key)
 	}
 
 -- a state monad for running Stats in
@@ -84,7 +84,7 @@ stat :: String -> StatState String -> Stat
 stat desc a = return $ Just (desc, a)
 
 nostat :: Stat
-nostat = return $ Nothing
+nostat = return Nothing
 
 showStat :: Stat -> StatState ()
 showStat s = calc =<< s
@@ -144,7 +144,7 @@ cachedKeysPresent = do
 	case keysPresentCache s of
 		Just v -> return v
 		Nothing -> do
-			keys <- lift $ getKeysPresent
+			keys <- lift getKeysPresent
 			let v = sizeList keys
 			put s { keysPresentCache = Just v }
 			return v
@@ -155,7 +155,7 @@ cachedKeysReferenced = do
 	case keysReferencedCache s of
 		Just v -> return v
 		Nothing -> do
-			keys <- lift $ Command.Unused.getKeysReferenced
+			keys <- lift Command.Unused.getKeysReferenced
 			-- A given key may be referenced repeatedly.
 			-- nub does not seem too slow (yet)..
 			let v = sizeList $ nub keys
@@ -164,9 +164,9 @@ cachedKeysReferenced = do
 
 keySizeSum :: SizeList Key -> StatState String
 keySizeSum (keys, len) = do
-	let knownsize = catMaybes $ map keySize keys
-	let total = roughSize storageUnits False $ foldl (+) 0 knownsize
-	let missing = len - genericLength knownsize
+	let knownsizes = mapMaybe keySize keys
+	let total = roughSize storageUnits False $ sum knownsizes
+	let missing = len - genericLength knownsizes
 	return $ total ++
 		if missing > 0
 			then aside $ "but " ++ show missing ++ " keys have unknown size"
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 1497bbfd19..8b8d7e364d 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -52,8 +52,9 @@ cleanup = do
 	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
 	-- avoid normal shutdown
 	saveState
-	liftIO $ Git.run g "branch" [Param "-D", Param Branch.name]
-	liftIO $ exitSuccess
+	liftIO $ do
+		Git.run g "branch" [Param "-D", Param Branch.name]
+		exitSuccess
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 3f51e2c2c2..870c993f18 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -7,7 +7,7 @@
 
 module Command.Unused where
 
-import Control.Monad (filterM, unless, forM_, when)
+import Control.Monad (filterM, unless, forM_)
 import Control.Monad.State (liftIO)
 import qualified Data.Set as S
 import Data.Maybe
@@ -55,9 +55,9 @@ checkUnused = do
 	where
 		list file msg l c = do
 			let unusedlist = number c l
-			when (not $ null l) $ do
+			unless (null l) $ do
 				showLongNote $ msg unusedlist
-				showLongNote $ "\n"
+				showLongNote "\n"
 			writeUnusedFile file unusedlist
 			return $ c + length l
 
@@ -68,7 +68,7 @@ checkRemoteUnused name = do
 
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
-	showNote $ "checking for unused data..."
+	showNote "checking for unused data..."
 	referenced <- getKeysReferenced
 	remotehas <- filterM isthere =<< loggedKeys
 	let remoteunused = remotehas `exclude` referenced
@@ -76,7 +76,7 @@ checkRemoteUnused' r = do
 	writeUnusedFile "" list
 	unless (null remoteunused) $ do
 		showLongNote $ remoteUnusedMsg r list
-		showLongNote $ "\n"
+		showLongNote "\n"
 	where
 		isthere k = do
 			us <- keyLocations k
@@ -90,14 +90,14 @@ writeUnusedFile prefix l = do
 		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
 
 table :: [(Int, Key)] -> [String]
-table l = ["  NUMBER  KEY"] ++ map cols l
+table l = "  NUMBER  KEY" : map cols l
 	where
 		cols (n,k) = "  " ++ pad 6 (show n) ++ "  " ++ show k
 		pad n s = s ++ replicate (n - length s) ' '
 
 number :: Int -> [a] -> [(Int, a)]
 number _ [] = []
-number n (x:xs) = (n+1, x):(number (n+1) xs)
+number n (x:xs) = (n+1, x) : number (n+1) xs
 
 staleTmpMsg :: [(Int, Key)] -> String
 staleTmpMsg t = unlines $ 
@@ -210,4 +210,4 @@ staleKeys dirspec = do
 			contents <- liftIO $ getDirectoryContents dir
 			files <- liftIO $ filterM doesFileExist $
 				map (dir ) contents
-			return $ catMaybes $ map (fileKey . takeFileName) files
+			return $ mapMaybe (fileKey . takeFileName) files
diff --git a/Command/Version.hs b/Command/Version.hs
index bb7acd12dd..2392c9bf6b 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -31,4 +31,4 @@ start = do
 	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
 	stop
 	where
-		vs l = join " " l
+		vs = join " "
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 0e4858f8b7..05748e8d60 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -30,11 +30,11 @@ perform key = do
 	uuids <- keyLocations key
 	let num = length uuids
 	showNote $ show num ++ " " ++ copiesplural num
-	if null $ uuids
+	if null uuids
 		then stop
 		else do
 			pp <- prettyPrintUUIDs uuids
-			showLongNote $ pp
+			showLongNote pp
 			showProgress	
 			next $ return True
 	where
diff --git a/Locations.hs b/Locations.hs
index 2dbf2f55ea..942b687bbf 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -167,8 +167,8 @@ display_32bits_as_dir w = trim $ swap_pairs cs
 		-- a real word, use letters that appear less frequently.
 		chars = ['0'..'9'] ++ "zqjxkmvwgpfZQJXKMVWGPF"
 		cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7]
-		getc n = chars !! (fromIntegral n)
+		getc n = chars !! fromIntegral n
 		swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs
 		swap_pairs _ = []
 		-- Last 2 will always be 00, so omit.
-		trim s = take 6 s
+		trim = take 6
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 5a44397f0d..4ea455226e 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -8,7 +8,8 @@
 module Remote.Bup (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import IO
+import System.IO
+import System.IO.Error
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Control.Monad (when)
@@ -16,6 +17,7 @@ import Control.Monad.State (liftIO)
 import System.Process
 import System.Exit
 import System.FilePath
+import Data.Maybe
 import Data.List.Utils
 import System.Cmd.Utils
 
@@ -68,7 +70,7 @@ gen r u c = do
 bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 bupSetup u c = do
 	-- verify configuration is sane
-	let buprepo = maybe (error "Specify buprepo=") id $
+	let buprepo = fromMaybe (error "Specify buprepo=") $
 		M.lookup "buprepo" c
 	c' <- encryptionSetup c
 
@@ -87,7 +89,7 @@ bupSetup u c = do
 
 bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam]
 bupParams command buprepo params = 
-	(Param command) : [Param "-r", Param buprepo] ++ params
+	Param command : [Param "-r", Param buprepo] ++ params
 
 bup :: String -> BupRepo -> [CommandParam] -> Annex Bool
 bup command buprepo params = do
@@ -123,8 +125,8 @@ storeEncrypted r buprepo (cipher, enck) k = do
 	g <- Annex.gitRepo
 	let src = gitAnnexLocation g k
 	params <- bupSplitParams r buprepo enck (Param "-")
-	liftIO $ catchBool $ do
-		withEncryptedHandle cipher (L.readFile src) $ \h -> do
+	liftIO $ catchBool $
+		withEncryptedHandle cipher (L.readFile src) $ \h ->
 			pipeBup params (Just h) Nothing
 
 retrieve :: BupRepo -> Key -> FilePath -> Annex Bool
@@ -184,7 +186,7 @@ onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [
 onBupRemote r a command params = do
 	let dir = shellEscape (Git.workTree r)
 	sshparams <- sshToRepo r [Param $
-			"cd " ++ dir ++ " && " ++ (unwords $ command : toCommand params)]
+			"cd " ++ dir ++ " && " ++ unwords (command : toCommand params)]
 	liftIO $ a "ssh" sshparams
 
 {- Allow for bup repositories on removable media by checking
@@ -215,20 +217,20 @@ bup2GitRemote "" = do
 	Git.repoFromAbsPath $ h  ".bup"
 bup2GitRemote r
 	| bupLocal r = 
-		if r !! 0 == '/'
+		if head r == '/'
 			then Git.repoFromAbsPath r
 			else error "please specify an absolute path"
 	| otherwise = Git.repoFromUrl $ "ssh://" ++ host ++ slash dir
 		where
 			bits = split ":" r
-			host = bits !! 0
+			host = head bits
 			dir = join ":" $ drop 1 bits
 			-- "host:~user/dir" is not supported specially by bup;
 			-- "host:dir" is relative to the home directory;
 			-- "host:" goes in ~/.bup
 			slash d
 				| d == "" = "/~/.bup"
-				| d !! 0 == '/' = d
+				| head d == '/' = d
 				| otherwise = "/~/" ++ d
 
 bupLocal :: BupRepo -> Bool
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 05d42136f1..235f613000 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -8,13 +8,14 @@
 module Remote.Directory (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import IO
+import System.IO.Error
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory hiding (copyFile)
 import System.FilePath
+import Data.Maybe
 
 import Types
 import Types.Remote
@@ -60,7 +61,7 @@ gen r u c = do
 directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 directorySetup u c = do
 	-- verify configuration is sane
-	let dir = maybe (error "Specify directory=") id $
+	let dir = fromMaybe (error "Specify directory=") $
 		M.lookup "directory" c
 	liftIO $ doesDirectoryExist dir
 		>>! error $ "Directory does not exist: " ++ dir
diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs
index 443f5cf83d..66e1738ac2 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Encryptable.hs
@@ -56,10 +56,10 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 	where
 		store k = cip k >>= maybe
 			(storeKey r k)
-			(\x -> storeKeyEncrypted x k)
+			(`storeKeyEncrypted` k)
 		retrieve k f = cip k >>= maybe
 			(retrieveKeyFile r k f)
-			(\x -> retrieveKeyFileEncrypted x f)
+			(`retrieveKeyFileEncrypted` f)
 		withkey a k = cip k >>= maybe (a k) (a . snd)
 		cip = cipherKey c
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index fb85123825..1f22ad11c6 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -57,7 +57,7 @@ gen r u _ = do
 	let defcst = if cheap then cheapRemoteCost else expensiveRemoteCost
 	cst <- remoteCost r' defcst
 
-	return $ Remote {
+	return Remote {
 		uuid = u',
 		cost = cst,
 		name = Git.repoDescribe r',
@@ -81,7 +81,7 @@ tryGitConfigRead r
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.
 		safely a = do
-			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
+			result <- liftIO (try a :: IO (Either SomeException Git.Repo))
 			case result of
 				Left _ -> return r
 				Right r' -> return r'
@@ -154,7 +154,7 @@ copyToRemote r key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
-rsyncHelper :: [CommandParam] -> Annex (Bool)
+rsyncHelper :: [CommandParam] -> Annex Bool
 rsyncHelper p = do
 	showProgress -- make way for progress bar
 	res <- liftIO $ rsync p
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 86a7bca560..f0e4d5bfbc 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -17,6 +17,7 @@ import System.Posix.IO
 import System.IO
 import System.IO.Error (try)
 import System.Exit
+import Data.Maybe
 
 import Types
 import Types.Remote
@@ -61,7 +62,7 @@ gen r u c = do
 
 hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 hookSetup u c = do
-	let hooktype = maybe (error "Specify hooktype=") id $
+	let hooktype = fromMaybe (error "Specify hooktype=") $
 		M.lookup "hooktype" c
 	c' <- encryptionSetup c
 	gitConfigSpecialRemote u c' "hooktype" hooktype
@@ -73,12 +74,13 @@ hookEnv k f = Just $ fileenv f ++ keyenv
 		env s v = ("ANNEX_" ++ s, v)
 		keyenv =
 			[ env "KEY" (show k)
-			, env "HASH_1" (hashbits !! 0)
-			, env "HASH_2" (hashbits !! 1)
+			, env "HASH_1" hash_1
+			, env "HASH_2" hash_2
 			]
 		fileenv Nothing = []
 		fileenv (Just file) =  [env "FILE" file]
-		hashbits = map takeDirectory $ splitPath $ hashDirMixed k
+		[hash_1, hash_2, _rest] =
+			map takeDirectory $ splitPath $ hashDirMixed k
 
 lookupHook :: String -> String -> Annex (Maybe String)
 lookupHook hooktype hook =do
@@ -127,7 +129,7 @@ retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
 		return True
 
 remove :: String -> Key -> Annex Bool
-remove h k = runHook h "remove" k Nothing $ do return True
+remove h k = runHook h "remove" k Nothing $ return True
 
 checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool)
 checkPresent r h k = do
@@ -135,7 +137,7 @@ checkPresent r h k = do
 	v <- lookupHook h "checkpresent"
 	liftIO (try (check v) ::IO (Either IOException Bool))
 	where
-		findkey s = (show k) `elem` (lines s)
+		findkey s = show k `elem` lines s
 		env = hookEnv k Nothing
 		check Nothing = error "checkpresent hook misconfigured"
 		check (Just hook) = do
@@ -150,5 +152,5 @@ checkPresent r h k = do
 			hClose fromh
 			s <- getProcessStatus True False pid
 			case s of
-				Just (Exited (ExitSuccess)) -> return $ findkey reply
+				Just (Exited ExitSuccess) -> return $ findkey reply
 				_ -> error "checkpresent hook failed"
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 80e194fed1..ca4236276f 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -15,6 +15,7 @@ import System.FilePath
 import System.Directory
 import System.Posix.Files
 import System.Posix.Process
+import Data.Maybe
 
 import Types
 import Types.Remote
@@ -82,7 +83,7 @@ genRsyncOpts r = do
 rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 rsyncSetup u c = do
 	-- verify configuration is sane
-	let url = maybe (error "Specify rsyncurl=") id $
+	let url = fromMaybe (error "Specify rsyncurl=") $
 		M.lookup "rsyncurl" c
 	c' <- encryptionSetup c
 
@@ -160,10 +161,10 @@ partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial"
 withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool
 withRsyncScratchDir a = do
 	g <- Annex.gitRepo
-	pid <- liftIO $ getProcessID
+	pid <- liftIO getProcessID
 	let tmp = gitAnnexTmpDir g  "rsynctmp"  show pid
 	nuke tmp
-	liftIO $ createDirectoryIfMissing True $ tmp
+	liftIO $ createDirectoryIfMissing True tmp
 	res <- a tmp
 	nuke tmp
 	return res
@@ -189,15 +190,14 @@ rsyncRemote o params = do
 rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool
 rsyncSend o k src = withRsyncScratchDir $ \tmp -> do
 	let dest = tmp  hashDirMixed k  f  f
-	liftIO $ createDirectoryIfMissing True $ parentDir $ dest
+	liftIO $ createDirectoryIfMissing True $ parentDir dest
 	liftIO $ createLink src dest
-	res <- rsyncRemote o
+	rsyncRemote o
 		[ Param "--recursive"
 		, partialParams
  		  -- tmp/ to send contents of tmp dir
 		, Param $ addTrailingPathSeparator tmp
 		, Param $ rsyncUrl o
 		]
-	return res
 	where
 		f = keyFile k
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 52d1ed1be1..cbd3ef6225 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -52,7 +52,7 @@ gen r u c = do
 	cst <- remoteCost r expensiveRemoteCost
 	return $ gen' r u c cst
 gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex
-gen' r u c cst = do
+gen' r u c cst =
 	encryptableRemote c
 		(storeEncrypted this)
 		(retrieveEncrypted this)
@@ -85,7 +85,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 		
 		handlehost Nothing = defaulthost
 		handlehost (Just h)
-			| ".archive.org" `isSuffixOf` (map toLower h) = archiveorg
+			| ".archive.org" `isSuffixOf` map toLower h = archiveorg
 			| otherwise = defaulthost
 
 		use fullconfig = do
@@ -99,7 +99,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 			use fullconfig
 
 		archiveorg = do
-			showNote $ "Internet Archive mode"
+			showNote "Internet Archive mode"
 			maybe (error "specify bucket=") (const $ return ()) $
 				M.lookup "bucket" archiveconfig
 			use archiveconfig
@@ -203,10 +203,8 @@ s3Error :: ReqError -> a
 s3Error e = error $ prettyReqError e
 
 s3Bool :: AWSResult () -> Annex Bool
-s3Bool res = do
-	case res of
-		Right _ -> return True
-		Left e -> s3Warning e
+s3Bool (Right _) = return True
+s3Bool (Left e) = s3Warning e
 
 s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a
 s3Action r noconn action = do
@@ -219,7 +217,7 @@ s3Action r noconn action = do
 		_ -> return noconn
 
 bucketFile :: Remote Annex -> Key -> FilePath
-bucketFile r k = (munge $ show k)
+bucketFile r = munge . show
 	where
 		munge s = case M.lookup "mungekeys" $ fromJust $ config r of
 			Just "ia" -> iaMunge s
@@ -271,8 +269,8 @@ s3Connection c = do
 			warning $ "Set both " ++ s3AccessKey ++ " and " ++ s3SecretKey  ++ " to use S3"
 			return Nothing
 	where
-		host = fromJust $ (M.lookup "host" c)
-		port = let s = fromJust $ (M.lookup "port" c) in
+		host = fromJust $ M.lookup "host" c
+		port = let s = fromJust $ M.lookup "port" c in
 			case reads s of
 			[(p, _)] -> p
 			_ -> error $ "bad S3 port value: " ++ s
@@ -283,7 +281,7 @@ s3GetCreds :: RemoteConfig -> Annex (Maybe (String, String))
 s3GetCreds c = do
 	ak <- getEnvKey s3AccessKey
 	sk <- getEnvKey s3SecretKey
-	if (null ak || null sk)
+	if null ak || null sk
 		then do
 			mcipher <- remoteCipher c
 			case (M.lookup "s3creds" c, mcipher) of
@@ -291,9 +289,7 @@ s3GetCreds c = do
 					s <- liftIO $ withDecryptedContent cipher
 						(return $ L.pack $ fromB64 encrypted)
 						(return . L.unpack)
-					let line = lines s
-					let ak' = line !! 0
-					let sk' = line !! 1
+					let [ak', sk', _rest] = lines s
 					liftIO $ do
 						setEnv s3AccessKey ak True
 						setEnv s3SecretKey sk True
diff --git a/Remote/Special.hs b/Remote/Special.hs
index 9a00dbd82e..d6f362ce30 100644
--- a/Remote/Special.hs
+++ b/Remote/Special.hs
@@ -38,7 +38,7 @@ gitConfigSpecialRemote u c k v = do
 	g <- Annex.gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
-		Git.run g "config" [Param (configsetting $ "annex-uuid"), Param u]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
 	where
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Remote/Ssh.hs b/Remote/Ssh.hs
index 0d4842a1ab..fe4e6dfc1a 100644
--- a/Remote/Ssh.hs
+++ b/Remote/Ssh.hs
@@ -39,7 +39,7 @@ git_annex_shell r command params
 	where
 		dir = Git.workTree r
 		shellcmd = "git-annex-shell"
-		shellopts = (Param command):(File dir):params
+		shellopts = Param command : File dir : params
 		sshcmd = shellcmd ++ " " ++ 
 			unwords (map shellEscape $ toCommand shellopts)
 
diff --git a/Remote/Web.hs b/Remote/Web.hs
index d3d140d73b..60f64cfe02 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -52,7 +52,7 @@ webUUID = "00000000-0000-0000-0000-000000000001"
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
-	return $ Remote {
+	return Remote {
 		uuid = webUUID,
 		cost = expensiveRemoteCost,
 		name = Git.repoDescribe r,
@@ -111,7 +111,7 @@ checkKey' (u:us) = do
 	if e then return e else checkKey' us
 
 urlexists :: URLString -> IO Bool
-urlexists url = do
+urlexists url =
 	case parseURI url of
 		Nothing -> return False
 		Just u -> do
diff --git a/Touch.hsc b/Touch.hsc
index 4f26855d21..dd0c38984e 100644
--- a/Touch.hsc
+++ b/Touch.hsc
@@ -15,6 +15,7 @@ module Touch (
 
 import Foreign
 import Foreign.C
+import Control.Monad (when)
 
 newtype TimeSpec = TimeSpec CTime
 
@@ -66,9 +67,7 @@ touchBoth file atime mtime follow =
 	withCString file $ \f -> do
 		pokeArray ptr [atime, mtime]
 		r <- c_utimensat at_fdcwd f ptr flags
-		if (r /= 0)
-			then throwErrno "touchBoth"
-			else return ()
+		when (r /= 0) $ throwErrno "touchBoth"
 	where
 		flags = if follow
 			then 0
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 5ee4a91df7..2e06dd92bb 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -20,10 +20,8 @@ copyFile src dest = do
 		removeFile dest
 	boolSystem "cp" [params, File src, File dest]
 	where
-		params = if SysConfig.cp_reflink_auto
-			then Params "--reflink=auto"
-			else if SysConfig.cp_a
-				then Params "-a"
-				else if SysConfig.cp_p
-					then Params "-p"
-					else Params ""
+		params
+			| SysConfig.cp_reflink_auto = Params "--reflink=auto"
+			| SysConfig.cp_a = Params "-a"
+			| SysConfig.cp_p = Params "-p"
+			| otherwise = Params ""
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index 7af2eadafb..f2bc333ea0 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -106,7 +106,7 @@ oldSchoolUnits = map mingle $ zip storageUnits memoryUnits
 {- approximate display of a particular number of bytes -}
 roughSize :: [Unit] -> Bool -> ByteSize -> String
 roughSize units abbrev i
-	| i < 0 = "-" ++ findUnit units' (negate i)
+	| i < 0 = '-' : findUnit units' (negate i)
 	| otherwise = findUnit units' i
 	where
 		units' = reverse $ sort units -- largest first
@@ -139,10 +139,10 @@ readSize :: [Unit] -> String -> Maybe ByteSize
 readSize units input
 	| null parsednum = Nothing
 	| null parsedunit = Nothing
-	| otherwise = Just $ round $ number * (fromIntegral multiplier)
+	| otherwise = Just $ round $ number * fromIntegral multiplier
 	where
 		(number, rest) = head parsednum
-		multiplier = head $ parsedunit
+		multiplier = head parsedunit
 		unitname = takeWhile isAlpha $ dropWhile isSpace rest
 
 		parsednum = reads input :: [(Double, String)]
diff --git a/Utility/Dot.hs b/Utility/Dot.hs
index 8696849963..83f52a3cc1 100644
--- a/Utility/Dot.hs
+++ b/Utility/Dot.hs
@@ -20,13 +20,13 @@ graphNode nodeid desc = label desc $ quote nodeid
 
 {- an edge between two nodes -}
 graphEdge :: String -> String -> Maybe String -> String
-graphEdge fromid toid desc = indent $ maybe edge (\d -> label d edge) desc
+graphEdge fromid toid desc = indent $ maybe edge (`label` edge) desc
 	where
 		edge = quote fromid ++ " -> " ++ quote toid
 
 {- adds a label to a node or edge -}
 label :: String -> String -> String
-label l s = attr "label" l s
+label = attr "label"
 
 {- adds an attribute to a node or edge
  - (can be called multiple times for multiple attributes) -}
@@ -35,7 +35,7 @@ attr a v s = s ++ " [ " ++ a ++ "=" ++ quote v ++ " ]"
 
 {- fills a node with a color -}
 fillColor :: String -> String -> String
-fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s
+fillColor color s = attr "fillcolor" color $ attr "style" "filled" s
 
 {- apply to graphNode to put the node in a labeled box -}
 subGraph :: String -> String -> String -> String -> String
@@ -52,10 +52,10 @@ subGraph subid l color s =
 		setlabel = "label=" ++ quote l
 		setfilled = "style=" ++ quote "filled"
 		setcolor = "fillcolor=" ++ quote color
-		ii x = (indent $ indent x) ++ "\n"
+		ii x = indent (indent x) ++ "\n"
 
 indent ::String -> String
-indent s = "\t" ++ s
+indent s = '\t' : s
 
 quote :: String -> String
 quote s = "\"" ++ s' ++ "\""
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index c68909d2dc..6e21ba0632 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -19,7 +19,7 @@ rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand comman
 		{- rsync requires some weird, non-shell like quoting in
                  - here. A doubled single quote inside the single quoted
                  - string is a single quote. -}
-		escape s = "'" ++  (join "''" $ split "'" s) ++ "'"
+		escape s = "'" ++  join "''" (split "'" s) ++ "'"
 
 {- Runs rsync in server mode to send a file, and exits. -}
 rsyncServerSend :: FilePath -> IO ()

From 7fa7601490c1346ab7dfcd645fe9b822eddee68a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 12:51:06 -0400
Subject: [PATCH 2068/2835] response

---
 doc/bugs/unannex_command_doesn__39__t_all_files.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn b/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
index c3128c0fc2..8094ed9182 100644
--- a/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
+++ b/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
@@ -19,3 +19,8 @@
     total 72
     -rw-r--r-- 1 simons users 65536 Jul 15 17:29 file1
     lrwxrwxrwx 1 simons users   132 Jul 15 17:29 file2 -> .git/annex/objects/jp/Fk/SHA1-s65536--795b58cc4e5190b02e7026fd9e94a10c98c6475f/SHA1-s65536--795b58cc4e5190b02e7026fd9e94a10c98c6475f
+
+> This was recently discussed in
+> [[annex_unannex__47__uninit_should_handle_copies]] and `unannex --fast`
+> added to leave contents behind in the annex, which allows handling
+> copies. But needs manual cleanup later with dropunused. --[[Joey]] 

From 39ae33386d0b2b3f55a766839e1eabfded3a28e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 12:53:58 -0400
Subject: [PATCH 2069/2835] update

---
 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
index 249a175055..3fd1a8082c 100644
--- a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -13,3 +13,7 @@ See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html
 
 >> Even if there is nothing it can _do_, knowing that the data is intact,
 >> or not, is valuable in and as of itself. -- RichiH
+
+>>> While storing the data is no longer an issue in bare repos, fsck would
+>>> need a special mode that examines all the location logs, since it
+>>> cannot run thru the checked out files. --[[Joey]] 

From 13187b1ba75f800d28a3667aa80a252c9f1a5e29 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 12:55:05 -0400
Subject: [PATCH 2070/2835] close

---
 doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn
index 608704d9d7..3c9374100c 100644
--- a/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn
+++ b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn
@@ -1 +1,4 @@
 I have files with very long filenames on an xfs at home. On my laptop the annex should have been checked out on an encfs, but there filenames can't be as long as on the xfs. So perhaps it would be good to limit the keysize to a sane substring of the filename e.g. use only the first 120 characters.
+
+> Since there seems no strong argument for a WORM100, and better options
+> exist, closing. [[done]] --[[Joey]] 

From 6a58f61d2870d6631f8a039a6fdc07f262a7e2af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 16 Jul 2011 15:09:57 -0400
Subject: [PATCH 2071/2835] typo

---
 Utility.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility.hs b/Utility.hs
index 786c6a1770..dab3b4d0ea 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -288,6 +288,6 @@ unlessM c a = c >>= flip unless a
 (>>!) :: Monad m => m Bool -> m () -> m ()
 (>>!) = unlessM
 
--- low fixity allows eg, foo bar <|> error $ "failed " ++ meep
+-- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
 infixr 0 >>?
 infixr 0 >>!

From ec9e9343d9fa99b0786ee93ff142484e2402d3c8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jul 2011 19:05:50 -0400
Subject: [PATCH 2072/2835] add closure for new bug that I already fixed

---
 debian/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/changelog b/debian/changelog
index 9a7c3e7b0f..3a6c731fe4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110708) UNRELEASED; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+    Closes: #634233
   * Bugfix: Make add ../ work.
   * Support the standard git -c name=value
   * unannex: Clean up use of git commit -a.

From 00153eed48a2328969cc08688ef674a4c19c2014 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jul 2011 14:07:23 -0400
Subject: [PATCH 2073/2835] unify elipsis handling

And add a simple dots-based progress display, currently only used in v2
upgrade.
---
 AnnexQueue.hs         |  2 +-
 Backend/SHA.hs        |  2 +-
 Branch.hs             |  2 +-
 Command.hs            |  2 +-
 Command/AddUrl.hs     |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Get.hs        |  2 +-
 Command/Map.hs        |  8 ++++----
 Command/Move.hs       | 14 +++++++-------
 Command/Unlock.hs     |  2 +-
 Command/Unused.hs     |  4 ++--
 Command/Whereis.hs    |  2 +-
 Messages.hs           | 39 ++++++++++++++++++++++++---------------
 Remote/Bup.hs         | 10 +++++-----
 Remote/Git.hs         |  4 ++--
 Remote/Hook.hs        |  4 ++--
 Remote/Rsync.hs       |  4 ++--
 Remote/S3real.hs      |  6 +++---
 Remote/Web.hs         |  4 ++--
 Upgrade/V0.hs         |  2 +-
 Upgrade/V1.hs         |  8 ++++----
 Upgrade/V2.hs         | 13 +++++++++----
 22 files changed, 76 insertions(+), 62 deletions(-)

diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index b1678df07f..79116c48af 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -38,7 +38,7 @@ flush silent = do
 	q <- getState repoqueue
 	unless (0 == Git.Queue.size q) $ do
 		unless silent $
-			showSideAction "Recording state in git..."
+			showSideAction "Recording state in git"
 		g <- gitRepo
 		q' <- liftIO $ Git.Queue.flush g q
 		store q'
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index dc27b30003..c1d7136485 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -72,7 +72,7 @@ shaNameE size = shaName size ++ "E"
 
 shaN :: SHASize -> FilePath -> Annex String
 shaN size file = do
-	showNote "checksum..."
+	showAction "checksum"
 	liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do
 		line <- hGetLine h
 		let bits = split " " line
diff --git a/Branch.hs b/Branch.hs
index c8e6bc2bb4..35e3050936 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -190,7 +190,7 @@ updateRef ref
 		if null diffs
 			then return Nothing
 			else do
-				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name ++ "..."
+				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name
 				-- By passing only one ref, it is actually
 				-- merged into the index, preserving any
 				-- changes that may already be staged.
diff --git a/Command.hs b/Command.hs
index 729e442fc4..02bbd29d44 100644
--- a/Command.hs
+++ b/Command.hs
@@ -102,7 +102,7 @@ doCommand = start
 		stage a b = b >>= a
 		success = return True
 		failure = do
-			showProgress
+			showOutput -- avoid clutter around error message
 			showEndFail
 			return False
 
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e80fe9621b..1b12362e9f 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -43,7 +43,7 @@ start s = do
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
 	g <- Annex.gitRepo
-	showNote $ "downloading " ++ url
+	showAction $ "downloading " ++ url ++ " "
 	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index a01e08ab51..41bcd6aa78 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -61,7 +61,7 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 	where
 		dropremote name = do
 			r <- Remote.byName name
-			showNote $ "from " ++ Remote.name r ++ "..."
+			showAction $ "from " ++ Remote.name r
 			next $ Command.Move.fromCleanup r True key
 		droplocal = Command.Drop.perform key (Just 0) -- force drop
 
diff --git a/Command/Get.hs b/Command/Get.hs
index cc780cb6a3..e0436a8680 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -75,7 +75,7 @@ getKeyFile key file = do
 						Left _ -> return False
 				else return True
 		docopy r continue = do
-			showNote $ "from " ++ Remote.name r ++ "..."
+			showAction $ "from " ++ Remote.name r
 			copied <- Remote.retrieveKeyFile r key file
 			if copied
 				then return True
diff --git a/Command/Map.hs b/Command/Map.hs
index 557ae27871..07f127f14e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -44,7 +44,7 @@ start = do
 
 	liftIO $ writeFile file (drawMap rs umap trusted)
 	showLongNote $ "running: dot -Tx11 " ++ file
-	showProgress
+	showOutput
 	r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file]
 	next $ next $ return r
 	where
@@ -176,7 +176,7 @@ scan r = do
 			showEndOk
 			return r'
 		Nothing -> do
-			showProgress
+			showOutput
 			showEndFail
 			return r
 
@@ -224,5 +224,5 @@ tryScan r
 				ok -> return ok
 
 		sshnote = do
-			showNote "sshing..."
-			showProgress
+			showAction "sshing"
+			showOutput
diff --git a/Command/Move.hs b/Command/Move.hs
index a98276e7ec..a081a863f2 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -44,9 +44,9 @@ start move file = do
 			fromStart src move file
 		(_ ,  _) -> error "only one of --from or --to can be specified"
 
-showAction :: Bool -> FilePath -> Annex ()
-showAction True file = showStart "move" file
-showAction False file = showStart "copy" file
+showMoveAction :: Bool -> FilePath -> Annex ()
+showMoveAction True file = showStart "move" file
+showMoveAction False file = showStart "copy" file
 
 {- Used to log a change in a remote's having a key. The change is logged
  - in the local repo, not on the remote. The process of transferring the
@@ -77,7 +77,7 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 	if not ishere || u == Remote.uuid dest
 		then stop -- not here, so nothing to do
 		else do
-			showAction move file
+			showMoveAction move file
 			next $ toPerform dest move key
 toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
@@ -97,7 +97,7 @@ toPerform dest move key = do
 			showNote $ show err
 			stop
 		Right False -> do
-			showNote $ "to " ++ Remote.name dest ++ "..."
+			showAction $ "to " ++ Remote.name dest
 			ok <- Remote.storeKey dest key
 			if ok
 				then next $ toCleanup dest move key
@@ -127,7 +127,7 @@ fromStart src move file = isAnnexed file $ \(key, _) -> do
 	if u == Remote.uuid src || not (any (== src) remotes)
 		then stop
 		else do
-			showAction move file
+			showMoveAction move file
 			next $ fromPerform src move key
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = do
@@ -135,7 +135,7 @@ fromPerform src move key = do
 	if ishere
 		then next $ fromCleanup src move key
 		else do
-			showNote $ "from " ++ Remote.name src ++ "..."
+			showAction $ "from " ++ Remote.name src
 			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
 			if ok
 				then next $ fromCleanup src move key
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index d189545f5d..280eff9de7 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -45,7 +45,7 @@ perform dest key = do
 	let src = gitAnnexLocation g key
 	let tmpdest = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
-	showNote "copying..."
+	showAction "copying"
 	ok <- liftIO $ copyFile src tmpdest
         if ok
                 then do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 870c993f18..e7065b3c36 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -68,7 +68,7 @@ checkRemoteUnused name = do
 
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
-	showNote "checking for unused data..."
+	showAction "checking for unused data"
 	referenced <- getKeysReferenced
 	remotehas <- filterM isthere =<< loggedKeys
 	let remoteunused = remotehas `exclude` referenced
@@ -152,7 +152,7 @@ unusedKeys = do
 			bad <- staleKeys gitAnnexBadDir
 			return ([], bad, tmp)
 		else do
-			showNote "checking for unused data..."
+			showAction "checking for unused data"
 			present <- getKeysPresent
 			referenced <- getKeysReferenced
 			let unused = present `exclude` referenced
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 05748e8d60..314fef7826 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -35,7 +35,7 @@ perform key = do
 		else do
 			pp <- prettyPrintUUIDs uuids
 			showLongNote pp
-			showProgress	
+			showOutput
 			next $ return True
 	where
 		copiesplural 1 = "copy"
diff --git a/Messages.hs b/Messages.hs
index 5f150aafb4..36f0b89c5c 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -20,21 +20,29 @@ verbose a = do
 	q <- Annex.getState Annex.quiet
 	unless q a
 
-showSideAction :: String -> Annex ()
-showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")"
-
 showStart :: String -> String -> Annex ()
-showStart command file = verbose $ do
-	liftIO $ putStr $ command ++ " " ++ file ++ " "
-	liftIO $ hFlush stdout
+showStart command file = verbose $ liftIO $ do
+	putStr $ command ++ " " ++ file ++ " "
+	hFlush stdout
 
 showNote :: String -> Annex ()
-showNote s = verbose $ do
-	liftIO $ putStr $ "(" ++ s ++ ") "
-	liftIO $ hFlush stdout
+showNote s = verbose $ liftIO $ do
+	putStr $ "(" ++ s ++ ") "
+	hFlush stdout
+
+showAction :: String -> Annex ()
+showAction s = showNote $ s ++ "..."
 
 showProgress :: Annex ()
-showProgress = verbose $ liftIO $ putStr "\n"
+showProgress = verbose $ liftIO $ do
+	putStr "."
+	hFlush stdout
+
+showSideAction :: String -> Annex ()
+showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ "...)"
+
+showOutput :: Annex ()
+showOutput = verbose $ liftIO $ putStr "\n"
 
 showLongNote :: String -> Annex ()
 showLongNote s = verbose $ liftIO $ putStr $ '\n' : indent s
@@ -50,15 +58,16 @@ showEndResult True = showEndOk
 showEndResult False = showEndFail
 
 showErr :: (Show a) => a -> Annex ()
-showErr e = do
-	liftIO $ hFlush stdout
-	liftIO $ hPutStrLn stderr $ "git-annex: " ++ show e
+showErr e = liftIO $ do
+	hFlush stdout
+	hPutStrLn stderr $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
 warning w = do
 	verbose $ liftIO $ putStr "\n"
-	liftIO $ hFlush stdout
-	liftIO $ hPutStrLn stderr $ indent w
+	liftIO $ do
+		hFlush stdout
+		hPutStrLn stderr $ indent w
 
 indent :: String -> String
 indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 4ea455226e..1023cda186 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -76,7 +76,7 @@ bupSetup u c = do
 
 	-- bup init will create the repository.
 	-- (If the repository already exists, bup init again appears safe.)
-	showNote "bup init"
+	showAction "bup init"
 	bup "init" buprepo [] >>! error "bup init failed"
 
 	storeBupUUID u buprepo
@@ -93,7 +93,7 @@ bupParams command buprepo params =
 
 bup :: String -> BupRepo -> [CommandParam] -> Annex Bool
 bup command buprepo params = do
-	showProgress -- make way for bup output
+	showOutput -- make way for bup output
 	liftIO $ boolSystem "bup" $ bupParams command buprepo params
 
 pipeBup :: [CommandParam] -> Maybe Handle -> Maybe Handle -> IO Bool
@@ -109,7 +109,7 @@ bupSplitParams :: Git.Repo -> BupRepo -> Key -> CommandParam -> Annex [CommandPa
 bupSplitParams r buprepo k src = do
 	o <- getConfig r "bup-split-options" ""
 	let os = map Param $ words o
-	showProgress -- make way for bup output
+	showOutput -- make way for bup output
 	return $ bupParams "split" buprepo 
 		(os ++ [Param "-n", Param (show k), src])
 
@@ -157,7 +157,7 @@ remove _ = do
 checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either IOException Bool)
 checkPresent r bupr k
 	| Git.repoIsUrl bupr = do
-		showNote ("checking " ++ Git.repoDescribe r ++ "...")
+		showAction $ "checking " ++ Git.repoDescribe r
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
 	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine bupr params
@@ -172,7 +172,7 @@ storeBupUUID u buprepo = do
 	r <- liftIO $ bup2GitRemote buprepo
 	if Git.repoIsUrl r
 		then do
-			showNote "storing uuid"
+			showAction "storing uuid"
 			onBupRemote r boolSystem "git"
 				[Params $ "config annex.uuid " ++ u]
 					>>! error "ssh failed"
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 1f22ad11c6..de51c891e2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -115,7 +115,7 @@ inAnnex r key = if Git.repoIsUrl r
 			a <- Annex.new r
 			Annex.eval a (Content.inAnnex key)
 		checkremote = do
-			showNote ("checking " ++ Git.repoDescribe r ++ "...")
+			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
 				[Param (show key)]
 			return $ Right inannex
@@ -156,7 +156,7 @@ copyToRemote r key
 
 rsyncHelper :: [CommandParam] -> Annex Bool
 rsyncHelper p = do
-	showProgress -- make way for progress bar
+	showOutput -- make way for progress bar
 	res <- liftIO $ rsync p
 	if res
 		then return res
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index f0e4d5bfbc..87f86ffe4f 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -98,7 +98,7 @@ runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool
 runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype hook
 	where
 		run command = do
-			showProgress -- make way for hook output
+			showOutput -- make way for hook output
 			res <- liftIO $ boolSystemEnv
 				"sh" [Param "-c", Param command] $ hookEnv k f
 			if res
@@ -133,7 +133,7 @@ remove h k = runHook h "remove" k Nothing $ return True
 
 checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool)
 checkPresent r h k = do
-	showNote ("checking " ++ Git.repoDescribe r ++ "...")
+	showAction $ "checking " ++ Git.repoDescribe r
 	v <- lookupHook h "checkpresent"
 	liftIO (try (check v) ::IO (Either IOException Bool))
 	where
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index ca4236276f..f073e7bd79 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -141,7 +141,7 @@ remove o k = withRsyncScratchDir $ \tmp -> do
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool)
 checkPresent r o k = do
-	showNote ("checking " ++ Git.repoDescribe r ++ "...")
+	showAction $ "checking " ++ Git.repoDescribe r
 	-- note: Does not currently differnetiate between rsync failing
 	-- to connect, and the file not being present.
 	res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd]
@@ -174,7 +174,7 @@ withRsyncScratchDir a = do
 
 rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool
 rsyncRemote o params = do
-	showProgress -- make way for progress bar
+	showOutput -- make way for progress bar
 	res <- liftIO $ rsync $ rsyncOptions o ++ defaultParams ++ params
 	if res
 		then return res
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index cbd3ef6225..e4dcc2a71d 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -185,7 +185,7 @@ remove r k = s3Action r False $ \(conn, bucket) -> do
 
 checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
 checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
-	showNote ("checking " ++ name r ++ "...")
+	showAction $ "checking " ++ name r
 	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k
 	case res of
 		Right _ -> return $ Right True
@@ -241,13 +241,13 @@ iaMunge = (>>= munge)
 genBucket :: RemoteConfig -> Annex ()
 genBucket c = do
 	conn <- s3ConnectionRequired c
-	showNote "checking bucket"
+	showAction "checking bucket"
 	loc <- liftIO $ getBucketLocation conn bucket 
 	case loc of
 		Right _ -> return ()
 		Left err@(NetworkError _) -> s3Error err
 		Left (AWSError _ _) -> do
-			showNote $ "creating bucket in " ++ datacenter
+			showAction $ "creating bucket in " ++ datacenter
 			res <- liftIO $ createBucketIn conn bucket datacenter
 			case res of
 				Right _ -> return ()
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 60f64cfe02..2f8fac23b5 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -106,7 +106,7 @@ checkKey key = do
 checkKey' :: [URLString] -> Annex Bool
 checkKey' [] = return False
 checkKey' (u:us) = do
-	showNote ("checking " ++ u)
+	showAction $ "checking " ++ u
 	e <- liftIO $ urlexists u
 	if e then return e else checkKey' us
 
@@ -129,6 +129,6 @@ urlexists url =
 download :: [URLString] -> FilePath -> Annex Bool
 download [] _ = return False
 download (url:us) file = do
-	showProgress -- make way for curl progress bar
+	showOutput -- make way for curl progress bar
 	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
 	if ok then return ok else download us file
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index 071fd12ee1..3aabe07700 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -23,7 +23,7 @@ import qualified Upgrade.V1
 
 upgrade :: Annex Bool
 upgrade = do
-	showNote "v0 to v1..."
+	showAction "v0 to v1"
 	g <- Annex.gitRepo
 
 	-- do the reorganisation of the key files
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 8a3d37a642..c41310880f 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -58,7 +58,7 @@ import qualified Upgrade.V2
 
 upgrade :: Annex Bool
 upgrade = do
-	showNote "v1 to v2"
+	showAction "v1 to v2"
 
 	g <- Annex.gitRepo
 	if Git.repoIsLocalBare g
@@ -77,7 +77,7 @@ upgrade = do
 
 moveContent :: Annex ()
 moveContent = do
-	showNote "moving content..."
+	showAction "moving content"
 	files <- getKeyFilesPresent1
 	forM_ files move
 	where
@@ -91,7 +91,7 @@ moveContent = do
 
 updateSymlinks :: Annex ()
 updateSymlinks = do
-	showNote "updating symlinks..."
+	showAction "updating symlinks"
 	g <- Annex.gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	forM_ files fixlink
@@ -108,7 +108,7 @@ updateSymlinks = do
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
-	showNote "moving location logs..."
+	showAction "moving location logs"
 	logkeys <- oldlocationlogs
 	forM_ logkeys move
 		where
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 99c7806d27..0b1d69f8e1 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -45,21 +45,25 @@ olddir g
  -}
 upgrade :: Annex Bool
 upgrade = do
-	showNote "v2 to v3"
+	showAction "v2 to v3"
 	g <- Annex.gitRepo
 	let bare = Git.repoIsLocalBare g
 
 	Branch.create
+	showProgress
+
 	e <- liftIO $ doesDirectoryExist (olddir g)
 	when e $ do
 		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
 		mapM_ (\f -> inject f f) =<< logFiles (olddir g)
 
 	saveState
+	showProgress
 
 	when e $ liftIO $ do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
 		unless bare $ gitAttributesUnWrite g
+	showProgress
 
 	unless bare push
 
@@ -83,6 +87,7 @@ inject source dest = do
 	new <- liftIO (readFile $ olddir g  source)
 	prev <- Branch.get dest
 	Branch.change dest $ unlines $ nub $ lines prev ++ lines new
+	showProgress
 
 logFiles :: FilePath -> Annex [FilePath]
 logFiles dir = return . filter (".log" `isSuffixOf`)
@@ -105,8 +110,8 @@ push = do
 			-- "git push" will from then on
 			-- automatically push it
 			Branch.update -- just in case
-			showNote "pushing new git-annex branch to origin"
-			showProgress
+			showAction "pushing new git-annex branch to origin"
+			showOutput
 			g <- Annex.gitRepo
 			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]
 		_ -> do
@@ -116,7 +121,7 @@ push = do
 			showLongNote $
 				"git-annex branch created\n" ++
 				"Be sure to push this branch when pushing to remotes.\n"
-			showProgress
+			showOutput
 
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]

From a8a71b9d915f6c274fb07636e0840bcc26ab6731 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jul 2011 23:52:09 -0400
Subject: [PATCH 2074/2835] releasing version 3.20110719

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 3a6c731fe4..fab4927cdd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110708) UNRELEASED; urgency=low
+git-annex (3.20110719) unstable; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
     Closes: #634233
@@ -6,7 +6,7 @@ git-annex (3.20110708) UNRELEASED; urgency=low
   * Support the standard git -c name=value
   * unannex: Clean up use of git commit -a.
 
- -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
+ -- Joey Hess   Tue, 19 Jul 2011 23:39:53 -0400
 
 git-annex (3.20110707) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index bbbcbf9fb4..807158f3eb 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110708
+Version: 3.20110719
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From d8f21b7955c48f121924d187b1be07f53edbff93 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jul 2011 23:54:36 -0400
Subject: [PATCH 2075/2835] add news item for git-annex 3.20110719

---
 doc/news/version_0.20110610.mdwn | 6 ------
 doc/news/version_3.20110719.mdwn | 7 +++++++
 2 files changed, 7 insertions(+), 6 deletions(-)
 delete mode 100644 doc/news/version_0.20110610.mdwn
 create mode 100644 doc/news/version_3.20110719.mdwn

diff --git a/doc/news/version_0.20110610.mdwn b/doc/news/version_0.20110610.mdwn
deleted file mode 100644
index 9ab9e09076..0000000000
--- a/doc/news/version_0.20110610.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex 0.20110610 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Add --numcopies option.
-   * Add --trust, --untrust, and --semitrust options.
-   * get --from is the same as copy --from
-   * Bugfix: Fix fsck to not think all SHAnE keys are bad."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110719.mdwn b/doc/news/version_3.20110719.mdwn
new file mode 100644
index 0000000000..5beae32270
--- /dev/null
+++ b/doc/news/version_3.20110719.mdwn
@@ -0,0 +1,7 @@
+git-annex 3.20110719 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+     Closes: #[634233](http://bugs.debian.org/634233)
+   * Bugfix: Make add ../ work.
+   * Support the standard git -c name=value
+   * unannex: Clean up use of git commit -a."""]]
\ No newline at end of file

From 1d88b966e29e0b064a142a712d6a350b3f19e2d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 24 Jul 2011 15:15:43 +0200
Subject: [PATCH 2076/2835] fix broken img

---
 doc/index.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 8975c82de7..4b7159cd53 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -12,7 +12,7 @@ To get a feel for it, see the [[walkthrough]].
 * [[forum]]
 * [[comments]]
 * [[contact]]
-* Flattr this
+* Flattr this
 
 [[News]]:
 

From 50edbb03eb0582dd568a6fb83fcdb410828c1ea1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkvSZ1AFJdY_1FeutZr_KWeqtzjZta1PNE"
 
Date: Thu, 28 Jul 2011 17:40:53 +0000
Subject: [PATCH 2077/2835]

---
 ...es_which_have_names_containing_spaces.mdwn | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn

diff --git a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
new file mode 100644
index 0000000000..464c06b442
--- /dev/null
+++ b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
@@ -0,0 +1,48 @@
+    ~$ mkdir test annex
+    ~$ cd test
+    ~$ git init
+    Initialized empty Git repository in /home/user/test/.git/
+    ~$ git annex init test
+    init test ok
+    ~$ git annex initremote localrsync encryption=none type=rsync rsyncurl=localhost:annex/
+    initremote localrsync ok
+    ~$ cp /home/user/Music/Charming\ Hostess/Eat/03\ Mi\ Nuera.ogg ./
+    ~$ git annex add 03\ Mi\ Nuera.ogg
+    add 03 Mi Nuera.ogg ok
+    (Recording state in git...)
+    ~$ git commit -m "add ogg"
+    fatal: No HEAD commit to compare with (yet)
+    fatal: No HEAD commit to compare with (yet)
+    [master (root-commit) 12608af] add ogg
+     1 files changed, 1 insertions(+), 0 deletions(-)
+     create mode 120000 03 Mi Nuera.ogg
+    ~$ git annex move 03\ Mi\ Nuera.ogg --to localrsync
+    move 03 Mi Nuera.ogg (checking localrsync...) (to localrsync...) 
+    sending incremental file list
+    1X/
+    1X/39/
+    1X/39/WORM-s6296772-m1311874383--03 Mi Nuera.ogg/
+    1X/39/WORM-s6296772-m1311874383--03 Mi Nuera.ogg/WORM-s6296772-m1311874383--03 Mi Nuera.ogg
+         6296772 100%   42.98MB/s    0:00:00 (xfer#1, to-check=0/5)
+    
+    sent 6297754 bytes  received 43 bytes  4198531.33 bytes/sec
+    total size is 6296772  speedup is 1.00
+    ok
+    ~$ git annex get 03\ Mi\ Nuera.ogg
+    get 03 Mi Nuera.ogg (from localrsync...) 
+    rsync: link_stat "/home/user/annex/1X/39/WORM-s6296772-m1311874383--03" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Mi" failed: No such file or directory (2)
+    rsync: change_dir "/home/user/Nuera.ogg" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Mi" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Nuera.ogg" failed: No such file or directory (2)
+    
+    sent 8 bytes  received 12 bytes  13.33 bytes/sec
+    total size is 0  speedup is 0.00
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1526) [Receiver=3.0.7]
+    
+      rsync failed -- run git annex again to resume file transfer
+      Unable to access these remotes: localrsync
+      Try making some of these repositories available:
+            b8b1ea7a-b93f-11e0-b712-d7bffb6e61e6  -- localrsync
+    failed
+    git-annex: 1 failed

From 45bbf210a1210172c7c7b87879ed74f7c8ccbdba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 29 Jul 2011 15:28:21 +0200
Subject: [PATCH 2078/2835] Fix shell escaping in rsync special remote.

---
 Remote/Rsync.hs                                        | 10 ++++++----
 debian/changelog                                       |  6 ++++++
 ..._96___files_which_have_names_containing_spaces.mdwn |  2 ++
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index f073e7bd79..9535376097 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -93,10 +93,13 @@ rsyncSetup u c = do
 	return c'
 
 rsyncKey :: RsyncOpts -> Key -> String
-rsyncKey o k = rsyncUrl o  hashDirMixed k  f  f
+rsyncKey o k = rsyncUrl o  hashDirMixed k  shellEscape (f  f)
         where
                 f = keyFile k
 
+rsyncKeyDir :: RsyncOpts -> Key -> String
+rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  shellEscape (keyFile k)
+
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = do
 	g <- Annex.gitRepo
@@ -136,7 +139,7 @@ remove o k = withRsyncScratchDir $ \tmp -> do
 		[ Params "--delete --recursive"
 		, partialParams
 		, Param $ addTrailingPathSeparator dummy
-		, Param $ parentDir $ rsyncKey o k
+		, Param $ rsyncKeyDir o k
 		]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool)
@@ -147,8 +150,7 @@ checkPresent r o k = do
 	res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd]
 	return $ Right res
 	where
-		cmd = "rsync --quiet " ++ testfile ++ " 2>/dev/null"
-		testfile = shellEscape $ rsyncKey o k
+		cmd = "rsync --quiet " ++ shellEscape (rsyncKey o k) ++ " 2>/dev/null"
 
 {- Rsync params to enable resumes of sending files safely,
  - ensure that files are only moved into place once complete
diff --git a/debian/changelog b/debian/changelog
index fab4927cdd..748a945d7b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110720) UNRELEASED; urgency=low
+
+  * Fix shell escaping in rsync special remote.
+
+ -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
+
 git-annex (3.20110719) unstable; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
diff --git a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
index 464c06b442..040d86bb87 100644
--- a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
+++ b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
@@ -46,3 +46,5 @@
             b8b1ea7a-b93f-11e0-b712-d7bffb6e61e6  -- localrsync
     failed
     git-annex: 1 failed
+
+> [[fixed|done]] --[[Joey]] 

From 16c55f61f2d4f060f20094335fdfcac827f7d508 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Sat, 30 Jul 2011 17:28:42 +0000
Subject: [PATCH 2079/2835]

---
 doc/install/OSX.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 23cb1b62e2..2d376d053f 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -1,5 +1,7 @@
+Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. The version provided by Macports is too old to work with current versions of git-annex. Then execute
+
 
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils pcre
+sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string
@@ -21,7 +23,7 @@ make
 sudo make install
 

 
-Originally posted by Jon at  --[[Joey]]
+Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]
 
 See also:
 

From d9cdb5eb457c5069d20c0738158de2423770b3e4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Sat, 30 Jul 2011 18:04:25 +0000
Subject: [PATCH 2080/2835]

---
 doc/install/OSX.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 2d376d053f..983ac3a2fd 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -2,6 +2,8 @@ Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. T
 
 
 sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
+
+sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string

From ad4528cce83709eb075158ef15fb1fd6faeb1171 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 31 Jul 2011 01:20:35 +0000
Subject: [PATCH 2081/2835]

---
 doc/bugs/Prevent_accidental_merges.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/Prevent_accidental_merges.mdwn

diff --git a/doc/bugs/Prevent_accidental_merges.mdwn b/doc/bugs/Prevent_accidental_merges.mdwn
new file mode 100644
index 0000000000..9b0c8a3de6
--- /dev/null
+++ b/doc/bugs/Prevent_accidental_merges.mdwn
@@ -0,0 +1,9 @@
+With the storage layout v3, pulling the git-annex branch into the master branch is... less than ideal.
+
+The fact that the two branches contain totally different data make an accidental merge worse, arguably.
+
+Adding a tiny binary file called .gitnomerge to both branches would solve that without any noticeable overhead.
+
+Yes, there is an argument to be made that this is too much hand-holding, but I still think it's worth it.
+
+-- Richard

From 344f2c3ec18baaefe6de14a32fc1209459470bf5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Sun, 31 Jul 2011 15:24:29 +0000
Subject: [PATCH 2082/2835] Added a comment: Solution

---
 ...nt_2_f5ebb7f43dcef861ecc13373fb1e263f._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
new file mode 100644
index 0000000000..9601003798
--- /dev/null
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
+ nickname="Kristian"
+ subject="Solution"
+ date="2011-07-31T15:24:25Z"
+ content="""
+Yes, it can read id3-tags and guess titles from movie filenames but it sometimes gets confused by the filename metadata provided by the WORM-backend.
+
+I think I have a good enough solution to this problem. It's not efficient when it comes to renames but handles adding and deletion just fine
+
+    rsync -vaL --delete source dest
+
+The -L flag looks at symbolic links and copies the actual data they are pointing to. Of course \"source\" must have all data locally for this to work.
+
+"""]]

From b48fec3846ac3e32b473499c007c98865236848c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 1 Aug 2011 09:52:22 +0000
Subject: [PATCH 2083/2835]

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing.mdwn

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
new file mode 100644
index 0000000000..03d31ea34f
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -0,0 +1,8 @@
+Just issuing the command `cabal install` results in the following error message.
+
+    Command/Add.hs:54:3:
+        No instance for (Control.Monad.IO.Control.MonadControlIO
+                           (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+          arising from a use of `handle' at Command/Add.hs:54:3-24
+
+Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?

From a746623a334fa1673179621fe5c9c9d6d407851f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 1 Aug 2011 09:54:51 +0000
Subject: [PATCH 2084/2835]

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
index 03d31ea34f..cf4b138128 100644
--- a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -5,4 +5,5 @@ Just issuing the command `cabal install` results in the following error message.
                            (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
           arising from a use of `handle' at Command/Add.hs:54:3-24
 
-Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?
+Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?  
+-- Thomas

From d3f6f4fe34e812fafbc487c969e6aaa0d49de33f Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Thu, 4 Aug 2011 15:25:21 +0000
Subject: [PATCH 2085/2835] fixed internal link

---
 doc/special_remotes/web.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index 68df31ef4d..6991ce4e44 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -1,5 +1,5 @@
 git-annex can use the WWW as a special remote, downloading urls to files.
-See [[walkthrough/using_web_web]] for usage examples.
+See [[walkthrough/using_the_web]] for usage examples.
 
 ## notes
 

From 578e66996446f3c29f431fc6da0ce9a525c15135 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 5 Aug 2011 10:27:22 -0400
Subject: [PATCH 2086/2835] response

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
index cf4b138128..b5213b8aa5 100644
--- a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -7,3 +7,8 @@ Just issuing the command `cabal install` results in the following error message.
 
 Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?  
 -- Thomas
+
+> No, it's already satisfied by `monad-control` being listed as a
+> dependency in the cabal file. Your system might be old/new/or broken,
+> perhaps it's time to provide some details about the version of haskell
+> and of `monad-control` you have installed? --[[Joey]] 

From 46ddb4e66aa1ba2a569eb8734c8ca0f40f3930d4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 5 Aug 2011 10:29:21 -0400
Subject: [PATCH 2087/2835] response

---
 doc/bugs/Prevent_accidental_merges.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Prevent_accidental_merges.mdwn b/doc/bugs/Prevent_accidental_merges.mdwn
index 9b0c8a3de6..3e30e02235 100644
--- a/doc/bugs/Prevent_accidental_merges.mdwn
+++ b/doc/bugs/Prevent_accidental_merges.mdwn
@@ -7,3 +7,8 @@ Adding a tiny binary file called .gitnomerge to both branches would solve that w
 Yes, there is an argument to be made that this is too much hand-holding, but I still think it's worth it.
 
 -- Richard
+
+> It should be as easy to undo such an accidential merge
+> as it is to undo any other git commit, right? I quite like that git-annex 
+> no longer adds any clutter to the master branch, and would be reluctant
+> to change that. --[[Joey]]

From d2492f990fdfa13a04429edaa5373a39e75cc9f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawliqfHEW134uawIUPwyKiyOdoF-oI5TxnQ"
 
Date: Fri, 5 Aug 2011 22:35:48 +0000
Subject: [PATCH 2088/2835]

---
 doc/cheatsheet.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 9f7c146c82..9ccb22e3e2 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -1,4 +1,4 @@
-A suppliment to the [[walkthrough]].
+A supplement to the [[walkthrough]].
 
 [[!toc]]
 

From 3ffc0bb4f57e20350f59c0e331656e54877916aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 12:50:20 -0400
Subject: [PATCH 2089/2835] foo

---
 Backend.hs     | 2 +-
 Backend/SHA.hs | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 3429e8f42c..6942692e83 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -65,7 +65,7 @@ orderedList = do
 genKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
 genKey file trybackend = do
 	bs <- orderedList
-	let bs' = maybe bs (:bs) trybackend
+	let bs' = maybe bs (: bs) trybackend
 	genKey' bs' file
 genKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
 genKey' [] _ = return Nothing
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index c1d7136485..bae19be003 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -32,7 +32,7 @@ sizes :: [Int]
 sizes = [1, 256, 512, 224, 384]
 
 backends :: [Backend Annex]
--- order is slightly significant; want sha1 first ,and more general
+-- order is slightly significant; want sha1 first, and more general
 -- sizes earlier
 backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
@@ -107,7 +107,7 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 				then "" -- probably not really an extension
 				else naiveextension
 
--- A key's checksum is checked during fsck.
+{- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
 	g <- Annex.gitRepo

From dd8e649f49212c46df23f329299c64e13a3c90e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 14:45:58 -0400
Subject: [PATCH 2090/2835] fix file name for web remote log files

The key name was not being sufficiently escaped, although it didn't break
anything due to luck. Switch to properly escaped key names for the log
filename, with a fallback to the buggy old name.
---
 Remote/Web.hs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 2f8fac23b5..cd028a06d9 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -67,10 +67,17 @@ gen r _ _ =
 {- The urls for a key are stored in remote/web/hash/key.log 
  - in the git-annex branch. -}
 urlLog :: Key -> FilePath
-urlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
+urlLog key = "remote/web"  hashDirLower key  keyFile key ++ ".log"
+oldurlLog :: Key -> FilePath
+{- A bug used to store the urls elsewhere. -}
+oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
 
 getUrls :: Key -> Annex [URLString]
-getUrls key = currentLog (urlLog key)
+getUrls key = do
+	us <- currentLog (urlLog key)
+	if null us
+		then currentLog (oldurlLog key)
+		else return us
 
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()

From dede05171bc9431778da72e5e1235c69db9fa38e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 14:57:22 -0400
Subject: [PATCH 2091/2835] addurl: --fast can be used to avoid immediately
 downloading the url.

The tricky part about this is that to generate a key, the file must be
present already. Worked around by adding (back) an URL key type, which
is used for addurl --fast.
---
 Backend.hs         |  3 ++-
 Backend/URL.hs     | 28 ++++++++++++++++++++++++++++
 Command/Add.hs     | 21 ++++++++++++---------
 Command/AddUrl.hs  | 27 +++++++++++++++++++--------
 Command/Migrate.hs |  2 +-
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  2 ++
 7 files changed, 65 insertions(+), 19 deletions(-)
 create mode 100644 Backend/URL.hs

diff --git a/Backend.hs b/Backend.hs
index 6942692e83..0bb9f4b575 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -32,9 +32,10 @@ import Messages
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
 import qualified Backend.SHA
+import qualified Backend.URL
 
 list :: [Backend Annex]
-list = Backend.WORM.backends ++ Backend.SHA.backends
+list = Backend.WORM.backends ++ Backend.SHA.backends ++ Backend.URL.backends
 
 {- List of backends in the order to try them when storing a new key. -}
 orderedList :: Annex [Backend Annex]
diff --git a/Backend/URL.hs b/Backend/URL.hs
new file mode 100644
index 0000000000..f20aa1f95e
--- /dev/null
+++ b/Backend/URL.hs
@@ -0,0 +1,28 @@
+{- git-annex "URL" backend -- keys whose content is available from urls.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Backend.URL (
+	backends,
+	fromUrl
+) where
+
+import Types.Backend
+import Types.Key
+import Types
+
+backends :: [Backend Annex]
+backends = [backend]
+
+backend :: Backend Annex
+backend = Types.Backend.Backend {
+	name = "URL",
+	getKey = const (return Nothing),
+	fsckKey = const (return True)
+}
+
+fromUrl :: String -> Key
+fromUrl url = stubKey { keyName = url, keyBackendName = "URL" }
diff --git a/Command/Add.hs b/Command/Add.hs
index 58c0143dd0..d8947fb07c 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -8,6 +8,7 @@
 module Command.Add where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (when)
 import System.Posix.Files
 import System.Directory
 import Control.Exception.Control (handle)
@@ -52,7 +53,7 @@ perform (file, backend) = do
 		Nothing -> stop
 		Just (key, _) -> do
 			handle (undo file key) $ moveAnnex key file
-			next $ cleanup file key
+			next $ cleanup file key True
 
 {- On error, put the file back so it doesn't seem to have vanished.
  - This can be called before or after the symlink is in place. -}
@@ -72,18 +73,20 @@ undo file key e = do
 			g <- Annex.gitRepo
 			liftIO $ renameFile (gitAnnexLocation g key) file
 
-cleanup :: FilePath -> Key -> CommandCleanup
-cleanup file key = do
+cleanup :: FilePath -> Key -> Bool -> CommandCleanup
+cleanup file key hascontent = do
 	handle (undo file key) $ do
 		link <- calcGitLink file key
 		liftIO $ createSymbolicLink link file
-		logStatus key InfoPresent
+
+		when hascontent $ do
+			logStatus key InfoPresent
 	
-		-- touch the symlink to have the same mtime as the
-		-- file it points to
-		s <- liftIO $ getFileStatus file
-		let mtime = modificationTime s
-		liftIO $ touch file (TimeSpec mtime) False
+			-- touch the symlink to have the same mtime as the
+			-- file it points to
+			s <- liftIO $ getFileStatus file
+			let mtime = modificationTime s
+			liftIO $ touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 1b12362e9f..e87de384b3 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -17,10 +17,10 @@ import qualified Backend
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
+import qualified Backend.URL
 import Messages
 import Content
 import PresenceLog
-import Types.Key
 import Locations
 import Utility
 
@@ -42,9 +42,14 @@ start s = do
 			
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
+	fast <- Annex.getState Annex.fast
+	if fast then nodownload url file else download url file
+
+download :: String -> FilePath -> CommandPerform
+download url file = do
 	g <- Annex.gitRepo
 	showAction $ "downloading " ++ url ++ " "
-	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }
+	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	ok <- Remote.Web.download [url] tmp
@@ -57,9 +62,16 @@ perform url file = do
 				Just (key, _) -> do
 					moveAnnex key tmp
 					Remote.Web.setUrl key url InfoPresent
-					next $ Command.Add.cleanup file key
+					next $ Command.Add.cleanup file key True
 		else stop
 
+nodownload :: String -> FilePath -> CommandPerform
+nodownload url file = do
+	let key = Backend.URL.fromUrl url
+	Remote.Web.setUrl key url InfoPresent
+	
+	next $ Command.Add.cleanup file key False
+
 url2file :: URI -> IO FilePath
 url2file url = do
 	let parts = filter safe $ split "/" $ uriPath url
@@ -75,8 +87,7 @@ url2file url = do
 			e <- doesFileExist file
 			when e $ error "already have this url"
 			return file
-		safe s
-			| null s = False
-			| s == "." = False
-			| s == ".." = False
-			| otherwise = True
+		safe "" = False
+		safe "." = False
+		safe ".." = False
+		safe _ = True
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 495bf9fb63..5ae8354406 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -72,7 +72,7 @@ perform file oldkey newbackend = do
 				then do
 					-- Update symlink to use the new key.
 					liftIO $ removeFile file
-					next $ Command.Add.cleanup file newkey
+					next $ Command.Add.cleanup file newkey True
 				else stop
 	where
 		cleantmp t = whenM (doesFileExist t) $ removeFile t
diff --git a/debian/changelog b/debian/changelog
index 748a945d7b..fcc0b58b89 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110720) UNRELEASED; urgency=low
 
   * Fix shell escaping in rsync special remote.
+  * addurl: --fast can be used to avoid immediately downloading the url.
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 11f617f1ba..2865c8af5d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -282,6 +282,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Downloads each url to a file, which is added to the annex.
 
+  To avoid immediately downloading the url, specify --fast
+
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file

From 614d8f98566ba8acc2bfeb661410b5d5eaec624c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 8 Aug 2011 09:04:21 +0000
Subject: [PATCH 2092/2835] Added a comment

---
 ..._14be660aa57fadec0d81b32a8b52c66f._comment | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment
new file mode 100644
index 0000000000..8e38205f00
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment
@@ -0,0 +1,75 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
+ nickname="Thomas"
+ subject="comment 1"
+ date="2011-08-08T09:04:20Z"
+ content="""
+I use Debian Squeeze, I have the Debian package cabal-install 0.8.0-1 installed.
+
+    $ git clone git://git-annex.branchable.com/
+    $ cd git-annex.branchable.com
+    $ cabal update
+    $ cabal install cabal-install
+
+This installed: Cabal-1.10.2.0, zlib-0.5.3.1, cabal-install 0.10.2.
+No version of monad-control or monadIO installed.
+
+    $ ~/.cabal/bin/cabal install
+    Registering QuickCheck-2.4.1.1...
+    Registering Crypto-4.2.3...
+    Registering base-unicode-symbols-0.2.2.1...
+    Registering deepseq-1.1.0.2...
+    Registering hxt-charproperties-9.1.0...
+    Registering hxt-regex-xmlschema-9.0.0...
+    Registering hxt-unicode-9.0.1...
+    Registering hxt-9.1.2...
+    Registering stm-2.2.0.1...
+    Registering hS3-0.5.6...
+    Registering transformers-0.2.2.0...
+    Registering monad-control-0.2.0.1...
+    [1 of 1] Compiling Main             ( Setup.hs, dist/setup/Main.o )
+    Linking ./dist/setup/setup ...
+    ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make configure
+    [1 of 2] Compiling TestConfig       ( TestConfig.hs, TestConfig.o )
+    [2 of 2] Compiling Main             ( configure.hs, configure.o )
+    Linking configure ...
+    ./configure
+      checking version... 3.20110720
+      checking cp -a... yes
+      checking cp -p... yes
+      checking cp --reflink=auto... yes
+      checking uuid generator... uuid
+      checking xargs -0... yes
+      checking rsync... yes
+      checking curl... yes
+      checking bup... yes
+      checking gpg... yes
+      checking sha1... sha1sum
+      checking sha256... sha256sum
+      checking sha512... sha512sum
+      checking sha224... sha224sum
+      checking sha384... sha384sum
+
+    ...
+
+    Command/Add.hs:54:3:
+        No instance for (Control.Monad.IO.Control.MonadControlIO
+                           (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+          arising from a use of `handle' at Command/Add.hs:54:3-24
+        Possible fix:
+          add an instance declaration for
+          (Control.Monad.IO.Control.MonadControlIO
+             (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+        In the first argument of `($)', namely `handle (undo file key)'
+        In a stmt of a 'do' expression:
+              handle (undo file key) $ moveAnnex key file
+        In the expression:
+            do { handle (undo file key) $ moveAnnex key file;
+                 next $ cleanup file key }
+    cabal: Error: some packages failed to install:
+    git-annex-3.20110719 failed during the building phase. The exception was:
+    ExitFailure 1
+
+After I added a depencency for monadIO to the git-annex.cabal file, it installed correctly.  
+-- Thomas
+"""]]

From 065e1a507b74e8e41017a1a5368fc2b8aaac7f19 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:00:21 +0000
Subject: [PATCH 2093/2835]

---
 .../--git-dir_and_--work-tree_options.mdwn    | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/bugs/--git-dir_and_--work-tree_options.mdwn

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
new file mode 100644
index 0000000000..2e5d1cf0c8
--- /dev/null
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -0,0 +1,28 @@
+git-annex does not take into account the --git-dir and --work-tree command line options (while they can be useful when scripting).
+
+> mkdir /tmp/test
+> cd /tmp/test
+> git init
+Initialized empty Git repository in /tmp/test/.git/
+> git annex init test
+init test ok
+> touch foo
+> cd
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
+git-annex: Not in a git repository.
+
+regular git add works:
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
+# On branch master
+#
+# Initial commit
+#
+# Changes to be committed:
+#   (use "git rm --cached ..." to unstage)
+#
+#       new file:   foo
+#
+
+git-annex version: 3.20110702
+

From 1e934c29d66f44ff074a9d02004e2bd6abb90462 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:02:32 +0000
Subject: [PATCH 2094/2835]

---
 .../--git-dir_and_--work-tree_options.mdwn    | 43 ++++++++++---------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
index 2e5d1cf0c8..ebae56b049 100644
--- a/doc/bugs/--git-dir_and_--work-tree_options.mdwn
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -1,28 +1,29 @@
 git-annex does not take into account the --git-dir and --work-tree command line options (while they can be useful when scripting).
 
-> mkdir /tmp/test
-> cd /tmp/test
-> git init
-Initialized empty Git repository in /tmp/test/.git/
-> git annex init test
-init test ok
-> touch foo
-> cd
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
-git-annex: Not in a git repository.
+    > mkdir /tmp/test
+    > cd /tmp/test
+    > git init
+    Initialized empty Git repository in /tmp/test/.git/
+    > git annex init test
+    init test ok
+    > touch foo
+    > cd
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
+    git-annex: Not in a git repository.
 
 regular git add works:
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
-# On branch master
-#
-# Initial commit
-#
-# Changes to be committed:
-#   (use "git rm --cached ..." to unstage)
-#
-#       new file:   foo
-#
+
+    > git --git-dir =/tmp/test/.git --work-tree=/tmp/test add foo
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
+    # On branch master
+    #
+    # Initial commit
+    #
+    # Changes to be committed:
+    #   (use "git rm --cached ..." to unstage)
+    #
+    #       new file:   foo
+    #
 
 git-annex version: 3.20110702
 

From 20a8f8c85bbda5d8af90782a596abbb1b0ac1e86 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:04:34 +0000
Subject: [PATCH 2095/2835]

---
 doc/bugs/--git-dir_and_--work-tree_options.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
index ebae56b049..d76a42bfff 100644
--- a/doc/bugs/--git-dir_and_--work-tree_options.mdwn
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -13,7 +13,7 @@ git-annex does not take into account the --git-dir and --work-tree command line
 
 regular git add works:
 
-    > git --git-dir =/tmp/test/.git --work-tree=/tmp/test add foo
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
     > git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
     # On branch master
     #

From f8282b55cece7561cd04788637627e79d2f29e17 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Wed, 10 Aug 2011 16:56:51 +0000
Subject: [PATCH 2096/2835] problems with version 3 upgrade instructions

---
 doc/forum/version_3_upgrade.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/version_3_upgrade.mdwn

diff --git a/doc/forum/version_3_upgrade.mdwn b/doc/forum/version_3_upgrade.mdwn
new file mode 100644
index 0000000000..476b8c235f
--- /dev/null
+++ b/doc/forum/version_3_upgrade.mdwn
@@ -0,0 +1,9 @@
+after upgrading to git-annex 3, i'm stuck with diverging git-annex branches -- i didn't manage to follow this line in the directions:
+
+> After this upgrade, you should make sure you include the git-annex branch when git pushing and pulling.
+
+could you explain how to do that in a littel more detail? git pull seems to only merge master, although i have these ``.git/config`` settings:
+
+    [branch "git-annex"]
+    	remote = poseidon
+    	merge = git-annex

From 8b847517a810d384a79178124b9766141b89bc17 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Wed, 10 Aug 2011 17:00:09 +0000
Subject: [PATCH 2097/2835] (i don't have a remote origin, but it's more common
 and doesn't change the example)

---
 doc/forum/version_3_upgrade.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/version_3_upgrade.mdwn b/doc/forum/version_3_upgrade.mdwn
index 476b8c235f..7fdbcbc805 100644
--- a/doc/forum/version_3_upgrade.mdwn
+++ b/doc/forum/version_3_upgrade.mdwn
@@ -5,5 +5,5 @@ after upgrading to git-annex 3, i'm stuck with diverging git-annex branches -- i
 could you explain how to do that in a littel more detail? git pull seems to only merge master, although i have these ``.git/config`` settings:
 
     [branch "git-annex"]
-    	remote = poseidon
+    	remote = origin
     	merge = git-annex

From 18012c7ec8e8bcbcdc8712be29046c12adeaf6d2 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Fri, 12 Aug 2011 13:09:11 +0000
Subject: [PATCH 2098/2835] enhancement suggestion

---
 ...e_commit_messages_in_git-annex_branch.mdwn | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn

diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
new file mode 100644
index 0000000000..68b24c2fba
--- /dev/null
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -0,0 +1,26 @@
+as of git-annex version 3.20110719, all git-annex commits only contain the word "update" as a commit message. given that the contents of the commit are pretty non-descriptive (SHA1 hashes for file names, uuids for repository names), i suggest to have more descriptive commit messages, as shown here:
+
+    /mnt/usb_disk/photos/2011$ git annex get
+    /mnt/usb_disk/photos/2011$ git show git-annex
+    [...]
+    usb-disk-photos: get 2011
+    
+    * 10 files retrieved from 2 sources (9 from local-harddisk, 1 from my-server)
+    * 120 files were already present
+    * 2 files could not be retrieved
+    /mnt/usb_disk/photos/2011$ cd ~/photos/2011/07
+    ~/photos/2011/07$ git copy --to my-server
+    ~/photos/2011/07$ git show git-annex
+    [...]
+    local-harddisk: copy 2011/07 to my-server
+    
+    * 20 files pushed
+    ~/photos/2011/07$
+
+in my opinion, the messages should at least contain
+
+* what command was used
+* in which repository they were executed
+* which files or directories they affected (not necessarily all files, but what was given on command line or implicitly from the working directory)
+
+--[[chrysn]]

From 829ea31b72b1a9c42d770815363ef350ec2bda31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 15 Aug 2011 13:30:59 -0400
Subject: [PATCH 2099/2835] typo

---
 doc/internals.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index a4ec5c417b..e80ecbac0d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -22,7 +22,7 @@ deleting or changing the file contents.
 This branch is managed by git-annex, with the contents listed below.
 
 The file `.git/annex/index` is a separate git index file it uses
-to accumlate changes for the git-annex. Also, `.git/annex/journal/` is used
+to accumulate changes for the git-annex. Also, `.git/annex/journal/` is used
 to record changes before they are added to git.
 
 Note that for speed reasons, git-annex assumes only it will modify this

From 9f719e5674280f49b71c4d58dcc0b0002845f852 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkHscTHMCNvjJ6nLI1VpsBrJFI5FTwhUT4"
 
Date: Mon, 15 Aug 2011 18:18:22 +0000
Subject: [PATCH 2100/2835]

---
 ...ith_the_annex_directory_exposed_to_http.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn

diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
new file mode 100644
index 0000000000..c1b790be1e
--- /dev/null
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -0,0 +1,17 @@
+Let's say that http://people.collabora.com/~alsuren/git/fate-suite.git/ is a bare git repo. It has been 'git update-server-info'd so that it can be served on a dumb http server.
+
+The repo is also a git annex remote, created using the following commands:
+
+* git remote add alsuren git+ssh://people.collabora.co.uk/user/alsuren/public_html/fate-suite.git
+* git push alsuren --all
+* git annex copy --to=alsuren
+
+so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though
+
+I would like to be able to use the following commands to get a clone of the repo:
+
+* git clone http://people.collabora.com/~alsuren/git/fate-suite.git/
+* cd fate-suite
+* git annex get
+
+This would allow contributors to quickly get a copy of our upstream repo and start contributing with minimal bandwidth/effort.

From c3f74c6959cf67ad511dc9b6bed93112c754c247 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkHscTHMCNvjJ6nLI1VpsBrJFI5FTwhUT4"
 
Date: Mon, 15 Aug 2011 18:19:47 +0000
Subject: [PATCH 2101/2835]

---
 ...git_repo__44___with_the_annex_directory_exposed_to_http.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
index c1b790be1e..a2e344c115 100644
--- a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -6,7 +6,7 @@ The repo is also a git annex remote, created using the following commands:
 * git push alsuren --all
 * git annex copy --to=alsuren
 
-so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though
+so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though listing dirs is forbidden, so you need to know the filenames ahead of time).
 
 I would like to be able to use the following commands to get a clone of the repo:
 

From c04f1f4c30d5334b06b606c89fcb8e693bd88701 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 16:11:55 -0400
Subject: [PATCH 2102/2835] response

---
 ...escriptive_commit_messages_in_git-annex_branch.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
index 68b24c2fba..aad119ffd3 100644
--- a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -24,3 +24,13 @@ in my opinion, the messages should at least contain
 * which files or directories they affected (not necessarily all files, but what was given on command line or implicitly from the working directory)
 
 --[[chrysn]]
+
+> The implementation of the git-annex branch precludes more descriptive
+> commit messages, since a single commit can include changes that were
+> previously staged to the branch's index file, or spooled to its journal
+> by other git-annex commands (either concurrently running or
+> interrupted commands, or even changes needed to automatically merge
+> other git-annex branches).
+> 
+> It would be possible to make it *less* verbose, with an empty commit
+> message. :) --[[Joey]] 

From cfcd7805b441c48e404826903480113a82cff9cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 19:23:56 -0400
Subject: [PATCH 2103/2835] add repoIsHttp

---
 Git.hs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Git.hs b/Git.hs
index 9b7ac7ea91..b226bce0f2 100644
--- a/Git.hs
+++ b/Git.hs
@@ -17,6 +17,7 @@ module Git (
 	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
+	repoIsHttp,
 	repoIsLocalBare,
 	repoDescribe,
 	repoLocation,
@@ -206,6 +207,13 @@ repoIsSsh Repo { location = Url url }
 	| otherwise = False
 repoIsSsh _ = False
 
+repoIsHttp :: Repo -> Bool
+repoIsHttp Repo { location = Url url } 
+	| uriScheme url == "http:" = True
+	| uriScheme url == "https:" = True
+	| otherwise = False
+repoIsHttp _ = False
+
 configAvail ::Repo -> Bool
 configAvail Repo { config = c } = c /= M.empty
 

From 354c5f349bcd8c43b45191983dce4a6c7489e9ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:45:58 -0400
Subject: [PATCH 2104/2835] add withTempFile

This is essentially the same as withSystemTempFile from System.IO.Temp,
but that library is not packaged for Debian, and may not be widely used.
I see various other withTempFile implementations here and there, none canonical.
Sigh.
---
 Utility.hs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/Utility.hs b/Utility.hs
index dab3b4d0ea..511350898a 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -23,6 +23,7 @@ module Utility (
 	unsetFileMode,
 	readMaybe,
 	viaTmp,
+	withTempFile,
 	dirContains,
 	dirContents,
 	myHomeDir,
@@ -38,6 +39,7 @@ module Utility (
 	prop_relPathDirToFile_basics
 ) where
 
+import IO (bracket)
 import System.IO
 import System.Exit
 import qualified System.Posix.Process
@@ -253,6 +255,18 @@ viaTmp a file content = do
 	a tmpfile content
 	renameFile tmpfile file
 
+{- Runs an action with a temp file, then removes the file. -}
+withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
+withTempFile template action = bracket create remove use
+	where
+		create = do
+			tmpdir <- catch getTemporaryDirectory (const $ return ".")
+			openTempFile tmpdir template
+		remove (name, handle) = do
+			hClose handle
+			catchBool (removeFile name >> return True)
+		use (name, handle) = action name handle
+
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]

From 5000aba76e6f066fd310d9635ea4369f07684b86 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:47:48 -0400
Subject: [PATCH 2105/2835] avoid namespace conflict

---
 Command.hs        | 2 --
 Command/SetKey.hs | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/Command.hs b/Command.hs
index 02bbd29d44..d3c1640ee0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -178,8 +178,6 @@ withKeys :: CommandSeekKeys
 withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
-withTempFile :: CommandSeekStrings
-withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index f2a5259bac..807cbd5b93 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -20,7 +20,7 @@ command = [repoCommand "setkey" paramPath seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
-seek = [withTempFile start]
+seek = [withStrings start]
 
 {- Sets cached content for a key. -}
 start :: CommandStartString

From 07f2e7ee726f3d7f60cd478e928afc69db60c0c8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:48:11 -0400
Subject: [PATCH 2106/2835] support reading git config from http remotes

The config file is downloaded to a temp file, and git-config run on that
to parse it.
---
 Remote/Git.hs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index de51c891e2..c8facb47a7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -12,6 +12,7 @@ import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
 import System.Posix.Files
+import System.IO
 
 import Types
 import Types.Remote
@@ -24,7 +25,8 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Remote.Ssh
+import Remote.Helper.Ssh
+import qualified Remote.Helper.Url as Url
 import Config
 
 remote :: RemoteType Annex
@@ -75,6 +77,7 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
 	| not $ M.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
+	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ Git.configRead r
 	where
@@ -85,9 +88,19 @@ tryGitConfigRead r
 			case result of
 				Left _ -> return r
 				Right r' -> return r'
+
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
 				Git.hConfigRead r
+
+		geturlconfig = do
+			s <- Url.get (Git.repoLocation r ++ "/config")
+			withTempFile "git-annex.tmp" $ \tmpfile -> \h -> do
+				hPutStr h s
+				hClose h
+				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
+					Git.hConfigRead r
+
 		store a = do
 			r' <- a
 			g <- Annex.gitRepo
@@ -95,6 +108,7 @@ tryGitConfigRead r
 			let g' = Git.remotesAdd g $ exchange l r'
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 			return r'
+
 		exchange [] _ = []
 		exchange (old:ls) new =
 			if Git.repoRemoteName old == Git.repoRemoteName new

From 4545a0e78cf675c6bbbcdd86b5c06bf99bb0c7e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:49:04 -0400
Subject: [PATCH 2107/2835] split out generic url stuff into a helper library
 from Remote.Web

---
 Command/AddUrl.hs    |  3 +-
 Remote/Helper/Url.hs | 66 ++++++++++++++++++++++++++++++++++++++++++++
 Remote/Web.hs        | 42 ++++++----------------------
 3 files changed, 77 insertions(+), 34 deletions(-)
 create mode 100644 Remote/Helper/Url.hs

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e87de384b3..add71c8209 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -14,6 +14,7 @@ import System.Directory
 
 import Command
 import qualified Backend
+import qualified Remote.Helper.Url
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
@@ -52,7 +53,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Remote.Web.download [url] tmp
+	ok <- Remote.Helper.Url.download url tmp
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
diff --git a/Remote/Helper/Url.hs b/Remote/Helper/Url.hs
new file mode 100644
index 0000000000..d3aea56220
--- /dev/null
+++ b/Remote/Helper/Url.hs
@@ -0,0 +1,66 @@
+{- Url downloading for remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Helper.Url (
+	exists,
+	download,
+	get
+) where
+
+import Control.Monad (liftM)
+import Control.Monad.State (liftIO)
+import qualified Network.Browser as Browser
+import Network.HTTP
+import Network.URI
+
+import Types
+import Messages
+import Utility
+
+type URLString = String
+
+{- Checks that an url exists and could be successfully downloaded. -}
+exists :: URLString -> IO Bool
+exists url =
+	case parseURI url of
+		Nothing -> return False
+		Just u -> do
+			r <- request u HEAD
+			case rspCode r of
+				(2,_,_) -> return True
+				_ -> return False
+
+{- Used to download large files, such as the contents of keys.
+ - Uses curl program for its progress bar. -}
+download :: URLString -> FilePath -> Annex Bool
+download url file = do
+	showOutput -- make way for curl progress bar
+	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
+
+{- Downloads a small file. -}
+get :: URLString -> IO String
+get url =
+	case parseURI url of
+		Nothing -> error "url parse error"
+		Just u -> do
+			r <- request u GET
+			case rspCode r of
+				(2,_,_) -> return $ rspBody r
+				_ -> error $ rspReason r
+
+{- Makes a http request of an url. For example, HEAD can be used to
+ - check if the url exists, or GET used to get the url content (best for
+ - small urls). -}
+request :: URI -> RequestMethod -> IO (Response String)
+request url requesttype = Browser.browse $ do
+	Browser.setErrHandler ignore
+	Browser.setOutHandler ignore
+	Browser.setAllowRedirects True
+	liftM snd $ Browser.request
+		(mkRequest requesttype url :: Request_String)
+	where
+		ignore = const $ return ()
diff --git a/Remote/Web.hs b/Remote/Web.hs
index cd028a06d9..cc96d5306d 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -7,28 +7,24 @@
 
 module Remote.Web (
 	remote,
-	setUrl,
-	download
+	setUrl
 ) where
 
 import Control.Monad.State (liftIO)
 import Control.Exception
 import System.FilePath
-import Network.Browser
-import Network.HTTP
-import Network.URI
 
 import Types
 import Types.Remote
 import qualified Git
 import qualified Annex
 import Messages
-import Utility
 import UUID
 import Config
 import PresenceLog
 import LocationLog
 import Locations
+import qualified Remote.Helper.Url as Url
 
 type URLString = String
 
@@ -90,9 +86,12 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = do
-	us <- getUrls key
-	download us file
+downloadKey key file = iter =<< getUrls key
+	where
+		iter [] = return False
+		iter (url:urls) = do
+			ok <- Url.download url file
+			if ok then return ok else iter urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
@@ -114,28 +113,5 @@ checkKey' :: [URLString] -> Annex Bool
 checkKey' [] = return False
 checkKey' (u:us) = do
 	showAction $ "checking " ++ u
-	e <- liftIO $ urlexists u
+	e <- liftIO $ Url.exists u
 	if e then return e else checkKey' us
-
-urlexists :: URLString -> IO Bool
-urlexists url =
-	case parseURI url of
-		Nothing -> return False
-		Just u -> do
-			(_, r) <- Network.Browser.browse $ do
-				setErrHandler ignore
-				setOutHandler ignore
-				setAllowRedirects True
-				request (mkRequest HEAD u :: Request_String)
-			case rspCode r of
-				(2,_,_) -> return True
-				_ -> return False
-	where
-		ignore = const $ return ()
-
-download :: [URLString] -> FilePath -> Annex Bool
-download [] _ = return False
-download (url:us) file = do
-	showOutput -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
-	if ok then return ok else download us file

From a55faff08fd9173edaf22a1de46cf7fafe89ebb7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:49:54 -0400
Subject: [PATCH 2108/2835] reorg Remote/*

---
 Command/Map.hs                     | 2 +-
 Remote/Bup.hs                      | 6 +++---
 Remote/Directory.hs                | 4 ++--
 Remote/{ => Helper}/Encryptable.hs | 2 +-
 Remote/{ => Helper}/Special.hs     | 2 +-
 Remote/{ => Helper}/Ssh.hs         | 2 +-
 Remote/Hook.hs                     | 4 ++--
 Remote/Rsync.hs                    | 4 ++--
 Remote/S3real.hs                   | 4 ++--
 9 files changed, 15 insertions(+), 15 deletions(-)
 rename Remote/{ => Helper}/Encryptable.hs (98%)
 rename Remote/{ => Helper}/Special.hs (97%)
 rename Remote/{ => Helper}/Ssh.hs (98%)

diff --git a/Command/Map.hs b/Command/Map.hs
index 07f127f14e..75c5b0b550 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -22,7 +22,7 @@ import Types
 import Utility
 import UUID
 import Trust
-import Remote.Ssh
+import Remote.Helper.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 1023cda186..c82f84745d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -30,9 +30,9 @@ import Locations
 import Config
 import Utility
 import Messages
-import Remote.Ssh
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Ssh
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 
 type BupRepo = String
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 235f613000..fd227f85d8 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -27,8 +27,8 @@ import Utility.CopyFile
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 
 remote :: RemoteType Annex
diff --git a/Remote/Encryptable.hs b/Remote/Helper/Encryptable.hs
similarity index 98%
rename from Remote/Encryptable.hs
rename to Remote/Helper/Encryptable.hs
index 66e1738ac2..04041c6553 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Encryptable where
+module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
 import Control.Monad.State (liftIO)
diff --git a/Remote/Special.hs b/Remote/Helper/Special.hs
similarity index 97%
rename from Remote/Special.hs
rename to Remote/Helper/Special.hs
index d6f362ce30..c302a0ff5e 100644
--- a/Remote/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Special where
+module Remote.Helper.Special where
 
 import qualified Data.Map as M
 import Data.Maybe
diff --git a/Remote/Ssh.hs b/Remote/Helper/Ssh.hs
similarity index 98%
rename from Remote/Ssh.hs
rename to Remote/Helper/Ssh.hs
index fe4e6dfc1a..478b018812 100644
--- a/Remote/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Ssh where
+module Remote.Helper.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 87f86ffe4f..ef52d0482b 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -28,8 +28,8 @@ import Locations
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Messages
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9535376097..eba67e3fd6 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -26,8 +26,8 @@ import Locations
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Messages
 import Utility.RsyncFile
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index e4dcc2a71d..456a77f0e4 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -33,8 +33,8 @@ import UUID
 import Messages
 import Locations
 import Config
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Content
 import Utility.Base64

From 5ccb926b51b0a270c8b1d754dac78d2074e07bdf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:04:23 -0400
Subject: [PATCH 2109/2835] support for getting files from http git remotes

---
 Remote/Git.hs | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index c8facb47a7..1adf8cfeb7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -119,9 +119,10 @@ tryGitConfigRead r
  - If the remote cannot be accessed, returns a Left error.
  -}
 inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
-inAnnex r key = if Git.repoIsUrl r
-		then checkremote
-		else liftIO (try checklocal ::IO (Either IOException Bool))
+inAnnex r key
+	| Git.repoIsHttp r = safely checkhttp
+	| Git.repoIsUrl r = checkremote
+	| otherwise = safely checklocal
 	where
 		checklocal = do
 			-- run a local check inexpensively,
@@ -133,7 +134,12 @@ inAnnex r key = if Git.repoIsUrl r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
 				[Param (show key)]
 			return $ Right inannex
-	
+		checkhttp = Url.exists $ keyUrl r key
+		safely a = liftIO (try a ::IO (Either IOException Bool))
+
+keyUrl :: Git.Repo -> Key -> String
+keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
+
 dropKey :: Git.Repo -> Key -> Annex Bool
 dropKey r key = 
 	onRemote r (boolSystem, False) "dropkey"
@@ -146,8 +152,9 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| otherwise = error "copying from non-ssh repo not supported"
-		
+	| Git.repoIsHttp r = Url.download (keyUrl r key) file
+	| otherwise = error "copying from non-ssh, non-http repo not supported"
+
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key

From e6752cc06424df18f3e14406674f9a630c3387ef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:12:48 -0400
Subject: [PATCH 2110/2835] Added support for getting content from git remotes
 using http (and https).

---
 debian/changelog                                             | 1 +
 ..._repo__44___with_the_annex_directory_exposed_to_http.mdwn | 3 +++
 doc/special_remotes.mdwn                                     | 5 +++--
 doc/special_remotes/web.mdwn                                 | 4 ++++
 4 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index fcc0b58b89..89cc794e7e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110720) UNRELEASED; urgency=low
 
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
+  * Added support for getting content from git remotes using http (and https).
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
index a2e344c115..ba7dcad300 100644
--- a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -15,3 +15,6 @@ I would like to be able to use the following commands to get a clone of the repo
 * git annex get
 
 This would allow contributors to quickly get a copy of our upstream repo and start contributing with minimal bandwidth/effort.
+
+> This is now supported.. I look forward to seeing your project using it!
+> --[[Joey]] [[!tag done]]
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index afc6a2cf23..55cd1f1a0f 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -1,6 +1,7 @@
 Most [[backends]] can transfer data to and from configured git remotes.
-Normally those remotes are normal git repositories (bare and non-bare),
-that store the file contents in their own git annex directory.
+Normally those remotes are normal git repositories (bare and non-bare;
+local and remote), that store the file contents in their own git annex
+directory.
 
 But, git-annex also extends git's concept of remotes, with these special
 types of remotes. These can be used just like any normal remote by git-annex.
diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index 6991ce4e44..a969fb071d 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -5,3 +5,7 @@ See [[walkthrough/using_the_web]] for usage examples.
 
 Currently git-annex only supports downloading content from the web; 
 it cannot upload to it or remove content.
+
+This special remote uses arbitrary urls on the web as the source for content.
+git-annex can also download content from a normal git remote, accessible by
+http.

From 790b0f38795ce9b62deda416c458376f1e7b7016 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:16:23 -0400
Subject: [PATCH 2111/2835] update

---
 doc/transferring_data.mdwn | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
index f6ae9bfcde..57873f6f0e 100644
--- a/doc/transferring_data.mdwn
+++ b/doc/transferring_data.mdwn
@@ -1,8 +1,12 @@
 git-annex can transfer data to or from any of a repository's git remotes.
 Depending on where the remote is, the data transfer is done using rsync
-(over ssh, with automatic resume), or plain cp (with copy-on-write
-optimisations on supported filesystems). Some [[special_remotes]]
-are also supported that are not traditional git remotes.
+(over ssh or locally), or plain cp (with copy-on-write
+optimisations on supported filesystems), or using curl (for repositories
+on the web). Some [[special_remotes]] are also supported that are not
+traditional git remotes.
+
+If a data transfer is interrupted, git-annex retains the partial transfer
+to allow it to be automatically resumed later.
 
 It's equally easy to transfer a single file to or from a repository,
 or to launch a retrievel of a massive pile of files from whatever

From f5449aae16af431ce6474080d123a930209b2cde Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:20:14 -0400
Subject: [PATCH 2112/2835] error out when dropping from http repo

---
 Remote/Git.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 1adf8cfeb7..d4847d6105 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -141,8 +141,9 @@ keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
 
 dropKey :: Git.Repo -> Key -> Annex Bool
-dropKey r key = 
-	onRemote r (boolSystem, False) "dropkey"
+dropKey r key
+	| Git.repoIsHttp r = error "dropping from http repo not supported"
+	| otherwise = onRemote r (boolSystem, False) "dropkey"
 		[ Params "--quiet --force"
 		, Param $ show key
 		]

From dffbf49d90236aedb559a3db26a96e2ae7904ece Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 01:33:08 +0000
Subject: [PATCH 2113/2835] Added a comment

---
 ...ment_1_05fc9c9cad26c520bebb98c852c71e35._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment

diff --git a/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
new file mode 100644
index 0000000000..18746225e9
--- /dev/null
+++ b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-17T01:33:08Z"
+ content="""
+It's ok that `git pull` does not merge the git-annex branch. You can merge it with `git annex merge`, or it will be done
+automatically when you use other git-annex commands.
+
+If you use `git pull` and `git push` without any options, the defaults will make git pull and push the git-annex branch automatically.
+
+But if you're in the habit of doing `git push origin master`, that won't cause the git-annex branch to be pushed (use `git push origin git-annex` to manually push it then). Similarly, `git pull origin master` won't pull it. And also, the `remote.origin.fetch` setting in `.git/config` can be modified in ways that make `git pull` not automatically pull the git-annex branch. So those are the things to avoid after upgrade to v3, basically.
+"""]]

From 4a023dd1aafb725ba5ccff5f09adc7055a2643ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 22:22:00 -0400
Subject: [PATCH 2114/2835] Added curl to Debian package dependencies.

---
 debian/changelog | 1 +
 debian/control   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 89cc794e7e..dc03647a72 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20110720) UNRELEASED; urgency=low
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
   * Added support for getting content from git remotes using http (and https).
+  * Added curl to Debian package dependencies.
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/debian/control b/debian/control
index d519c42b71..975faf5ea5 100644
--- a/debian/control
+++ b/debian/control
@@ -31,6 +31,7 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	git | git-core,
 	uuid,
 	rsync,
+	curl,
 	openssh-client
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git

From cde8221051546902ef24a4fa51f2108177b67a4a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 04:56:30 +0000
Subject: [PATCH 2115/2835] Added a comment

---
 .../comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment
new file mode 100644
index 0000000000..adf7a34e66
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-08-17T04:56:30Z"
+ content="""
+Finally got a chance to try to reproduce this. I followed your recipe exactly in a clean squeeze chroot. monadIO was not installed, but git-annex built ok, using monad-control.
+"""]]

From f0c21307001ec03437cca066d21f35f541b66de9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 01:34:15 -0400
Subject: [PATCH 2116/2835] releasing version 3.20110817

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index dc03647a72..3eab43578a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,11 @@
-git-annex (3.20110720) UNRELEASED; urgency=low
+git-annex (3.20110817) unstable; urgency=low
 
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
   * Added support for getting content from git remotes using http (and https).
   * Added curl to Debian package dependencies.
 
- -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
+ -- Joey Hess   Wed, 17 Aug 2011 01:29:02 -0400
 
 git-annex (3.20110719) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 807158f3eb..14188cd350 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110719
+Version: 3.20110817
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 00cc34a4c3a496f4d8fb5dfc5de826e321e6ac56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 01:34:43 -0400
Subject: [PATCH 2117/2835] add news item for git-annex 3.20110817

---
 doc/news/version_3.20110624.mdwn | 33 --------------------------------
 doc/news/version_3.20110817.mdwn |  6 ++++++
 2 files changed, 6 insertions(+), 33 deletions(-)
 delete mode 100644 doc/news/version_3.20110624.mdwn
 create mode 100644 doc/news/version_3.20110817.mdwn

diff --git a/doc/news/version_3.20110624.mdwn b/doc/news/version_3.20110624.mdwn
deleted file mode 100644
index 6204673bd2..0000000000
--- a/doc/news/version_3.20110624.mdwn
+++ /dev/null
@@ -1,33 +0,0 @@
-News for git-annex 3.20110624:
-
-There has been another change to the git-annex data store.
-Use `git annex upgrade` to migrate your repositories to the new
-layout. See [[upgrades]].
-
-The significant change this time is that the .git-annex/ directory
-is gone; instead there is a git-annex branch that is automatically
-maintained by git-annex, and encapsulates all its state nicely out
-of your way.
-
-You should make sure you include the git-annex branch when
-git pushing and pulling.
-
-git-annex 3.20110624 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
-   * git-annex now stores its logs in a git-annex branch.
-   * merge: New subcommand. Auto-merges the new git-annex branch.
-   * Improved handling of bare git repos with annexes. Many more commands will
-     work in them.
-   * git-annex is now more robust; it will never leave state files
-     uncommitted when some other git process comes along and locks the index
-     at an inconvenient time.
-   * rsync is now used when copying files from repos on other filesystems.
-     cp is still used when copying file from repos on the same filesystem,
-     since --reflink=auto can make it significantly faster on filesystems
-     such as btrfs.
-   * Allow --trust etc to specify a repository by name, for temporarily
-     trusting repositories that are not configured remotes.
-   * unlock: Made atomic.
-   * git-union-merge: New git subcommand, that does a generic union merge
-     operation, and operates efficiently without touching the working tree."""]]
diff --git a/doc/news/version_3.20110817.mdwn b/doc/news/version_3.20110817.mdwn
new file mode 100644
index 0000000000..51388f3c78
--- /dev/null
+++ b/doc/news/version_3.20110817.mdwn
@@ -0,0 +1,6 @@
+git-annex 3.20110817 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix shell escaping in rsync special remote.
+   * addurl: --fast can be used to avoid immediately downloading the url.
+   * Added support for getting content from git remotes using http (and https).
+   * Added curl to Debian package dependencies."""]]
\ No newline at end of file

From e950947cc9af07e03d4d2c335154790bc2943016 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
 
Date: Wed, 17 Aug 2011 12:34:48 +0000
Subject: [PATCH 2118/2835] Added a comment: squeeze-backports update?

---
 .../comment_1_029486088d098c2d4f1099f2f0e701a9._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment

diff --git a/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment b/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment
new file mode 100644
index 0000000000..9a4ed7c31d
--- /dev/null
+++ b/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
+ nickname="Matthias"
+ subject="squeeze-backports update?"
+ date="2011-08-17T12:34:46Z"
+ content="""
+Is there going to be an update of git-annex in debian squeeze-backports to a version that supports repository version 3?
+Thx
+"""]]

From a131fabc0dd73a50e4f9a994e876ed1bca608f19 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 15:34:29 +0000
Subject: [PATCH 2119/2835] Added a comment: Re: squeeze-backports update?

---
 .../comment_2_648e3467e260cdf233acdb0b53313ce0._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment

diff --git a/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment b/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment
new file mode 100644
index 0000000000..b8b3d68f33
--- /dev/null
+++ b/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="Re: squeeze-backports update?"
+ date="2011-08-17T15:34:29Z"
+ content="""
+Yes, I uploaded it last night.
+"""]]

From 3b5f7221305ba6e768711d8326a01b78fb1e4f79 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 13:07:59 -0400
Subject: [PATCH 2120/2835] init no longer creates .gitattributes

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 2865c8af5d..9c202e6689 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -122,7 +122,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * init description
 
   Initializes git-annex with a description of the git repository,
-  and sets up `.gitattributes` and the pre-commit hook.
+  and sets up the pre-commit hook.
 
 * describe repository description
 

From 56f6923ccbc6fb1932137b53458a4cece47e69b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:14:43 -0400
Subject: [PATCH 2121/2835] Now "git annex init" only has to be run once

when a git repository is first being created. Clones will automatically
notice that git-annex is in use and automatically perform a basic
initalization. It's still recommended to run "git annex init" in any
clones, to describe them.
---
 Branch.hs         | 32 ++++++++++++++++------
 CmdLine.hs        | 22 +++++++++++----
 Command/Init.hs   | 38 +++-----------------------
 Command/Uninit.hs | 17 ++----------
 Init.hs           | 69 +++++++++++++++++++++++++++++++++++++++++++++++
 Version.hs        |  6 ++---
 debian/changelog  |  9 +++++++
 7 files changed, 127 insertions(+), 66 deletions(-)
 create mode 100644 Init.hs

diff --git a/Branch.hs b/Branch.hs
index 35e3050936..fd4ce4cede 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -14,6 +14,7 @@ module Branch (
 	files,
 	refExists,
 	hasOrigin,
+	hasSomeBranch,
 	name	
 ) where
 
@@ -124,7 +125,7 @@ getCache file = getState >>= handle
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM (refExists fullname) $ do
+create = unlessM hasBranch $ do
 	g <- Annex.gitRepo
 	e <- hasOrigin
 	if e
@@ -154,19 +155,14 @@ update = do
 		 -}
 		staged <- stageJournalFiles
 
-		g <- Annex.gitRepo
-		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-		let refs = map (last . words) (lines r)
+		refs <- siblingBranches
 		updated <- catMaybes `liftM` mapM updateRef refs
+		g <- Annex.gitRepo
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
-{- Does origin/git-annex exist? -}
-hasOrigin :: Annex Bool
-hasOrigin = refExists originname
-
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
 refExists ref = do
@@ -174,6 +170,26 @@ refExists ref = do
 	liftIO $ Git.runBool g "show-ref"
 		[Param "--verify", Param "-q", Param ref]
 
+{- Does the main git-annex branch exist? -}
+hasBranch :: Annex Bool
+hasBranch = refExists fullname
+
+{- Does origin/git-annex exist? -}
+hasOrigin :: Annex Bool
+hasOrigin = refExists originname
+
+{- Does the git-annex branch or a foo/git-annex branch exist? -}
+hasSomeBranch :: Annex Bool
+hasSomeBranch = liftM (not . null) siblingBranches
+
+{- List of all git-annex branches, including the main one and any
+ - from remotes. -}
+siblingBranches :: Annex [String]
+siblingBranches = do
+	g <- Annex.gitRepo
+	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+	return $ map (last . words) (lines r)
+
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: GitRef -> Annex (Maybe String)
 updateRef ref
diff --git a/CmdLine.hs b/CmdLine.hs
index c33c497856..ff1758f0dc 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,13 +19,14 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
+import qualified Branch
 import Content
 import Types
 import Command
 import Version
 import Options
 import Messages
-import UUID
+import Init
 
 {- Runs the passed command line. -}
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
@@ -45,7 +46,7 @@ parseCmd argv header cmds options = do
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			when (cmdusesrepo command) checkVersion
+			checkCmdEnviron command
 			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
@@ -57,6 +58,19 @@ parseCmd argv header cmds options = do
 		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds
 		usagemsg = "\n\n" ++ usage header cmds options
 
+{- Checks that the command can be run in the current environment. -}
+checkCmdEnviron :: Command -> Annex ()
+checkCmdEnviron command = do
+	when (cmdusesrepo command) $ checkVersion $ do
+		{- Automatically initialize if there is already a git-annex
+		   branch from somewhere. Otherwise, require a manual init
+		   to avoid git-annex accidentially being run in git
+		   repos that did not intend to use it. -}
+		annexed <- Branch.hasSomeBranch
+		if annexed
+			then initialize
+			else error "First run: git-annex init"
+
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
 usage header cmds options =
@@ -95,9 +109,7 @@ tryRun' errnum _ [] = when (errnum > 0) $ error $ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
-startup = do
-	prepUUID
-	return True
+startup = return True
 
 {- Cleanup actions. -}
 shutdown :: Annex Bool
diff --git a/Command/Init.hs b/Command/Init.hs
index 71e87050d8..0191060511 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,19 +7,13 @@
 
 module Command.Init where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when, unless)
-import System.Directory
+import Control.Monad (when)
 
 import Command
 import qualified Annex
-import qualified Git
-import qualified Branch
 import UUID
-import Version
 import Messages
-import Types
-import Utility
+import Init
 	
 command :: [Command]
 command = [standaloneCommand "init" paramDesc seek
@@ -39,34 +33,8 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
-	Branch.create
+	initialize
 	g <- Annex.gitRepo
 	u <- getUUID g
-	setVersion
 	describeUUID u description
-	unless (Git.repoIsLocalBare g) $
-		gitPreCommitHookWrite g
 	next $ return True
-
-{- set up a git pre-commit hook, if one is not already present -}
-gitPreCommitHookWrite :: Git.Repo -> Annex ()
-gitPreCommitHookWrite repo = do
-	exists <- liftIO $ doesFileExist hook
-	if exists
-		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
-		else liftIO $ do
-			viaTmp writeFile hook preCommitScript
-			p <- getPermissions hook
-			setPermissions hook $ p {executable = True}
-	where
-		hook = preCommitHook repo
-
-preCommitHook :: Git.Repo -> FilePath
-preCommitHook repo = 
-	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
-
-preCommitScript :: String
-preCommitScript = 
-		"#!/bin/sh\n" ++
-		"# automatically configured by git-annex\n" ++ 
-		"git annex pre-commit .\n"
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 8b8d7e364d..195246aa84 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,13 +12,11 @@ import System.Directory
 import System.Exit
 
 import Command
-import Messages
-import Types
 import Utility
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
-import qualified Command.Init
+import Init
 import qualified Branch
 import Content
 import Locations
@@ -47,7 +45,7 @@ perform = next cleanup
 cleanup :: CommandCleanup
 cleanup = do
 	g <- Annex.gitRepo
-	gitPreCommitHookUnWrite g
+	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
 	-- avoid normal shutdown
@@ -55,14 +53,3 @@ cleanup = do
 	liftIO $ do
 		Git.run g "branch" [Param "-D", Param Branch.name]
 		exitSuccess
-
-gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
-gitPreCommitHookUnWrite repo = do
-	let hook = Command.Init.preCommitHook repo
-	whenM (liftIO $ doesFileExist hook) $ do
-		c <- liftIO $ readFile hook
-		if c == Command.Init.preCommitScript
-			then liftIO $ removeFile hook
-			else warning $ "pre-commit hook (" ++ hook ++ 
-				") contents modified; not deleting." ++
-				" Edit it to remove call to git annex."
diff --git a/Init.hs b/Init.hs
new file mode 100644
index 0000000000..41256a9530
--- /dev/null
+++ b/Init.hs
@@ -0,0 +1,69 @@
+{- git-annex repository initialization
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Init (initialize, uninitialize) where
+
+import Control.Monad.State (liftIO)
+import Control.Monad (unless)
+import System.Directory
+
+import qualified Annex
+import qualified Git
+import qualified Branch
+import Version
+import Messages
+import Types
+import Utility
+import UUID
+
+initialize :: Annex ()
+initialize = do
+	prepUUID
+	Branch.create
+	setVersion
+	g <- Annex.gitRepo
+	unless (Git.repoIsLocalBare g) $
+		gitPreCommitHookWrite g
+
+uninitialize :: Annex ()
+uninitialize = do
+	g <- Annex.gitRepo
+	gitPreCommitHookUnWrite g
+
+{- set up a git pre-commit hook, if one is not already present -}
+gitPreCommitHookWrite :: Git.Repo -> Annex ()
+gitPreCommitHookWrite repo = do
+	exists <- liftIO $ doesFileExist hook
+	if exists
+		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
+		else liftIO $ do
+			viaTmp writeFile hook preCommitScript
+			p <- getPermissions hook
+			setPermissions hook $ p {executable = True}
+	where
+		hook = preCommitHook repo
+
+gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
+gitPreCommitHookUnWrite repo = do
+	let hook = preCommitHook repo
+	whenM (liftIO $ doesFileExist hook) $ do
+		c <- liftIO $ readFile hook
+		if c == preCommitScript
+			then liftIO $ removeFile hook
+			else warning $ "pre-commit hook (" ++ hook ++ 
+				") contents modified; not deleting." ++
+				" Edit it to remove call to git annex."
+
+preCommitHook :: Git.Repo -> FilePath
+preCommitHook repo = 
+	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
+
+preCommitScript :: String
+preCommitScript = 
+		"#!/bin/sh\n" ++
+		"# automatically configured by git-annex\n" ++ 
+		"git annex pre-commit .\n"
diff --git a/Version.hs b/Version.hs
index 7e6910fbe4..44fd2e9de1 100644
--- a/Version.hs
+++ b/Version.hs
@@ -39,10 +39,10 @@ getVersion = do
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
-checkVersion :: Annex ()
-checkVersion = getVersion >>= handle
+checkVersion :: Annex () -> Annex ()
+checkVersion initaction = getVersion >>= handle
 	where
-		handle Nothing = error "First run: git-annex init"
+		handle Nothing = initaction
 		handle (Just v) = unless (v `elem` supportedVersions) $
 			error $ "Repository version " ++ v ++ 
 				" is not supported. " ++
diff --git a/debian/changelog b/debian/changelog
index 3eab43578a..9230c00218 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+git-annex (3.20110818) UNRELEASED; urgency=low
+
+  * Now "git annex init" only has to be run once, when a git repository
+    is first being created. Clones will automatically notice that git-annex
+    is in use and automatically perform a basic initalization. It's
+    still recommended to run "git annex init" in any clones, to describe them.
+
+ -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
+
 git-annex (3.20110817) unstable; urgency=low
 
   * Fix shell escaping in rsync special remote.

From cf33eff684de5193379e99745d83c80fd2fb09c0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:17:12 -0400
Subject: [PATCH 2122/2835] git-annex-shell configlist should not be standalone

This makes it initialize the repository with a uuid, and list the uuid,
allowing automatic setup of bare repositories when git-annex is used.
---
 Command/ConfigList.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index d8dbff03af..1b1bb3c34b 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -14,7 +14,7 @@ import Command
 import UUID
 
 command :: [Command]
-command = [standaloneCommand "configlist" paramNothing seek
+command = [repoCommand "configlist" paramNothing seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]

From 32f27cc3e839a3c243641b953fd4bd0f15dda08a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:36:20 -0400
Subject: [PATCH 2123/2835] when reading configs of local repos, first
 initializeSafe

This auto-generates a uuid if the local repo does not already have one.
---
 CmdLine.hs    | 12 +-----------
 Init.hs       | 17 ++++++++++++++++-
 Remote/Git.hs | 30 +++++++++++++++++-------------
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index ff1758f0dc..0590f11124 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,7 +19,6 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
-import qualified Branch
 import Content
 import Types
 import Command
@@ -60,16 +59,7 @@ parseCmd argv header cmds options = do
 
 {- Checks that the command can be run in the current environment. -}
 checkCmdEnviron :: Command -> Annex ()
-checkCmdEnviron command = do
-	when (cmdusesrepo command) $ checkVersion $ do
-		{- Automatically initialize if there is already a git-annex
-		   branch from somewhere. Otherwise, require a manual init
-		   to avoid git-annex accidentially being run in git
-		   repos that did not intend to use it. -}
-		annexed <- Branch.hasSomeBranch
-		if annexed
-			then initialize
-			else error "First run: git-annex init"
+checkCmdEnviron command = when (cmdusesrepo command) $ checkVersion $ initializeSafe
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
diff --git a/Init.hs b/Init.hs
index 41256a9530..36d3ed0fac 100644
--- a/Init.hs
+++ b/Init.hs
@@ -5,7 +5,11 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Init (initialize, uninitialize) where
+module Init (
+	initialize,
+	initializeSafe,
+	uninitialize
+) where
 
 import Control.Monad.State (liftIO)
 import Control.Monad (unless)
@@ -34,6 +38,17 @@ uninitialize = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
 
+{- Call to automatically initialize if there is already a git-annex
+   branch from somewhere. Otherwise, require a manual init
+   to avoid git-annex accidentially being run in git
+   repos that did not intend to use it. -}
+initializeSafe :: Annex ()
+initializeSafe = do
+	annexed <- Branch.hasSomeBranch
+	if annexed
+		then initialize
+		else error "First run: git-annex init"
+
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()
 gitPreCommitHookWrite repo = do
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d4847d6105..c588cc73bb 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -28,6 +28,7 @@ import Utility.RsyncFile
 import Remote.Helper.Ssh
 import qualified Remote.Helper.Url as Url
 import Config
+import Init
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -79,7 +80,9 @@ tryGitConfigRead r
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
-	| otherwise = store $ safely $ Git.configRead r
+	| otherwise = store $ safely $ do
+		onLocal r initializeSafe
+		Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.
@@ -124,11 +127,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = safely checklocal
 	where
-		checklocal = do
-			-- run a local check inexpensively,
-			-- by making an Annex monad using the remote
-			a <- Annex.new r
-			Annex.eval a (Content.inAnnex key)
+		checklocal = onLocal r (Content.inAnnex key)
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
@@ -137,6 +136,13 @@ inAnnex r key
 		checkhttp = Url.exists $ keyUrl r key
 		safely a = liftIO (try a ::IO (Either IOException Bool))
 
+{- Runs an action on a local repository inexpensively, by making an annex
+ - monad using that repository. -}
+onLocal :: Git.Repo -> Annex a -> IO a
+onLocal r a = do
+	annex <- Annex.new r
+	Annex.eval annex a
+
 keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
 
@@ -163,13 +169,11 @@ copyToRemote r key
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
-		liftIO $ do
-			a <- Annex.new r
-			Annex.eval a $ do
-				ok <- Content.getViaTmp key $
-					rsyncOrCopyFile r keysrc
-				Content.saveState
-				return ok
+		liftIO $ onLocal r $ do
+			ok <- Content.getViaTmp key $
+				rsyncOrCopyFile r keysrc
+			Content.saveState
+			return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key

From 228a724d1d1e1d2f27763bd5f71473acc4d022d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:43:38 -0400
Subject: [PATCH 2124/2835] improve docs for init

---
 doc/git-annex.mdwn | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9c202e6689..9d63ca156d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -121,8 +121,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * init description
 
-  Initializes git-annex with a description of the git repository,
-  and sets up the pre-commit hook.
+  Initializes git-annex with a description of the git repository.
+
+  Until a repository (or one of its remotes) has been initialized,
+  git-annex will refuse to operate on it, to avoid accidentially
+  using it in a repository that was not intended to have an annex.
+
+  It's useful, but not mandatory, to initialize each new clone
+  of a repository with its own description.
 
 * describe repository description
 

From b7a4ff1c3185404d36d34b372b016be052394a95 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:38:26 -0400
Subject: [PATCH 2125/2835] optimise initialized check

Avoid running external command if annex.version is set.
---
 CmdLine.hs    |  3 +--
 Init.hs       | 20 ++++++++++++--------
 Remote/Git.hs |  2 +-
 Version.hs    | 19 +++++++------------
 4 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 0590f11124..2652e5b8fc 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,7 +22,6 @@ import qualified Git
 import Content
 import Types
 import Command
-import Version
 import Options
 import Messages
 import Init
@@ -59,7 +58,7 @@ parseCmd argv header cmds options = do
 
 {- Checks that the command can be run in the current environment. -}
 checkCmdEnviron :: Command -> Annex ()
-checkCmdEnviron command = when (cmdusesrepo command) $ checkVersion $ initializeSafe
+checkCmdEnviron command = when (cmdusesrepo command) ensureInitialized
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
diff --git a/Init.hs b/Init.hs
index 36d3ed0fac..ae92998bb9 100644
--- a/Init.hs
+++ b/Init.hs
@@ -6,8 +6,8 @@
  -}
 
 module Init (
+	ensureInitialized,
 	initialize,
-	initializeSafe,
 	uninitialize
 ) where
 
@@ -38,16 +38,20 @@ uninitialize = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
 
-{- Call to automatically initialize if there is already a git-annex
+{- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
    to avoid git-annex accidentially being run in git
    repos that did not intend to use it. -}
-initializeSafe :: Annex ()
-initializeSafe = do
-	annexed <- Branch.hasSomeBranch
-	if annexed
-		then initialize
-		else error "First run: git-annex init"
+ensureInitialized :: Annex ()
+ensureInitialized = do
+	v <- getVersion
+	case v of
+		Just version -> checkVersion version
+		Nothing -> do
+			annexed <- Branch.hasSomeBranch
+			if annexed
+				then initialize
+				else error "First run: git-annex init"
 
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()
diff --git a/Remote/Git.hs b/Remote/Git.hs
index c588cc73bb..d8ecd33c4a 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -81,7 +81,7 @@ tryGitConfigRead r
 	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ do
-		onLocal r initializeSafe
+		onLocal r ensureInitialized
 		Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
diff --git a/Version.hs b/Version.hs
index 44fd2e9de1..fcf6bc4d1d 100644
--- a/Version.hs
+++ b/Version.hs
@@ -7,8 +7,6 @@
 
 module Version where
 
-import Control.Monad (unless)
-
 import Types
 import qualified Annex
 import qualified Git
@@ -39,14 +37,11 @@ getVersion = do
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
-checkVersion :: Annex () -> Annex ()
-checkVersion initaction = getVersion >>= handle
+checkVersion :: Version -> Annex ()
+checkVersion v
+	| v `elem` supportedVersions = return ()
+	| v `elem` upgradableVersions = err "Upgrade this repository: git-annex upgrade"
+	| otherwise = err "Upgrade git-annex."
 	where
-		handle Nothing = initaction
-		handle (Just v) = unless (v `elem` supportedVersions) $
-			error $ "Repository version " ++ v ++ 
-				" is not supported. " ++
-				msg v
-		msg v
-			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"
-			| otherwise = "Upgrade git-annex."
+		err msg = error $ "Repository version " ++ v ++
+			" is not supported. " ++ msg

From 9e763954ae96201ce577cb51f904bf7756597f14 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:42:49 -0400
Subject: [PATCH 2126/2835] tweak

---
 Init.hs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/Init.hs b/Init.hs
index ae92998bb9..a11a7c51c9 100644
--- a/Init.hs
+++ b/Init.hs
@@ -1,6 +1,6 @@
 {- git-annex repository initialization
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -43,11 +43,9 @@ uninitialize = do
    to avoid git-annex accidentially being run in git
    repos that did not intend to use it. -}
 ensureInitialized :: Annex ()
-ensureInitialized = do
-	v <- getVersion
-	case v of
-		Just version -> checkVersion version
-		Nothing -> do
+ensureInitialized = getVersion >>= maybe needsinit checkVersion
+	where
+		needsinit = do
 			annexed <- Branch.hasSomeBranch
 			if annexed
 				then initialize

From 0c53ccc675b52d4995b3406d5814116605fe6ec7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:52:58 -0400
Subject: [PATCH 2127/2835] tweak

---
 Init.hs | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/Init.hs b/Init.hs
index a11a7c51c9..4708701fa7 100644
--- a/Init.hs
+++ b/Init.hs
@@ -29,14 +29,11 @@ initialize = do
 	prepUUID
 	Branch.create
 	setVersion
-	g <- Annex.gitRepo
-	unless (Git.repoIsLocalBare g) $
-		gitPreCommitHookWrite g
+	gitPreCommitHookWrite
 
 uninitialize :: Annex ()
 uninitialize = do
-	g <- Annex.gitRepo
-	gitPreCommitHookUnWrite g
+	gitPreCommitHookUnWrite
 
 {- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
@@ -52,8 +49,9 @@ ensureInitialized = getVersion >>= maybe needsinit checkVersion
 				else error "First run: git-annex init"
 
 {- set up a git pre-commit hook, if one is not already present -}
-gitPreCommitHookWrite :: Git.Repo -> Annex ()
-gitPreCommitHookWrite repo = do
+gitPreCommitHookWrite :: Annex ()
+gitPreCommitHookWrite = unlessBare $ do
+	hook <- preCommitHook
 	exists <- liftIO $ doesFileExist hook
 	if exists
 		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
@@ -61,12 +59,10 @@ gitPreCommitHookWrite repo = do
 			viaTmp writeFile hook preCommitScript
 			p <- getPermissions hook
 			setPermissions hook $ p {executable = True}
-	where
-		hook = preCommitHook repo
 
-gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
-gitPreCommitHookUnWrite repo = do
-	let hook = preCommitHook repo
+gitPreCommitHookUnWrite :: Annex ()
+gitPreCommitHookUnWrite = unlessBare $ do
+	hook <- preCommitHook
 	whenM (liftIO $ doesFileExist hook) $ do
 		c <- liftIO $ readFile hook
 		if c == preCommitScript
@@ -75,9 +71,15 @@ gitPreCommitHookUnWrite repo = do
 				") contents modified; not deleting." ++
 				" Edit it to remove call to git annex."
 
-preCommitHook :: Git.Repo -> FilePath
-preCommitHook repo = 
-	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
+unlessBare :: Annex () -> Annex ()
+unlessBare a = do
+	g <- Annex.gitRepo
+	unless (Git.repoIsLocalBare g) a
+
+preCommitHook :: Annex FilePath
+preCommitHook = do
+	g <- Annex.gitRepo
+	return $ Git.workTree g ++ "/" ++ Git.gitDir g ++ "/hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 

From 8a2197adfab9d8425f92000621a802dce37e124c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 18 Aug 2011 12:20:47 -0400
Subject: [PATCH 2128/2835] Added annex-cost-command configuration, which can
 be used to vary the cost of a remote based on the output of a shell command.

Also avoided crashing if the user specified cost value cannot be parsed.
---
 Config.hs          | 21 ++++++++++++++++-----
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  6 ++++++
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/Config.hs b/Config.hs
index 9cbf2d52fd..6f2014483b 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,6 +9,8 @@ module Config where
 
 import Data.Maybe
 import Control.Monad.State (liftIO)
+import Control.Monad (liftM)
+import System.Cmd.Utils
 
 import qualified Git
 import qualified Annex
@@ -40,14 +42,23 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex
 {- Calculates cost for a remote.
  -
  - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost
+ - repositories; it can also be configured by remote..annex-cost,
+ - or if remote..annex-cost-command is set and prints a number, that
+ - is used.
  -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
-	c <- getConfig r "cost" ""
-	if not $ null c
-		then return $ read c
-		else return def
+	cmd <- getConfig r "cost-command" ""
+	return . safeparse =<< if not $ null cmd
+			then liftM snd $ liftIO $ pipeFrom "sh" ["-c", cmd]
+			else getConfig r "cost" ""
+	where
+		safeparse v
+			| null ws || null ps = def
+			| otherwise = (fst . head) ps
+			where
+				ws = words v
+				ps = reads $ head ws
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/debian/changelog b/debian/changelog
index 9230c00218..1ace861a46 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     is first being created. Clones will automatically notice that git-annex
     is in use and automatically perform a basic initalization. It's
     still recommended to run "git annex init" in any clones, to describe them.
+  * Added annex-cost-command configuration, which can be used to vary the
+    cost of a remote based on the output of a shell command.
 
  -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9d63ca156d..a262d465fd 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -424,6 +424,12 @@ Here are all the supported configuration settings.
   The default cost is 100 for local repositories, and 200 for remote
   repositories.
 
+* `remote..annex-cost-command`
+
+  If set, the command is run, and the number it outputs is used as the cost.
+  This allows varying the cost based on eg, the current network. The
+  cost-command can be any shell command line.
+
 * `remote..annex-ignore`
 
   If set to `true`, prevents git-annex

From 7cedd28ab00d93b4a8d70810511a7c28cae65fd1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 18 Aug 2011 12:26:28 -0400
Subject: [PATCH 2129/2835] tweak

---
 Config.hs | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Config.hs b/Config.hs
index 6f2014483b..568eb71386 100644
--- a/Config.hs
+++ b/Config.hs
@@ -39,12 +39,9 @@ getConfig r key def = do
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 
-{- Calculates cost for a remote.
- -
- - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost,
- - or if remote..annex-cost-command is set and prints a number, that
- - is used.
+{- Calculates cost for a remote. Either the default, or as configured 
+ - by remote..annex-cost, or if remote..annex-cost-command
+ - is set and prints a number, that is used.
  -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do

From e97fede8cd86d0eb804ced6d2877f617ba15b1a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 12:59:07 -0400
Subject: [PATCH 2130/2835] make gitDir absolute

---
 Git.hs             | 12 ++++++------
 Init.hs            |  2 +-
 git-union-merge.hs |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Git.hs b/Git.hs
index b226bce0f2..168fe4154e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -247,11 +247,11 @@ attributes repo
 	| configBare repo = workTree repo ++ "/info/.gitattributes"
 	| otherwise = workTree repo ++ "/.gitattributes"
 
-{- Path to a repository's .git directory, relative to its workTree. -}
+{- Path to a repository's .git directory. -}
 gitDir :: Repo -> String
 gitDir repo
-	| configBare repo = ""
-	| otherwise = ".git"
+	| configBare repo = workTree repo
+	| otherwise = workTree repo  ".git"
 
 {- Path to a repository's --work-tree, that is, its top.
  -
@@ -345,10 +345,10 @@ urlAuthPart _ repo = assertUrl repo $ error "internal"
 
 {- Constructs a git command line operating on the specified repo. -}
 gitCommandLine :: Repo -> [CommandParam] -> [CommandParam]
-gitCommandLine repo@(Repo { location = Dir d} ) params =
+gitCommandLine repo@(Repo { location = Dir _ } ) params =
 	-- force use of specified repo via --git-dir and --work-tree
-	[ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo)
-	, Param ("--work-tree=" ++ d)
+	[ Param ("--git-dir=" ++ gitDir repo)
+	, Param ("--work-tree=" ++ workTree repo)
 	] ++ params
 gitCommandLine repo _ = assertLocal repo $ error "internal"
 
diff --git a/Init.hs b/Init.hs
index 4708701fa7..a469657a11 100644
--- a/Init.hs
+++ b/Init.hs
@@ -79,7 +79,7 @@ unlessBare a = do
 preCommitHook :: Annex FilePath
 preCommitHook = do
 	g <- Annex.gitRepo
-	return $ Git.workTree g ++ "/" ++ Git.gitDir g ++ "/hooks/pre-commit"
+	return $ Git.gitDir g ++ "/hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e763376077..4e1a932b45 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -20,7 +20,7 @@ usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
 tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
+tmpIndex g = Git.gitDir g  "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
 setup g = cleanup g -- idempotency

From 021e8e1e0e40543f3861ed8fb0fc12cd2be46d46 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 14:28:07 -0400
Subject: [PATCH 2131/2835] make Annex an opaque data type

Was a type alias; using newtype has the benefit that type errors will
show "Annex foo" rather than two lines of internal type nonsense. Yay!
There should be no other effects to size or runtime.

I've tried to do this at least twice before (each time I read RWH chapter 10);
finally understood how to this time.. sorta.
---
 Annex.hs | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index f7e3e29f82..fd7e3b391c 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+
 module Annex (
 	Annex,
 	AnnexState(..),
@@ -17,6 +19,7 @@ module Annex (
 ) where
 
 import Control.Monad.State
+import Control.Monad.IO.Control
 
 import qualified Git
 import Git.Queue
@@ -28,7 +31,14 @@ import Types.TrustLevel
 import Types.UUID
 
 -- git-annex's monad
-type Annex = StateT AnnexState IO
+newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
+	deriving (
+		Functor,
+		Monad,
+		MonadIO,
+		MonadControlIO,
+		MonadState AnnexState
+	)
 
 -- internal state storage
 data AnnexState = AnnexState
@@ -78,9 +88,9 @@ new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
-run = flip runStateT
+run s a = runStateT (runAnnex a) s
 eval :: AnnexState -> Annex a -> IO a
-eval = flip evalStateT
+eval s a = evalStateT (runAnnex a) s
 
 {- Gets a value from the internal state, selected by the passed value
  - constructor. -}

From e3ca08fad89d6f3b5951da74a8af3631c5ef9604 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 14:36:52 -0400
Subject: [PATCH 2132/2835] drop an unnecessart liftIO

the liftM on its own can lift all the way into IO.
---
 Annex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex.hs b/Annex.hs
index fd7e3b391c..46dfc9df56 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -84,7 +84,7 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
+new gitrepo = newState `liftM` Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)

From 01cd775d9271dfe96917c5557efe93ce0e6f52c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:05:08 -0400
Subject: [PATCH 2133/2835] Fix broken upgrade from V1 repository. Closes:
 #638584

Had forgotten to keep several old versions of functions needed during this
upgrade.
---
 PresenceLog.hs   |  4 +++-
 Upgrade/V1.hs    | 33 ++++++++++++++++++++++++++-------
 debian/changelog |  1 +
 3 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/PresenceLog.hs b/PresenceLog.hs
index 9c516a8db1..ccb75ff5be 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -15,10 +15,12 @@ module PresenceLog (
 	LogStatus(..),
 	addLog,
 	readLog,
+	parseLog,
 	writeLog,
 	logNow,
 	compactLog,
-	currentLog
+	currentLog,
+	LogLine
 ) where
 
 import Data.Time.Clock.POSIX
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c41310880f..160d9309fe 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -22,7 +22,7 @@ import Types.Key
 import Content
 import Types
 import Locations
-import LocationLog
+import PresenceLog
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
@@ -123,7 +123,7 @@ moveLocationLogs = do
 					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
-				let dest = logFile k
+				let dest = logFile2 g k
 				let dir = Upgrade.V2.gitStateDir g
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
@@ -131,9 +131,9 @@ moveLocationLogs = do
 				-- log files that are not checked into git,
 				-- as well as merging with already upgraded
 				-- logs that have been pulled from elsewhere
-				old <- readLog f
-				new <- readLog dest
-				writeLog dest (old++new)
+				old <- liftIO $ readLog1 f
+				new <- liftIO $ readLog1 dest
+				liftIO $ writeLog1 dest (old++new)
 				AnnexQueue.add "add" [Param "--"] [dest]
 				AnnexQueue.add "add" [Param "--"] [f]
 				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
@@ -186,8 +186,11 @@ fileKey1 :: FilePath -> Key
 fileKey1 file = readKey1 $
 	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file
 
-logFile1 :: Git.Repo -> Key -> String
-logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
+writeLog1 :: FilePath -> [LogLine] -> IO ()
+writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
+
+readLog1 :: FilePath -> IO [LogLine]
+readLog1 file = catch (return . parseLog =<< readFileStrict file) (const $ return [])
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
@@ -230,3 +233,19 @@ getKeyFilesPresent1' dir = do
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
+
+logFile1 :: Git.Repo -> Key -> String
+logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
+
+logFile2 :: Git.Repo -> Key -> String
+logFile2 = logFile' hashDirLower
+
+logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
+logFile' hasher repo key =
+	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
+
+stateDir :: FilePath
+stateDir = addTrailingPathSeparator $ ".git-annex"
+
+gitStateDir :: Git.Repo -> FilePath
+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
diff --git a/debian/changelog b/debian/changelog
index 1ace861a46..7d3315103e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     still recommended to run "git annex init" in any clones, to describe them.
   * Added annex-cost-command configuration, which can be used to vary the
     cost of a remote based on the output of a shell command.
+  * Fix broken upgrade from V1 repository. Closes: #638584
 
  -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
 

From 3786f8d3487fe5a460d40045d3bcaaa69f3f84d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:38:36 -0400
Subject: [PATCH 2134/2835] releasing version 3.20110819

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 7d3315103e..a8280a156d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110818) UNRELEASED; urgency=low
+git-annex (3.20110819) unstable; urgency=low
 
   * Now "git annex init" only has to be run once, when a git repository
     is first being created. Clones will automatically notice that git-annex
@@ -8,7 +8,7 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     cost of a remote based on the output of a shell command.
   * Fix broken upgrade from V1 repository. Closes: #638584
 
- -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
+ -- Joey Hess   Fri, 19 Aug 2011 20:34:09 -0400
 
 git-annex (3.20110817) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 14188cd350..a37c187c90 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110817
+Version: 3.20110819
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 71215263855e2d5e940bb0086efd7691aa0139d8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:39:03 -0400
Subject: [PATCH 2135/2835] add news item for git-annex 3.20110819

---
 doc/news/version_3.20110702.mdwn | 22 ----------------------
 doc/news/version_3.20110819.mdwn |  9 +++++++++
 2 files changed, 9 insertions(+), 22 deletions(-)
 delete mode 100644 doc/news/version_3.20110702.mdwn
 create mode 100644 doc/news/version_3.20110819.mdwn

diff --git a/doc/news/version_3.20110702.mdwn b/doc/news/version_3.20110702.mdwn
deleted file mode 100644
index a5bb925552..0000000000
--- a/doc/news/version_3.20110702.mdwn
+++ /dev/null
@@ -1,22 +0,0 @@
-News for git-annex 3.20110702:
-
-The URL backend has been removed. Instead the new web remote can be used.
-
-git-annex 3.20110702 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Now the web can be used as a special remote.
-     This feature replaces the old URL backend.
-   * addurl: New command to download an url and store it in the annex.
-   * Sped back up fsck, copy --from, and other commands that often
-     have to read a lot of information from the git-annex branch. Such
-     commands are now faster than they were before introduction of the
-     git-annex branch.
-   * Always ensure git-annex branch exists.
-   * Modify location log parser to allow future expansion.
-   * --force will cause add, etc, to operate on ignored files.
-   * Avoid mangling encoding when storing the description of repository
-     and other content.
-   * cabal can now be used to build git-annex. This is substantially
-     slower than using make, does not build or install documentation,
-     does not run the test suite, and is not particularly recommended,
-     but could be useful to some."""]]
diff --git a/doc/news/version_3.20110819.mdwn b/doc/news/version_3.20110819.mdwn
new file mode 100644
index 0000000000..bbc6abdbc7
--- /dev/null
+++ b/doc/news/version_3.20110819.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110819 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Now "git annex init" only has to be run once, when a git repository
+     is first being created. Clones will automatically notice that git-annex
+     is in use and automatically perform a basic initalization. It's
+     still recommended to run "git annex init" in any clones, to describe them.
+   * Added annex-cost-command configuration, which can be used to vary the
+     cost of a remote based on the output of a shell command.
+   * Fix broken upgrade from V1 repository. Closes: #[638584](http://bugs.debian.org/638584)"""]]
\ No newline at end of file

From ec746c511f5666fc214eba1a477d1ababfe9d367 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 20 Aug 2011 12:52:29 -0400
Subject: [PATCH 2136/2835] note about why curl -# is used

I'd rather use wget really, but as git-annex uses libcurl elsewhere, it
seems best to stick with curl. And making this configurable seems
overboard.
---
 Remote/Helper/Url.hs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Remote/Helper/Url.hs b/Remote/Helper/Url.hs
index d3aea56220..af1fee8f04 100644
--- a/Remote/Helper/Url.hs
+++ b/Remote/Helper/Url.hs
@@ -39,6 +39,10 @@ exists url =
 download :: URLString -> FilePath -> Annex Bool
 download url file = do
 	showOutput -- make way for curl progress bar
+	-- Uses the -# progress display, because the normal one is very
+	-- confusing when resuming, showing the remainder to download
+	-- as the whole file, and not indicating how much percent was
+	-- downloaded before the resume.
 	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
 
 {- Downloads a small file. -}

From 737b5d14c91101d46e20999e33461e9059dd9f28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 20 Aug 2011 16:11:42 -0400
Subject: [PATCH 2137/2835] moved files around

---
 .gitignore                           |  2 +-
 Backend/SHA.hs                       |  2 +-
 TestConfig.hs => Build/TestConfig.hs |  8 ++++----
 Command/Add.hs                       |  2 +-
 Command/AddUrl.hs                    |  4 ++--
 Command/Map.hs                       |  2 +-
 Command/Version.hs                   |  2 +-
 Content.hs                           |  2 +-
 Makefile                             | 10 ++++------
 Remote/Bup.hs                        |  2 +-
 Remote/Git.hs                        |  4 ++--
 Remote/Web.hs                        |  2 +-
 UUID.hs                              |  2 +-
 Utility/CopyFile.hs                  |  2 +-
 {Remote/Helper => Utility}/Ssh.hs    |  2 +-
 StatFS.hsc => Utility/StatFS.hsc     |  2 +-
 Touch.hsc => Utility/Touch.hsc       |  2 +-
 {Remote/Helper => Utility}/Url.hs    |  4 ++--
 configure.hs                         |  2 +-
 19 files changed, 28 insertions(+), 30 deletions(-)
 rename TestConfig.hs => Build/TestConfig.hs (94%)
 rename {Remote/Helper => Utility}/Ssh.hs (98%)
 rename StatFS.hsc => Utility/StatFS.hsc (98%)
 rename Touch.hsc => Utility/Touch.hsc (99%)
 rename {Remote/Helper => Utility}/Url.hs (96%)

diff --git a/.gitignore b/.gitignore
index d5bf54c813..1fdad216dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,7 @@
 *.o
 test
 configure
-SysConfig.hs
+Build/SysConfig.hs
 git-annex
 git-annex-shell
 git-union-merge
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index bae19be003..b8a0d254b7 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -24,7 +24,7 @@ import Types
 import Types.Backend
 import Types.Key
 import Utility
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
 
diff --git a/TestConfig.hs b/Build/TestConfig.hs
similarity index 94%
rename from TestConfig.hs
rename to Build/TestConfig.hs
index 8cfae7f0c7..e8a0d13368 100644
--- a/TestConfig.hs
+++ b/Build/TestConfig.hs
@@ -1,6 +1,6 @@
-{- Tests the system and generates SysConfig.hs. -}
+{- Tests the system and generates Build.SysConfig.hs. -}
 
-module TestConfig where
+module Build.TestConfig where
 
 import System.IO
 import System.Cmd
@@ -33,12 +33,12 @@ instance Show Config where
 			valuetype (MaybeStringConfig _) = "Maybe String"
 
 writeSysConfig :: [Config] -> IO ()
-writeSysConfig config = writeFile "SysConfig.hs" body
+writeSysConfig config = writeFile "Build/SysConfig.hs" body
 	where
 		body = unlines $ header ++ map show config ++ footer
 		header = [
 			  "{- Automatically generated. -}"
-			, "module SysConfig where"
+			, "module Build.SysConfig where"
 			, ""
 			]
 		footer = []
diff --git a/Command/Add.hs b/Command/Add.hs
index d8947fb07c..407a36093f 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -24,7 +24,7 @@ import Types
 import Content
 import Messages
 import Utility
-import Touch
+import Utility.Touch
 import Locations
 
 command :: [Command]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index add71c8209..72a6b671dd 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -14,7 +14,7 @@ import System.Directory
 
 import Command
 import qualified Backend
-import qualified Remote.Helper.Url
+import qualified Utility.Url as Url
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
@@ -53,7 +53,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Remote.Helper.Url.download url tmp
+	ok <- Url.download url tmp
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
diff --git a/Command/Map.hs b/Command/Map.hs
index 75c5b0b550..3fd6e42a10 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -22,7 +22,7 @@ import Types
 import Utility
 import UUID
 import Trust
-import Remote.Helper.Ssh
+import Utility.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Command/Version.hs b/Command/Version.hs
index 2392c9bf6b..1ff829a22a 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -12,7 +12,7 @@ import Data.String.Utils
 import Data.Maybe
 
 import Command
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 import Version
 
 command :: [Command]
diff --git a/Content.hs b/Content.hs
index c63042dfb5..e89f4c45af 100644
--- a/Content.hs
+++ b/Content.hs
@@ -41,7 +41,7 @@ import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
 import Utility
-import StatFS
+import Utility.StatFS
 import Types.Key
 import Utility.DataUnits
 import Config
diff --git a/Makefile b/Makefile
index ccf98f6252..fa087c1ae2 100644
--- a/Makefile
+++ b/Makefile
@@ -8,12 +8,11 @@ GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell git-union-merge
 mans=git-annex.1 git-annex-shell.1 git-union-merge.1
+sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
 
 all: $(bins) $(mans) docs
 
-sources: SysConfig.hs StatFS.hs Touch.hs Remote/S3.hs
-
-SysConfig.hs: configure.hs TestConfig.hs
+Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	$(GHCMAKE) configure
 	./configure
 
@@ -30,7 +29,7 @@ Remote/S3.o: Remote/S3.hs
 		echo "** building without S3 support"; \
 	fi
 
-$(bins): SysConfig.hs Touch.hs StatFS.hs Remote/S3.o
+$(bins): $(sources)
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
@@ -82,8 +81,7 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \
-		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs
+	rm -rf build $(bins) $(mans) test configure  *.tix .hpc $(sources)
 	rm -rf doc/.ikiwiki html dist
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index c82f84745d..069209792d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -30,7 +30,7 @@ import Locations
 import Config
 import Utility
 import Messages
-import Remote.Helper.Ssh
+import Utility.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d8ecd33c4a..80ba8a153d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -25,8 +25,8 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Remote.Helper.Ssh
-import qualified Remote.Helper.Url as Url
+import Utility.Ssh
+import qualified Utility.Url as Url
 import Config
 import Init
 
diff --git a/Remote/Web.hs b/Remote/Web.hs
index cc96d5306d..5bc6a204b9 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -24,7 +24,7 @@ import Config
 import PresenceLog
 import LocationLog
 import Locations
-import qualified Remote.Helper.Url as Url
+import qualified Utility.Url as Url
 
 type URLString = String
 
diff --git a/UUID.hs b/UUID.hs
index cfa519baa3..fa71bed391 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -33,7 +33,7 @@ import qualified Branch
 import Types
 import Types.UUID
 import qualified Annex
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 import Config
 
 configkey :: String
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 2e06dd92bb..befb00f8f2 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -10,7 +10,7 @@ module Utility.CopyFile (copyFile) where
 import System.Directory (doesFileExist, removeFile)
 
 import Utility
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
  - and because this allows easy access to features like cp --reflink. -}
diff --git a/Remote/Helper/Ssh.hs b/Utility/Ssh.hs
similarity index 98%
rename from Remote/Helper/Ssh.hs
rename to Utility/Ssh.hs
index 478b018812..6cbc362a03 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Helper.Ssh where
+module Utility.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/StatFS.hsc b/Utility/StatFS.hsc
similarity index 98%
rename from StatFS.hsc
rename to Utility/StatFS.hsc
index feb82aa9af..d3e4a507e5 100644
--- a/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -45,7 +45,7 @@
 {-# LANGUAGE CPP, ForeignFunctionInterface, EmptyDataDecls #-}
 
 
-module StatFS ( FileSystemStats(..), getFileSystemStats ) where
+module Utility.StatFS ( FileSystemStats(..), getFileSystemStats ) where
 
 import Foreign
 import Foreign.C.Types
diff --git a/Touch.hsc b/Utility/Touch.hsc
similarity index 99%
rename from Touch.hsc
rename to Utility/Touch.hsc
index dd0c38984e..f27ac31360 100644
--- a/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -7,7 +7,7 @@
 
 {-# LANGUAGE ForeignFunctionInterface #-}
 
-module Touch (
+module Utility.Touch (
 	TimeSpec(..),
 	touchBoth,
 	touch
diff --git a/Remote/Helper/Url.hs b/Utility/Url.hs
similarity index 96%
rename from Remote/Helper/Url.hs
rename to Utility/Url.hs
index af1fee8f04..5954e0ff7e 100644
--- a/Remote/Helper/Url.hs
+++ b/Utility/Url.hs
@@ -1,11 +1,11 @@
-{- Url downloading for remotes.
+{- Url downloading.
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Helper.Url (
+module Utility.Url (
 	exists,
 	download,
 	get
diff --git a/configure.hs b/configure.hs
index bfdfa32dd5..9f7179c539 100644
--- a/configure.hs
+++ b/configure.hs
@@ -3,7 +3,7 @@
 import System.Directory
 import Data.List
 
-import TestConfig
+import Build.TestConfig
 
 tests :: [TestCase]
 tests =

From 1c25df3c1b19ad8406cbe2aef9dfd215ef9f5ce8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 12:59:49 -0400
Subject: [PATCH 2138/2835] outdated comment

---
 AnnexQueue.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 79116c48af..56d71f3238 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -19,8 +19,7 @@ import Messages
 import qualified Git.Queue
 import Utility
 
-{- Adds a git command to the queue, possibly running previously queued
- - actions if enough have accumulated. -}
+{- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
 add command params files = do
 	q <- getState repoqueue

From 06ce5741365ae4602ca64204729e6a23705059ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 13:17:12 -0400
Subject: [PATCH 2139/2835] tweak

---
 Config.hs  | 5 ++---
 Utility.hs | 6 +++---
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/Config.hs b/Config.hs
index 568eb71386..493a907009 100644
--- a/Config.hs
+++ b/Config.hs
@@ -51,11 +51,10 @@ remoteCost r def = do
 			else getConfig r "cost" ""
 	where
 		safeparse v
-			| null ws || null ps = def
-			| otherwise = (fst . head) ps
+			| null ws = def
+			| otherwise = fromMaybe def $ readMaybe $ head ws
 			where
 				ws = words v
-				ps = reads $ head ws
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/Utility.hs b/Utility.hs
index 511350898a..8a332601b7 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -75,7 +75,7 @@ toCommand = (>>= unwrap)
 		unwrap (Params s) = filter (not . null) (split " " s)
 		-- Files that start with a dash are modified to avoid
 		-- the command interpreting them as options.
-		unwrap (File ('-':s)) = ["./-" ++ s]
+		unwrap (File s@('-':_)) = ["./" ++ s]
 		unwrap (File s) = [s]
 
 {- Run a system command, and returns True or False
@@ -257,7 +257,7 @@ viaTmp a file content = do
 
 {- Runs an action with a temp file, then removes the file. -}
 withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
-withTempFile template action = bracket create remove use
+withTempFile template a = bracket create remove use
 	where
 		create = do
 			tmpdir <- catch getTemporaryDirectory (const $ return ".")
@@ -265,7 +265,7 @@ withTempFile template action = bracket create remove use
 		remove (name, handle) = do
 			hClose handle
 			catchBool (removeFile name >> return True)
-		use (name, handle) = action name handle
+		use (name, handle) = a name handle
 
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}

From 06f509854af4e019434ddca73309fcd2a5b47463 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 13:19:33 -0400
Subject: [PATCH 2140/2835] file moved

---
 debian/copyright | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/copyright b/debian/copyright
index 112ad54aaa..a8a38913e4 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -8,7 +8,7 @@ License: GPL-3+
  this package's source, or in /usr/share/common-licenses/GPL-3 on
  Debian systems.
 
-Files: StatFS.hsc
+Files: Utility/StatFS.hsc
 Copyright: Jose A Ortega Ruiz 
 License: BSD-3-clause
  -- All rights reserved.

From 4c73d77b42e97ad740d5731ad73c40a31c0c84f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 14:59:34 -0400
Subject: [PATCH 2141/2835] avoid the functor

fmap = liftM
---
 Annex.hs  | 1 -
 Branch.hs | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 46dfc9df56..07316bd370 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -33,7 +33,6 @@ import Types.UUID
 -- git-annex's monad
 newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 	deriving (
-		Functor,
 		Monad,
 		MonadIO,
 		MonadControlIO,
diff --git a/Branch.hs b/Branch.hs
index fd4ce4cede..6abf6d9c2c 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -321,7 +321,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = fmap (map fileJournal) getJournalFilesRaw
+getJournalFiles = liftM (map fileJournal) getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do

From 203148363f459635b1be40b8c6ed376073230dda Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 22 Aug 2011 16:14:12 -0400
Subject: [PATCH 2142/2835] split groups of related functions out of Utility

---
 .gitignore               |   4 +-
 AnnexQueue.hs            |   2 +-
 Backend/SHA.hs           |   2 +-
 Branch.hs                |   2 +
 Command/Add.hs           |   3 +-
 Command/AddUrl.hs        |   2 +-
 Command/Drop.hs          |   1 +
 Command/DropUnused.hs    |   2 +-
 Command/Find.hs          |   2 +-
 Command/Fix.hs           |   3 +-
 Command/FromKey.hs       |   3 +-
 Command/Fsck.hs          |   1 +
 Command/Lock.hs          |   2 +-
 Command/Map.hs           |   2 +-
 Command/Migrate.hs       |   2 +-
 Command/RecvKey.hs       |   2 +-
 Command/SendKey.hs       |   2 +-
 Command/SetKey.hs        |   2 +-
 Command/Unannex.hs       |   3 +-
 Command/Uninit.hs        |   2 +-
 Command/Unlock.hs        |   3 +-
 Config.hs                |   1 +
 Content.hs               |   2 +
 Crypto.hs                |   1 +
 Git.hs                   |   3 +
 Git/LsFiles.hs           |   2 +-
 Git/Queue.hs             |   8 +-
 Git/UnionMerge.hs        |   2 +-
 Init.hs                  |   1 +
 Remote/Bup.hs            |   2 +
 Remote/Directory.hs      |   2 +
 Remote/Git.hs            |   2 +
 Remote/Helper/Special.hs |   2 +-
 Remote/Hook.hs           |   1 +
 Remote/Rsync.hs          |   3 +
 Upgrade/V1.hs            |   2 +
 Upgrade/V2.hs            |   2 +
 Utility.hs               | 215 +--------------------------------------
 Utility/Conditional.hs   |  26 +++++
 Utility/CopyFile.hs      |   3 +-
 Utility/Path.hs          |  92 +++++++++++++++++
 Utility/RsyncFile.hs     |   2 +-
 Utility/SafeCommand.hs   | 104 +++++++++++++++++++
 Utility/Ssh.hs           |   2 +-
 Utility/Url.hs           |   2 +-
 git-annex-shell.hs       |   3 +-
 test.hs                  |  45 ++++----
 47 files changed, 312 insertions(+), 265 deletions(-)
 create mode 100644 Utility/Conditional.hs
 create mode 100644 Utility/Path.hs
 create mode 100644 Utility/SafeCommand.hs

diff --git a/.gitignore b/.gitignore
index 1fdad216dc..7d2504de6f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,7 +13,7 @@ doc/.ikiwiki
 html
 *.tix
 .hpc
-Touch.hs
-StatFS.hs
+Utility/Touch.hs
+Utility/StatFS.hs
 Remote/S3.hs
 dist
diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 56d71f3238..d155a7b81f 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -17,7 +17,7 @@ import Control.Monad (when, unless)
 import Annex
 import Messages
 import qualified Git.Queue
-import Utility
+import Utility.SafeCommand
 
 {- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index b8a0d254b7..ed2a47db9b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -23,7 +23,7 @@ import Content
 import Types
 import Types.Backend
 import Types.Key
-import Utility
+import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
diff --git a/Branch.hs b/Branch.hs
index 6abf6d9c2c..d5bfe1b09f 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -37,6 +37,8 @@ import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Types
 import Messages
 import Locations
diff --git a/Command/Add.hs b/Command/Add.hs
index 407a36093f..579c4171b1 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -23,8 +23,9 @@ import LocationLog
 import Types
 import Content
 import Messages
-import Utility
+import Utility.Conditional
 import Utility.Touch
+import Utility.SafeCommand
 import Locations
 
 command :: [Command]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 72a6b671dd..55e51100ce 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -23,7 +23,7 @@ import Messages
 import Content
 import PresenceLog
 import Locations
-import Utility
+import Utility.Path
 
 command :: [Command]
 command = [repoCommand "addurl" paramPath seek "add urls to annex"]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 14f098349e..6e688d6632 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -15,6 +15,7 @@ import Types
 import Content
 import Messages
 import Utility
+import Utility.Conditional
 import Trust
 import Config
 
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 41bcd6aa78..4ad2aa85bb 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -22,7 +22,7 @@ import qualified Command.Move
 import qualified Remote
 import qualified Git
 import Types.Key
-import Utility
+import Utility.Conditional
 
 type UnusedMap = M.Map String Key
 
diff --git a/Command/Find.hs b/Command/Find.hs
index 9d760ff5a8..0716c5297e 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -11,7 +11,7 @@ import Control.Monad.State (liftIO)
 
 import Command
 import Content
-import Utility
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 47b0c4c9a0..b24f8e33c2 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -13,7 +13,8 @@ import System.Directory
 
 import Command
 import qualified AnnexQueue
-import Utility
+import Utility.Path
+import Utility.SafeCommand
 import Content
 import Messages
 
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index d59f1de397..89c3f4e912 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -14,10 +14,11 @@ import Control.Monad (unless)
 
 import Command
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
 import Content
 import Messages
 import Types.Key
+import Utility.Path
 
 command :: [Command]
 command = [repoCommand "fromkey" paramPath seek
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0d3ecb58f1..6ccec05fb3 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -27,6 +27,7 @@ import LocationLog
 import Locations
 import Trust
 import Utility.DataUnits
+import Utility.Path
 import Config
 
 command :: [Command]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index d39df5f335..77d1ff94f9 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -13,7 +13,7 @@ import System.Directory
 import Command
 import Messages
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
 	
 command :: [Command]
 command = [repoCommand "lock" paramPath seek "undo unlock command"]
diff --git a/Command/Map.hs b/Command/Map.hs
index 3fd6e42a10..ef8e04d909 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -19,7 +19,7 @@ import qualified Annex
 import qualified Git
 import Messages
 import Types
-import Utility
+import Utility.SafeCommand
 import UUID
 import Trust
 import Utility.Ssh
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 5ae8354406..25227ae162 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -20,7 +20,7 @@ import Locations
 import Types
 import Content
 import Messages
-import Utility
+import Utility.Conditional
 import qualified Command.Add
 
 command :: [Command]
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index e2f7c74abb..be6163558f 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -13,8 +13,8 @@ import System.Exit
 import Command
 import CmdLine
 import Content
-import Utility
 import Utility.RsyncFile
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 02fedb349e..f676ae947a 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -14,8 +14,8 @@ import Locations
 import qualified Annex
 import Command
 import Content
-import Utility
 import Utility.RsyncFile
+import Utility.Conditional
 import Messages
 
 command :: [Command]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 807cbd5b93..2f6f9ea9ee 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -10,7 +10,7 @@ module Command.SetKey where
 import Control.Monad.State (liftIO)
 
 import Command
-import Utility
+import Utility.SafeCommand
 import LocationLog
 import Content
 import Messages
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 960f99722f..54ef2fc68e 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -16,7 +16,8 @@ import Command
 import qualified Command.Drop
 import qualified Annex
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
+import Utility.Path
 import LocationLog
 import Types
 import Content
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 195246aa84..fadae0e5a9 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,7 +12,7 @@ import System.Directory
 import System.Exit
 
 import Command
-import Utility
+import Utility.SafeCommand
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 280eff9de7..0daf1b3218 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -16,8 +16,9 @@ import Types
 import Messages
 import Locations
 import Content
+import Utility.Conditional
 import Utility.CopyFile
-import Utility
+import Utility.Path
 
 command :: [Command]
 command =
diff --git a/Config.hs b/Config.hs
index 493a907009..12f6480470 100644
--- a/Config.hs
+++ b/Config.hs
@@ -16,6 +16,7 @@ import qualified Git
 import qualified Annex
 import Types
 import Utility
+import Utility.SafeCommand
 
 type ConfigKey = String
 
diff --git a/Content.hs b/Content.hs
index e89f4c45af..ba99f1330d 100644
--- a/Content.hs
+++ b/Content.hs
@@ -41,7 +41,9 @@ import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
 import Utility
+import Utility.Conditional
 import Utility.StatFS
+import Utility.Path
 import Types.Key
 import Utility.DataUnits
 import Config
diff --git a/Crypto.hs b/Crypto.hs
index 4fc41ede04..ed29747aa6 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -48,6 +48,7 @@ import Types.Key
 import Types.Remote
 import Utility
 import Utility.Base64
+import Utility.SafeCommand
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
diff --git a/Git.hs b/Git.hs
index 168fe4154e..7155b26343 100644
--- a/Git.hs
+++ b/Git.hs
@@ -85,6 +85,9 @@ import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
 import Utility
+import Utility.Path
+import Utility.Conditional
+import Utility.SafeCommand
 
 {- There are two types of repositories; those on local disk and those
  - accessed via an URL. -}
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 23b383a09d..1ecbb029b5 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -16,7 +16,7 @@ module Git.LsFiles (
 ) where
 
 import Git
-import Utility
+import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 0016be4727..25b9ffad08 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -19,15 +19,15 @@ import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
 import Control.Monad (forM_)
-import Utility
+import Utility.SafeCommand
 
 import Git
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
-data Action = Action {
-		getSubcommand :: String,
-		getParams :: [CommandParam]
+data Action = Action
+	{ getSubcommand :: String
+	, getParams :: [CommandParam]
 	} deriving (Show, Eq, Ord)
 
 {- A queue of actions to perform (in any order) on a git repository,
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index b0da071703..a5bcbeac4a 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -18,7 +18,7 @@ import Data.Maybe
 import Data.String.Utils
 
 import Git
-import Utility
+import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
diff --git a/Init.hs b/Init.hs
index a469657a11..2067c524cf 100644
--- a/Init.hs
+++ b/Init.hs
@@ -22,6 +22,7 @@ import Version
 import Messages
 import Types
 import Utility
+import Utility.Conditional
 import UUID
 
 initialize :: Annex ()
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 069209792d..ebb4b10a5b 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -29,6 +29,8 @@ import UUID
 import Locations
 import Config
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Messages
 import Utility.Ssh
 import Remote.Helper.Special
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index fd227f85d8..7ddb60462f 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -27,6 +27,8 @@ import Utility.CopyFile
 import Config
 import Content
 import Utility
+import Utility.Conditional
+import Utility.Path
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 80ba8a153d..fadd48a036 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -26,6 +26,8 @@ import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
+import Utility.SafeCommand
+import Utility.Path
 import qualified Utility.Url as Url
 import Config
 import Init
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index c302a0ff5e..b842588c0c 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -17,7 +17,7 @@ import Types.Remote
 import qualified Git
 import qualified Annex
 import UUID
-import Utility
+import Utility.SafeCommand
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index ef52d0482b..54b9806ffc 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -28,6 +28,7 @@ import Locations
 import Config
 import Content
 import Utility
+import Utility.SafeCommand
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index eba67e3fd6..3707966ad6 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -26,11 +26,14 @@ import Locations
 import Config
 import Content
 import Utility
+import Utility.Conditional
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
 import Messages
 import Utility.RsyncFile
+import Utility.SafeCommand
+import Utility.Path
 
 type RsyncUrl = String
 
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 160d9309fe..b4567a0b71 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -31,6 +31,8 @@ import Backend
 import Messages
 import Version
 import Utility
+import Utility.SafeCommand
+import Utility.Path
 import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 0b1d69f8e1..ffd0f06535 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -20,6 +20,8 @@ import qualified Git
 import qualified Branch
 import Messages
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import LocationLog
 import Content
 
diff --git a/Utility.hs b/Utility.hs
index 8a332601b7..788dc41030 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -6,20 +6,8 @@
  -}
 
 module Utility (
-	CommandParam(..),
-	toCommand,
 	hGetContentsStrict,
 	readFileStrict,
-	parentDir,
-	absPath,
-	absPathFrom,
-	relPathCwdToFile,
-	relPathDirToFile,
-	boolSystem,
-	boolSystemEnv,
-	executeFile,
-	shellEscape,
-	shellUnEscape,
 	unsetFileMode,
 	readMaybe,
 	viaTmp,
@@ -27,125 +15,19 @@ module Utility (
 	dirContains,
 	dirContents,
 	myHomeDir,
-	catchBool,
-	whenM,
-	(>>?),
-	unlessM,
-	(>>!),
-	
-	prop_idempotent_shellEscape,
-	prop_idempotent_shellEscape_multiword,
-	prop_parentDir_basics,
-	prop_relPathDirToFile_basics
+	catchBool
 ) where
 
 import IO (bracket)
 import System.IO
-import System.Exit
-import qualified System.Posix.Process
 import System.Posix.Process hiding (executeFile)
-import System.Posix.Signals
 import System.Posix.Files
 import System.Posix.Types
 import System.Posix.User
-import Data.String.Utils
-import System.Path
 import System.FilePath
 import System.Directory
 import Foreign (complement)
-import Data.List
-import Data.Maybe
-import Control.Monad (liftM2, when, unless)
-import System.Log.Logger
-
-{- A type for parameters passed to a shell command. A command can
- - be passed either some Params (multiple parameters can be included,
- - whitespace-separated, or a single Param (for when parameters contain
- - whitespace), or a File.
- -}
-data CommandParam = Params String | Param String | File FilePath
-	deriving (Eq, Show, Ord)
-
-{- Used to pass a list of CommandParams to a function that runs
- - a command and expects Strings. -}
-toCommand :: [CommandParam] -> [String]
-toCommand = (>>= unwrap)
-	where
-		unwrap (Param s) = [s]
-		unwrap (Params s) = filter (not . null) (split " " s)
-		-- Files that start with a dash are modified to avoid
-		-- the command interpreting them as options.
-		unwrap (File s@('-':_)) = ["./" ++ s]
-		unwrap (File s) = [s]
-
-{- Run a system command, and returns True or False
- - if it succeeded or failed.
- -
- - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
- -}
-boolSystem :: FilePath -> [CommandParam] -> IO Bool
-boolSystem command params = boolSystemEnv command params Nothing
-
-boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
-boolSystemEnv command params env = do
-	-- Going low-level because all the high-level system functions
-	-- block SIGINT etc. We need to block SIGCHLD, but allow
-	-- SIGINT to do its default program termination.
-	let sigset = addSignal sigCHLD emptySignalSet
-	oldint <- installHandler sigINT Default Nothing
-	oldset <- getSignalMask
-	blockSignals sigset
-	childpid <- forkProcess $ childaction oldint oldset
-	mps <- getProcessStatus True False childpid
-	restoresignals oldint oldset
-	case mps of
-		Just (Exited ExitSuccess) -> return True
-		_ -> return False
-	where
-		restoresignals oldint oldset = do
-			_ <- installHandler sigINT oldint Nothing
-			setSignalMask oldset
-		childaction oldint oldset = do
-			restoresignals oldint oldset
-			executeFile command True (toCommand params) env
-
-{- executeFile with debug logging -}
-executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
-executeFile c path p e = do
-	debugM "Utility.executeFile" $
-		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
-	System.Posix.Process.executeFile c path p e
-
-{- Escapes a filename or other parameter to be safely able to be exposed to
- - the shell. -}
-shellEscape :: String -> String
-shellEscape f = "'" ++ escaped ++ "'"
-	where
-		-- replace ' with '"'"'
-		escaped = join "'\"'\"'" $ split "'" f
-
-{- Unescapes a set of shellEscaped words or filenames. -}
-shellUnEscape :: String -> [String]
-shellUnEscape [] = []
-shellUnEscape s = word : shellUnEscape rest
-	where
-		(word, rest) = findword "" s
-		findword w [] = (w, "")
-		findword w (c:cs)
-			| c == ' ' = (w, cs)
-			| c == '\'' = inquote c w cs
-			| c == '"' = inquote c w cs
-			| otherwise = findword (w++[c]) cs
-		inquote _ w [] = (w, "")
-		inquote q w (c:cs)
-			| c == q = findword w cs
-			| otherwise = inquote q (w++[c]) cs
-
-{- For quickcheck. -}
-prop_idempotent_shellEscape :: String -> Bool
-prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
-prop_idempotent_shellEscape_multiword :: [String] -> Bool
-prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
+import Utility.Path
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -156,82 +38,6 @@ hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict f = readFile f >>= \s -> length s `seq` return s
 
-{- Returns the parent directory of a path. Parent of / is "" -}
-parentDir :: FilePath -> FilePath
-parentDir dir =
-	if not $ null dirs
-	then slash ++ join s (take (length dirs - 1) dirs)
-	else ""
-		where
-			dirs = filter (not . null) $ split s dir
-			slash = if isAbsolute dir then s else ""
-			s = [pathSeparator]
-
-prop_parentDir_basics :: FilePath -> Bool
-prop_parentDir_basics dir
-	| null dir = True
-	| dir == "/" = parentDir dir == ""
-	| otherwise = p /= dir
-	where
-		p = parentDir dir
-
-{- Checks if the first FilePath is, or could be said to contain the second.
- - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
- - are all equivilant.
- -}
-dirContains :: FilePath -> FilePath -> Bool
-dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
-	where
-		norm p = fromMaybe "" $ absNormPath p "."
-		a' = norm a
-		b' = norm b
-
-{- Converts a filename into a normalized, absolute path. -}
-absPath :: FilePath -> IO FilePath
-absPath file = do
-	cwd <- getCurrentDirectory
-	return $ absPathFrom cwd file
-
-{- Converts a filename into a normalized, absolute path
- - from the specified cwd. -}
-absPathFrom :: FilePath -> FilePath -> FilePath
-absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
-	where
-		bad = error $ "unable to normalize " ++ file
-
-{- Constructs a relative path from the CWD to a file.
- -
- - For example, assuming CWD is /tmp/foo/bar:
- -    relPathCwdToFile "/tmp/foo" == ".."
- -    relPathCwdToFile "/tmp/foo/bar" == "" 
- -}
-relPathCwdToFile :: FilePath -> IO FilePath
-relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
-
-{- Constructs a relative path from a directory to a file.
- -
- - Both must be absolute, and normalized (eg with absNormpath).
- -}
-relPathDirToFile :: FilePath -> FilePath -> FilePath
-relPathDirToFile from to = path
-	where
-		s = [pathSeparator]
-		pfrom = split s from
-		pto = split s to
-		common = map fst $ filter same $ zip pfrom pto
-		same (c,d) = c == d
-		uncommon = drop numcommon pto
-		dotdots = replicate (length pfrom - numcommon) ".."
-		numcommon = length common
-		path = join s $ dotdots ++ uncommon
-
-prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
-prop_relPathDirToFile_basics from to
-	| from == to = null r
-	| otherwise = not (null r)
-	where
-		r = relPathDirToFile from to 
-
 {- Removes a FileMode from a file.
  - For example, call with otherWriteMode to chmod o-w -}
 unsetFileMode :: FilePath -> FileMode -> IO ()
@@ -288,20 +94,3 @@ myHomeDir = do
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
-
-{- when with a monadic conditional -}
-whenM :: Monad m => m Bool -> m () -> m ()
-whenM c a = c >>= flip when a
-
-unlessM :: Monad m => m Bool -> m () -> m ()
-unlessM c a = c >>= flip unless a
-
-(>>?) :: Monad m => m Bool -> m () -> m ()
-(>>?) = whenM
-
-(>>!) :: Monad m => m Bool -> m () -> m ()
-(>>!) = unlessM
-
--- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
-infixr 0 >>?
-infixr 0 >>!
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
new file mode 100644
index 0000000000..85e39ec64c
--- /dev/null
+++ b/Utility/Conditional.hs
@@ -0,0 +1,26 @@
+{- monadic conditional operators
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Conditional where
+
+import Control.Monad (when, unless)
+
+whenM :: Monad m => m Bool -> m () -> m ()
+whenM c a = c >>= flip when a
+
+unlessM :: Monad m => m Bool -> m () -> m ()
+unlessM c a = c >>= flip unless a
+
+(>>?) :: Monad m => m Bool -> m () -> m ()
+(>>?) = whenM
+
+(>>!) :: Monad m => m Bool -> m () -> m ()
+(>>!) = unlessM
+
+-- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
+infixr 0 >>?
+infixr 0 >>!
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index befb00f8f2..9019357196 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -9,7 +9,8 @@ module Utility.CopyFile (copyFile) where
 
 import System.Directory (doesFileExist, removeFile)
 
-import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
diff --git a/Utility/Path.hs b/Utility/Path.hs
new file mode 100644
index 0000000000..517c175bc4
--- /dev/null
+++ b/Utility/Path.hs
@@ -0,0 +1,92 @@
+{- path manipulation
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Path where
+
+import Data.String.Utils
+import System.Path
+import System.FilePath
+import System.Directory
+import Data.List
+import Data.Maybe
+import Control.Monad (liftM2)
+
+{- Returns the parent directory of a path. Parent of / is "" -}
+parentDir :: FilePath -> FilePath
+parentDir dir =
+	if not $ null dirs
+	then slash ++ join s (take (length dirs - 1) dirs)
+	else ""
+		where
+			dirs = filter (not . null) $ split s dir
+			slash = if isAbsolute dir then s else ""
+			s = [pathSeparator]
+
+prop_parentDir_basics :: FilePath -> Bool
+prop_parentDir_basics dir
+	| null dir = True
+	| dir == "/" = parentDir dir == ""
+	| otherwise = p /= dir
+	where
+		p = parentDir dir
+
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
+	where
+		norm p = fromMaybe "" $ absNormPath p "."
+		a' = norm a
+		b' = norm b
+
+{- Converts a filename into a normalized, absolute path. -}
+absPath :: FilePath -> IO FilePath
+absPath file = do
+	cwd <- getCurrentDirectory
+	return $ absPathFrom cwd file
+
+{- Converts a filename into a normalized, absolute path
+ - from the specified cwd. -}
+absPathFrom :: FilePath -> FilePath -> FilePath
+absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
+	where
+		bad = error $ "unable to normalize " ++ file
+
+{- Constructs a relative path from the CWD to a file.
+ -
+ - For example, assuming CWD is /tmp/foo/bar:
+ -    relPathCwdToFile "/tmp/foo" == ".."
+ -    relPathCwdToFile "/tmp/foo/bar" == "" 
+ -}
+relPathCwdToFile :: FilePath -> IO FilePath
+relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
+
+{- Constructs a relative path from a directory to a file.
+ -
+ - Both must be absolute, and normalized (eg with absNormpath).
+ -}
+relPathDirToFile :: FilePath -> FilePath -> FilePath
+relPathDirToFile from to = path
+	where
+		s = [pathSeparator]
+		pfrom = split s from
+		pto = split s to
+		common = map fst $ filter same $ zip pfrom pto
+		same (c,d) = c == d
+		uncommon = drop numcommon pto
+		dotdots = replicate (length pfrom - numcommon) ".."
+		numcommon = length common
+		path = join s $ dotdots ++ uncommon
+
+prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
+prop_relPathDirToFile_basics from to
+	| from == to = null r
+	| otherwise = not (null r)
+	where
+		r = relPathDirToFile from to 
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index 6e21ba0632..b6c2267e87 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -9,7 +9,7 @@ module Utility.RsyncFile where
 
 import Data.String.Utils
 
-import Utility
+import Utility.SafeCommand
 
 {- Generates parameters to make rsync use a specified command as its remote
  - shell. -}
diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs
new file mode 100644
index 0000000000..ba9362603e
--- /dev/null
+++ b/Utility/SafeCommand.hs
@@ -0,0 +1,104 @@
+{- safely running shell commands
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.SafeCommand where
+
+import System.Exit
+import qualified System.Posix.Process
+import System.Posix.Process hiding (executeFile)
+import System.Posix.Signals
+import Data.String.Utils
+import System.Log.Logger
+
+{- A type for parameters passed to a shell command. A command can
+ - be passed either some Params (multiple parameters can be included,
+ - whitespace-separated, or a single Param (for when parameters contain
+ - whitespace), or a File.
+ -}
+data CommandParam = Params String | Param String | File FilePath
+	deriving (Eq, Show, Ord)
+
+{- Used to pass a list of CommandParams to a function that runs
+ - a command and expects Strings. -}
+toCommand :: [CommandParam] -> [String]
+toCommand = (>>= unwrap)
+	where
+		unwrap (Param s) = [s]
+		unwrap (Params s) = filter (not . null) (split " " s)
+		-- Files that start with a dash are modified to avoid
+		-- the command interpreting them as options.
+		unwrap (File s@('-':_)) = ["./" ++ s]
+		unwrap (File s) = [s]
+
+{- Run a system command, and returns True or False
+ - if it succeeded or failed.
+ -
+ - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
+ -}
+boolSystem :: FilePath -> [CommandParam] -> IO Bool
+boolSystem command params = boolSystemEnv command params Nothing
+
+boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
+boolSystemEnv command params env = do
+	-- Going low-level because all the high-level system functions
+	-- block SIGINT etc. We need to block SIGCHLD, but allow
+	-- SIGINT to do its default program termination.
+	let sigset = addSignal sigCHLD emptySignalSet
+	oldint <- installHandler sigINT Default Nothing
+	oldset <- getSignalMask
+	blockSignals sigset
+	childpid <- forkProcess $ childaction oldint oldset
+	mps <- getProcessStatus True False childpid
+	restoresignals oldint oldset
+	case mps of
+		Just (Exited ExitSuccess) -> return True
+		_ -> return False
+	where
+		restoresignals oldint oldset = do
+			_ <- installHandler sigINT oldint Nothing
+			setSignalMask oldset
+		childaction oldint oldset = do
+			restoresignals oldint oldset
+			executeFile command True (toCommand params) env
+
+{- executeFile with debug logging -}
+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
+executeFile c path p e = do
+	debugM "Utility.SafeCommand.executeFile" $
+		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
+	System.Posix.Process.executeFile c path p e
+
+{- Escapes a filename or other parameter to be safely able to be exposed to
+ - the shell. -}
+shellEscape :: String -> String
+shellEscape f = "'" ++ escaped ++ "'"
+	where
+		-- replace ' with '"'"'
+		escaped = join "'\"'\"'" $ split "'" f
+
+{- Unescapes a set of shellEscaped words or filenames. -}
+shellUnEscape :: String -> [String]
+shellUnEscape [] = []
+shellUnEscape s = word : shellUnEscape rest
+	where
+		(word, rest) = findword "" s
+		findword w [] = (w, "")
+		findword w (c:cs)
+			| c == ' ' = (w, cs)
+			| c == '\'' = inquote c w cs
+			| c == '"' = inquote c w cs
+			| otherwise = findword (w++[c]) cs
+		inquote _ w [] = (w, "")
+		inquote q w (c:cs)
+			| c == q = findword w cs
+			| otherwise = inquote q (w++[c]) cs
+
+{- For quickcheck. -}
+prop_idempotent_shellEscape :: String -> Bool
+prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
+prop_idempotent_shellEscape_multiword :: [String] -> Bool
+prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 6cbc362a03..05269552c7 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -10,7 +10,7 @@ module Utility.Ssh where
 import Control.Monad.State (liftIO)
 
 import qualified Git
-import Utility
+import Utility.SafeCommand
 import Types
 import Config
 
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 5954e0ff7e..69b53c34c0 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -19,7 +19,7 @@ import Network.URI
 
 import Types
 import Messages
-import Utility
+import Utility.SafeCommand
 
 type URLString = String
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 29ac63aea1..1fb928f9da 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -11,7 +11,8 @@ import Data.List
 import qualified Git
 import CmdLine
 import Command
-import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Options
 
 import qualified Command.ConfigList
diff --git a/test.hs b/test.hs
index 2352df36a6..e4e1fb131d 100644
--- a/test.hs
+++ b/test.hs
@@ -24,11 +24,12 @@ import qualified Data.Map as M
 import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
 
+import Utility.SafeCommand
+
 import qualified Annex
 import qualified Backend
 import qualified Git
 import qualified Locations
-import qualified Utility
 import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
@@ -42,6 +43,7 @@ import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Config
 import qualified Crypto
+import qualified Utility.Path
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -72,11 +74,12 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
-	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
-	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
+	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape
+	, qctest "prop_idempotent_shellEscape_multiword" Utility.SafeCommand.prop_idempotent_shellEscape_multiword
 	, qctest "prop_idempotent_configEscape" RemoteLog.prop_idempotent_configEscape
-	, qctest "prop_parentDir_basics" Utility.prop_parentDir_basics
-	, qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics
+	, qctest "prop_parentDir_basics" Utility.Path.prop_parentDir_basics
+
+	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
 	]
@@ -117,8 +120,8 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			git_annex "add" ["-q", annexedfile] @? "add failed"
 			annexed_present annexedfile
 			writeFile ingitfile $ content ingitfile
-			Utility.boolSystem "git" [Utility.Param "add", Utility.File ingitfile] @? "git add failed"
-			Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed"
+			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
+			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
 			unannexed ingitfile
 		sha1dup = TestCase $ intmpclonerepo $ do
@@ -145,7 +148,7 @@ test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
 	let key = show $ fromJust r
 	git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed"
 	git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed"
-	Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed"
+	boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 	annexed_present sha1annexedfile
 	where
 		tmp = "tmpfile"
@@ -172,7 +175,7 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do
 			git_annex "get" ["-q", annexedfile] @? "get failed"
-			Utility.boolSystem "git" [Utility.Params "remote rm origin"]
+			boolSystem "git" [Params "remote rm origin"]
 				@? "git remote rm origin failed"
 			r <- git_annex "drop" ["-q", annexedfile]
 			not r @? "drop wrongly succeeded with no known copy of file"
@@ -303,12 +306,12 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 			then do
 				-- pre-commit depends on the file being
 				-- staged, normally git commit does this
-				Utility.boolSystem "git" [Utility.Param "add", Utility.File annexedfile]
+				boolSystem "git" [Param "add", File annexedfile]
 					@? "git add of edited file failed"
 				git_annex "pre-commit" ["-q"]
 					@? "pre-commit failed"
 			else do
-				Utility.boolSystem "git" [Utility.Params "commit -q -a -m contentchanged"]
+				boolSystem "git" [Params "commit -q -a -m contentchanged"]
 					@? "git commit of edited file failed"
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
@@ -326,7 +329,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 	git_annex "fix" ["-q", annexedfile] @? "fix of present file failed"
 	annexed_present annexedfile
 	createDirectory subdir
-	Utility.boolSystem "git" [Utility.Param "mv", Utility.File annexedfile, Utility.File subdir]
+	boolSystem "git" [Param "mv", File annexedfile, File subdir]
 		@? "git mv failed"
 	git_annex "fix" ["-q", newfile] @? "fix of moved file failed"
 	runchecks [checklink, checkunwritable] newfile
@@ -364,9 +367,9 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
 			git_annex "fsck" ["-q"] @? "fsck failed"
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			fsck_should_fail "numcopies unsatisfied"
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 1"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
 		withlocaluntrusted = TestCase $ intmpclonerepo $ do
@@ -377,7 +380,7 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			git_annex "trust" ["-q", "."] @? "trust of current repo failed"
 			git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo"
 		withremoteuntrusted = TestCase $ intmpclonerepo $ do
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			git_annex "get" ["-q", annexedfile] @? "get failed"
 			git_annex "get" ["-q", sha1annexedfile] @? "get failed"
 			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
@@ -448,9 +451,9 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	git_annex "get" ["-q", annexedfile] @? "get of file failed"
 	git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
 	checkunused []
-	Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
 	checkunused [annexedfilekey]
-	Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File sha1annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed"
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also
@@ -526,10 +529,10 @@ setuprepo :: FilePath -> IO FilePath
 setuprepo dir = do
 	cleanup dir
 	ensuretmpdir
-	Utility.boolSystem "git" [Utility.Params "init -q", Utility.File dir] @? "git init failed"
+	boolSystem "git" [Params "init -q", File dir] @? "git init failed"
 	indir dir $ do
-		Utility.boolSystem "git" [Utility.Params "config user.name", Utility.Param "Test User"] @? "git config failed"
-		Utility.boolSystem "git" [Utility.Params "config user.email test@example.com"] @? "git config failed"
+		boolSystem "git" [Params "config user.name", Param "Test User"] @? "git config failed"
+		boolSystem "git" [Params "config user.email test@example.com"] @? "git config failed"
 	return dir
 
 -- clones are always done as local clones; we cannot test ssh clones
@@ -537,7 +540,7 @@ clonerepo :: FilePath -> FilePath -> IO FilePath
 clonerepo old new = do
 	cleanup new
 	ensuretmpdir
-	Utility.boolSystem "git" [Utility.Params "clone -q", Utility.File old, Utility.File new] @? "git clone failed"
+	boolSystem "git" [Params "clone -q", File old, File new] @? "git clone failed"
 	indir new $ git_annex "init" ["-q", new] @? "git annex init failed"
 	return new
 	

From 20259c2955e408a72e0960207fc8be4cbeec2e21 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 23 Aug 2011 13:41:32 -0400
Subject: [PATCH 2143/2835] Set EMAIL when running test suite so that git does
 not need to be configured first. Closes: #638998

---
 debian/changelog | 7 +++++++
 test.hs          | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index a8280a156d..bb06e1fba9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20110820) UNRELEASED; urgency=low
+
+  * Set EMAIL when running test suite so that git does not need to be
+    configured first. Closes: #638998
+
+ -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
+
 git-annex (3.20110819) unstable; urgency=low
 
   * Now "git annex init" only has to be run once, when a git repository
diff --git a/test.hs b/test.hs
index e4e1fb131d..4d751a707b 100644
--- a/test.hs
+++ b/test.hs
@@ -646,6 +646,9 @@ prepare = do
 	p <- getEnvDefault  "PATH" ""
 	setEnv "PATH" (cwd ++ ":" ++ p) True
 	setEnv "TOPDIR" cwd True
+	-- Avoid git complaining if it cannot determine the user's email
+	-- address.
+	setEnv "EMAIL" "git-annex test " True
 
 changeToTmpDir :: FilePath -> IO ()
 changeToTmpDir t = do

From 678726c10c13481c082743808a5188d28567e2b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 25 Aug 2011 00:28:55 -0400
Subject: [PATCH 2144/2835] code simplification thanks to applicative functors

---
 Annex.hs           |  7 +++++--
 Branch.hs          |  7 ++++---
 Command.hs         |  7 ++++---
 Command/Migrate.hs |  3 ++-
 Command/Status.hs  | 10 ++++------
 Config.hs          |  6 +++---
 Crypto.hs          |  3 ++-
 Git.hs             |  3 ++-
 LocationLog.hs     |  3 ++-
 PresenceLog.hs     |  3 ++-
 Remote.hs          | 13 +++++++------
 RemoteLog.hs       |  3 ++-
 Upgrade/V1.hs      |  5 +++--
 Utility/Path.hs    |  4 ++--
 Utility/Url.hs     |  5 ++---
 15 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 07316bd370..287aed875e 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -20,6 +20,7 @@ module Annex (
 
 import Control.Monad.State
 import Control.Monad.IO.Control
+import Control.Applicative hiding (empty)
 
 import qualified Git
 import Git.Queue
@@ -36,7 +37,9 @@ newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 		Monad,
 		MonadIO,
 		MonadControlIO,
-		MonadState AnnexState
+		MonadState AnnexState,
+		Functor,
+		Applicative
 	)
 
 -- internal state storage
@@ -83,7 +86,7 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState `liftM` Git.configRead gitrepo
+new gitrepo = newState <$> Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
diff --git a/Branch.hs b/Branch.hs
index d5bfe1b09f..5008b2e200 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -20,6 +20,7 @@ module Branch (
 
 import Control.Monad (when, unless, liftM)
 import Control.Monad.State (liftIO)
+import Control.Applicative ((<$>))
 import System.FilePath
 import System.Directory
 import Data.String.Utils
@@ -158,7 +159,7 @@ update = do
 		staged <- stageJournalFiles
 
 		refs <- siblingBranches
-		updated <- catMaybes `liftM` mapM updateRef refs
+		updated <- catMaybes <$> mapM updateRef refs
 		g <- Annex.gitRepo
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
@@ -182,7 +183,7 @@ hasOrigin = refExists originname
 
 {- Does the git-annex branch or a foo/git-annex branch exist? -}
 hasSomeBranch :: Annex Bool
-hasSomeBranch = liftM (not . null) siblingBranches
+hasSomeBranch = not . null <$> siblingBranches
 
 {- List of all git-annex branches, including the main one and any
  - from remotes. -}
@@ -323,7 +324,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = liftM (map fileJournal) getJournalFilesRaw
+getJournalFiles = map fileJournal <$> getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do
diff --git a/Command.hs b/Command.hs
index d3c1640ee0..21c50f9c02 100644
--- a/Command.hs
+++ b/Command.hs
@@ -11,6 +11,7 @@ import Control.Monad.State (liftIO)
 import System.Directory
 import System.Posix.Files
 import Control.Monad (filterM, liftM, when)
+import Control.Applicative
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
@@ -183,7 +184,7 @@ withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
 backendPairs :: CommandSeekBackendFiles
-backendPairs a files = liftM (map a) $ Backend.chooseBackends files
+backendPairs a files = map a <$> Backend.chooseBackends files
 
 {- Filter out files those matching the exclude glob pattern,
  - if it was specified. -}
@@ -204,7 +205,7 @@ wildsRegex ws = compile regex []
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
-notSymlink f = liftM (not . isSymbolicLink) $ liftIO $ getSymbolicLinkStatus f
+notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
 
 {- Descriptions of params used in usage messages. -}
 paramRepeating :: String -> String
@@ -271,4 +272,4 @@ preserveOrder orig new = collect orig new
  - of git file list commands, that assumption tends to hold.
  -}
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
-runPreserveOrder a files = liftM (preserveOrder files) (a files)
+runPreserveOrder a files = preserveOrder files <$> a files
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 25227ae162..6ad7e239c9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,6 +8,7 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
+import Control.Applicative
 import System.Posix.Files
 import System.Directory
 import System.FilePath
@@ -39,7 +40,7 @@ start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = return . head =<< Backend.orderedList
+		choosebackend Nothing = head <$> Backend.orderedList
 		choosebackend (Just backend) = return backend
 
 {- Checks if a key is upgradable to a newer representation. -}
diff --git a/Command/Status.hs b/Command/Status.hs
index aef4df2329..5c82744b10 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -8,6 +8,7 @@
 module Command.Status where
 
 import Control.Monad.State
+import Control.Applicative
 import Data.Maybe
 import System.IO
 import Data.List
@@ -112,12 +113,10 @@ total_annex_size = stat "total annex size" $
 	cachedKeysReferenced >>= keySizeSum
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" $ 
-	return . show . snd =<< cachedKeysPresent
+local_annex_keys = stat "local annex keys" $ show . snd <$> cachedKeysPresent
 
 total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" $
-	return . show . snd =<< cachedKeysReferenced
+total_annex_keys = stat "total annex keys" $ show . snd <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -126,8 +125,7 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $
-	return . usage =<< cachedKeysReferenced
+backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 	where
 		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
 		splits :: [Key] -> [(String, Integer)]
diff --git a/Config.hs b/Config.hs
index 12f6480470..b4f4c0b922 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,7 +9,7 @@ module Config where
 
 import Data.Maybe
 import Control.Monad.State (liftIO)
-import Control.Monad (liftM)
+import Control.Applicative
 import System.Cmd.Utils
 
 import qualified Git
@@ -47,8 +47,8 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
 	cmd <- getConfig r "cost-command" ""
-	return . safeparse =<< if not $ null cmd
-			then liftM snd $ liftIO $ pipeFrom "sh" ["-c", cmd]
+	safeparse <$> if not $ null cmd
+			then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd]
 			else getConfig r "cost" ""
 	where
 		safeparse v
diff --git a/Crypto.hs b/Crypto.hs
index ed29747aa6..d789b44556 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -38,6 +38,7 @@ import System.IO
 import System.Posix.IO
 import System.Posix.Types
 import System.Posix.Process
+import Control.Applicative
 import Control.Concurrent
 import Control.Exception (finally)
 import System.Exit
@@ -136,7 +137,7 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
 decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
 decryptCipher _ (EncryptedCipher encipher _) = 
-	return . Cipher =<< gpgPipeStrict decrypt encipher
+	Cipher <$> gpgPipeStrict decrypt encipher
 	where
 		decrypt = [ Param "--decrypt" ]
 
diff --git a/Git.hs b/Git.hs
index 7155b26343..ab43504e1a 100644
--- a/Git.hs
+++ b/Git.hs
@@ -63,6 +63,7 @@ module Git (
 ) where
 
 import Control.Monad (unless, when)
+import Control.Applicative
 import System.Directory
 import System.FilePath
 import System.Posix.Directory
@@ -446,7 +447,7 @@ commit g message newref parentrefs = do
 		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
 	run g "update-ref" [Param newref, Param sha]
 	where
-		ignorehandle a = return . snd =<< a
+		ignorehandle a = snd <$> a
 		ps = concatMap (\r -> ["-p", r]) parentrefs
 
 {- Reads null terminated output of a git command (as enabled by the -z 
diff --git a/LocationLog.hs b/LocationLog.hs
index 768483fa1b..fa660c8b67 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -24,6 +24,7 @@ module LocationLog (
 
 import System.FilePath
 import Control.Monad (when)
+import Control.Applicative
 import Data.Maybe
 
 import qualified Git
@@ -49,7 +50,7 @@ keyLocations key = currentLog $ logFile key
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Annex [Key]
-loggedKeys = return . mapMaybe (logFileKey . takeFileName) =<< Branch.files
+loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Branch.files
 
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
diff --git a/PresenceLog.hs b/PresenceLog.hs
index ccb75ff5be..e0c8729979 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -28,6 +28,7 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as Map
 import Control.Monad.State (liftIO)
+import Control.Applicative
 
 import qualified Branch
 import Types
@@ -81,7 +82,7 @@ addLog file line = do
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
-readLog file = return . parseLog =<< Branch.get file
+readLog file = parseLog <$> Branch.get file
 
 parseLog :: String -> [LogLine]
 parseLog s = filter parsable $ map read $ lines s
diff --git a/Remote.hs b/Remote.hs
index 1a5006f6fb..2c883f1a8f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -29,11 +29,12 @@ module Remote (
 	forceTrust
 ) where
 
-import Control.Monad (filterM, liftM2)
+import Control.Monad (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
 import Data.Maybe
+import Control.Applicative
 
 import Types
 import Types.Remote
@@ -111,10 +112,10 @@ nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
-		Left e -> return . fromMaybe (error e) =<< byDescription
+		Left e -> fromMaybe (error e) <$> byDescription
 		Right r -> return $ uuid r
 	where
-		byDescription = return . M.lookup n . invertMap =<< uuidMap
+		byDescription = M.lookup n . invertMap <$> uuidMap
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
@@ -124,10 +125,10 @@ prettyPrintUUIDs uuids = do
 	here <- getUUID =<< Annex.gitRepo
 	-- Show descriptions from the uuid log, falling back to remote names,
 	-- as some remotes may not be in the uuid log
-	m <- liftM2 M.union uuidMap $
-		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList
+	m <- M.union <$> uuidMap <*> availMap
 	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
 	where
+		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		prettify m u here = base ++ ishere
 			where
 				base = if not $ null $ findlog m u
@@ -147,7 +148,7 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote Annex]
-keyPossibilities key = return . fst =<< keyPossibilities' False key
+keyPossibilities key = fst <$> keyPossibilities' False key
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 69a82f4987..620c0d8757 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -19,6 +19,7 @@ import Data.List
 import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
+import Control.Applicative
 
 import qualified Branch
 import Types
@@ -40,7 +41,7 @@ configSet u c = do
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
+readRemoteLog = remoteLogParse <$> Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index b4567a0b71..9c3fd99595 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -11,6 +11,7 @@ import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import Control.Monad (filterM, forM_, unless)
+import Control.Applicative
 import System.Posix.Files
 import System.FilePath
 import Data.String.Utils
@@ -192,7 +193,7 @@ writeLog1 :: FilePath -> [LogLine] -> IO ()
 writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
 
 readLog1 :: FilePath -> IO [LogLine]
-readLog1 file = catch (return . parseLog =<< readFileStrict file) (const $ return [])
+readLog1 file = catch (parseLog <$> readFileStrict file) (const $ return [])
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
@@ -201,7 +202,7 @@ lookupFile1 file = do
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = return . takeFileName =<< readSymbolicLink file
+		getsymlink = takeFileName <$> readSymbolicLink file
 		makekey l = case maybeLookupBackendName bname of
 			Nothing -> do
 				unless (null kname || null bname ||
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 517c175bc4..9b8041dad0 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -13,7 +13,7 @@ import System.FilePath
 import System.Directory
 import Data.List
 import Data.Maybe
-import Control.Monad (liftM2)
+import Control.Applicative
 
 {- Returns the parent directory of a path. Parent of / is "" -}
 parentDir :: FilePath -> FilePath
@@ -65,7 +65,7 @@ absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
  -    relPathCwdToFile "/tmp/foo/bar" == "" 
  -}
 relPathCwdToFile :: FilePath -> IO FilePath
-relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
+relPathCwdToFile f = relPathDirToFile <$> getCurrentDirectory <*> absPath f
 
 {- Constructs a relative path from a directory to a file.
  -
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 69b53c34c0..f678720ed8 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -11,7 +11,7 @@ module Utility.Url (
 	get
 ) where
 
-import Control.Monad (liftM)
+import Control.Applicative
 import Control.Monad.State (liftIO)
 import qualified Network.Browser as Browser
 import Network.HTTP
@@ -64,7 +64,6 @@ request url requesttype = Browser.browse $ do
 	Browser.setErrHandler ignore
 	Browser.setOutHandler ignore
 	Browser.setAllowRedirects True
-	liftM snd $ Browser.request
-		(mkRequest requesttype url :: Request_String)
+	snd <$> Browser.request (mkRequest requesttype url :: Request_String)
 	where
 		ignore = const $ return ()

From 9170e1b87d1eac000e9ea4d3944a73209ba9c718 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 26 Aug 2011 10:55:43 +0000
Subject: [PATCH 2145/2835]

---
 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn

diff --git a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
new file mode 100644
index 0000000000..7a579f8fe6
--- /dev/null
+++ b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
@@ -0,0 +1 @@
+When the test suite cannot be compiled, the build just fails silenty. This means that in automated builds there is no easy way to ensure that the generated binaries have passed the test suite, because it may not even have been run! IMHO, "make test" should fail (i.e. return a non-zero exit code) when it can't succeeed.

From f82da1d9dca0712cdd87e3fc0ed8a2c2e2440228 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 07:08:15 -0400
Subject: [PATCH 2146/2835] show a message if asked to get something from the
 web that is not there

---
 Remote/Web.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 5bc6a204b9..0e4b757673 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -86,8 +86,12 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = iter =<< getUrls key
+downloadKey key file = get =<< getUrls key
 	where
+		get [] = do
+			warning "no known url"
+			return False
+		get a = iter a
 		iter [] = return False
 		iter (url:urls) = do
 			ok <- Url.download url file

From 6e750764b7d30d9cb0684cdaadd79ec091a4fda6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 12:31:50 -0400
Subject: [PATCH 2147/2835] The wget command will now be used in preference to
 curl, if available.

Got tired of curl's various ugly progress bars.
---
 Utility.hs       | 12 +++++++++++-
 Utility/Url.hs   | 24 +++++++++++++++++-------
 debian/changelog |  1 +
 debian/control   |  2 +-
 doc/install.mdwn |  2 +-
 5 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/Utility.hs b/Utility.hs
index 788dc41030..451b1b44fc 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -15,7 +15,8 @@ module Utility (
 	dirContains,
 	dirContents,
 	myHomeDir,
-	catchBool
+	catchBool,
+	inPath
 ) where
 
 import IO (bracket)
@@ -94,3 +95,12 @@ myHomeDir = do
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
+
+{- Checks if a command is available in PATH. -}
+inPath :: String -> IO Bool
+inPath command = search =<< getSearchPath
+	where
+		search [] = return False
+		search (d:ds) = do
+			e <- doesFileExist $ d  command
+			if e then return True else search ds
diff --git a/Utility/Url.hs b/Utility/Url.hs
index f678720ed8..6ddeecc14f 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -20,6 +20,7 @@ import Network.URI
 import Types
 import Messages
 import Utility.SafeCommand
+import Utility
 
 type URLString = String
 
@@ -35,15 +36,24 @@ exists url =
 				_ -> return False
 
 {- Used to download large files, such as the contents of keys.
- - Uses curl program for its progress bar. -}
+ - Uses wget or curl program for its progress bar. (Wget has a better one,
+ - so is preferred.) -}
 download :: URLString -> FilePath -> Annex Bool
 download url file = do
-	showOutput -- make way for curl progress bar
-	-- Uses the -# progress display, because the normal one is very
-	-- confusing when resuming, showing the remainder to download
-	-- as the whole file, and not indicating how much percent was
-	-- downloaded before the resume.
-	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
+	showOutput -- make way for program's progress bar
+	e <- liftIO $ inPath "wget"
+	if e
+		then
+			liftIO $ boolSystem "wget"
+				[Params "-c -O", File file, File url]
+		else
+			-- Uses the -# progress display, because the normal
+			-- one is very confusing when resuming, showing
+			-- the remainder to download as the whole file,
+			-- and not indicating how much percent was
+			-- downloaded before the resume.
+			liftIO $ boolSystem "curl" 
+				[Params "-L -C - -# -o", File file, File url]
 
 {- Downloads a small file. -}
 get :: URLString -> IO String
diff --git a/debian/changelog b/debian/changelog
index bb06e1fba9..54fdbd835c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
+  * The wget command will now be used in preference to curl, if available.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/debian/control b/debian/control
index 975faf5ea5..63488dc68d 100644
--- a/debian/control
+++ b/debian/control
@@ -31,7 +31,7 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	git | git-core,
 	uuid,
 	rsync,
-	curl,
+	wget | curl,
 	openssh-client
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 49ddd913f0..ac521da187 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -34,7 +34,7 @@ To build and use git-annex, you will need:
     (or `uuidgen` from util-linux)
   * [xargs](http://savannah.gnu.org/projects/findutils/)
   * [rsync](http://rsync.samba.org/)
-  * [curl](http://http://curl.haxx.se/) (optional, but recommended)
+  * [wget](http://www.gnu.org/software/wget/) or [curl](http://http://curl.haxx.se/) (optional, but recommended)
   * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
     a sha1 command will also do)
   * [gpg](http://gnupg.org/) (optional; needed for encryption)

From b26ee162f33858aa4ef82fbe3d56eecc00590755 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 12:34:02 -0400
Subject: [PATCH 2148/2835] guess not everyone configures their shell to show
 nonzero exit codes..

---
 Makefile                                              | 4 +++-
 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index fa087c1ae2..20d1347140 100644
--- a/Makefile
+++ b/Makefile
@@ -53,7 +53,9 @@ test: $(bins)
 	@if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
-		./test; \
+		if ! ./test; then \
+			echo "** test suite failed!" >&2; \
+		fi; \
 	fi
 
 testcoverage: $(bins)
diff --git a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
index 7a579f8fe6..2f486ad652 100644
--- a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
+++ b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
@@ -1 +1,3 @@
 When the test suite cannot be compiled, the build just fails silenty. This means that in automated builds there is no easy way to ensure that the generated binaries have passed the test suite, because it may not even have been run! IMHO, "make test" should fail (i.e. return a non-zero exit code) when it can't succeeed.
+
+> Ok, fixed. --[[Joey]] [[done]]

From 999d5df90b013a7cc8a390c940785118400faf8a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 28 Aug 2011 15:46:49 -0400
Subject: [PATCH 2149/2835] factor out firstM and anyM

Control.Monad.Loops has these, but has no Debian package yet.
---
 Remote/Web.hs |  7 ++-----
 Utility.hs    | 28 ++++++++++++++++++++++------
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 0e4b757673..3695bb1ab3 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -24,6 +24,7 @@ import Config
 import PresenceLog
 import LocationLog
 import Locations
+import Utility
 import qualified Utility.Url as Url
 
 type URLString = String
@@ -91,11 +92,7 @@ downloadKey key file = get =<< getUrls key
 		get [] = do
 			warning "no known url"
 			return False
-		get a = iter a
-		iter [] = return False
-		iter (url:urls) = do
-			ok <- Url.download url file
-			if ok then return ok else iter urls
+		get urls = anyM (`Url.download` file) urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
diff --git a/Utility.hs b/Utility.hs
index 451b1b44fc..ce17363488 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -16,7 +16,9 @@ module Utility (
 	dirContents,
 	myHomeDir,
 	catchBool,
-	inPath
+	inPath,
+	firstM,
+	anyM
 ) where
 
 import IO (bracket)
@@ -29,6 +31,8 @@ import System.FilePath
 import System.Directory
 import Foreign (complement)
 import Utility.Path
+import Data.Maybe
+import Control.Monad (liftM)
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -96,11 +100,23 @@ myHomeDir = do
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
 
+{- Return the first value from a list, if any, satisfying the given
+ - predicate -}
+firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
+firstM _ [] = return Nothing
+firstM p (x:xs) = do
+	q <- p x
+	if q
+		then return (Just x)
+		else firstM p xs
+
+{- Returns true if any value in the list satisfies the preducate,
+ - stopping once one is found. -}
+anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
+anyM p = liftM isJust . firstM p
+
 {- Checks if a command is available in PATH. -}
 inPath :: String -> IO Bool
-inPath command = search =<< getSearchPath
+inPath command = getSearchPath >>= anyM indir
 	where
-		search [] = return False
-		search (d:ds) = do
-			e <- doesFileExist $ d  command
-			if e then return True else search ds
+		indir d = doesFileExist $ d  command

From bbba6c19bd03f2b4a4ce8a38a2423c794826b1c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 28 Aug 2011 16:28:38 -0400
Subject: [PATCH 2150/2835] update documentation for new, neutered key-value
 backends

Backends are now only used to generate keys (and check them); they
are not arbitrary key-value stores for data, because it turned out such
a store is better modeled as a special remote. Updated docs to not
imply backends do more than they do now.

Sometimes I'm tempted to rename "backend" to "keytype" or something,
which would really be more clear. But it would be an annoying transition
for users, with annex.backends etc.
---
 doc/backends.mdwn                             | 22 +++++-----------
 doc/copies.mdwn                               |  6 ++---
 doc/git-annex.mdwn                            | 26 +++++++++----------
 .../fsck:_verifying_your_data.mdwn            | 10 +++----
 doc/walkthrough/unused_data.mdwn              |  2 +-
 5 files changed, 29 insertions(+), 37 deletions(-)

diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 9e698032d8..f2f1c5580c 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -1,24 +1,16 @@
-git-annex uses a key-value abstraction layer to allow file contents to be
-stored in different ways. In theory, any key-value storage system could be
-used to store file contents.
-
 When a file is annexed, a key is generated from its content and/or metadata.
 The file checked into git symlinks to the key. This key can later be used
 to retrieve the file's content (its value).
 
-Multiple pluggable backends are supported, and a single repository
-can use different backends for different files.
+Multiple pluggable key-value backends are supported, and a single repository
+can use different ones for different files.
 
-These backends can transfer file contents between configured git remotes.
-It's also possible to use [[special_remotes]], such as Amazon S3 with
-these backends.
-
-* `WORM` ("Write Once, Read Many") This backend assumes that any file with
-  the same basename, size, and modification time has the same content. So with
-  this backend, files can be moved around, but should never be added to
+* `WORM` ("Write Once, Read Many") This assumes that any file with
+  the same basename, size, and modification time has the same content. So
+  files can be moved around, but should never be added to
   or changed. This is the default, and the least expensive backend.
-* `SHA1` -- This backend uses a key based on a sha1 checksum. This backend
-  allows modifications of files to be tracked. Its need to generate checksums
+* `SHA1` -- This uses a key based on a sha1 checksum. This allows
+  modifications of files to be tracked. Its need to generate checksums
   can make it slower for large files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
diff --git a/doc/copies.mdwn b/doc/copies.mdwn
index 16eba19c81..93cbd8ea80 100644
--- a/doc/copies.mdwn
+++ b/doc/copies.mdwn
@@ -1,8 +1,8 @@
-The WORM and SHA1 key-value [[backends]] store data inside
-your git repository's `.git` directory, not in some external data store.
+Annexed data is stored inside  your git repository's `.git/annex` directory.
+Some [[special_remotes]] can store annexed data elsewhere.
 
 It's important that data not get lost by an ill-considered `git annex drop`
-command.  So, then using those backends, git-annex can be configured to try
+command.  So, git-annex can be configured to try
 to keep N copies of a file's content available across all repositories. 
 (Although [[untrusted_repositories|trust]] don't count toward this total.)
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a262d465fd..52599611ed 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -72,15 +72,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * get [path ...]
 
-  Makes the content of annexed files available in this repository. Depending
-  on the backend used, this will involve copying them from another repository,
-  or downloading them, or transferring them from some kind of key-value store.
+  Makes the content of annexed files available in this repository. This
+  will involve copying them from another repository, or downloading them,
+  or transferring them from some kind of key-value store.
 
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 
 
-  git-annex may refuse to drop content if the backend does not think
+  git-annex may refuse to drop content if it does not think
   it is safe to do so, typically because of the setting of annex.numcopies.
 
 * move [path ...]
@@ -207,14 +207,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * migrate [path ...]
 
-  Changes the specified annexed files to store their content in the
-  default backend (or the one specified with --backend). Only files whose
-  content is currently available are migrated.
+  Changes the specified annexed files to use the default key-value backend
+  (or the one specified with --backend). Only files whose content
+  is currently available are migrated.
 
-  Note that the content is not removed from the backend it was previously in.
-  Use `git annex unused` to find and remove such content.
+  Note that the content is also still available using the old key after
+  migration. Use `git annex unused` to find and remove the old key.
 
-  Normally, nothing will be done to files already in the backend.
+  Normally, nothing will be done to files already using the new backend.
   However, if a backend changes the information it uses to construct a key,
   this can also be used to migrate files to use the new key format.
 
@@ -293,7 +293,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file
-  to link to a specified key in the key-value backend.
+  in the git repository to link to a specified key.
 
 * dropkey [key ...]
 
@@ -500,8 +500,8 @@ Here are all the supported configuration settings.
 
 # CONFIGURATION VIA .gitattributes
 
-The backend used when adding a new file to the annex can be configured
-on a per-file-type basis via `.gitattributes` files. In the file,
+The key-value backend used when adding a new file to the annex can be
+configured on a per-file-type basis via `.gitattributes` files. In the file,
 the `annex.backend` attribute can be set to the name of the backend to
 use. For example, this here's how to use the WORM backend by default,
 but the SHA1 backend for ogg files:
diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn
index 7e05469a12..d036332fb3 100644
--- a/doc/walkthrough/fsck:_verifying_your_data.mdwn
+++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn
@@ -1,8 +1,8 @@
-You can use the fsck subcommand to check for problems in your data.
-What can be checked depends on the [[backend|backends]] you've used to store
-the data. For example, when you use the SHA1 backend, fsck will verify that
-the checksums of your files are good. Fsck also checks that the annex.numcopies
-setting is satisfied for all files.
+You can use the fsck subcommand to check for problems in your data. What
+can be checked depends on the key-value [[backend|backends]] you've used
+for the data. For example, when you use the SHA1 backend, fsck will verify
+that the checksums of your files are good. Fsck also checks that the
+annex.numcopies setting is satisfied for all files.
 
 	# git annex fsck
 	fsck some_file (checksum...) ok
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index fb84193034..e142b576c0 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -2,7 +2,7 @@ It's possible for data to accumulate in the annex that no files point to
 anymore. One way it can happen is if you `git rm` a file without 
 first calling `git annex drop`. And, when you modify an annexed file, the old
 content of the file remains in the annex. Another way is when migrating
-between backends.
+between key-value [[backends|backend]].
 
 This might be historical data you want to preserve, so git-annex defaults to
 preserving it. So from time to time, you may want to check for such data and

From b6758746f6433a46ac14bad58f9f6e0220a42e3e Mon Sep 17 00:00:00 2001
From: cwg 
Date: Mon, 29 Aug 2011 12:35:19 +0000
Subject: [PATCH 2151/2835]

---
 doc/forum/advantages_of_SHA__42___over_WORM.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/forum/advantages_of_SHA__42___over_WORM.mdwn

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
new file mode 100644
index 0000000000..b959db550c
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
@@ -0,0 +1,6 @@
+Thanks for creating git-annex.
+
+I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.
+
+Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
+If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, is there actually any advantage in using SHA*?

From eb4607aafc193073468da09a3cac5d54f6c16735 Mon Sep 17 00:00:00 2001
From: cwg 
Date: Mon, 29 Aug 2011 12:37:23 +0000
Subject: [PATCH 2152/2835]

---
 doc/forum/advantages_of_SHA__42___over_WORM.mdwn | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
index b959db550c..5b544593f5 100644
--- a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
+++ b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
@@ -1,6 +1,5 @@
 Thanks for creating git-annex.
 
-I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.
+I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM, files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.  Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
 
-Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
-If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, is there actually any advantage in using SHA*?
+If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, and if it is unlikely that several files will be identical, is there actually any advantage in using SHA*?

From 2a76ad41673968793a479f79ed33ba83a0b10773 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 29 Aug 2011 16:10:38 +0000
Subject: [PATCH 2153/2835] Added a comment

---
 .../comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
new file mode 100644
index 0000000000..218027ca53
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-29T16:10:38Z"
+ content="""
+You're right -- as long as nothing changes a file without letting the modification time update, editing WORM files is safe.
+"""]]

From 025e66e3d35d89fc3c3dd2c81e2f618941130a26 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 12:08:54 -0400
Subject: [PATCH 2154/2835] update to not overstate the danger or WORM

---
 doc/backends.mdwn | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index f2f1c5580c..ebcdedc2a7 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -6,11 +6,11 @@ Multiple pluggable key-value backends are supported, and a single repository
 can use different ones for different files.
 
 * `WORM` ("Write Once, Read Many") This assumes that any file with
-  the same basename, size, and modification time has the same content. So
-  files can be moved around, but should never be added to
-  or changed. This is the default, and the least expensive backend.
+  the same basename, size, and modification time has the same content.
+  This is the default, and the least expensive backend.
 * `SHA1` -- This uses a key based on a sha1 checksum. This allows
-  modifications of files to be tracked. Its need to generate checksums
+  verifying that the file content is right, and can avoid duplicates of
+  files with the same content. Its need to generate checksums
   can make it slower for large files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
@@ -28,7 +28,7 @@ files, the `.gitattributes` file can be used. The `annex.backend`
 attribute can be set to the name of the backend to use for matching files.
 
 For example, to use the SHA1 backend for sound files, which tend to be
-smallish and might be modified over time, you could set in
+smallish and might be modified or copied over time, you could set in
 `.gitattributes`:
 
 	*.mp3 annex.backend=SHA1

From 676c467801051e71ea7668615985301f26ab0f3e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 12:49:38 -0400
Subject: [PATCH 2155/2835] close

---
 ...ude_repo_description_and__47__or_UUID_in_commit_message.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
index 9ca61bff55..be7e2dacc8 100644
--- a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
+++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
@@ -9,3 +9,5 @@ I'm not sure that the above suggestion is going down a path that really
 makes sense. If you want a list of repository UUIDs and descriptions,
 it's there in machine-usable form in `.git-annex/uuid.log`, there is no
 need to try to pull this info out of git commit messages. --[[Joey]]
+
+[[done]]

From b2c5639dcc53f6e734643878f9696405fa6cae64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 13:29:39 -0400
Subject: [PATCH 2156/2835] update

---
 doc/todo/smudge.mdwn | 46 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index c51662b282..f78b215ac4 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -19,6 +19,26 @@ add` files, and just being able to use `git add` or `git commit -a`,
 and have it use git-annex when .gitattributes says to. Also, annexed
 files can be directly modified without having to `git annex unlock`.
 
+### design
+
+In .gitattributes, the user would put something like "* filter=git-annex".
+This way they could control which files are annexed vs added normally.
+
+(git-annex could have further controls to allow eg, passing small files
+through to regular processing. At least .gitattributes is a special case,
+it should never be annexed...)
+
+For files not configured this way, git-annex could continue to use
+its symlink method -- this would preserve backwards compatability,
+and even allow mixing the two methods in a repo as desired.
+
+To find files in the repository that are annexed, git-annex would do
+`ls-files` as now, but would check if found files have the appropriate
+filter, rather than the current symlink checks. To determine the key
+of a file, rather than reading its symlink, git-annex would need to
+look up the git blob associated with the file -- this can be done
+efficiently using the existing code in `Branch.catFile`.
+
 ### efficiency
 
 The trick is doing it efficiently. Since git a2b665d, v1.7.4.1,
@@ -30,12 +50,16 @@ This avoids it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
 
+(The smudge script can also be provided a filename with %f, but it
+cannot directly write to the file or git gets unhappy.)
+
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
 do nothing, file content is already cached.
 
 SHA1 has a harder job. Would not want to re-sha1 the file every time,
-probably. So it'd need a cache of file stat info, mapped to known objects.
+probably. So it'd need a local cache of file stat info, mapped to known
+objects.
 
 ### dealing with partial content availability
 
@@ -59,9 +83,10 @@ huge-smudge:
 
 #!/bin/sh
 read sha1
+file="$1"
 echo "smudging $sha1" >&2
 if [ -e ~/$sha1 ]; then
-	cat ~/$sha1
+	cat ~/$sha1 # possibly expensive copy here
 else
 	echo "$sha1 not available"
 fi
@@ -71,16 +96,15 @@ huge-clean:
 
 
 #!/bin/sh
-cat >temp
-if grep -q 'not available' temp; then
-	awk '{print $1}' temp # provide what we would if the content were avail!
-	rm temp
+temp="$1"
+if grep -q 'not available' "$temp"; then
+	awk '{print $1}' "$temp" # provide what we would if the content were avail!
 	exit 0
 fi
-sha1=`sha1sum temp | cut -d' ' -f1`
+sha1=`sha1sum "$temp" | cut -d' ' -f1`
 echo "cleaning $sha1" >&2
-ls -l temp >&2
-mv temp ~/$sha1
+ls -l "$temp" >&2
+ln -f "$temp" ~/$sha1 # can't delete temp file
 echo $sha1
 

 
@@ -94,6 +118,6 @@ in .git/config:
 
 
 [filter "huge"]
-        clean = huge-clean
-        smudge = huge-smudge
+        clean = huge-clean %f
+        smudge = huge-smudge %f
 

From d1154d0837f4dac18771a6a5c3ab96de4d5fcd90 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 14:13:38 -0400
Subject: [PATCH 2157/2835] init: Make description an optional parameter.

---
 Command/Init.hs    | 6 ++----
 debian/changelog   | 1 +
 doc/git-annex.mdwn | 4 +---
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 0191060511..2abe4c6615 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,7 +7,7 @@
 
 module Command.Init where
 
-import Control.Monad (when)
+import Control.Monad
 
 import Command
 import qualified Annex
@@ -17,15 +17,13 @@ import Init
 	
 command :: [Command]
 command = [standaloneCommand "init" paramDesc seek
-		"initialize git-annex with repository description"]
+		"initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
 start ws = do
-	when (null description) $
-		error "please specify a description of this repository\n"
 	showStart "init" description
 	next $ perform description
 	where
diff --git a/debian/changelog b/debian/changelog
index 54fdbd835c..7803c9a62a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
   * The wget command will now be used in preference to curl, if available.
+  * init: Make description an optional parameter.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 52599611ed..e7ac9adf7a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -119,9 +119,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
-* init description
-
-  Initializes git-annex with a description of the git repository.
+* init [description]
 
   Until a repository (or one of its remotes) has been initialized,
   git-annex will refuse to operate on it, to avoid accidentially

From cd199e442f31743ee7ddc7c545f390533ed30a9d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 16:31:47 -0400
Subject: [PATCH 2158/2835] update; showstopper issue with current git

developed a patch for git, we'll see if they like it..
---
 doc/todo/smudge.mdwn | 49 +++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index f78b215ac4..2f5d21d7e1 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -41,18 +41,17 @@ efficiently using the existing code in `Branch.catFile`.
 
 ### efficiency
 
+#### clean
+
 The trick is doing it efficiently. Since git a2b665d, v1.7.4.1,
 something like this works to provide a filename to the clean script:
 
 	git config --global filter.huge.clean huge-clean %f
 
-This avoids it needing to read all the current file content from stdin
+This could avoid it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
 
-(The smudge script can also be provided a filename with %f, but it
-cannot directly write to the file or git gets unhappy.)
-
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
 do nothing, file content is already cached.
@@ -61,6 +60,21 @@ SHA1 has a harder job. Would not want to re-sha1 the file every time,
 probably. So it'd need a local cache of file stat info, mapped to known
 objects.
 
+But: Even with %f, git actually passes the full file content to the clean
+filter, and if it fails to consume it all, it will crash (may only happen
+if the file is larger than some chunk size; tried with 500 mb file and 
+saw a SIGPIPE.) This means unnecessary works needs to be done, 
+and it slows down *everything*, from `git status` to `git commit`.
+**showstopper** I have sent a patch to the git mailing list to address
+this.
+
+#### smudge
+
+The smudge script can also be provided a filename with %f, but it
+cannot directly write to the file or git gets unhappy.
+
+
+
 ### dealing with partial content availability
 
 The smudge filter cannot be allowed to fail, that leaves the tree and
@@ -82,13 +96,13 @@ huge-smudge:
 
 
 #!/bin/sh
-read sha1
+read f
 file="$1"
-echo "smudging $sha1" >&2
-if [ -e ~/$sha1 ]; then
-	cat ~/$sha1 # possibly expensive copy here
+echo "smudging $f" >&2
+if [ -e ~/$f ]; then
+	cat ~/$f # possibly expensive copy here
 else
-	echo "$sha1 not available"
+	echo "$f not available"
 fi
 

 
@@ -96,16 +110,17 @@ huge-clean:
 
 
 #!/bin/sh
-temp="$1"
-if grep -q 'not available' "$temp"; then
-	awk '{print $1}' "$temp" # provide what we would if the content were avail!
+file="$1"
+# in real life, this should be done more efficiently, not trying to read
+# the whole file content!
+if grep -q 'not available' "$file"; then
+	awk '{print $1}' "$file" # provide what we would if the content were avail!
 	exit 0
 fi
-sha1=`sha1sum "$temp" | cut -d' ' -f1`
-echo "cleaning $sha1" >&2
-ls -l "$temp" >&2
-ln -f "$temp" ~/$sha1 # can't delete temp file
-echo $sha1
+echo "cleaning $file" >&2
+ls -l "$file" >&2
+ln -f "$file" ~/$file # can't delete temp file
+echo $file
 

 
 .gitattributes:

From 5ef11350aafe2aec48f66799743423a9c6e42556 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 16:41:47 -0400
Subject: [PATCH 2159/2835] link to patch

---
 doc/todo/smudge.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 2f5d21d7e1..7d8f9a136d 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -66,7 +66,7 @@ if the file is larger than some chunk size; tried with 500 mb file and
 saw a SIGPIPE.) This means unnecessary works needs to be done, 
 and it slows down *everything*, from `git status` to `git commit`.
 **showstopper** I have sent a patch to the git mailing list to address
-this.
+this. 
 
 #### smudge
 

From 9e135a6ee2b6b7ac31bbf76d0d590a0415f9f512 Mon Sep 17 00:00:00 2001
From: "http://www.schleptet.net/~cfm/" 
Date: Tue, 30 Aug 2011 14:31:38 +0000
Subject: [PATCH 2160/2835] Added a comment

---
 ...ment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment

diff --git a/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
new file mode 100644
index 0000000000..1148a87cab
--- /dev/null
+++ b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://www.schleptet.net/~cfm/"
+ ip="64.30.148.100"
+ subject="comment 1"
+ date="2011-08-30T14:31:36Z"
+ content="""
+You can also use Homebrew instead of MacPorts.  Homebrew's `haskell-platform` is up-to-date, too:
+
+    brew install haskell-platform git ossp-uuid md5sha1sum coreutils pcre
+    ln -s /usr/local/include/pcre.h /usr/include/pcre.h
+
+As of this writing, however, Homebrew's `md5sha1sum` has a broken mirror.  I wound up getting that from MacPorts anyway.
+"""]]

From a64c16bf7d6cf3ba5295a23496e10ed70c1656dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 13:23:21 -0400
Subject: [PATCH 2161/2835] tweak

---
 Backend.hs           | 7 +++----
 Utility/DataUnits.hs | 3 +--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 0bb9f4b575..75327de80b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -122,8 +122,7 @@ lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: String -> Maybe (Backend Annex)
-maybeLookupBackendName s =
-	if 1 /= length matches
-		then Nothing
-		else Just $ head matches
+maybeLookupBackendName s
+	| length matches == 1 = Just $ head matches
+	| otherwise = Nothing
 	where matches = filter (\b -> s == B.name b) list
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index f2bc333ea0..0baa5dd896 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -137,8 +137,7 @@ compareSizes units abbrev old new
 {- Parses strings like "10 kilobytes" or "0.5tb". -}
 readSize :: [Unit] -> String -> Maybe ByteSize
 readSize units input
-	| null parsednum = Nothing
-	| null parsedunit = Nothing
+	| null parsednum || null parsedunit = Nothing
 	| otherwise = Just $ round $ number * fromIntegral multiplier
 	where
 		(number, rest) = head parsednum

From b96443364e80720172638f0fcd9caa5b9948555d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 13:29:07 -0400
Subject: [PATCH 2162/2835] smudge update: Not practical.

---
 doc/todo/smudge.mdwn | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 7d8f9a136d..55de5f5789 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -1,5 +1,18 @@
 git-annex should use smudge/clean filters.
 
+----
+
+Update: Currently, this does not look likely to work. In particular,
+the clean filter needs to consume all stdin from git, which consists of the
+entire content of the file. It cannot optimise by directly accessing
+the file in the repository, because git may be cleaning a different
+version of the file during a merge. 
+
+So every `git status` would need to read the entire content of all
+available files, and checksum them, which is too expensive.
+
+----
+
 The clean filter is run when files are staged for commit. So a user could copy
 any file into the annex, git add it, and git-annex's clean filter causes
 the file's key to be staged, while its value is added to the annex.
@@ -8,7 +21,7 @@ The smudge filter is run when files are checked out. Since git annex
 repos have partial content, this would not git annex get the file content.
 Instead, if the content is not currently available, it would need to do
 something like return empty file content. (Sadly, it cannot create a
-symlink, as git still wants to write the file afterwards.
+symlink, as git still wants to write the file afterwards.)
 
 So the nice current behavior of unavailable files being clearly missing due
 to dangling symlinks, would be lost when using smudge/clean filters.
@@ -39,7 +52,13 @@ of a file, rather than reading its symlink, git-annex would need to
 look up the git blob associated with the file -- this can be done
 efficiently using the existing code in `Branch.catFile`.
 
-### efficiency
+The clean filter would inject the file's content into the annex, and hard
+link from the annex to the file. Avoiding duplication of data.
+
+The smudge filter can't do that, so to avoid duplication of data, it
+might always create an empty file. To get the content, `git annex get`
+could be used (which would hard link it). A `post-checkout` hook might
+be used to set up hard links for all currently available content.
 
 #### clean
 
@@ -51,6 +70,8 @@ something like this works to provide a filename to the clean script:
 This could avoid it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
+(Update: No, doesn't work; git may be cleaning a different file content
+than is currently on disk, and git requires all stdin be consumed too.)
 
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
@@ -66,15 +87,14 @@ if the file is larger than some chunk size; tried with 500 mb file and
 saw a SIGPIPE.) This means unnecessary works needs to be done, 
 and it slows down *everything*, from `git status` to `git commit`.
 **showstopper** I have sent a patch to the git mailing list to address
-this. 
+this.  (Update: apparently
+can't be fixed.)
 
 #### smudge
 
 The smudge script can also be provided a filename with %f, but it
 cannot directly write to the file or git gets unhappy.
 
-
-
 ### dealing with partial content availability
 
 The smudge filter cannot be allowed to fail, that leaves the tree and
@@ -111,15 +131,15 @@ huge-clean:
 
 #!/bin/sh
 file="$1"
+cat >/tmp/file
 # in real life, this should be done more efficiently, not trying to read
 # the whole file content!
-if grep -q 'not available' "$file"; then
-	awk '{print $1}' "$file" # provide what we would if the content were avail!
+if grep -q 'not available' /tmp/file; then
+	awk '{print $1}' /tmp/file # provide what we would if the content were avail!
 	exit 0
 fi
 echo "cleaning $file" >&2
-ls -l "$file" >&2
-ln -f "$file" ~/$file # can't delete temp file
+# XXX store file content here
 echo $file
 

 

From ea7b1828d48a5dce42393d252f4bbda5cba10d24 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 15:16:34 -0400
Subject: [PATCH 2163/2835] unused, status: Sped up by avoiding unnecessary
 stats of annexed files.

Statting files returned by dirContents to see if they exist and are regular
files seems pretty useless. This code was originally part of fsck, and
perhaps the idea then was to avoid things returned by dirContents that were
not files. But it's certianly not needed in the current use cases for
getKeysPresent.
---
 Content.hs       | 9 +--------
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/Content.hs b/Content.hs
index ba99f1330d..1c2475240b 100644
--- a/Content.hs
+++ b/Content.hs
@@ -254,15 +254,8 @@ getKeysPresent' dir = do
 			levela <- dirContents dir
 			levelb <- mapM dirContents levela
 			contents <- mapM dirContents (concat levelb)
-			files <- filterM present (concat contents)
+			let files = concat contents
 			return $ mapMaybe (fileKey . takeFileName) files
-	where
-		present d = do
-			result <- try $
-				getFileStatus $ d  takeFileName d
-			case result of
-				Right s -> return $ isRegularFile s
-				Left _ -> return False
 
 {- Things to do to record changes to content. -}
 saveState :: Annex ()
diff --git a/debian/changelog b/debian/changelog
index 7803c9a62a..c8c60579e2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
     configured first. Closes: #638998
   * The wget command will now be used in preference to curl, if available.
   * init: Make description an optional parameter.
+  * unused, status: Sped up by avoiding unnecessary stats of annexed files.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 

From baa43cb9c0b38135e136a8ff807c4caeb909b538 Mon Sep 17 00:00:00 2001
From: "http://jefferai.org/" 
Date: Tue, 30 Aug 2011 20:56:44 +0000
Subject: [PATCH 2164/2835]

---
 doc/bugs/add_script-friendly_output_options.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/add_script-friendly_output_options.mdwn

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
new file mode 100644
index 0000000000..efcd54553f
--- /dev/null
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -0,0 +1,3 @@
+I have a need to use git-annex from a larger program. It'd be great if the information output by some of the commands that is descriptive (for example, whereis) could be sent to stdout in a machine-readable format like (preferably) JSON, or XML. That way I can simply read in the output of the command and use the data directly instead of having to parse it via regexes or other such string manipulation.
+
+This could perhaps be triggered by a --json or --xml flag to the relevant commands.

From f600444ab6d00826cc16c4a5cc67aca84cc1f9e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 31 Aug 2011 19:13:02 -0400
Subject: [PATCH 2165/2835] unused --remote: Reduced memory use to 1/4th what
 was used before.

Using a single strictness annotation, in just the right place.
Tried several others, none of which helped and some of which potentially
hurt. This is only the second time I've really had to deal with this in
a year of using haskell, which is, I suppose not that bad.
---
 Command/Unused.hs                          | 5 ++++-
 debian/changelog                           | 1 +
 doc/todo/git-annex_unused_eats_memory.mdwn | 6 ------
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index e7065b3c36..27f5af1b45 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Command.Unused where
 
 import Control.Monad (filterM, unless, forM_)
@@ -80,7 +82,8 @@ checkRemoteUnused' r = do
 	where
 		isthere k = do
 			us <- keyLocations k
-			return $ uuid `elem` us
+			let !there = uuid `elem` us
+			return there
 		uuid = Remote.uuid r
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
diff --git a/debian/changelog b/debian/changelog
index c8c60579e2..4ee0b80f2e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * The wget command will now be used in preference to curl, if available.
   * init: Make description an optional parameter.
   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
+  * unused --remote: Reduced memory use to 1/4th what was used before.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index 6ce7140045..fcb09a1af7 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -17,9 +17,3 @@ currently present in the repository (possibly using a bloom filter again),
 and that would yield a shortlist of keys that are probably not used. 
 Then scan thru all files in the repo to make sure that none point to keys
 on the shortlist.
-
-----
-
-`git annex unused --from remote` is much worse, using hundreds of mb of
-memory. It has not been profiled at all yet, and can probably be improved
-somewhat by fixing whatever memory leak it (probably) has.

From 55783d886d3300555d4b68ff7323e2365928e059 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 31 Aug 2011 19:50:08 -0400
Subject: [PATCH 2166/2835] add explanation for why strictness is needed here

---
 Command/Unused.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 27f5af1b45..6a62cde5f8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -80,6 +80,8 @@ checkRemoteUnused' r = do
 		showLongNote $ remoteUnusedMsg r list
 		showLongNote "\n"
 	where
+		{- This should run strictly to avoid the filterM
+		 - building many thunks containing keyLocations data. -}
 		isthere k = do
 			us <- keyLocations k
 			let !there = uuid `elem` us

From 57dd34c6be5dbc01286108fd943ff9e02956e8aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 13:35:07 -0400
Subject: [PATCH 2167/2835] generalize quiet flag to output type

This will allow adding other styles of output.
---
 Annex.hs        | 7 +++++--
 Command/Init.hs | 2 --
 Content.hs      | 3 +--
 Messages.hs     | 7 ++++---
 Options.hs      | 6 +++---
 5 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 287aed875e..fac5d27e49 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -10,6 +10,7 @@
 module Annex (
 	Annex,
 	AnnexState(..),
+	OutputType(..),
 	new,
 	run,
 	eval,
@@ -48,7 +49,7 @@ data AnnexState = AnnexState
 	, backends :: [Backend Annex]
 	, remotes :: [Remote Annex]
 	, repoqueue :: Queue
-	, quiet :: Bool
+	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
 	, branchstate :: BranchState
@@ -63,13 +64,15 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
+data OutputType = NormalOutput | QuietOutput
+
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
 	, repoqueue = empty
-	, quiet = False
+	, output = NormalOutput
 	, force = False
 	, fast = False
 	, branchstate = startBranchState
diff --git a/Command/Init.hs b/Command/Init.hs
index 2abe4c6615..6ba7df6829 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,8 +7,6 @@
 
 module Command.Init where
 
-import Control.Monad
-
 import Command
 import qualified Annex
 import UUID
diff --git a/Content.hs b/Content.hs
index 1c2475240b..e4bbee5282 100644
--- a/Content.hs
+++ b/Content.hs
@@ -23,11 +23,10 @@ module Content (
 	saveState
 ) where
 
-import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import System.Path
-import Control.Monad (when, filterM)
+import Control.Monad
 import System.Posix.Files
 import System.FilePath
 import Data.Maybe
diff --git a/Messages.hs b/Messages.hs
index 36f0b89c5c..b2c871ede1 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -9,7 +9,6 @@ module Messages where
 
 import Control.Monad.State (liftIO)
 import System.IO
-import Control.Monad (unless)
 import Data.String.Utils
 
 import Types
@@ -17,8 +16,10 @@ import qualified Annex
 
 verbose :: Annex () -> Annex ()
 verbose a = do
-	q <- Annex.getState Annex.quiet
-	unless q a
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> a
+		_ -> return ()
 
 showStart :: String -> String -> Annex ()
 showStart command file = verbose $ liftIO $ do
diff --git a/Options.hs b/Options.hs
index 7f78f44f62..768a1c289d 100644
--- a/Options.hs
+++ b/Options.hs
@@ -26,9 +26,9 @@ commonOptions =
 		"allow actions that may lose annexed data"
 	, Option ['F'] ["fast"] (NoArg (setfast True))
 		"avoid slow operations"
-	, Option ['q'] ["quiet"] (NoArg (setquiet True))
+	, Option ['q'] ["quiet"] (NoArg (setoutput Annex.QuietOutput))
 		"avoid verbose output"
-	, Option ['v'] ["verbose"] (NoArg (setquiet False))
+	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
 		"allow verbose output (default)"
 	, Option ['d'] ["debug"] (NoArg (setdebug))
 		"show debug messages"
@@ -38,7 +38,7 @@ commonOptions =
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
-		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
+		setoutput v = Annex.changeState $ \s -> s { Annex.output = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG

From 2f4d4d1c4552a93a5f26a8a0a713e3916612329e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 15:16:31 -0400
Subject: [PATCH 2168/2835] basic json support

This includes a generic JSONStream library built on top of Text.JSON
(somewhat hackishly).

It would be possible to stream out a single json document describing
all actions, but it's probably better for consumers if they can expect
one json document per line, so I did it that way instead.

Output from external programs used for transferring files is not
currently hidden when outputting json, which probably makes it not very
useful there. This may be dealt with if there is demand for json
output for --get or --move to be parsable.

The version, status, and find subcommands have hand-crafted output and
don't do json. The whereis subcommand needs to be modified to produce
useful json.
---
 Annex.hs              |  2 +-
 Messages.hs           | 61 +++++++++++++++++++++++++++++--------------
 Messages/JSON.hs      | 23 ++++++++++++++++
 Options.hs            |  2 ++
 Utility/JSONStream.hs | 44 +++++++++++++++++++++++++++++++
 debian/changelog      |  1 +
 debian/control        |  1 +
 doc/git-annex.mdwn    | 10 +++++--
 doc/install.mdwn      |  1 +
 9 files changed, 123 insertions(+), 22 deletions(-)
 create mode 100644 Messages/JSON.hs
 create mode 100644 Utility/JSONStream.hs

diff --git a/Annex.hs b/Annex.hs
index fac5d27e49..f5c3e4de45 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -64,7 +64,7 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-data OutputType = NormalOutput | QuietOutput
+data OutputType = NormalOutput | QuietOutput | JSONOutput
 
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
diff --git a/Messages.hs b/Messages.hs
index b2c871ede1..87d414f172 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -5,7 +5,22 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Messages where
+module Messages (
+	showStart,
+	showNote,
+	showAction,
+	showProgress,
+	showSideAction,
+	showOutput,
+	showLongNote,
+	showEndOk,
+	showEndFail,
+	showEndResult,
+	showErr,
+	warning,
+	indent,
+	setupConsole
+) where
 
 import Control.Monad.State (liftIO)
 import System.IO
@@ -13,21 +28,15 @@ import Data.String.Utils
 
 import Types
 import qualified Annex
-
-verbose :: Annex () -> Annex ()
-verbose a = do
-	output <- Annex.getState Annex.output
-	case output of
-		Annex.NormalOutput -> a
-		_ -> return ()
+import qualified Messages.JSON as JSON
 
 showStart :: String -> String -> Annex ()
-showStart command file = verbose $ liftIO $ do
+showStart command file = handle (JSON.start command file) $ do
 	putStr $ command ++ " " ++ file ++ " "
 	hFlush stdout
 
 showNote :: String -> Annex ()
-showNote s = verbose $ liftIO $ do
+showNote s = handle (JSON.note s) $ do
 	putStr $ "(" ++ s ++ ") "
 	hFlush stdout
 
@@ -35,28 +44,31 @@ showAction :: String -> Annex ()
 showAction s = showNote $ s ++ "..."
 
 showProgress :: Annex ()
-showProgress = verbose $ liftIO $ do
+showProgress = handle q $ do
 	putStr "."
 	hFlush stdout
 
 showSideAction :: String -> Annex ()
-showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ "...)"
+showSideAction s = handle q $ putStrLn $ "(" ++ s ++ "...)"
 
 showOutput :: Annex ()
-showOutput = verbose $ liftIO $ putStr "\n"
+showOutput = handle q $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = verbose $ liftIO $ putStr $ '\n' : indent s
+showLongNote s = handle (JSON.note s) $ putStr $ '\n' : indent s
 
 showEndOk :: Annex ()
-showEndOk = verbose $ liftIO $ putStrLn "ok"
+showEndOk = showEndResult True
 
 showEndFail :: Annex ()
-showEndFail = verbose $ liftIO $ putStrLn "failed"
+showEndFail = showEndResult False
 
 showEndResult :: Bool -> Annex ()
-showEndResult True = showEndOk
-showEndResult False = showEndFail
+showEndResult b = handle (JSON.end b) $ putStrLn msg
+	where
+		msg
+			| b = "ok"
+			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
 showErr e = liftIO $ do
@@ -65,7 +77,7 @@ showErr e = liftIO $ do
 
 warning :: String -> Annex ()
 warning w = do
-	verbose $ liftIO $ putStr "\n"
+	handle q $ putStr "\n"
 	liftIO $ do
 		hFlush stdout
 		hPutStrLn stderr $ indent w
@@ -85,3 +97,14 @@ setupConsole :: IO ()
 setupConsole = do
 	hSetBinaryMode stdout True
 	hSetBinaryMode stderr True
+
+handle :: IO () -> IO () -> Annex ()
+handle json normal = do
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> liftIO normal
+		Annex.QuietOutput -> q
+		Annex.JSONOutput -> liftIO json
+
+q :: Monad m => m ()
+q = return ()
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
new file mode 100644
index 0000000000..ee6ea34a32
--- /dev/null
+++ b/Messages/JSON.hs
@@ -0,0 +1,23 @@
+{- git-annex JSON output
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Messages.JSON (
+	start,
+	end,
+	note
+) where
+
+import qualified Utility.JSONStream as Stream
+
+start :: String -> String -> IO ()
+start command file = putStr $ Stream.start [("command", command), ("file", file)]
+
+end :: Bool -> IO ()
+end b = putStr $ Stream.add [("success", b)] ++ Stream.end
+
+note :: String -> IO ()
+note s = putStr $ Stream.add [("note", s)]
diff --git a/Options.hs b/Options.hs
index 768a1c289d..e0ca48c01b 100644
--- a/Options.hs
+++ b/Options.hs
@@ -30,6 +30,8 @@ commonOptions =
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
 		"allow verbose output (default)"
+	, Option ['j'] ["json"] (NoArg (setoutput Annex.JSONOutput))
+		"enable JSON output"
 	, Option ['d'] ["debug"] (NoArg (setdebug))
 		"show debug messages"
 	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
diff --git a/Utility/JSONStream.hs b/Utility/JSONStream.hs
new file mode 100644
index 0000000000..af3766948f
--- /dev/null
+++ b/Utility/JSONStream.hs
@@ -0,0 +1,44 @@
+{- Streaming JSON output.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.JSONStream (
+	start,
+	add,
+	end
+) where
+
+import Text.JSON
+
+{- Text.JSON does not support building up a larger JSON document piece by
+   piece as a stream. To support streaming, a hack. The JSObject is converted
+   to a string with its final "}" is left off, allowing it to be added to
+   later. -}
+start :: JSON a => [(String, a)] -> String
+start l
+	| last s == endchar = take (length s - 1)  s
+	| otherwise = bad s
+	where
+		s = encodeStrict $ toJSObject l
+
+add :: JSON a => [(String, a)] -> String
+add l
+	| head s == startchar = ',' : drop 1 s
+	| otherwise = bad s
+	where
+		s = start l
+
+end :: String
+end = [endchar, '\n']
+
+startchar :: Char
+startchar = '{'
+
+endchar :: Char
+endchar = '}'
+
+bad :: String -> a
+bad s = error $ "Text.JSON returned unexpected string: " ++ s
diff --git a/debian/changelog b/debian/changelog
index 4ee0b80f2e..ca23ab4735 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * init: Make description an optional parameter.
   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
   * unused --remote: Reduced memory use to 1/4th what was used before.
+  * Add --json switch, to produce machine-consumable output.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/debian/control b/debian/control
index 63488dc68d..cb5a8212ab 100644
--- a/debian/control
+++ b/debian/control
@@ -14,6 +14,7 @@ Build-Depends:
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
 	libghc-monad-control-dev,
+	libghc-json-dev,
 	ikiwiki,
 	perlmagick,
 	git | git-core,
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e7ac9adf7a..0a484a3842 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -337,12 +337,18 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * --quiet
 
-  Avoid the default verbose logging of what is done; only show errors
+  Avoid the default verbose display of what is done; only show errors
   and progress displays.
 
 * --verbose
 
-  Enable verbose logging.
+  Enable verbose display.
+
+* --json
+
+  Rather than the normal output, generate JSON. This is intended to be
+  parsed by programs that use git-annex. Each line of output is a JSON
+  object.
 
 * --debug
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index ac521da187..cd51b96d23 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -28,6 +28,7 @@ To build and use git-annex, you will need:
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [HTTP](http://hackage.haskell.org/package/HTTP)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
+  * [json](http://hackage.haskell.org/package/json)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)

From 5bc32c7f3438a329878f4da7ad0b514c12a54332 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:02:01 -0400
Subject: [PATCH 2169/2835] add json formatted list of remotes

Wherever a list of remotes is shown, --json now enables a json formatted
list.
---
 Command/Fsck.hs    |  2 +-
 Command/Whereis.hs |  2 +-
 Remote.hs          | 35 +++++++++++++++++++++++++----------
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 6ccec05fb3..bad60d30dd 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -131,7 +131,7 @@ checkKeyNumCopies key file numcopies = do
 	let present = length safelocations
 	if present < needed
 		then do
-			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
+			ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations
 			warning $ missingNote (filename file key) present needed ppuuids
 			return False
 		else return True
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 314fef7826..f80c823b70 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -33,7 +33,7 @@ perform key = do
 	if null uuids
 		then stop
 		else do
-			pp <- prettyPrintUUIDs uuids
+			pp <- prettyPrintUUIDs "whereis" uuids
 			showLongNote pp
 			showOutput
 			next $ return True
diff --git a/Remote.hs b/Remote.hs
index 2c883f1a8f..ad1768da64 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -30,11 +30,14 @@ module Remote (
 ) where
 
 import Control.Monad (filterM)
+import Control.Monad.State (liftIO)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
 import Data.Maybe
 import Control.Applicative
+import Text.JSON
+import Text.JSON.Generic
 
 import Types
 import Types.Remote
@@ -44,6 +47,7 @@ import Config
 import Trust
 import LocationLog
 import Messages
+import qualified Utility.JSONStream
 import RemoteLog
 
 import qualified Remote.Git
@@ -119,23 +123,34 @@ nameToUUID n = do
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
-{- Pretty-prints a list of UUIDs of remotes. -}
-prettyPrintUUIDs :: [UUID] -> Annex String
-prettyPrintUUIDs uuids = do
+{- Pretty-prints a list of UUIDs of remotes, for human display.
+ -
+ - Shows descriptions from the uuid log, falling back to remote names,
+ - as some remotes may not be in the uuid log.
+ -
+ - When JSON is enabled, also generates a machine-readable description
+ - of the UUIDs. -}
+prettyPrintUUIDs :: String -> [UUID] -> Annex String
+prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
-	-- Show descriptions from the uuid log, falling back to remote names,
-	-- as some remotes may not be in the uuid log
 	m <- M.union <$> uuidMap <*> availMap
-	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
+	liftIO . putStr $ Utility.JSONStream.add
+		[(desc, map (jsonify m here) uuids)]
+	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
 		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
-		prettify m u here = base ++ ishere
+		findlog m u = M.findWithDefault "" u m
+		prettify m here u = base ++ ishere
 			where
 				base = if not $ null $ findlog m u
 					then u ++ "  -- " ++ findlog m u
 					else u
 				ishere = if here == u then " <-- here" else ""
-		findlog m u = M.findWithDefault "" u m
+		jsonify m here u = toJSObject
+			[ ("uuid", toJSON u)
+			, ("description", toJSON $ findlog m u)
+			, ("here", toJSON $ here == u)
+			]
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
@@ -186,8 +201,8 @@ showLocations key exclude = do
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
-	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
-	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
+	ppuuidswanted <- Remote.prettyPrintUUIDs "wanted" uuidswanted
+	ppuuidsskipped <- Remote.prettyPrintUUIDs "skipped" uuidsskipped
 	showLongNote $ message ppuuidswanted ppuuidsskipped
 	where
 		filteruuids l x = filter (`notElem` x) l

From e4a74c0dc533922ce1e7cd56dfa7ac08efc064cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:11:21 -0400
Subject: [PATCH 2170/2835] close bug with some caveats

---
 doc/bugs/add_script-friendly_output_options.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
index efcd54553f..08f7e6c7b8 100644
--- a/doc/bugs/add_script-friendly_output_options.mdwn
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -1,3 +1,18 @@
 I have a need to use git-annex from a larger program. It'd be great if the information output by some of the commands that is descriptive (for example, whereis) could be sent to stdout in a machine-readable format like (preferably) JSON, or XML. That way I can simply read in the output of the command and use the data directly instead of having to parse it via regexes or other such string manipulation.
 
 This could perhaps be triggered by a --json or --xml flag to the relevant commands.
+
+> This is [[done]], --json is supported by all commands, more or less.
+> 
+> Caveats: 
+> 
+>  * the version, status, and find commands produce custom output and so
+>    no json. This could change for version and status; find needs to just
+>    be a simple list of files, I think 
+>  * The "note" fields may repeat multiple times per object with different
+>    notes and are of course not machine readable, and subject to change.
+>  * Output of helper commands like rsync is not diverted away, and
+>    could clutter up the json output badly. Should only affect commands
+>    that transfer data.
+> 
+> --[[Joey]] 

From cb5dacfd403a5ca4e3bdeb3eb3d0009136966f91 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:15:57 -0400
Subject: [PATCH 2171/2835] rsync and wget use stderr for progress, so no
 problem

---
 doc/bugs/add_script-friendly_output_options.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
index 08f7e6c7b8..7d7bdfc51a 100644
--- a/doc/bugs/add_script-friendly_output_options.mdwn
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -13,6 +13,7 @@ This could perhaps be triggered by a --json or --xml flag to the relevant comman
 >    notes and are of course not machine readable, and subject to change.
 >  * Output of helper commands like rsync is not diverted away, and
 >    could clutter up the json output badly. Should only affect commands
->    that transfer data.
+>    that transfer data. And AFAICS, wget and rsync both output their
+>    progress displays to stderr, so shouldn't be a problem.
 > 
 > --[[Joey]] 

From d1dd40c49bb95416adec5b33e4dc1e7672a2f7f7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 16:44:04 -0400
Subject: [PATCH 2172/2835] avoid showing json lists of remotes when not in
 json mode

---
 Messages.hs      |  6 ++++++
 Messages/JSON.hs | 10 ++++++++--
 Remote.hs        |  5 +----
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/Messages.hs b/Messages.hs
index 87d414f172..faa4dbcdec 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -19,12 +19,14 @@ module Messages (
 	showErr,
 	warning,
 	indent,
+	maybeShowJSON,
 	setupConsole
 ) where
 
 import Control.Monad.State (liftIO)
 import System.IO
 import Data.String.Utils
+import Text.JSON
 
 import Types
 import qualified Annex
@@ -106,5 +108,9 @@ handle json normal = do
 		Annex.QuietOutput -> q
 		Annex.JSONOutput -> liftIO json
 
+{- Shows a JSON value only when in json mode. -}
+maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
+maybeShowJSON v = handle (JSON.add v) q
+
 q :: Monad m => m ()
 q = return ()
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
index ee6ea34a32..fb95f550e8 100644
--- a/Messages/JSON.hs
+++ b/Messages/JSON.hs
@@ -8,9 +8,12 @@
 module Messages.JSON (
 	start,
 	end,
-	note
+	note,
+	add
 ) where
 
+import Text.JSON
+
 import qualified Utility.JSONStream as Stream
 
 start :: String -> String -> IO ()
@@ -20,4 +23,7 @@ end :: Bool -> IO ()
 end b = putStr $ Stream.add [("success", b)] ++ Stream.end
 
 note :: String -> IO ()
-note s = putStr $ Stream.add [("note", s)]
+note s = add [("note", s)]
+
+add :: JSON a => [(String, a)] -> IO ()
+add v = putStr $ Stream.add v
diff --git a/Remote.hs b/Remote.hs
index ad1768da64..e54d2e2334 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -30,7 +30,6 @@ module Remote (
 ) where
 
 import Control.Monad (filterM)
-import Control.Monad.State (liftIO)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
@@ -47,7 +46,6 @@ import Config
 import Trust
 import LocationLog
 import Messages
-import qualified Utility.JSONStream
 import RemoteLog
 
 import qualified Remote.Git
@@ -134,8 +132,7 @@ prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
 	m <- M.union <$> uuidMap <*> availMap
-	liftIO . putStr $ Utility.JSONStream.add
-		[(desc, map (jsonify m here) uuids)]
+	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
 		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList

From d238bbd9d9f5e96334b9214c7aeca7875054e99e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 21:32:05 -0400
Subject: [PATCH 2173/2835] releasing version 3.20110902

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ca23ab4735..35144f678d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110820) UNRELEASED; urgency=low
+git-annex (3.20110902) unstable; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
@@ -8,7 +8,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * unused --remote: Reduced memory use to 1/4th what was used before.
   * Add --json switch, to produce machine-consumable output.
 
- -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
+ -- Joey Hess   Fri, 02 Sep 2011 21:20:37 -0400
 
 git-annex (3.20110819) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index a37c187c90..7850f88a77 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110819
+Version: 3.20110902
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From ea621d0c2d418c6dfe247b0aa3f78976c002b2b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 21:32:21 -0400
Subject: [PATCH 2174/2835] add news item for git-annex 3.20110902

---
 doc/news/version_3.20110705.mdwn | 9 ---------
 doc/news/version_3.20110902.mdwn | 9 +++++++++
 2 files changed, 9 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_3.20110705.mdwn
 create mode 100644 doc/news/version_3.20110902.mdwn

diff --git a/doc/news/version_3.20110705.mdwn b/doc/news/version_3.20110705.mdwn
deleted file mode 100644
index bb4665c047..0000000000
--- a/doc/news/version_3.20110705.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 3.20110705 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * uninit: Delete the git-annex branch and .git/annex/
-   * unannex: In --fast mode, file content is left in the annex, and a
-     hard link made to it.
-   * uninit: Use unannex in --fast mode, to support unannexing multiple
-     files that link to the same content.
-   * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
-   * Fix a pipeline stall when upgrading (caused by #624389)."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110902.mdwn b/doc/news/version_3.20110902.mdwn
new file mode 100644
index 0000000000..e354874ea3
--- /dev/null
+++ b/doc/news/version_3.20110902.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110902 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Set EMAIL when running test suite so that git does not need to be
+     configured first. Closes: #[638998](http://bugs.debian.org/638998)
+   * The wget command will now be used in preference to curl, if available.
+   * init: Make description an optional parameter.
+   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
+   * unused --remote: Reduced memory use to 1/4th what was used before.
+   * Add --json switch, to produce machine-consumable output."""]]
\ No newline at end of file

From c5c525d930480ccbf230420efb3b07cfd3ea78d7 Mon Sep 17 00:00:00 2001
From: DavidEdmondson 
Date: Mon, 5 Sep 2011 15:43:26 +0000
Subject: [PATCH 2175/2835] Added a comment: Is it necessary to commit after
 the 'drop'?

---
 .../comment_1_cb65e7c510b75be1c51f655b058667c6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment

diff --git a/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
new file mode 100644
index 0000000000..1c8719cecd
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="DavidEdmondson"
+ subject="Is it necessary to commit after the 'drop'?"
+ date="2011-09-05T15:43:25Z"
+ content="""
+In fact is it possible? Nothing changed as far as git is concerned.
+
+"""]]

From f0777d9b5a90326c495d557f79c8420e0c7d284a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 5 Sep 2011 15:59:27 +0000
Subject: [PATCH 2176/2835] Added a comment

---
 .../comment_2_64709ea4558915edd5c8ca4486965b07._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment

diff --git a/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
new file mode 100644
index 0000000000..f5fb8dc7f5
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-05T15:59:27Z"
+ content="""
+Good catch. It used to be necessary before there was a git-annex branch, but not now.
+"""]]

From dbef6a045c3f80510af3dd5ce80d94371a21a229 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Sep 2011 11:56:30 -0400
Subject: [PATCH 2177/2835] remove now-unnecessary commit after drop

---
 doc/walkthrough/removing_files.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/walkthrough/removing_files.mdwn b/doc/walkthrough/removing_files.mdwn
index 85a7d50a6b..0df6e82a1f 100644
--- a/doc/walkthrough/removing_files.mdwn
+++ b/doc/walkthrough/removing_files.mdwn
@@ -3,4 +3,3 @@ has the file before removing it.
 
 	# git annex drop iso/debian.iso
 	drop iso/Debian_5.0.iso ok
-	# git commit -a -m "freed up space"

From 14f75ced75aa73d4f8dbc45cdc697bac1535fb37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Sep 2011 11:57:58 -0400
Subject: [PATCH 2178/2835] remove now unnecessary commit after get

---
 doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
index 936d088f1f..cfb70aaf9a 100644
--- a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
+++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
@@ -15,4 +15,3 @@ it:
 	# sudo mount /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
 	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
-	# git commit -a -m "got a video I want to rewatch on the plane"

From 07125dca53ece4896a83fad00b7f43101855ce9c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 13:46:08 -0400
Subject: [PATCH 2179/2835] Improve display of newlines around error and
 warning messages.

---
 CmdLine.hs                |  2 +-
 Command.hs                |  1 -
 Messages.hs               | 11 ++++++-----
 debian/changelog          |  6 ++++++
 doc/bugs/fsck_output.mdwn | 10 ++++++++++
 5 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 2652e5b8fc..4ccd2c2c2f 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -89,8 +89,8 @@ tryRun' errnum state (a:as) = do
 	case result of
 		Left err -> do
 			Annex.eval state $ do
-				showEndFail
 				showErr err
+				showEndFail
 			tryRun' (errnum + 1) state as
 		Right (True,state') -> tryRun' errnum state' as
 		Right (False,state') -> tryRun' (errnum + 1) state' as
diff --git a/Command.hs b/Command.hs
index 21c50f9c02..78f9823fb3 100644
--- a/Command.hs
+++ b/Command.hs
@@ -103,7 +103,6 @@ doCommand = start
 		stage a b = b >>= a
 		success = return True
 		failure = do
-			showOutput -- avoid clutter around error message
 			showEndFail
 			return False
 
diff --git a/Messages.hs b/Messages.hs
index faa4dbcdec..4922519819 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -73,16 +73,17 @@ showEndResult b = handle (JSON.end b) $ putStrLn msg
 			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
-showErr e = liftIO $ do
-	hFlush stdout
-	hPutStrLn stderr $ "git-annex: " ++ show e
+showErr e = warning' $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
-warning w = do
+warning w = warning' (indent w)
+
+warning' :: String -> Annex ()
+warning' w = do
 	handle q $ putStr "\n"
 	liftIO $ do
 		hFlush stdout
-		hPutStrLn stderr $ indent w
+		hPutStrLn stderr w
 
 indent :: String -> String
 indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
diff --git a/debian/changelog b/debian/changelog
index 35144f678d..9ab3b4fd3d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110903) UNRELEASED; urgency=low
+
+  * Improve display of newlines around error and warning messages.
+
+ -- Joey Hess   Tue, 06 Sep 2011 13:45:16 -0400
+
 git-annex (3.20110902) unstable; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 90af1600d8..1b00dd7b37 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -34,3 +34,13 @@ The newline is in the wrong place and confuses the user. It should be printed _a
 > failed
 > 
 > --[[Joey]] 
+
+>> Well, I fixed this in all cases except a thrown non-IO error (last
+>> example aboce), which output is printed by haskell's runtime. I'd
+>> have to add a second error handler to handle those, and it's not
+>> clear what it would do. Often an error will occur before anything
+>> else is printed, and then the current behavior is right; if something
+>> has been printed it would be nice to have a newline before the error,
+>> but by the time the error is caught we'd be out of the annex monad
+>> and not really have any way to know if something has been printed.
+>> I think my fix is good enough [[done]] --[[Joey]] 

From fcfd2776cb010f35557a6e617d30327f4274e0ff Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl0-EtQjVUNysjom6sTlQxRUwkwD6uPx88"
 
Date: Tue, 6 Sep 2011 18:27:08 +0000
Subject: [PATCH 2180/2835]

---
 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn

diff --git a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
new file mode 100644
index 0000000000..61c87b56a0
--- /dev/null
+++ b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
@@ -0,0 +1,9 @@
+While following the instructions given at the OSX build page , I get this error:
+
+$ make
+ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make git-annex
+
+Utility/JSONStream.hs:14:8:
+    Could not find module `Text.JSON':
+      Use -v to see a list of the files searched for.
+make: *** [git-annex] Error 1

From ca4eb842a4c7aa491c266e79ee1e1b340615ba01 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:36:21 -0400
Subject: [PATCH 2181/2835] add json to build scripts

---
 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn | 2 ++
 doc/install/Fedora.mdwn                   | 1 +
 doc/install/OSX.mdwn                      | 1 +
 3 files changed, 4 insertions(+)

diff --git a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
index 61c87b56a0..43fb0323c4 100644
--- a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
+++ b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
@@ -7,3 +7,5 @@ Utility/JSONStream.hs:14:8:
     Could not find module `Text.JSON':
       Use -v to see a list of the files searched for.
 make: *** [git-annex] Error 1
+
+> Updated the instructions. [[done]] --[[Joey]] 
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 0c1da2e6a5..17b8635218 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -11,6 +11,7 @@ sudo cabal install SHA
 sudo cabal install dataenc
 sudo cabal install monad-control
 sudo cabal install HTTP
+sudo cabal install json
 sudo cabal install hS3
 
 git clone git://git-annex.branchable.com/
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 983ac3a2fd..12502e0900 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -13,6 +13,7 @@ sudo cabal install SHA
 sudo cabal install dataenc
 sudo cabal install monad-control
 sudo cabal install HTTP
+sudo cabal install json
 sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)

From 7722147e1625e0a16cfcf13fd967a4a961decaf7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:36:51 -0400
Subject: [PATCH 2182/2835] add josn build dep

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 7850f88a77..48ac48bb2d 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5, monad-control
+   base < 5, monad-control, json
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From 3778e8897dcd8b2a5a54d076cc5d3a6d0a0bb689 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:48:15 -0400
Subject: [PATCH 2183/2835] switch to using cabal-install git-annex

---
 doc/install/Fedora.mdwn | 19 +------------------
 doc/install/OSX.mdwn    | 18 ++----------------
 2 files changed, 3 insertions(+), 34 deletions(-)

diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 17b8635218..068d5c111c 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -3,22 +3,5 @@ Installation recipe for Fedora 14.
 
 sudo yum install ghc cabal-install
 sudo cabal update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo cabal install pcre-light
-sudo cabal install quickcheck
-sudo cabal install SHA
-sudo cabal install dataenc
-sudo cabal install monad-control
-sudo cabal install HTTP
-sudo cabal install json
-sudo cabal install hS3
-
-git clone git://git-annex.branchable.com/
-
-cd git-annex
-sudo make   # For some reason you need to use sudo here as otherwise the cabal installed packages doesn't seem to be there...
-sudo install git-annex
+sudo cabal install git-annex
 

-
-Originally posted by Jon at  --[[Joey]]
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 12502e0900..680c331ee6 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -4,26 +4,12 @@ Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. T
 sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 
 sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
-sudo cabal update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo cabal install pcre-light
-sudo cabal install quickcheck  
-sudo cabal install SHA
-sudo cabal install dataenc
-sudo cabal install monad-control
-sudo cabal install HTTP
-sudo cabal install json
-sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
-git clone  git://git-annex.branchable.com/
-
-cd git-annex
-make
-sudo make install
+sudo cabal update
+sudo cabal install git-annex
 

 
 Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]

From ebb92221fd2d42e360bf24a72b53971129908f01 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 15:35:13 -0400
Subject: [PATCH 2184/2835] Fix Makefile to work with cabal again.

---
 Makefile         | 4 +++-
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 20d1347140..3f36928afb 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,9 @@ Remote/S3.o: Remote/S3.hs
 		echo "** building without S3 support"; \
 	fi
 
-$(bins): $(sources)
+sources: $(sources)
+
+$(bins): sources
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
diff --git a/debian/changelog b/debian/changelog
index 9ab3b4fd3d..973623e200 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110903) UNRELEASED; urgency=low
 
   * Improve display of newlines around error and warning messages.
+  * Fix Makefile to work with cabal again.
 
  -- Joey Hess   Tue, 06 Sep 2011 13:45:16 -0400
 

From 6fd0df7c2ff20710491923abfbb7a1bc4d3750bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 15:54:21 -0400
Subject: [PATCH 2185/2835] releasing version 3.20110906

---
 debian/changelog | 2 +-
 git-annex.cabal  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 973623e200..0fca41ee5c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110903) UNRELEASED; urgency=low
+git-annex (3.20110906) unstable; urgency=low
 
   * Improve display of newlines around error and warning messages.
   * Fix Makefile to work with cabal again.
diff --git a/git-annex.cabal b/git-annex.cabal
index 48ac48bb2d..34c8e76222 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110902
+Version: 3.20110906
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From b7bcd942c57631f50d686cb8d45a83eaf377c33a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 15:54:43 -0400
Subject: [PATCH 2186/2835] add news item for git-annex 3.20110906

---
 doc/news/version_3.20110707.mdwn | 8 --------
 doc/news/version_3.20110906.mdwn | 4 ++++
 2 files changed, 4 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_3.20110707.mdwn
 create mode 100644 doc/news/version_3.20110906.mdwn

diff --git a/doc/news/version_3.20110707.mdwn b/doc/news/version_3.20110707.mdwn
deleted file mode 100644
index 3f489ae5d6..0000000000
--- a/doc/news/version_3.20110707.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 3.20110707 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix sign bug in disk free space checking.
-   * Bugfix: Forgot to de-escape keys when upgrading. Could result in
-     bad location log data for keys that contain [&:%] in their names.
-     (A workaround for this problem is to run git annex fsck.)
-   * add: Avoid a failure mode that resulted in the file seemingly being
-     deleted (content put in the annex but no symlink present)."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110906.mdwn b/doc/news/version_3.20110906.mdwn
new file mode 100644
index 0000000000..d91776ccaf
--- /dev/null
+++ b/doc/news/version_3.20110906.mdwn
@@ -0,0 +1,4 @@
+git-annex 3.20110906 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Improve display of newlines around error and warning messages.
+   * Fix Makefile to work with cabal again."""]]
\ No newline at end of file

From 6f98fd53914b2490e866a2613e86b93f689034bf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 16:59:53 -0400
Subject: [PATCH 2187/2835] whereis: Show untrusted locations separately and do
 not include in location count.

---
 Command/Whereis.hs | 27 ++++++++++++++++++---------
 debian/changelog   |  7 +++++++
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index f80c823b70..d5c090e493 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,11 +7,15 @@
 
 module Command.Whereis where
 
+import Control.Monad
+import Data.List
+
 import LocationLog
 import Command
 import Messages
 import Remote
 import Types
+import Trust
 
 command :: [Command]
 command = [repoCommand "whereis" (paramOptional $ paramRepeating paramPath) seek
@@ -27,16 +31,21 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: Key -> CommandPerform
 perform key = do
-	uuids <- keyLocations key
-	let num = length uuids
+	locations <- keyLocations key
+	untrusted <- trustGet UnTrusted
+	let untrustedlocations = intersect untrusted locations
+	let safelocations = filter (`notElem` untrusted) locations
+	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
-	if null uuids
-		then stop
-		else do
-			pp <- prettyPrintUUIDs "whereis" uuids
-			showLongNote pp
-			showOutput
-			next $ return True
+	pp <- prettyPrintUUIDs "whereis" safelocations
+	unless (null safelocations) $
+		showLongNote pp
+	pp' <- prettyPrintUUIDs "untrusted" untrustedlocations
+	unless (null untrustedlocations) $
+		showLongNote $ untrustedheader ++ pp'
+	unless (null locations) showOutput
+	if null safelocations then stop else next $ return True
 	where
 		copiesplural 1 = "copy"
 		copiesplural _ = "copies"
+		untrustedheader = "The following untrusted locations may also have copies:\n"
diff --git a/debian/changelog b/debian/changelog
index 0fca41ee5c..d900ab1840 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20110907) UNRELEASED; urgency=low
+
+  * whereis: Show untrusted locations separately and do not include in
+    location count.
+
+ -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
+
 git-annex (3.20110906) unstable; urgency=low
 
   * Improve display of newlines around error and warning messages.

From 3623d831d193d029a35aac81571d67768b176534 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 17:19:29 -0400
Subject: [PATCH 2188/2835] refactor

---
 Command/Fsck.hs    |  6 +-----
 Command/Whereis.hs |  5 +----
 Remote.hs          |  2 +-
 Trust.hs           | 12 ++++++++++--
 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index bad60d30dd..529a5015a9 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -10,7 +10,6 @@ module Command.Fsck where
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory
-import Data.List
 import System.Posix.Files
 
 import Command
@@ -124,10 +123,7 @@ checkKeySize key = do
 checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
-	locations <- keyLocations key
-	untrusted <- trustGet UnTrusted
-	let untrustedlocations = intersect untrusted locations
-	let safelocations = filter (`notElem` untrusted) locations
+	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< keyLocations key
 	let present = length safelocations
 	if present < needed
 		then do
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index d5c090e493..850975048a 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -8,7 +8,6 @@
 module Command.Whereis where
 
 import Control.Monad
-import Data.List
 
 import LocationLog
 import Command
@@ -32,9 +31,7 @@ start file = isAnnexed file $ \(key, _) -> do
 perform :: Key -> CommandPerform
 perform key = do
 	locations <- keyLocations key
-	untrusted <- trustGet UnTrusted
-	let untrustedlocations = intersect untrusted locations
-	let safelocations = filter (`notElem` untrusted) locations
+	(untrustedlocations, safelocations) <- trustPartition UnTrusted locations
 	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
 	pp <- prettyPrintUUIDs "whereis" safelocations
diff --git a/Remote.hs b/Remote.hs
index e54d2e2334..429f9058b6 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -181,7 +181,7 @@ keyPossibilities' withtrusted key = do
 	let validuuids = filter (/= u) uuids
 
 	-- note that validuuids is assumed to not have dups
-	let validtrusteduuids = intersect validuuids trusted
+	let validtrusteduuids = validuuids `intersect` trusted
 
 	-- remotes that match uuids that have the key
 	allremotes <- genList
diff --git a/Trust.hs b/Trust.hs
index 365186da22..232eea6a5d 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -9,11 +9,13 @@ module Trust (
 	TrustLevel(..),
 	trustLog,
 	trustGet,
-	trustSet
+	trustSet,
+	trustPartition
 ) where
 
 import Control.Monad.State
 import qualified Data.Map as M
+import Data.List
 
 import Types.TrustLevel
 import qualified Branch
@@ -32,7 +34,7 @@ trustGet level = do
 	return $ M.keys $ M.filter (== level) m
 
 {- Read the trustLog into a map, overriding with any
- - values from forcetrust -}
+ - values from forcetrust. The map is cached for speed. -}
 trustMap :: Annex TrustMap
 trustMap = do
 	cached <- Annex.getState Annex.trustmap
@@ -70,3 +72,9 @@ trustSet uuid level = do
         where
                 serialize m = unlines $ map showpair $ M.toList m
 		showpair (u, t) = u ++ " " ++ show t
+
+{- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
+trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
+trustPartition level ls = do
+	candidates <- trustGet level
+	return $ partition (`elem` candidates) ls

From 72b54d617006fd5ddce92ee577c52f2bff279310 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Sep 2011 10:21:19 -0400
Subject: [PATCH 2189/2835] Fix build without S3.

---
 Makefile         | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3f36928afb..ff58362766 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@ Remote/S3.o: Remote/S3.hs
 
 sources: $(sources)
 
-$(bins): sources
+$(bins): sources Remote/S3.o
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
diff --git a/debian/changelog b/debian/changelog
index d900ab1840..e79685a9ab 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110907) UNRELEASED; urgency=low
 
   * whereis: Show untrusted locations separately and do not include in
     location count.
+  * Fix build without S3.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 

From 7c768c09841d7346444d65721b132d144835fc99 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Sep 2011 18:57:38 -0400
Subject: [PATCH 2190/2835] simplify

---
 Git.hs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/Git.hs b/Git.hs
index ab43504e1a..cd6cdfbfd0 100644
--- a/Git.hs
+++ b/Git.hs
@@ -335,10 +335,8 @@ urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r
 
 {- The full authority portion an URL repo. (ie, "user@host:port") -}
 urlAuthority :: Repo -> String
-urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName' a ++ uriPort a
-	where
-		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
-urlAuthority repo = assertUrl repo $ error "internal"
+urlAuthority r = flip urlAuthPart r $ \a ->
+	uriUserInfo a ++ uriRegName' a ++ uriPort a
 
 {- Applies a function to extract part of the uriAuthority of an URL repo. -}
 urlAuthPart :: (URIAuth -> a) -> Repo -> a

From 03d6209e1ccee4a8df7d1b0336c1d5587a2b3ff6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Sep 2011 19:04:51 -0400
Subject: [PATCH 2191/2835] addurl: Always use whole url as destination
 filename, rather than only its file component.

First, this ensures that git annex addurl, when run repeatedly with the
same url, doesn't create duplicate files, which it did before when it
fell back to the longer filename.

Secondly, the file part of an url is frequently not very descriptive on its
own.

The uri scheme, auth, and port is intentionally left out, as clutter.
---
 Command/AddUrl.hs                  | 26 +++++++++-----------------
 debian/changelog                   |  2 ++
 doc/walkthrough/using_the_web.mdwn | 12 ++++++------
 3 files changed, 17 insertions(+), 23 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 55e51100ce..9fc68ca037 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -7,9 +7,10 @@
 
 module Command.AddUrl where
 
-import Control.Monad.State (liftIO, when)
+import Control.Monad.State
 import Network.URI
 import Data.String.Utils
+import Data.Maybe
 import System.Directory
 
 import Command
@@ -24,6 +25,7 @@ import Content
 import PresenceLog
 import Locations
 import Utility.Path
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "addurl" paramPath seek "add urls to annex"]
@@ -75,20 +77,10 @@ nodownload url file = do
 
 url2file :: URI -> IO FilePath
 url2file url = do
-	let parts = filter safe $ split "/" $ uriPath url
-	if null parts
-		then fallback
-		else do
-			let file = last parts
-			e <- doesFileExist file
-			if e then fallback else return file
+	whenM (doesFileExist file) $
+		error $ "already have this url in " ++ file
+	return file
 	where
-		fallback = do
-			let file = replace "/" "_" $ show url
-			e <- doesFileExist file
-			when e $ error "already have this url"
-			return file
-		safe "" = False
-		safe "." = False
-		safe ".." = False
-		safe _ = True
+		file = escape $ uriRegName auth ++ uriPath url ++ uriQuery url
+		escape = replace "/?" $ repeat '_'
+		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url
diff --git a/debian/changelog b/debian/changelog
index e79685a9ab..9ff745566f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20110907) UNRELEASED; urgency=low
   * whereis: Show untrusted locations separately and do not include in
     location count.
   * Fix build without S3.
+  * addurl: Always use whole url as destination filename, rather than
+    only its file component.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 
diff --git a/doc/walkthrough/using_the_web.mdwn b/doc/walkthrough/using_the_web.mdwn
index 9d5525758a..8009927a49 100644
--- a/doc/walkthrough/using_the_web.mdwn
+++ b/doc/walkthrough/using_the_web.mdwn
@@ -1,20 +1,20 @@
 The web can be used as a [[special_remote|special_remotes]] too.
 
 	# git annex addurl http://example.com/video.mpeg
-	addurl video.mpeg (downloading http://example.com/video.mpeg)
+	addurl example.com_video.mpeg (downloading http://example.com/video.mpeg)
 	########################################################## 100.0%
 	ok
 
 Now the file is downloaded, and has been added to the annex like any other
-file. So it can be copied to other repositories, and so on.
+file. So it can be renamed, copied to other repositories, and so on.
 
 Note that git-annex assumes that, if the web site does not 404, the file is
 still present on the web, and this counts as one [[copy|copies]] of the
 file. So it will let you remove your last copy, trusting it can be
 downloaded again:
 
-	# git annex drop video.mpeg
-	drop video.mpeg (checking http://example.com/video.mpeg) ok
+	# git annex drop example.com_video.mpeg
+	drop example.com_video.mpeg (checking http://example.com/video.mpeg) ok
 
 If you don't [[trust]] the web to this degree, just let git-annex know:
 
@@ -23,8 +23,8 @@ If you don't [[trust]] the web to this degree, just let git-annex know:
 
 With the result that it will hang onto files:
 
-	# git annex drop video.mpeg
-	drop video.mpeg (unsafe) 
+	# git annex drop example.com_video.mpeg
+	drop example.com_video.mpeg (unsafe) 
 	  Could only verify the existence of 0 out of 1 necessary copies
 	  Also these untrusted repositories may contain the file:
 	  	00000000-0000-0000-0000-000000000001  -- web

From e4ba0934c2992f00275bd24e8c4d25d6b9ea10ff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Sep 2011 00:11:32 -0400
Subject: [PATCH 2192/2835] fix / escape

---
 Command/AddUrl.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 9fc68ca037..b4e3ad9bdf 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -79,8 +79,9 @@ url2file :: URI -> IO FilePath
 url2file url = do
 	whenM (doesFileExist file) $
 		error $ "already have this url in " ++ file
+	liftIO $ print file
 	return file
 	where
 		file = escape $ uriRegName auth ++ uriPath url ++ uriQuery url
-		escape = replace "/?" $ repeat '_'
+		escape = replace "/" "_" . replace "?" "_"
 		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url

From e296da4bfef21c55fc23376300becb17a08eea83 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Sep 2011 01:45:41 -0400
Subject: [PATCH 2193/2835] more newline fixes

Adds a missing newline when a longnote is followed by a endresult.

Multiple longnotes in a row will now be separated by a blank line, which
could be a bug or a feature depending on taste.

Removed several places where newlines were explicitly displayed after
longnotes.
---
 Command/Unused.hs  | 8 ++------
 Command/Whereis.hs | 1 -
 Messages.hs        | 2 +-
 Upgrade/V2.hs      | 1 -
 4 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 6a62cde5f8..960b2e1dfb 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -57,9 +57,7 @@ checkUnused = do
 	where
 		list file msg l c = do
 			let unusedlist = number c l
-			unless (null l) $ do
-				showLongNote $ msg unusedlist
-				showLongNote "\n"
+			unless (null l) $ showLongNote $ msg unusedlist
 			writeUnusedFile file unusedlist
 			return $ c + length l
 
@@ -76,9 +74,7 @@ checkRemoteUnused' r = do
 	let remoteunused = remotehas `exclude` referenced
 	let list = number 0 remoteunused
 	writeUnusedFile "" list
-	unless (null remoteunused) $ do
-		showLongNote $ remoteUnusedMsg r list
-		showLongNote "\n"
+	unless (null remoteunused) $ showLongNote $ remoteUnusedMsg r list
 	where
 		{- This should run strictly to avoid the filterM
 		 - building many thunks containing keyLocations data. -}
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 850975048a..74660f2834 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -40,7 +40,6 @@ perform key = do
 	pp' <- prettyPrintUUIDs "untrusted" untrustedlocations
 	unless (null untrustedlocations) $
 		showLongNote $ untrustedheader ++ pp'
-	unless (null locations) showOutput
 	if null safelocations then stop else next $ return True
 	where
 		copiesplural 1 = "copy"
diff --git a/Messages.hs b/Messages.hs
index 4922519819..c663c17c20 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -57,7 +57,7 @@ showOutput :: Annex ()
 showOutput = handle q $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = handle (JSON.note s) $ putStr $ '\n' : indent s
+showLongNote s = handle (JSON.note s) $ putStrLn $ '\n' : indent s
 
 showEndOk :: Annex ()
 showEndOk = showEndResult True
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index ffd0f06535..e99a7cf817 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -123,7 +123,6 @@ push = do
 			showLongNote $
 				"git-annex branch created\n" ++
 				"Be sure to push this branch when pushing to remotes.\n"
-			showOutput
 
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]

From 1ac6217c74b63b9b154d5ee14ed72df8b5aa9268 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 13:32:46 -0400
Subject: [PATCH 2194/2835] shorten synopsis

---
 Command/Merge.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Merge.hs b/Command/Merge.hs
index 04328e8c5a..faaef906b5 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -13,7 +13,7 @@ import Messages
 
 command :: [Command]
 command = [repoCommand "merge" paramNothing seek
-		"auto-merges remote changes into the git-annex branch"]
+		"auto-merge remote changes into git-annex branch"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]

From 949b3f69d0f2b2a5c32a00d05d09a0b312fad35a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 13:47:22 -0400
Subject: [PATCH 2195/2835] optimize: A new subcommand that either gets or
 drops file content as needed to work toward meeting the configured numcopies
 setting.

This is currently rather simplistic, though still useful.
In the future, it could become smarter about what content is stored where,
etc.
---
 Command.hs                                   |  2 ++
 Command/Fsck.hs                              |  2 +-
 Command/Optimize.hs                          | 35 ++++++++++++++++++++
 GitAnnex.hs                                  |  2 ++
 debian/changelog                             |  2 ++
 doc/git-annex.mdwn                           |  5 +++
 doc/walkthrough.mdwn                         |  1 +
 doc/walkthrough/optimizing_repositories.mdwn | 13 ++++++++
 test.hs                                      | 12 +++++++
 9 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 Command/Optimize.hs
 create mode 100644 doc/walkthrough/optimizing_repositories.mdwn

diff --git a/Command.hs b/Command.hs
index 78f9823fb3..75c3b4412a 100644
--- a/Command.hs
+++ b/Command.hs
@@ -131,6 +131,8 @@ withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	liftM (map a) $ liftIO $ Git.checkAttr repo attr files
+withNumCopies :: CommandSeekAttrFiles
+withNumCopies = withAttrFilesInGit "annex.numcopies"
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 529a5015a9..cdc68581ee 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -34,7 +34,7 @@ command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
 	"check for problems"]
 
 seek :: [CommandSeek]
-seek = [withAttrFilesInGit "annex.numcopies" start]
+seek = [withNumCopies start]
 
 start :: CommandStartAttrFile
 start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
diff --git a/Command/Optimize.hs b/Command/Optimize.hs
new file mode 100644
index 0000000000..40625fc2f0
--- /dev/null
+++ b/Command/Optimize.hs
@@ -0,0 +1,35 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Optimize where
+
+import Command
+import Utility
+import LocationLog
+import Trust
+import Config
+import qualified Command.Get
+import qualified Command.Drop
+
+command :: [Command]
+command = [repoCommand "optimize" (paramOptional $ paramRepeating paramPath) seek
+	"get or drop content to best use available space"]
+
+seek :: [CommandSeek]
+seek = [withNumCopies start]
+
+start :: CommandStartAttrFile
+start p@(file, attr) = notBareRepo $ isAnnexed file $ \(key, _) -> do
+	needed <- getNumCopies numcopies
+	(_, safelocations) <- trustPartition UnTrusted =<< keyLocations key
+	dispatch needed (length safelocations)
+	where
+		dispatch needed present
+			| present < needed = Command.Get.start file
+			| present > needed = Command.Drop.start p
+			| otherwise = stop
+		numcopies = readMaybe attr :: Maybe Int
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 6f4e5d4921..8b9e557500 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -34,6 +34,7 @@ import qualified Command.Init
 import qualified Command.Describe
 import qualified Command.InitRemote
 import qualified Command.Fsck
+import qualified Command.Optimize
 import qualified Command.Unused
 import qualified Command.DropUnused
 import qualified Command.Unlock
@@ -77,6 +78,7 @@ cmds = concat
 	, Command.SetKey.command
 	, Command.Fix.command
 	, Command.Fsck.command
+	, Command.Optimize.command
 	, Command.Unused.command
 	, Command.DropUnused.command
 	, Command.Find.command
diff --git a/debian/changelog b/debian/changelog
index 9ff745566f..b02f6a15b2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (3.20110907) UNRELEASED; urgency=low
   * Fix build without S3.
   * addurl: Always use whole url as destination filename, rather than
     only its file component.
+  * optimize: A new subcommand that either gets or drops file content
+    as needed to work toward meeting the configured numcopies setting.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0a484a3842..8264c31b3a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -157,6 +157,11 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   To avoid expensive checksum calculations, specify --fast
 
+* optimize [path ...]
+  
+  Either gets or drops file content, as needed, to work toward meeting the
+  configured numcopies setting.
+
 * unused
 
   Checks the annex for data that does not correspond to any files present
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index eaae6b455c..b0eb258153 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -18,5 +18,6 @@ A walkthrough of the basic features of git-annex.
 	fsck:_verifying_your_data
 	fsck:_when_things_go_wrong
 	backups
+	optimizing_repositories
 	more
 """]]
diff --git a/doc/walkthrough/optimizing_repositories.mdwn b/doc/walkthrough/optimizing_repositories.mdwn
new file mode 100644
index 0000000000..0f17f1deae
--- /dev/null
+++ b/doc/walkthrough/optimizing_repositories.mdwn
@@ -0,0 +1,13 @@
+Once you have multiple repositories, and have perhaps configured numcopies,
+any given file can have many more copies than is needed, or perhaps fewer
+than you would like. Fsck can detect the latter problem, but there's another
+command that can help deal with both problems.
+
+The optimize subcommand either gets or drops file content, as needed,
+to work toward meeting the configured numcopies setting.
+
+	# git annex optimize
+	get my_cool_big_file (from laptop...) ok
+	drop other_file ok
+	# git annex optimize --numcopies=2
+	get other_file ok
diff --git a/test.hs b/test.hs
index 4d751a707b..bd2e1e46c5 100644
--- a/test.hs
+++ b/test.hs
@@ -93,6 +93,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_unannex
 	, test_drop
 	, test_get
+	, test_optimize
 	, test_move
 	, test_copy
 	, test_lock
@@ -216,6 +217,17 @@ test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do
 	inmainrepo $ unannexed ingitfile
 	unannexed ingitfile
 
+test_optimize :: Test
+test_optimize = "git-annex optimize" ~: TestCase $ intmpclonerepo $ do
+	inmainrepo $ annexed_present annexedfile
+	annexed_notpresent annexedfile
+	git_annex "optimize" ["-q", annexedfile, "--numcopies=2"] @? "optimize of file failed"
+	inmainrepo $ annexed_present annexedfile
+	annexed_present annexedfile
+	git_annex "optimize" ["-q", annexedfile] @? "optimize of file failed"
+	inmainrepo $ annexed_present annexedfile
+	annexed_notpresent annexedfile
+
 test_move :: Test
 test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do
 	annexed_notpresent annexedfile

From 5a1f10325f218bb2289e263ce1bdfd4a504467e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 13:48:39 -0400
Subject: [PATCH 2196/2835] refactor

---
 Command/Drop.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index 6e688d6632..8afffd3cb2 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -24,7 +24,7 @@ command = [repoCommand "drop" paramPath seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
-seek = [withAttrFilesInGit "annex.numcopies" start]
+seek = [withNumCopies start]
 
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}

From 11994ebb3dc50a74dea9601907493950ac3540a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 15:33:21 -0400
Subject: [PATCH 2197/2835] tweak

---
 Command/Whereis.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 74660f2834..ef4303b3ae 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -30,8 +30,7 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: Key -> CommandPerform
 perform key = do
-	locations <- keyLocations key
-	(untrustedlocations, safelocations) <- trustPartition UnTrusted locations
+	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< keyLocations key
 	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
 	pp <- prettyPrintUUIDs "whereis" safelocations

From 59fe0b29a6f15a61276702e1594b9476e1363fd9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 16:01:40 -0400
Subject: [PATCH 2198/2835] simplify

---
 Command/Drop.hs     | 4 +---
 Command/Fsck.hs     | 4 +---
 Command/Optimize.hs | 3 +--
 3 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index 8afffd3cb2..a18729940f 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -34,10 +34,8 @@ start (file, attr) = isAnnexed file $ \(key, _) -> do
 	if present
 		then do
 			showStart "drop" file
-			next $ perform key numcopies
+			next $ perform key $ readMaybe attr
 		else stop
-	where
-		numcopies = readMaybe attr :: Maybe Int
 
 perform :: Key -> Maybe Int -> CommandPerform
 perform key numcopies = do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index cdc68581ee..08bf2b97f4 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -39,9 +39,7 @@ seek = [withNumCopies start]
 start :: CommandStartAttrFile
 start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
-	next $ perform key file backend numcopies
-	where
-		numcopies = readMaybe attr :: Maybe Int
+	next $ perform key file backend $ readMaybe attr
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
 perform key file backend numcopies = do
diff --git a/Command/Optimize.hs b/Command/Optimize.hs
index 40625fc2f0..1a2b2237ff 100644
--- a/Command/Optimize.hs
+++ b/Command/Optimize.hs
@@ -24,7 +24,7 @@ seek = [withNumCopies start]
 
 start :: CommandStartAttrFile
 start p@(file, attr) = notBareRepo $ isAnnexed file $ \(key, _) -> do
-	needed <- getNumCopies numcopies
+	needed <- getNumCopies $ readMaybe attr
 	(_, safelocations) <- trustPartition UnTrusted =<< keyLocations key
 	dispatch needed (length safelocations)
 	where
@@ -32,4 +32,3 @@ start p@(file, attr) = notBareRepo $ isAnnexed file $ \(key, _) -> do
 			| present < needed = Command.Get.start file
 			| present > needed = Command.Drop.start p
 			| otherwise = stop
-		numcopies = readMaybe attr :: Maybe Int

From 5c96411314837db235b5cd1d412243806b83aa8a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 12:36:27 -0400
Subject: [PATCH 2199/2835] fix synopsis

---
 Command.hs        | 2 ++
 Command/AddUrl.hs | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command.hs b/Command.hs
index 75c3b4412a..710f36ea3e 100644
--- a/Command.hs
+++ b/Command.hs
@@ -221,6 +221,8 @@ paramKey :: String
 paramKey = "KEY"
 paramDesc :: String
 paramDesc = "DESC"
+paramUrl :: String
+paramUrl = "URL"
 paramNumber :: String
 paramNumber = "NUMBER"
 paramRemote :: String
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index b4e3ad9bdf..c78569ffc8 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -28,7 +28,8 @@ import Utility.Path
 import Utility.Conditional
 
 command :: [Command]
-command = [repoCommand "addurl" paramPath seek "add urls to annex"]
+command = [repoCommand "addurl" (paramRepeating $ paramUrl) seek
+	"add urls to annex"]
 
 seek :: [CommandSeek]
 seek = [withStrings start]

From 81984e60acb920fffc969818e63604060636aa09 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 12:37:27 -0400
Subject: [PATCH 2200/2835] better var name

---
 Command/Drop.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index a18729940f..8f886f283c 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -29,12 +29,12 @@ seek = [withNumCopies start]
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}
 start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, _) -> do
+start (file, numcopies) = isAnnexed file $ \(key, _) -> do
 	present <- inAnnex key
 	if present
 		then do
 			showStart "drop" file
-			next $ perform key $ readMaybe attr
+			next $ perform key $ readMaybe numcopies
 		else stop
 
 perform :: Key -> Maybe Int -> CommandPerform

From 984c9fc0523bcd3bfcd7de83f4f7974daa6872bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 13:30:04 -0400
Subject: [PATCH 2201/2835] remove optimize subcommand; use --auto instead

get, drop: Added --auto option, which decides whether to get/drop content
as needed to work toward the configured numcopies.

The problem with bundling it up in optimize was that I then found I wanted
to run an optmize that did not drop files, only got them. Considered adding
a --only-get switch to it, but that seemed wrong. Instead, let's make
existing subcommands optionally smarter.

Note that the only actual difference between drop and drop --auto is that
the latter does not even try to drop a file if it knows of not enough
copies, and does not print any error messages about files it was unable to
drop.

It might be nice to make get avoid asking git for attributes when not in
auto mode. For now it always asks for attributes.
---
 Annex.hs                                      |  2 +
 Command.hs                                    | 19 ++++++++++
 Command/Drop.hs                               |  8 ++--
 Command/Get.hs                                | 11 ++++--
 Command/Optimize.hs                           | 34 -----------------
 GitAnnex.hs                                   |  2 -
 Options.hs                                    |  3 ++
 debian/changelog                              |  4 +-
 doc/git-annex.mdwn                            | 22 +++++++----
 doc/walkthrough.mdwn                          |  2 +-
 .../automatically_managing_content.mdwn       | 38 +++++++++++++++++++
 doc/walkthrough/optimizing_repositories.mdwn  | 13 -------
 test.hs                                       | 12 ------
 13 files changed, 92 insertions(+), 78 deletions(-)
 delete mode 100644 Command/Optimize.hs
 create mode 100644 doc/walkthrough/automatically_managing_content.mdwn
 delete mode 100644 doc/walkthrough/optimizing_repositories.mdwn

diff --git a/Annex.hs b/Annex.hs
index f5c3e4de45..2c8ea1d61d 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -52,6 +52,7 @@ data AnnexState = AnnexState
 	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
+	, auto :: Bool
 	, branchstate :: BranchState
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
@@ -75,6 +76,7 @@ newState gitrepo = AnnexState
 	, output = NormalOutput
 	, force = False
 	, fast = False
+	, auto = False
 	, branchstate = startBranchState
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
diff --git a/Command.hs b/Command.hs
index 710f36ea3e..9d85fbe9d7 100644
--- a/Command.hs
+++ b/Command.hs
@@ -26,6 +26,9 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Utility
 import Types.Key
+import Trust
+import LocationLog
+import Config
 
 {- A command runs in four stages.
  -
@@ -276,3 +279,19 @@ preserveOrder orig new = collect orig new
  -}
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
 runPreserveOrder a files = preserveOrder files <$> a files
+
+{- Used for commands that have an auto mode that checks the number of known
+ - copies of a key.
+ -
+ - In auto mode, first checks that the number of known
+ - copies of the key is > or < than the numcopies setting, before running
+ - the action. -}
+autoCopies :: Key -> (Int -> Int -> Bool) -> Maybe Int -> CommandStart -> CommandStart
+autoCopies key vs numcopiesattr a = do
+	auto <- Annex.getState Annex.auto
+	if auto
+		then do
+			needed <- getNumCopies numcopiesattr
+			(_, have) <- trustPartition UnTrusted =<< keyLocations key
+			if length have `vs` needed then a else stop
+		else a
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 8f886f283c..32cb81e808 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -29,13 +29,15 @@ seek = [withNumCopies start]
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}
 start :: CommandStartAttrFile
-start (file, numcopies) = isAnnexed file $ \(key, _) -> do
+start (file, attr) = isAnnexed file $ \(key, _) -> do
 	present <- inAnnex key
 	if present
-		then do
+		then autoCopies key (>) numcopies $ do
 			showStart "drop" file
-			next $ perform key $ readMaybe numcopies
+			next $ perform key numcopies
 		else stop
+	where
+		numcopies = readMaybe attr
 
 perform :: Key -> Maybe Int -> CommandPerform
 perform key numcopies = do
diff --git a/Command/Get.hs b/Command/Get.hs
index e0436a8680..1a50158e38 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -13,6 +13,7 @@ import qualified Remote
 import Types
 import Content
 import Messages
+import Utility
 import qualified Command.Move
 
 command :: [Command]
@@ -20,14 +21,14 @@ command = [repoCommand "get" paramPath seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withNumCopies start]
 
-start :: CommandStartString
-start file = isAnnexed file $ \(key, _) -> do
+start :: CommandStartAttrFile
+start (file, attr) = isAnnexed file $ \(key, _) -> do
 	inannex <- inAnnex key
 	if inannex
 		then stop
-		else do
+		else autoCopies key (<) numcopies $ do
 			showStart "get" file
 			from <- Annex.getState Annex.fromremote
 			case from of
@@ -35,6 +36,8 @@ start file = isAnnexed file $ \(key, _) -> do
 				Just name -> do
 					src <- Remote.byName name
 					next $ Command.Move.fromPerform src False key
+	where
+		numcopies = readMaybe attr
 
 perform :: Key -> CommandPerform
 perform key = do
diff --git a/Command/Optimize.hs b/Command/Optimize.hs
deleted file mode 100644
index 1a2b2237ff..0000000000
--- a/Command/Optimize.hs
+++ /dev/null
@@ -1,34 +0,0 @@
-{- git-annex command
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Command.Optimize where
-
-import Command
-import Utility
-import LocationLog
-import Trust
-import Config
-import qualified Command.Get
-import qualified Command.Drop
-
-command :: [Command]
-command = [repoCommand "optimize" (paramOptional $ paramRepeating paramPath) seek
-	"get or drop content to best use available space"]
-
-seek :: [CommandSeek]
-seek = [withNumCopies start]
-
-start :: CommandStartAttrFile
-start p@(file, attr) = notBareRepo $ isAnnexed file $ \(key, _) -> do
-	needed <- getNumCopies $ readMaybe attr
-	(_, safelocations) <- trustPartition UnTrusted =<< keyLocations key
-	dispatch needed (length safelocations)
-	where
-		dispatch needed present
-			| present < needed = Command.Get.start file
-			| present > needed = Command.Drop.start p
-			| otherwise = stop
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 8b9e557500..6f4e5d4921 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -34,7 +34,6 @@ import qualified Command.Init
 import qualified Command.Describe
 import qualified Command.InitRemote
 import qualified Command.Fsck
-import qualified Command.Optimize
 import qualified Command.Unused
 import qualified Command.DropUnused
 import qualified Command.Unlock
@@ -78,7 +77,6 @@ cmds = concat
 	, Command.SetKey.command
 	, Command.Fix.command
 	, Command.Fsck.command
-	, Command.Optimize.command
 	, Command.Unused.command
 	, Command.DropUnused.command
 	, Command.Find.command
diff --git a/Options.hs b/Options.hs
index e0ca48c01b..ee3ce66203 100644
--- a/Options.hs
+++ b/Options.hs
@@ -26,6 +26,8 @@ commonOptions =
 		"allow actions that may lose annexed data"
 	, Option ['F'] ["fast"] (NoArg (setfast True))
 		"avoid slow operations"
+	, Option ['a'] ["auto"] (NoArg (setauto True))
+		"automatic mode"
 	, Option ['q'] ["quiet"] (NoArg (setoutput Annex.QuietOutput))
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
@@ -40,6 +42,7 @@ commonOptions =
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
+		setauto v = Annex.changeState $ \s -> s { Annex.auto = v }
 		setoutput v = Annex.changeState $ \s -> s { Annex.output = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
diff --git a/debian/changelog b/debian/changelog
index b02f6a15b2..a3ae839abf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,8 +5,8 @@ git-annex (3.20110907) UNRELEASED; urgency=low
   * Fix build without S3.
   * addurl: Always use whole url as destination filename, rather than
     only its file component.
-  * optimize: A new subcommand that either gets or drops file content
-    as needed to work toward meeting the configured numcopies setting.
+  * get, drop: Added --auto option, which decides whether to get/drop
+    content as needed to work toward the configured numcopies.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 8264c31b3a..83bfc7e405 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -76,12 +76,19 @@ Many git-annex commands will stage changes for later `git commit` by you.
   will involve copying them from another repository, or downloading them,
   or transferring them from some kind of key-value store.
 
+  When the --auto switch is used, only gets content of files if needed
+  to satisfy the setting of annex.numcopies
+
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 
 
-  git-annex may refuse to drop content if it does not think
-  it is safe to do so, typically because of the setting of annex.numcopies.
+  git-annex will refuse to drop content if it cannot verify it is
+  safe to do so. At least one copy of content needs to exist in another
+  remote. This can be overridden with the --force switch.
+
+  When the --auto switch is used, only tries to drop content if
+  more than annex.numcopies copies exist.
 
 * move [path ...]
   
@@ -157,11 +164,6 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   To avoid expensive checksum calculations, specify --fast
 
-* optimize [path ...]
-  
-  Either gets or drops file content, as needed, to work toward meeting the
-  configured numcopies setting.
-
 * unused
 
   Checks the annex for data that does not correspond to any files present
@@ -340,6 +342,12 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Enables less expensive, but also less thorough versions of some commands.
   What is avoided depends on the command.
 
+* --auto
+
+  Enable automatic mode, in which git-annex decides whether to perform
+  actions on files. See descriptions of individual commands to see what
+  they do in automatic mode.
+
 * --quiet
 
   Avoid the default verbose display of what is done; only show errors
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index b0eb258153..68f94a6f27 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -18,6 +18,6 @@ A walkthrough of the basic features of git-annex.
 	fsck:_verifying_your_data
 	fsck:_when_things_go_wrong
 	backups
-	optimizing_repositories
+	automatically_managing_content
 	more
 """]]
diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
new file mode 100644
index 0000000000..74a4c1b5b7
--- /dev/null
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -0,0 +1,38 @@
+Once you have multiple repositories, and have perhaps configured numcopies,
+any given file can have many more copies than is needed, or perhaps fewer
+than you would like. How to manage this?
+
+The whereis subcommand can be used to see how many copies of a file are known,
+but then you have to decide what to get or drop. In this example, there
+are rather too many copies of `other_file` and perhaps not enough of the
+other file.
+
+	# cd /media/usbdrive
+	# git annex whereis
+	whereis my_cool_big_file (1 copy)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+	whereis other_file (3 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive <-- here
+		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
+
+What would be handy is some automated versions of get and drop, that only
+get a file if there are not yet enough copies of it, or only drop a file
+if there are too many copies. Well, these exist, just use the --auto
+option.
+
+	# git annex get --auto --numcopies=2
+	get my_cool_big_file (from laptop...) ok
+	# git annex drop --auto --numcopies=2
+	drop other_file ok
+
+With two quick commands, git-annex was able to decide for you how to
+work toward having two copies of your files.
+
+	# git annex whereis
+	whereis my_cool_big_file (2 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive <-- here
+	whereis other_file (2 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
diff --git a/doc/walkthrough/optimizing_repositories.mdwn b/doc/walkthrough/optimizing_repositories.mdwn
deleted file mode 100644
index 0f17f1deae..0000000000
--- a/doc/walkthrough/optimizing_repositories.mdwn
+++ /dev/null
@@ -1,13 +0,0 @@
-Once you have multiple repositories, and have perhaps configured numcopies,
-any given file can have many more copies than is needed, or perhaps fewer
-than you would like. Fsck can detect the latter problem, but there's another
-command that can help deal with both problems.
-
-The optimize subcommand either gets or drops file content, as needed,
-to work toward meeting the configured numcopies setting.
-
-	# git annex optimize
-	get my_cool_big_file (from laptop...) ok
-	drop other_file ok
-	# git annex optimize --numcopies=2
-	get other_file ok
diff --git a/test.hs b/test.hs
index bd2e1e46c5..4d751a707b 100644
--- a/test.hs
+++ b/test.hs
@@ -93,7 +93,6 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_unannex
 	, test_drop
 	, test_get
-	, test_optimize
 	, test_move
 	, test_copy
 	, test_lock
@@ -217,17 +216,6 @@ test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do
 	inmainrepo $ unannexed ingitfile
 	unannexed ingitfile
 
-test_optimize :: Test
-test_optimize = "git-annex optimize" ~: TestCase $ intmpclonerepo $ do
-	inmainrepo $ annexed_present annexedfile
-	annexed_notpresent annexedfile
-	git_annex "optimize" ["-q", annexedfile, "--numcopies=2"] @? "optimize of file failed"
-	inmainrepo $ annexed_present annexedfile
-	annexed_present annexedfile
-	git_annex "optimize" ["-q", annexedfile] @? "optimize of file failed"
-	inmainrepo $ annexed_present annexedfile
-	annexed_notpresent annexedfile
-
 test_move :: Test
 test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do
 	annexed_notpresent annexedfile

From 9fe3c6d21134638a29c6ad4ced6a11ed9b0242ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 14:33:37 -0400
Subject: [PATCH 2202/2835] clean up params in usage display

---
 Command.hs           | 14 ++++++++------
 Command/Add.hs       |  2 +-
 Command/Copy.hs      |  2 +-
 Command/Drop.hs      |  2 +-
 Command/Find.hs      |  3 +--
 Command/Fix.hs       |  2 +-
 Command/Fsck.hs      |  3 +--
 Command/Get.hs       |  2 +-
 Command/Lock.hs      |  2 +-
 Command/Migrate.hs   |  3 ++-
 Command/Move.hs      |  2 +-
 Command/PreCommit.hs |  2 +-
 Command/Unannex.hs   |  2 +-
 Command/Uninit.hs    |  2 +-
 Command/Unlock.hs    |  4 ++--
 Command/Whereis.hs   |  2 +-
 16 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/Command.hs b/Command.hs
index 9d85fbe9d7..c1069f0d92 100644
--- a/Command.hs
+++ b/Command.hs
@@ -212,12 +212,8 @@ notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
 
 {- Descriptions of params used in usage messages. -}
-paramRepeating :: String -> String
-paramRepeating s = s ++ " ..."
-paramOptional :: String -> String
-paramOptional s = "[" ++ s ++ "]"
-paramPair :: String -> String -> String
-paramPair a b = a ++ " " ++ b
+paramPaths :: String
+paramPaths = paramOptional $ paramRepeating paramPath -- most often used
 paramPath :: String
 paramPath = "PATH"
 paramKey :: String
@@ -240,6 +236,12 @@ paramKeyValue :: String
 paramKeyValue = "K=V"
 paramNothing :: String
 paramNothing = ""
+paramRepeating :: String -> String
+paramRepeating s = s ++ " ..."
+paramOptional :: String -> String
+paramOptional s = "[" ++ s ++ "]"
+paramPair :: String -> String -> String
+paramPair a b = a ++ " " ++ b
 
 {- The Key specified by the --key parameter. -}
 cmdlineKey :: Annex Key
diff --git a/Command/Add.hs b/Command/Add.hs
index 579c4171b1..f6dfd2da43 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -29,7 +29,7 @@ import Utility.SafeCommand
 import Locations
 
 command :: [Command]
-command = [repoCommand "add" paramPath seek "add files to annex"]
+command = [repoCommand "add" paramPaths seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 46d49dd058..7d40f438dc 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -11,7 +11,7 @@ import Command
 import qualified Command.Move
 
 command :: [Command]
-command = [repoCommand "copy" paramPath seek
+command = [repoCommand "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 -- A copy is just a move that does not delete the source file.
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 32cb81e808..1325e3dcbd 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -20,7 +20,7 @@ import Trust
 import Config
 
 command :: [Command]
-command = [repoCommand "drop" paramPath seek
+command = [repoCommand "drop" paramPaths seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/Find.hs b/Command/Find.hs
index 0716c5297e..0ff68b7ed1 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -14,8 +14,7 @@ import Content
 import Utility.Conditional
 
 command :: [Command]
-command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
-	"lists available files"]
+command = [repoCommand "find" paramPaths seek "lists available files"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index b24f8e33c2..0044052e9c 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -19,7 +19,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [repoCommand "fix" paramPath seek
+command = [repoCommand "fix" paramPaths seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 08bf2b97f4..4e8eeb0433 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -30,8 +30,7 @@ import Utility.Path
 import Config
 
 command :: [Command]
-command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
-	"check for problems"]
+command = [repoCommand "fsck" paramPaths seek "check for problems"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/Get.hs b/Command/Get.hs
index 1a50158e38..358105cac1 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -17,7 +17,7 @@ import Utility
 import qualified Command.Move
 
 command :: [Command]
-command = [repoCommand "get" paramPath seek
+command = [repoCommand "get" paramPaths seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 77d1ff94f9..07721e9370 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -16,7 +16,7 @@ import qualified AnnexQueue
 import Utility.SafeCommand
 	
 command :: [Command]
-command = [repoCommand "lock" paramPath seek "undo unlock command"]
+command = [repoCommand "lock" paramPaths seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 6ad7e239c9..ec570acb7b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -25,7 +25,8 @@ import Utility.Conditional
 import qualified Command.Add
 
 command :: [Command]
-command = [repoCommand "migrate" paramPath seek "switch data to different backend"]
+command = [repoCommand "migrate" paramPaths seek
+	"switch data to different backend"]
 
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
diff --git a/Command/Move.hs b/Command/Move.hs
index a081a863f2..88da92f0a4 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -20,7 +20,7 @@ import UUID
 import Messages
 
 command :: [Command]
-command = [repoCommand "move" paramPath seek
+command = [repoCommand "move" paramPaths seek
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 3046d35626..6884a4787c 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -12,7 +12,7 @@ import qualified Command.Add
 import qualified Command.Fix
 
 command :: [Command]
-command = [repoCommand "pre-commit" paramPath seek "run by git pre-commit hook"]
+command = [repoCommand "pre-commit" paramPaths seek "run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 54ef2fc68e..748da4066c 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -27,7 +27,7 @@ import Messages
 import Locations
 
 command :: [Command]
-command = [repoCommand "unannex" paramPath seek "undo accidential add command"]
+command = [repoCommand "unannex" paramPaths seek "undo accidential add command"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index fadae0e5a9..4c70ec80c4 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -22,7 +22,7 @@ import Content
 import Locations
 
 command :: [Command]
-command = [repoCommand "uninit" paramPath seek 
+command = [repoCommand "uninit" paramPaths seek 
         "de-initialize git-annex and clean out repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 0daf1b3218..ba6d023874 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -22,8 +22,8 @@ import Utility.Path
 
 command :: [Command]
 command =
-	[ repoCommand "unlock" paramPath seek "unlock files for modification"
-	, repoCommand "edit" paramPath seek "same as unlock"
+	[ repoCommand "unlock" paramPaths seek "unlock files for modification"
+	, repoCommand "edit" paramPaths seek "same as unlock"
 	]
 
 seek :: [CommandSeek]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index ef4303b3ae..7d0ab188c0 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -17,7 +17,7 @@ import Types
 import Trust
 
 command :: [Command]
-command = [repoCommand "whereis" (paramOptional $ paramRepeating paramPath) seek
+command = [repoCommand "whereis" paramPaths seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]

From fe5e4bdc647f779e65f935f42e6fe798ab036165 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 14:34:15 -0400
Subject: [PATCH 2203/2835] comment

---
 Command/Get.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Command/Get.hs b/Command/Get.hs
index 358105cac1..bc617c9fa1 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -34,6 +34,7 @@ start (file, attr) = isAnnexed file $ \(key, _) -> do
 			case from of
 				Nothing -> next $ perform key
 				Just name -> do
+					-- get --from = copy --from
 					src <- Remote.byName name
 					next $ Command.Move.fromPerform src False key
 	where

From 7b90cb72fc7d86a8f99d839378fba1ef1c86ff4a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:15:47 -0400
Subject: [PATCH 2204/2835] document --auto all in one place

---
 doc/git-annex.mdwn | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 83bfc7e405..0d3aebe2be 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -76,9 +76,6 @@ Many git-annex commands will stage changes for later `git commit` by you.
   will involve copying them from another repository, or downloading them,
   or transferring them from some kind of key-value store.
 
-  When the --auto switch is used, only gets content of files if needed
-  to satisfy the setting of annex.numcopies
-
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 
@@ -87,9 +84,6 @@ Many git-annex commands will stage changes for later `git commit` by you.
   safe to do so. At least one copy of content needs to exist in another
   remote. This can be overridden with the --force switch.
 
-  When the --auto switch is used, only tries to drop content if
-  more than annex.numcopies copies exist.
-
 * move [path ...]
   
   When used with the --from option, moves the content of annexed files
@@ -344,9 +338,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * --auto
 
-  Enable automatic mode, in which git-annex decides whether to perform
-  actions on files. See descriptions of individual commands to see what
-  they do in automatic mode.
+  Enables automatic mode. Commands that get, drop, or move file contents
+  will only do so when needed to help satisfy the setting of annex.numcopies.
 
 * --quiet
 

From a0d3a343b52fba63df523d49849b43217ce744ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:27:45 -0400
Subject: [PATCH 2205/2835] copy --auto

Only does copy when numcopies is not yet satisfied.
---
 Command/Copy.hs                                     | 13 +++++++++++--
 debian/changelog                                    |  4 ++--
 doc/walkthrough/automatically_managing_content.mdwn |  3 +++
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/Command/Copy.hs b/Command/Copy.hs
index 7d40f438dc..125e0bb9f8 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -9,11 +9,20 @@ module Command.Copy where
 
 import Command
 import qualified Command.Move
+import Utility
 
 command :: [Command]
 command = [repoCommand "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
--- A copy is just a move that does not delete the source file.
 seek :: [CommandSeek]
-seek = [withFilesInGit $ Command.Move.start False]
+seek = [withNumCopies start]
+
+-- A copy is just a move that does not delete the source file.
+-- However, --auto mode avoids unnecessary copies.
+start :: CommandStartAttrFile
+start (file, attr) = isAnnexed file $ \(key, _) ->
+	autoCopies key (<) numcopies $
+		Command.Move.start False file
+	where
+		numcopies = readMaybe attr
diff --git a/debian/changelog b/debian/changelog
index a3ae839abf..d9e606f829 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,8 +5,8 @@ git-annex (3.20110907) UNRELEASED; urgency=low
   * Fix build without S3.
   * addurl: Always use whole url as destination filename, rather than
     only its file component.
-  * get, drop: Added --auto option, which decides whether to get/drop
-    content as needed to work toward the configured numcopies.
+  * get, drop, copy: Added --auto option, which decides whether
+    to get/drop content as needed to work toward the configured numcopies.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 
diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
index 74a4c1b5b7..5107d2f6f8 100644
--- a/doc/walkthrough/automatically_managing_content.mdwn
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -36,3 +36,6 @@ work toward having two copies of your files.
 	whereis other_file (2 copies)
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
 		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
+
+The --auto option can also be used with the copy command,
+again this lets git-annex decide whether to actually copy content.

From e47d1fd43e2433966b7baa6fc179ec6b70774214 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:33:20 -0400
Subject: [PATCH 2206/2835] add error for move --auto

It probably does not make sense to enable auto mode for move. I cannot
think of a situation where it would make sense to try to use it.

A hypothetical auto mode for move would only differ from a normal
move in one case -- when both repositories have a file, move deletes it
from one, and this reduces the number of copies. So an auto mode would
either only let move work in that situation, or avoid removing the file
in that situation, depending on the number of copies. This would be
complex to implement, and is perhaps not a very obvious behavior.

The error is a good thing to have, so users don't expect it to do something
it does not.
---
 Command/Move.hs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Command/Move.hs b/Command/Move.hs
index 88da92f0a4..f4310a2b88 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -18,6 +18,7 @@ import Content
 import qualified Remote
 import UUID
 import Messages
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "move" paramPaths seek
@@ -32,6 +33,7 @@ seek = [withFilesInGit $ start True]
  - moving data in the key-value backend. -}
 start :: Bool -> CommandStartString
 start move file = do
+	noAuto
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
 	case (from, to) of
@@ -43,6 +45,9 @@ start move file = do
 			src <- Remote.byName name
 			fromStart src move file
 		(_ ,  _) -> error "only one of --from or --to can be specified"
+	where
+		noAuto = when move $ whenM (Annex.getState Annex.auto) $ error
+			"--auto is not supported for move"
 
 showMoveAction :: Bool -> FilePath -> Annex ()
 showMoveAction True file = showStart "move" file

From aedf84f7d638eb222811c13da1c642268ca06600 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:39:48 -0400
Subject: [PATCH 2207/2835] wording

---
 doc/walkthrough/automatically_managing_content.mdwn | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
index 5107d2f6f8..ba0cad6092 100644
--- a/doc/walkthrough/automatically_managing_content.mdwn
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -4,8 +4,8 @@ than you would like. How to manage this?
 
 The whereis subcommand can be used to see how many copies of a file are known,
 but then you have to decide what to get or drop. In this example, there
-are rather too many copies of `other_file` and perhaps not enough of the
-other file.
+are perhaps not enough copies of the first file, and too many of the second
+file.
 
 	# cd /media/usbdrive
 	# git annex whereis
@@ -17,9 +17,8 @@ other file.
 		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
 
 What would be handy is some automated versions of get and drop, that only
-get a file if there are not yet enough copies of it, or only drop a file
-if there are too many copies. Well, these exist, just use the --auto
-option.
+gets a file if there are not yet enough copies of it, or only drops a file
+if there are too many copies. Well, these exist, just use the --auto option.
 
 	# git annex get --auto --numcopies=2
 	get my_cool_big_file (from laptop...) ok

From d036cd590f5c3c4edcd025effcf57c3d16886559 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:44:32 -0400
Subject: [PATCH 2208/2835] bugfix: drop and fsck did not honor --exclude

---
 Command.hs       | 3 ++-
 debian/changelog | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command.hs b/Command.hs
index c1069f0d92..bcc05c03c9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -133,7 +133,8 @@ withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	liftM (map a) $ liftIO $ Git.checkAttr repo attr files
+	files' <- filterFiles files
+	liftM (map a) $ liftIO $ Git.checkAttr repo attr files'
 withNumCopies :: CommandSeekAttrFiles
 withNumCopies = withAttrFilesInGit "annex.numcopies"
 withBackendFilesInGit :: CommandSeekBackendFiles
diff --git a/debian/changelog b/debian/changelog
index d9e606f829..e940989da7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,7 @@ git-annex (3.20110907) UNRELEASED; urgency=low
     only its file component.
   * get, drop, copy: Added --auto option, which decides whether
     to get/drop content as needed to work toward the configured numcopies.
+  * bugfix: drop and fsck did not honor --exclude
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 

From 456b45b9b3982d9440a43ec014635dee15066f0e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 16:24:47 -0400
Subject: [PATCH 2209/2835] move annex.numcopies parsing into withNumCopies

---
 Command.hs      | 13 +++++++------
 Command/Copy.hs |  7 ++-----
 Command/Drop.hs |  7 ++-----
 Command/Fsck.hs |  7 +++----
 Command/Get.hs  |  7 ++-----
 5 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/Command.hs b/Command.hs
index bcc05c03c9..05b215ec22 100644
--- a/Command.hs
+++ b/Command.hs
@@ -59,9 +59,6 @@ type CommandStartKey = Key -> CommandStart
 type BackendFile = (FilePath, Maybe (Backend Annex))
 type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek
 type CommandStartBackendFile = BackendFile -> CommandStart
-type AttrFile = (FilePath, String)
-type CommandSeekAttrFiles = CommandStartAttrFile -> CommandSeek
-type CommandStartAttrFile = AttrFile -> CommandStart
 type CommandSeekNothing = CommandStart -> CommandSeek
 type CommandStartNothing = CommandStart
 
@@ -129,14 +126,18 @@ withFilesInGit a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	liftM (map a) $ filterFiles files
-withAttrFilesInGit :: String -> CommandSeekAttrFiles
+withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	files' <- filterFiles files
 	liftM (map a) $ liftIO $ Git.checkAttr repo attr files'
-withNumCopies :: CommandSeekAttrFiles
-withNumCopies = withAttrFilesInGit "annex.numcopies"
+withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
+withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
+	where
+		go (file, v) = do
+			let numcopies = readMaybe v
+			a file numcopies
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 125e0bb9f8..d7625ccdb0 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -9,7 +9,6 @@ module Command.Copy where
 
 import Command
 import qualified Command.Move
-import Utility
 
 command :: [Command]
 command = [repoCommand "copy" paramPaths seek
@@ -20,9 +19,7 @@ seek = [withNumCopies start]
 
 -- A copy is just a move that does not delete the source file.
 -- However, --auto mode avoids unnecessary copies.
-start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, _) ->
+start :: FilePath -> Maybe Int -> CommandStart
+start file numcopies = isAnnexed file $ \(key, _) ->
 	autoCopies key (<) numcopies $
 		Command.Move.start False file
-	where
-		numcopies = readMaybe attr
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 1325e3dcbd..4a7596921f 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -14,7 +14,6 @@ import LocationLog
 import Types
 import Content
 import Messages
-import Utility
 import Utility.Conditional
 import Trust
 import Config
@@ -28,16 +27,14 @@ seek = [withNumCopies start]
 
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}
-start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> Maybe Int -> CommandStart
+start file numcopies = isAnnexed file $ \(key, _) -> do
 	present <- inAnnex key
 	if present
 		then autoCopies key (>) numcopies $ do
 			showStart "drop" file
 			next $ perform key numcopies
 		else stop
-	where
-		numcopies = readMaybe attr
 
 perform :: Key -> Maybe Int -> CommandPerform
 perform key numcopies = do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 4e8eeb0433..142f755a7f 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -20,7 +20,6 @@ import qualified Types.Key
 import UUID
 import Types
 import Messages
-import Utility
 import Content
 import LocationLog
 import Locations
@@ -35,10 +34,10 @@ command = [repoCommand "fsck" paramPaths seek "check for problems"]
 seek :: [CommandSeek]
 seek = [withNumCopies start]
 
-start :: CommandStartAttrFile
-start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
+start :: FilePath -> Maybe Int -> CommandStart
+start file numcopies = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
-	next $ perform key file backend $ readMaybe attr
+	next $ perform key file backend numcopies
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
 perform key file backend numcopies = do
diff --git a/Command/Get.hs b/Command/Get.hs
index bc617c9fa1..4fd654f63e 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -13,7 +13,6 @@ import qualified Remote
 import Types
 import Content
 import Messages
-import Utility
 import qualified Command.Move
 
 command :: [Command]
@@ -23,8 +22,8 @@ command = [repoCommand "get" paramPaths seek
 seek :: [CommandSeek]
 seek = [withNumCopies start]
 
-start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> Maybe Int -> CommandStart
+start file numcopies = isAnnexed file $ \(key, _) -> do
 	inannex <- inAnnex key
 	if inannex
 		then stop
@@ -37,8 +36,6 @@ start (file, attr) = isAnnexed file $ \(key, _) -> do
 					-- get --from = copy --from
 					src <- Remote.byName name
 					next $ Command.Move.fromPerform src False key
-	where
-		numcopies = readMaybe attr
 
 perform :: Key -> CommandPerform
 perform key = do

From 35145202d2e463569989b710ab5b87f6d9a8fdc1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 16:50:49 -0400
Subject: [PATCH 2210/2835] remove command type definitions

These were a mistake, they make the type signatures harder to read and
less flexible. The CommandSeek, CommandStart, CommandPerform, and
CommandCleanup types were a good idea, but composing them with the
parameters expected is going too far.
---
 Command.hs            | 39 ++++++++++++++-------------------------
 Command/Add.hs        |  6 +++---
 Command/AddUrl.hs     |  2 +-
 Command/ConfigList.hs |  2 +-
 Command/Describe.hs   |  2 +-
 Command/DropKey.hs    |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  2 +-
 Command/FromKey.hs    |  2 +-
 Command/InAnnex.hs    |  3 ++-
 Command/Init.hs       |  2 +-
 Command/InitRemote.hs |  2 +-
 Command/Lock.hs       |  2 +-
 Command/Map.hs        |  2 +-
 Command/Merge.hs      |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/Move.hs       |  6 +++---
 Command/PreCommit.hs  |  4 ++--
 Command/RecvKey.hs    |  3 ++-
 Command/Semitrust.hs  |  2 +-
 Command/SendKey.hs    |  3 ++-
 Command/SetKey.hs     |  2 +-
 Command/Status.hs     |  2 +-
 Command/Trust.hs      |  2 +-
 Command/Unannex.hs    |  2 +-
 Command/Uninit.hs     |  4 ++--
 Command/Unlock.hs     |  2 +-
 Command/Untrust.hs    |  2 +-
 Command/Unused.hs     |  2 +-
 Command/Upgrade.hs    |  2 +-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  2 +-
 33 files changed, 55 insertions(+), 63 deletions(-)

diff --git a/Command.hs b/Command.hs
index 05b215ec22..6bd451a7e9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -48,19 +48,8 @@ type CommandPerform = Annex (Maybe CommandCleanup)
 {- 3. The cleanup stage is run only if the perform stage succeeds, and it
  -    returns the overall success/fail of the command. -}
 type CommandCleanup = Annex Bool
-{- Some helper functions are used to build up CommandSeek and CommandStart
- - functions. -}
-type CommandSeekStrings = CommandStartString -> CommandSeek
-type CommandStartString = String -> CommandStart
-type CommandSeekWords = CommandStartWords -> CommandSeek
-type CommandStartWords = [String] -> CommandStart
-type CommandSeekKeys = CommandStartKey -> CommandSeek
-type CommandStartKey = Key -> CommandStart
+
 type BackendFile = (FilePath, Maybe (Backend Annex))
-type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek
-type CommandStartBackendFile = BackendFile -> CommandStart
-type CommandSeekNothing = CommandStart -> CommandSeek
-type CommandStartNothing = CommandStart
 
 data Command = Command {
 	cmdusesrepo :: Bool,
@@ -121,7 +110,7 @@ notBareRepo a = do
 
 {- These functions find appropriate files or other things based on a
    user's parameters, and run a specified action on them. -}
-withFilesInGit :: CommandSeekStrings
+withFilesInGit :: (String -> CommandStart) -> CommandSeek
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
@@ -138,13 +127,13 @@ withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 		go (file, v) = do
 			let numcopies = readMaybe v
 			a file numcopies
-withBackendFilesInGit :: CommandSeekBackendFiles
+withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	files' <- filterFiles files
 	backendPairs a files'
-withFilesMissing :: CommandSeekStrings
+withFilesMissing :: (String -> CommandStart) -> CommandSeek
 withFilesMissing a params = do
 	files <- liftIO $ filterM missing params
 	liftM (map a) $ filterFiles files
@@ -152,27 +141,27 @@ withFilesMissing a params = do
 		missing f = do
 			e <- doesFileExist f
 			return $ not e
-withFilesNotInGit :: CommandSeekBackendFiles
+withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
 	force <- Annex.getState Annex.force
 	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
-withWords :: CommandSeekWords
+withWords :: ([String] -> CommandStart) -> CommandSeek
 withWords a params = return [a params]
-withStrings :: CommandSeekStrings
+withStrings :: (String -> CommandStart) -> CommandSeek
 withStrings a params = return $ map a params
-withFilesToBeCommitted :: CommandSeekStrings
+withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
 	tocommit <- liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
 	liftM (map a) $ filterFiles tocommit
-withFilesUnlocked :: CommandSeekBackendFiles
+withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
-withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles
+withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
-withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBackendFiles
+withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
 	repo <- Annex.gitRepo
@@ -181,15 +170,15 @@ withFilesUnlocked' typechanged a params = do
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
 	unlockedfiles' <- filterFiles unlockedfiles
 	backendPairs a unlockedfiles'
-withKeys :: CommandSeekKeys
+withKeys :: (Key -> CommandStart) -> CommandSeek
 withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
-withNothing :: CommandSeekNothing
+withNothing :: CommandStart -> CommandSeek
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
-backendPairs :: CommandSeekBackendFiles
+backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
 backendPairs a files = map a <$> Backend.chooseBackends files
 
 {- Filter out files those matching the exclude glob pattern,
diff --git a/Command/Add.hs b/Command/Add.hs
index f6dfd2da43..c6ab4d0ad7 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -38,14 +38,14 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
 {- The add subcommand annexes a file, storing it in a backend, and then
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
-start :: CommandStartBackendFile
-start pair@(file, _) = notAnnexed file $ do
+start :: BackendFile -> CommandStart
+start p@(file, _) = notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
 	if isSymbolicLink s || not (isRegularFile s)
 		then stop
 		else do
 			showStart "add" file
-			next $ perform pair
+			next $ perform p
 
 perform :: BackendFile -> CommandPerform
 perform (file, backend) = do
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index c78569ffc8..1fae358b2d 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -34,7 +34,7 @@ command = [repoCommand "addurl" (paramRepeating $ paramUrl) seek
 seek :: [CommandSeek]
 seek = [withStrings start]
 
-start :: CommandStartString
+start :: String -> CommandStart
 start s = do
 	let u = parseURI s
 	case u of
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 1b1bb3c34b..3de26c8925 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -20,7 +20,7 @@ command = [repoCommand "configlist" paramNothing seek
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	g <- Annex.gitRepo
 	u <- getUUID g
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 453e4ebafc..8d2f9071b0 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -19,7 +19,7 @@ command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	let (name, description) =
 		case ws of
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 16a3e35d64..b9938585e9 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -21,7 +21,7 @@ command = [repoCommand "dropkey" (paramRepeating paramKey) seek
 seek :: [CommandSeek]
 seek = [withKeys start]
 
-start :: CommandStartKey
+start :: Key -> CommandStart
 start key = do
 	present <- inAnnex key
 	force <- Annex.getState Annex.force
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 4ad2aa85bb..90fea050e8 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -41,7 +41,7 @@ withUnusedMaps params = do
 	unusedtmp <- readUnusedLog "tmp"
 	return $ map (start (unused, unusedbad, unusedtmp)) params
 
-start :: (UnusedMap, UnusedMap, UnusedMap) -> CommandStartString
+start :: (UnusedMap, UnusedMap, UnusedMap) -> FilePath -> CommandStart
 start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
 	[ (unused, perform)
 	, (unusedbad, performOther gitAnnexBadLocation)
diff --git a/Command/Find.hs b/Command/Find.hs
index 0ff68b7ed1..51849f6b85 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -20,7 +20,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
 {- Output a list of files. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	whenM (inAnnex key) $ liftIO $ putStrLn file
 	stop
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 0044052e9c..481da52f2b 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -26,7 +26,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
 {- Fixes the symlink to an annexed file. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	link <- calcGitLink file key
 	l <- liftIO $ readSymbolicLink file
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 89c3f4e912..9ff126a45e 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -27,7 +27,7 @@ command = [repoCommand "fromkey" paramPath seek
 seek :: [CommandSeek]
 seek = [withFilesMissing start]
 
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = notBareRepo $ do
 	key <- cmdlineKey
 	inbackend <- inAnnex key
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 24f7162ace..713492c2fe 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -12,6 +12,7 @@ import System.Exit
 
 import Command
 import Content
+import Types
 
 command :: [Command]
 command = [repoCommand "inannex" (paramRepeating paramKey) seek
@@ -20,7 +21,7 @@ command = [repoCommand "inannex" (paramRepeating paramKey) seek
 seek :: [CommandSeek]
 seek = [withKeys start]
 
-start :: CommandStartKey
+start :: Key -> CommandStart
 start key = do
 	present <- inAnnex key
 	if present
diff --git a/Command/Init.hs b/Command/Init.hs
index 6ba7df6829..2351763a95 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -20,7 +20,7 @@ command = [standaloneCommand "init" paramDesc seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	showStart "init" description
 	next $ perform description
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 9859308e56..671f945d22 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -30,7 +30,7 @@ command = [repoCommand "initremote"
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	when (null ws) needname
 
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 07721e9370..1c9a747f43 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -22,7 +22,7 @@ seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
 
 {- Undo unlock -}
-start :: CommandStartBackendFile
+start :: BackendFile -> CommandStart
 start (file, _) = do
 	showStart "lock" file
 	next $ perform file
diff --git a/Command/Map.hs b/Command/Map.hs
index ef8e04d909..7e23da774d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -34,7 +34,7 @@ command = [repoCommand "map" paramNothing seek "generate map of repositories"]
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	g <- Annex.gitRepo
 	rs <- spider g
diff --git a/Command/Merge.hs b/Command/Merge.hs
index faaef906b5..832cde5127 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -18,7 +18,7 @@ command = [repoCommand "merge" paramNothing seek
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	showStart "merge" "."
 	next perform
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index ec570acb7b..69fe61e1d0 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -31,7 +31,7 @@ command = [repoCommand "migrate" paramPaths seek
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
 
-start :: CommandStartBackendFile
+start :: BackendFile -> CommandStart
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
diff --git a/Command/Move.hs b/Command/Move.hs
index f4310a2b88..15dae39385 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -31,7 +31,7 @@ seek = [withFilesInGit $ start True]
  -
  - This only operates on the cached file content; it does not involve
  - moving data in the key-value backend. -}
-start :: Bool -> CommandStartString
+start :: Bool -> FilePath -> CommandStart
 start move file = do
 	noAuto
 	to <- Annex.getState Annex.toremote
@@ -74,7 +74,7 @@ remoteHasKey remote key present	= do
  - A file's content can be moved even if there are insufficient copies to
  - allow it to be dropped.
  -}
-toStart :: Remote.Remote Annex -> Bool -> CommandStartString
+toStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 toStart dest move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
@@ -124,7 +124,7 @@ toCleanup dest move key = do
  - If the current repository already has the content, it is still removed
  - from the remote.
  -}
-fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
+fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 6884a4787c..bcc1c943ee 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -20,8 +20,8 @@ seek :: [CommandSeek]
 seek = [withFilesToBeCommitted Command.Fix.start,
 	withFilesUnlockedToBeCommitted start]
 
-start :: CommandStartBackendFile
-start pair = next $ perform pair
+start :: BackendFile -> CommandStart
+start p = next $ perform p
 
 perform :: BackendFile -> CommandPerform
 perform pair@(file, _) = do
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index be6163558f..33792e5b6e 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -15,6 +15,7 @@ import CmdLine
 import Content
 import Utility.RsyncFile
 import Utility.Conditional
+import Types
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
@@ -23,7 +24,7 @@ command = [repoCommand "recvkey" paramKey seek
 seek :: [CommandSeek]
 seek = [withKeys start]
 
-start :: CommandStartKey
+start :: Key -> CommandStart
 start key = do
 	whenM (inAnnex key) $ error "key is already present in annex"
 	
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index b467861bf9..3b12bb747d 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -20,7 +20,7 @@ command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	let name = unwords ws
 	showStart "semitrust" name
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index f676ae947a..98d2573380 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -17,6 +17,7 @@ import Content
 import Utility.RsyncFile
 import Utility.Conditional
 import Messages
+import Types
 
 command :: [Command]
 command = [repoCommand "sendkey" paramKey seek
@@ -25,7 +26,7 @@ command = [repoCommand "sendkey" paramKey seek
 seek :: [CommandSeek]
 seek = [withKeys start]
 
-start :: CommandStartKey
+start :: Key -> CommandStart
 start key = do
 	g <- Annex.gitRepo
 	let file = gitAnnexLocation g key
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 2f6f9ea9ee..c03c5d0445 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -23,7 +23,7 @@ seek :: [CommandSeek]
 seek = [withStrings start]
 
 {- Sets cached content for a key. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = do
 	showStart "setkey" file
 	next $ perform file
diff --git a/Command/Status.hs b/Command/Status.hs
index 5c82744b10..76659a75e1 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -74,7 +74,7 @@ slowstats =
 	, backend_usage
 	]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	fast <- Annex.getState Annex.fast
 	let todo = if fast then faststats else faststats ++ slowstats
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 41eb17ccdd..5e25b519bc 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -20,7 +20,7 @@ command = [repoCommand "trust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	let name = unwords ws
 	showStart "trust" name
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 748da4066c..3dedd007ef 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -33,7 +33,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
 {- The unannex subcommand undoes an add. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	ishere <- inAnnex key
 	if ishere
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 4c70ec80c4..ce12665424 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -28,7 +28,7 @@ command = [repoCommand "uninit" paramPaths seek
 seek :: [CommandSeek]
 seek = [withFilesInGit startUnannex, withNothing start]
 
-startUnannex :: CommandStartString
+startUnannex :: FilePath -> CommandStart
 startUnannex file = do
 	-- Force fast mode before running unannex. This way, if multiple
 	-- files link to a key, it will be left in the annex and hardlinked
@@ -36,7 +36,7 @@ startUnannex file = do
 	Annex.changeState $ \s -> s { Annex.fast = True }
 	Command.Unannex.start file
 
-start :: CommandStartNothing
+start :: CommandStart
 start = next perform
 
 perform :: CommandPerform
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index ba6d023874..5817e8f221 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -31,7 +31,7 @@ seek = [withFilesInGit start]
 
 {- The unlock subcommand replaces the symlink with a copy of the file's
  - content. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	showStart "unlock" file
 	next $ perform file key
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index ea23208006..9f7e521987 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -20,7 +20,7 @@ command = [repoCommand "untrust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	let name = unwords ws
 	showStart "untrust" name
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 960b2e1dfb..535b9b33e8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -37,7 +37,7 @@ seek :: [CommandSeek]
 seek = [withNothing start]
 
 {- Finds unused content in the annex. -} 
-start :: CommandStartNothing
+start :: CommandStart
 start = notBareRepo $ do
 	from <- Annex.getState Annex.fromremote
 	let (name, action) = case from of
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index b79b13cd3c..5d9ed92fae 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -19,7 +19,7 @@ command = [standaloneCommand "upgrade" paramNothing seek
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	showStart "upgrade" "."
 	r <- upgrade
diff --git a/Command/Version.hs b/Command/Version.hs
index 1ff829a22a..af547949c6 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -21,7 +21,7 @@ command = [standaloneCommand "version" paramNothing seek "show version info"]
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion
 	v <- getVersion
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 7d0ab188c0..a414428f72 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -23,7 +23,7 @@ command = [repoCommand "whereis" paramPaths seek
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	showStart "whereis" file
 	next $ perform key

From 5ff04bf2afcc62d1762adbc1c2a0374952328c0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 16:57:02 -0400
Subject: [PATCH 2211/2835] tweak

---
 Backend.hs           | 9 ++++++---
 Command.hs           | 5 ++---
 Command/Add.hs       | 5 +++--
 Command/AddUrl.hs    | 2 +-
 Command/Lock.hs      | 3 ++-
 Command/Migrate.hs   | 3 ++-
 Command/PreCommit.hs | 3 ++-
 7 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 75327de80b..0c9ea8d0b7 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -6,6 +6,7 @@
  -}
 
 module Backend (
+	BackendFile,
 	list,
 	orderedList,
 	genKey,
@@ -101,20 +102,22 @@ lookupFile file = do
 				skip = "skipping " ++ file ++ 
 					" (unknown backend " ++ bname ++ ")"
 
+type BackendFile = (Maybe (Backend Annex), FilePath)
+
 {- Looks up the backends that should be used for each file in a list.
  - That can be configured on a per-file basis in the gitattributes file.
  -}
-chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))]
+chooseBackends :: [FilePath] -> Annex [BackendFile]
 chooseBackends fs = do
 	g <- Annex.gitRepo
 	forced <- Annex.getState Annex.forcebackend
 	if forced /= Nothing
 		then do
 			l <- orderedList
-			return $ map (\f -> (f, Just $ head l)) fs
+			return $ map (\f -> (Just $ head l, f)) fs
 		else do
 			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
-			return $ map (\(f,b) -> (f, maybeLookupBackendName b)) pairs
+			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend Annex
diff --git a/Command.hs b/Command.hs
index 6bd451a7e9..08087a5a5d 100644
--- a/Command.hs
+++ b/Command.hs
@@ -1,4 +1,4 @@
-{- git-annex commands
+{- git-annex command infrastructure
  -
  - Copyright 2010 Joey Hess 
  -
@@ -29,6 +29,7 @@ import Types.Key
 import Trust
 import LocationLog
 import Config
+import Backend
 
 {- A command runs in four stages.
  -
@@ -49,8 +50,6 @@ type CommandPerform = Annex (Maybe CommandCleanup)
  -    returns the overall success/fail of the command. -}
 type CommandCleanup = Annex Bool
 
-type BackendFile = (FilePath, Maybe (Backend Annex))
-
 data Command = Command {
 	cmdusesrepo :: Bool,
 	cmdname :: String,
diff --git a/Command/Add.hs b/Command/Add.hs
index c6ab4d0ad7..4b2ef24cd9 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -27,6 +27,7 @@ import Utility.Conditional
 import Utility.Touch
 import Utility.SafeCommand
 import Locations
+import Backend
 
 command :: [Command]
 command = [repoCommand "add" paramPaths seek "add files to annex"]
@@ -39,7 +40,7 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
 start :: BackendFile -> CommandStart
-start p@(file, _) = notAnnexed file $ do
+start p@(_, file) = notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
 	if isSymbolicLink s || not (isRegularFile s)
 		then stop
@@ -48,7 +49,7 @@ start p@(file, _) = notAnnexed file $ do
 			next $ perform p
 
 perform :: BackendFile -> CommandPerform
-perform (file, backend) = do
+perform (backend, file) = do
 	k <- Backend.genKey file backend
 	case k of
 		Nothing -> stop
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 1fae358b2d..d9fcc17e2b 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -59,7 +59,7 @@ download url file = do
 	ok <- Url.download url tmp
 	if ok
 		then do
-			[(_, backend)] <- Backend.chooseBackends [file]
+			[(backend, _)] <- Backend.chooseBackends [file]
 			k <- Backend.genKey tmp backend
 			case k of
 				Nothing -> stop
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 1c9a747f43..04d1bb94d6 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -14,6 +14,7 @@ import Command
 import Messages
 import qualified AnnexQueue
 import Utility.SafeCommand
+import Backend
 	
 command :: [Command]
 command = [repoCommand "lock" paramPaths seek "undo unlock command"]
@@ -23,7 +24,7 @@ seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
 
 {- Undo unlock -}
 start :: BackendFile -> CommandStart
-start (file, _) = do
+start (_, file) = do
 	showStart "lock" file
 	next $ perform file
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 69fe61e1d0..2be9108512 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -23,6 +23,7 @@ import Content
 import Messages
 import Utility.Conditional
 import qualified Command.Add
+import Backend
 
 command :: [Command]
 command = [repoCommand "migrate" paramPaths seek
@@ -32,7 +33,7 @@ seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
 
 start :: BackendFile -> CommandStart
-start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
+start (b, file) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
 	if (newbackend /= oldbackend || upgradableKey key) && exists
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index bcc1c943ee..b6323e2b79 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -10,6 +10,7 @@ module Command.PreCommit where
 import Command
 import qualified Command.Add
 import qualified Command.Fix
+import Backend
 
 command :: [Command]
 command = [repoCommand "pre-commit" paramPaths seek "run by git pre-commit hook"]
@@ -24,7 +25,7 @@ start :: BackendFile -> CommandStart
 start p = next $ perform p
 
 perform :: BackendFile -> CommandPerform
-perform pair@(file, _) = do
+perform pair@(_, file) = do
 	ok <- doCommand $ Command.Add.start pair
 	if ok
 		then next $ return True

From cbd9ade0757788506d48578466802d037f4791fb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 22:09:19 -0400
Subject: [PATCH 2212/2835] remove now unnecessary git commits

---
 doc/git-annex.mdwn | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0d3aebe2be..a5163a4b33 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -42,15 +42,12 @@ content from the key-value store.
 	# git remote add usbdrive /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
 	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
-	# git commit -a -m "got a video I want to rewatch on the plane"
 	
 	# git annex add iso
 	add iso/Debian_5.0.iso ok
-	# git commit -a -m "saving Debian CD for later"
 	
 	# git annex drop iso/Debian_4.0.iso
 	drop iso/Debian_4.0.iso ok
-	# git commit -a -m "freed up space"
 	
 	# git annex move iso --to=usbdrive
 	move iso/Debian_5.0.iso (moving to usbdrive...) ok
@@ -63,8 +60,6 @@ files in the directory. If no path is specified, most git-annex commands
 default to acting on all relevant files in the current directory (and
 subdirectories).
 
-Many git-annex commands will stage changes for later `git commit` by you.
-
 * add [path ...]
 
   Adds files in the path to the annex. Files that are already checked into

From 73769190b6f2df7e71e943af860ec904a1336367 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 22:22:43 -0400
Subject: [PATCH 2213/2835] grouped commands into related sections

---
 doc/git-annex.mdwn | 123 ++++++++++++++++++++++++---------------------
 1 file changed, 67 insertions(+), 56 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a5163a4b33..d587f763ca 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -52,11 +52,11 @@ content from the key-value store.
 	# git annex move iso --to=usbdrive
 	move iso/Debian_5.0.iso (moving to usbdrive...) ok
 
-# COMMANDS
+# COMMONLY USED COMMANDS
 
 Like many git commands, git-annex can be passed a path that 
 is either a file or a directory. In the latter case it acts on all relevant
-files in the directory. If no path is specified, most git-annex commands
+files in the directory. When no path is specified, most git-annex commands
 default to acting on all relevant files in the current directory (and
 subdirectories).
 
@@ -115,6 +115,14 @@ subdirectories).
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
+* addurl [url ...]
+
+  Downloads each url to a file, which is added to the annex.
+
+  To avoid immediately downloading the url, specify --fast
+
+# REPOSITORY SETUP COMMANDS
+
 * init [description]
 
   Until a repository (or one of its remotes) has been initialized,
@@ -144,6 +152,24 @@ subdirectories).
 
 	initremote mys3 type=S3 encryption=none datacenter=EU
 
+* trust [repository ...]
+
+  Records that a repository is trusted to not unexpectedly lose
+  content. Use with care.
+
+  To trust the current repository, use "."
+
+* untrust [repository ...]
+
+  Records that a repository is not trusted and could lose content
+  at any time.
+
+* semitrust [repository ...]
+
+  Returns a repository to the default semi trusted state.
+
+# REPOSITORY MAINTENANCE COMMANDS
+
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
@@ -170,6 +196,29 @@ subdirectories).
 
   To drop the data from a remote, specify --from.
 
+* merge
+
+  Automatically merges any changes from remotes into the git-annex branch.
+  While git-annex mostly handles keeping the git-annex branch merged
+  automatically, if you find you are unable to push the git-annex branch
+  due non-fast-forward, this will fix it.
+
+* fix [path ...]
+
+  Fixes up symlinks that have become broken to again point to annexed content.
+  This is useful to run if you have been moving the symlinks around,
+  but is done automatically when committing a change with git too.
+
+* upgrade
+
+  Upgrades the repository to current layout.
+
+# QUERY COMMANDS
+
+* version
+
+  Shows the version of git-annex, as well as repository version information.
+
 * find [path ...]
 
   Outputs a list of annexed files whose content is currently present.
@@ -182,13 +231,6 @@ subdirectories).
   Displays a list of repositories known to contain the content of the
   specified file or files.
 
-* merge
-
-  Automatically merges any changes from remotes into the git-annex branch.
-  While git-annex mostly handles keeping the git-annex branch merged
-  automatically, if you find you are unable to push the git-annex branch
-  due non-fast-forward, this will fix it.
-
 * status
 
   Displays some statistics and other information, including how much data
@@ -199,19 +241,6 @@ subdirectories).
   information you wanted to see. Or, use --fast to only display
   the first, fast(ish) statistics.
 
-* migrate [path ...]
-
-  Changes the specified annexed files to use the default key-value backend
-  (or the one specified with --backend). Only files whose content
-  is currently available are migrated.
-
-  Note that the content is also still available using the old key after
-  migration. Use `git annex unused` to find and remove the old key.
-
-  Normally, nothing will be done to files already using the new backend.
-  However, if a backend changes the information it uses to construct a key,
-  this can also be used to migrate files to use the new key format.
-
 * map
 
   Helps you keep track of your repositories, and the connections between them,
@@ -229,6 +258,21 @@ subdirectories).
   Note that this subcommand can be used to graph any git repository; it
   is not limited to git-annex repositories.
 
+# UTILITY COMMANDS
+
+* migrate [path ...]
+
+  Changes the specified annexed files to use the default key-value backend
+  (or the one specified with --backend). Only files whose content
+  is currently available are migrated.
+
+  Note that the content is also still available using the old key after
+  migration. Use `git annex unused` to find and remove the old key.
+
+  Normally, nothing will be done to files already using the new backend.
+  However, if a backend changes the information it uses to construct a key,
+  this can also be used to migrate files to use the new key format.
+
 * unannex [path ...]
 
   Use this to undo an accidental `git annex add` command. You can use
@@ -248,10 +292,7 @@ subdirectories).
   repository, and remove all of git-annex's other data, leaving you with a
   git repository plus the previously annexed files.
 
-* fix [path ...]
-
-  Fixes up symlinks that have become broken to again point to annexed content.
-  This is useful to run if you have been moving the symlinks around.
+# PLUMBING COMMANDS
 
 * pre-commit [path ...]
 
@@ -262,28 +303,6 @@ subdirectories).
   This is meant to be called from git's pre-commit hook. `git annex init`
   automatically creates a pre-commit hook using this.
 
-* trust [repository ...]
-
-  Records that a repository is trusted to not unexpectedly lose
-  content. Use with care.
-
-  To trust the current repository, use "."
-
-* untrust [repository ...]
-
-  Records that a repository is not trusted and could lose content
-  at any time.
-
-* semitrust [repository ...]
-
-  Returns a repository to the default semi trusted state.
-
-* addurl [url ...]
-
-  Downloads each url to a file, which is added to the annex.
-
-  To avoid immediately downloading the url, specify --fast
-
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file
@@ -310,14 +329,6 @@ subdirectories).
 
 	git annex setkey --key=WORM-s3-m1287765018--file /tmp/file
 
-* upgrade
-
-  Upgrades the repository to current layout.
-
-* version
-
-  Shows the version of git-annex, as well as repository version information.
-
 # OPTIONS
 
 * --force

From 3e73de4054338955ce0f923923a526e8225225aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 17 Sep 2011 09:21:09 -0400
Subject: [PATCH 2214/2835] releasing version 3.20110915

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e940989da7..dff945c287 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110907) UNRELEASED; urgency=low
+git-annex (3.20110915) unstable; urgency=low
 
   * whereis: Show untrusted locations separately and do not include in
     location count.
@@ -9,7 +9,7 @@ git-annex (3.20110907) UNRELEASED; urgency=low
     to get/drop content as needed to work toward the configured numcopies.
   * bugfix: drop and fsck did not honor --exclude
 
- -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
+ -- Joey Hess   Thu, 15 Sep 2011 22:25:46 -0400
 
 git-annex (3.20110906) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 34c8e76222..a518ca8cb9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110906
+Version: 3.20110915
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From b73d8225d5c95241eedb5969846f859a7f69db65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 17 Sep 2011 09:21:21 -0400
Subject: [PATCH 2215/2835] add news item for git-annex 3.20110915

---
 doc/news/version_3.20110719.mdwn |  7 -------
 doc/news/version_3.20110915.mdwn | 10 ++++++++++
 2 files changed, 10 insertions(+), 7 deletions(-)
 delete mode 100644 doc/news/version_3.20110719.mdwn
 create mode 100644 doc/news/version_3.20110915.mdwn

diff --git a/doc/news/version_3.20110719.mdwn b/doc/news/version_3.20110719.mdwn
deleted file mode 100644
index 5beae32270..0000000000
--- a/doc/news/version_3.20110719.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 3.20110719 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
-     Closes: #[634233](http://bugs.debian.org/634233)
-   * Bugfix: Make add ../ work.
-   * Support the standard git -c name=value
-   * unannex: Clean up use of git commit -a."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110915.mdwn b/doc/news/version_3.20110915.mdwn
new file mode 100644
index 0000000000..77a97242d9
--- /dev/null
+++ b/doc/news/version_3.20110915.mdwn
@@ -0,0 +1,10 @@
+git-annex 3.20110915 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * whereis: Show untrusted locations separately and do not include in
+     location count.
+   * Fix build without S3.
+   * addurl: Always use whole url as destination filename, rather than
+     only its file component.
+   * get, drop, copy: Added --auto option, which decides whether
+     to get/drop content as needed to work toward the configured numcopies.
+   * bugfix: drop and fsck did not honor --exclude"""]]
\ No newline at end of file

From 756338fb2e920be86f1d25fbe8be7e9bc88d2491 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 17 Sep 2011 18:37:38 -0400
Subject: [PATCH 2216/2835] idea

---
 doc/todo/exclude_files_on_a_given_remote.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/todo/exclude_files_on_a_given_remote.mdwn

diff --git a/doc/todo/exclude_files_on_a_given_remote.mdwn b/doc/todo/exclude_files_on_a_given_remote.mdwn
new file mode 100644
index 0000000000..fbb3f90bef
--- /dev/null
+++ b/doc/todo/exclude_files_on_a_given_remote.mdwn
@@ -0,0 +1,16 @@
+Say I have some files on remote A. But I'm away from it, and transferring
+files from B to C. I'd like to avoid transferring any files I already have
+on A. 
+
+Something like:
+
+	git annex copy --to C --exclude-on A
+
+This would not contact A, just use its cached location log info.
+
+I suppose I might also sometime want to only act on files that are
+thought/known to be on A.
+
+	git annex drop --only-on A
+
+--[[Joey]] 

From 6ebaa85bdb8d13a752d7994e0b35620a10744eaa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 16:02:12 -0400
Subject: [PATCH 2217/2835] a generic matcher

---
 Matcher.hs | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 Matcher.hs

diff --git a/Matcher.hs b/Matcher.hs
new file mode 100644
index 0000000000..32da9f95df
--- /dev/null
+++ b/Matcher.hs
@@ -0,0 +1,88 @@
+{- A generic matcher.
+ -
+ - Can be used to check if a user-supplied condition,
+ - like "foo and ( bar or not baz )" matches. The condition must already
+ - be tokenized, and can contain arbitrary operations.
+ -
+ - If operations are not separated by and/or, they are defaulted to being
+ - anded together, so "foo bar baz" all must match.
+ -
+ - Is forgiving about misplaced closing parens, so "foo and (bar or baz"
+ - will be handled, as will "foo and ( bar or baz ) )"
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Matcher (
+	Operation(..),
+	Token(..),
+	toMatcher,
+	match,
+	runMatch	
+) where
+
+import Control.Monad
+
+{- An Operation is a command and some parameters. -}
+data Operation = Operation String [String]
+	deriving (Show, Eq)
+
+{- A Token can either be a single word, or an Operation. -}
+data Token = Token String | TokenOp Operation
+	deriving (Show, Eq)
+
+data Matcher = Any
+	| And Matcher Matcher
+	| Or Matcher Matcher
+	| Not Matcher
+	| Op Operation
+	deriving (Show, Eq)
+
+{- Converts a list of Tokens into a Matcher. -}
+toMatcher :: [Token] -> Matcher
+toMatcher ts = toMatcher' Any ts
+toMatcher' :: Matcher -> [Token] -> Matcher
+toMatcher' m [] = m
+toMatcher' m ts = toMatcher' m' rest
+	where
+		(m', rest) = consume m ts
+
+{- Consumes one or more tokens, constructs a new Matcher,
+ - and returns unconsumed tokens. -}
+consume :: Matcher -> [Token] -> (Matcher, [Token])
+consume m [] = (m, [])
+consume m ((TokenOp o):ts) = (m `And` Op o, ts)
+consume m ((Token t):ts)
+	| t == "and" = cont $ m `And` next
+	| t == "or" = cont $ m `Or` next
+	| t == "not" = cont $ m `And` (Not next)
+	| t == "(" = let (n, r) = consume next rest in (m `And` n, r)
+	| t == ")" = (m, ts)
+	| otherwise = (m, ts) -- ignore unknown token
+	where
+		(next, rest) = consume Any ts
+		cont v = (v, rest)
+
+{- Checks if a Matcher matches, using a supplied function to check
+ - the value of Operations. -}
+match :: (Operation -> Bool) -> Matcher -> Bool
+match a = go
+	where
+		go Any = True
+		go (And m1 m2) = go m1 && go m2
+		go (Or m1 m2) = go m1 || go m2
+		go (Not m1) = not (go m1)
+		go (Op v) = a v
+
+{- Runs a Matcher in an arbitrary monadic contex, using a supplied
+ - action to evaluate Operations. -}
+runMatch :: Monad m => (Operation -> m Bool) -> Matcher -> m Bool
+runMatch a = go
+	where
+		go Any = return True
+		go (And m1 m2) = liftM2 (&&) (go m1) (go m2)
+		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
+		go (Not m1) = liftM not (go m1)
+		go (Op v) = a v

From 0e499e67be1ba32d30eb0a65537d2c6a0f4d6e05 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 16:32:39 -0400
Subject: [PATCH 2218/2835] convert to parameterized types, so the Operation
 can be any type the caller needs

Especially handy for running a match monadically.
---
 Matcher.hs | 52 +++++++++++++++++++++++-----------------------------
 1 file changed, 23 insertions(+), 29 deletions(-)

diff --git a/Matcher.hs b/Matcher.hs
index 32da9f95df..91e3bf6f33 100644
--- a/Matcher.hs
+++ b/Matcher.hs
@@ -16,44 +16,39 @@
  -}
 
 module Matcher (
-	Operation(..),
 	Token(..),
-	toMatcher,
+	generate,
 	match,
-	runMatch	
+	run
 ) where
 
 import Control.Monad
 
-{- An Operation is a command and some parameters. -}
-data Operation = Operation String [String]
+{- A Token can either be a single word, or an Operation of an arbitrary type. -}
+data Token op = Token String | Operation op
 	deriving (Show, Eq)
 
-{- A Token can either be a single word, or an Operation. -}
-data Token = Token String | TokenOp Operation
-	deriving (Show, Eq)
-
-data Matcher = Any
-	| And Matcher Matcher
-	| Or Matcher Matcher
-	| Not Matcher
-	| Op Operation
+data Matcher op = Any
+	| And (Matcher op) (Matcher op)
+	| Or (Matcher op) (Matcher op)
+	| Not (Matcher op)
+	| Op op
 	deriving (Show, Eq)
 
 {- Converts a list of Tokens into a Matcher. -}
-toMatcher :: [Token] -> Matcher
-toMatcher ts = toMatcher' Any ts
-toMatcher' :: Matcher -> [Token] -> Matcher
-toMatcher' m [] = m
-toMatcher' m ts = toMatcher' m' rest
+generate :: [Token op] -> Matcher op
+generate ts = generate' Any ts
+generate' :: Matcher op -> [Token op] -> Matcher op
+generate' m [] = m
+generate' m ts = generate' m' rest
 	where
 		(m', rest) = consume m ts
 
-{- Consumes one or more tokens, constructs a new Matcher,
- - and returns unconsumed tokens. -}
-consume :: Matcher -> [Token] -> (Matcher, [Token])
+{- Consumes one or more Tokens, constructs a new Matcher,
+ - and returns unconsumed Tokens. -}
+consume :: Matcher op -> [Token op] -> (Matcher op, [Token op])
 consume m [] = (m, [])
-consume m ((TokenOp o):ts) = (m `And` Op o, ts)
+consume m ((Operation o):ts) = (m `And` Op o, ts)
 consume m ((Token t):ts)
 	| t == "and" = cont $ m `And` next
 	| t == "or" = cont $ m `Or` next
@@ -67,7 +62,7 @@ consume m ((Token t):ts)
 
 {- Checks if a Matcher matches, using a supplied function to check
  - the value of Operations. -}
-match :: (Operation -> Bool) -> Matcher -> Bool
+match :: (op -> Bool) -> Matcher op -> Bool
 match a = go
 	where
 		go Any = True
@@ -76,13 +71,12 @@ match a = go
 		go (Not m1) = not (go m1)
 		go (Op v) = a v
 
-{- Runs a Matcher in an arbitrary monadic contex, using a supplied
- - action to evaluate Operations. -}
-runMatch :: Monad m => (Operation -> m Bool) -> Matcher -> m Bool
-runMatch a = go
+{- Runs a monadic Matcher, where Operations are actions in the monad. -}
+run :: Monad m => Matcher (m Bool) -> m Bool
+run = go
 	where
 		go Any = return True
 		go (And m1 m2) = liftM2 (&&) (go m1) (go m2)
 		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
 		go (Not m1) = liftM not (go m1)
-		go (Op v) = a v
+		go (Op o) = o -- run o

From 3e15187ac1082bb086c79109c8b35dd8c20017ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 16:36:30 -0400
Subject: [PATCH 2219/2835] move to Utility

---
 Matcher.hs => Utility/Matcher.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename Matcher.hs => Utility/Matcher.hs (98%)

diff --git a/Matcher.hs b/Utility/Matcher.hs
similarity index 98%
rename from Matcher.hs
rename to Utility/Matcher.hs
index 91e3bf6f33..0f589ec2c0 100644
--- a/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -15,7 +15,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Matcher (
+module Utility.Matcher (
 	Token(..),
 	generate,
 	match,

From 38c0f3eaf86b67d584d4ff30ab15ec2c725a7fad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 17:47:24 -0400
Subject: [PATCH 2220/2835] add a value to match against to match and matchM

---
 Utility/Matcher.hs | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 0f589ec2c0..08534fc304 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -17,9 +17,10 @@
 
 module Utility.Matcher (
 	Token(..),
+	Matcher,
 	generate,
 	match,
-	run
+	matchM
 ) where
 
 import Control.Monad
@@ -62,21 +63,21 @@ consume m ((Token t):ts)
 
 {- Checks if a Matcher matches, using a supplied function to check
  - the value of Operations. -}
-match :: (op -> Bool) -> Matcher op -> Bool
-match a = go
+match :: (op -> v -> Bool) -> Matcher op -> v -> Bool
+match a m v = go m
 	where
 		go Any = True
 		go (And m1 m2) = go m1 && go m2
 		go (Or m1 m2) = go m1 || go m2
 		go (Not m1) = not (go m1)
-		go (Op v) = a v
+		go (Op o) = a o v
 
 {- Runs a monadic Matcher, where Operations are actions in the monad. -}
-run :: Monad m => Matcher (m Bool) -> m Bool
-run = go
+matchM :: Monad m => Matcher (v -> m Bool) -> v -> m Bool
+matchM m v = go m
 	where
 		go Any = return True
 		go (And m1 m2) = liftM2 (&&) (go m1) (go m2)
 		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
 		go (Not m1) = liftM not (go m1)
-		go (Op o) = o -- run o
+		go (Op o) = o v

From 8a5a92480b9dcf691af1e8c4849cb71c4158b845 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 17:47:49 -0400
Subject: [PATCH 2221/2835] refactor --exclude to use Utility.Matcher

This should change no behavior, but opens the poissibility to use the
matcher for other sorts of limits on which files git-annex processes.
---
 Annex.hs    |  5 +++--
 Command.hs  | 21 +------------------
 GitAnnex.hs |  4 ++--
 Limit.hs    | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 24 deletions(-)
 create mode 100644 Limit.hs

diff --git a/Annex.hs b/Annex.hs
index 2c8ea1d61d..ad65e05ddf 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -31,6 +31,7 @@ import Types.Crypto
 import Types.BranchState
 import Types.TrustLevel
 import Types.UUID
+import qualified Utility.Matcher
 
 -- git-annex's monad
 newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
@@ -59,7 +60,7 @@ data AnnexState = AnnexState
 	, defaultkey :: Maybe String
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
-	, exclude :: [String]
+	, limit :: Either [Utility.Matcher.Token (FilePath -> Annex Bool)] (Utility.Matcher.Matcher (FilePath -> Annex Bool))
 	, forcetrust :: [(UUID, TrustLevel)]
 	, trustmap :: Maybe TrustMap
 	, cipher :: Maybe Cipher
@@ -83,7 +84,7 @@ newState gitrepo = AnnexState
 	, defaultkey = Nothing
 	, toremote = Nothing
 	, fromremote = Nothing
-	, exclude = []
+	, limit = Left []
 	, forcetrust = []
 	, trustmap = Nothing
 	, cipher = Nothing
diff --git a/Command.hs b/Command.hs
index 08087a5a5d..fd58ca801a 100644
--- a/Command.hs
+++ b/Command.hs
@@ -12,11 +12,8 @@ import System.Directory
 import System.Posix.Files
 import Control.Monad (filterM, liftM, when)
 import Control.Applicative
-import System.Path.WildMatch
-import Text.Regex.PCRE.Light.Char8
 import Data.List
 import Data.Maybe
-import Data.String.Utils
 
 import Types
 import qualified Backend
@@ -30,6 +27,7 @@ import Trust
 import LocationLog
 import Config
 import Backend
+import Limit
 
 {- A command runs in four stages.
  -
@@ -180,23 +178,6 @@ withNothing _ _ = error "This command takes no parameters."
 backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
 backendPairs a files = map a <$> Backend.chooseBackends files
 
-{- Filter out files those matching the exclude glob pattern,
- - if it was specified. -}
-filterFiles :: [FilePath] -> Annex [FilePath]
-filterFiles l = do
-	exclude <- Annex.getState Annex.exclude
-	if null exclude
-		then return l
-		else return $ filter (notExcluded $ wildsRegex exclude) l
-	where
-		notExcluded r f = isNothing $ match r f []
-
-wildsRegex :: [String] -> Regex
-wildsRegex ws = compile regex []
-	where
-		regex = "^(" ++ alternatives ++ ")"
-		alternatives = join "|" $ map wildToRegex ws
-
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 6f4e5d4921..bb0f85119c 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -19,6 +19,7 @@ import Types
 import Types.TrustLevel
 import qualified Annex
 import qualified Remote
+import qualified Limit
 
 import qualified Command.Add
 import qualified Command.Unannex
@@ -97,7 +98,7 @@ options = commonOptions ++
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
-	, Option ['x'] ["exclude"] (ReqArg addexclude paramGlob)
+	, Option ['x'] ["exclude"] (ReqArg (Limit.exclude) paramGlob)
 		"skip files matching the glob pattern"
 	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
@@ -113,7 +114,6 @@ options = commonOptions ++
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
-		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 		setgitconfig :: String -> Annex ()
diff --git a/Limit.hs b/Limit.hs
new file mode 100644
index 0000000000..324baee2e3
--- /dev/null
+++ b/Limit.hs
@@ -0,0 +1,59 @@
+{- user-specified limits on files to act on
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Limit where
+
+import Text.Regex.PCRE.Light.Char8
+import System.Path.WildMatch
+import Control.Monad (filterM)
+import Data.Maybe
+
+import Annex
+import qualified Utility.Matcher
+
+type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
+
+{- Filter out files not matching user-specified limits. -}
+filterFiles :: [FilePath] -> Annex [FilePath]
+filterFiles l = do
+	matcher <- getMatcher
+	filterM (Utility.Matcher.matchM matcher) l
+
+{- Gets a matcher for the user-specified limits. The matcher is cached for
+ - speed; once it's obtained the user-specified limits can't change. -}
+getMatcher :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
+getMatcher = do
+	m <- Annex.getState Annex.limit
+	case m of
+		Right r -> return r
+		Left l -> do
+			let matcher = Utility.Matcher.generate (reverse l)
+			Annex.changeState $ \s -> s { Annex.limit = Right matcher }
+			return matcher
+
+{- Adds something to the limit list. -}
+add :: Limit -> Annex ()
+add l = Annex.changeState $ \s -> s { Annex.limit = append $ Annex.limit s }
+	where
+		append (Left ls) = Left $ l:ls
+		append _ = error "internal"
+
+{- Adds a new limit. -}
+addl :: (FilePath -> Annex Bool) -> Annex ()
+addl = add . Utility.Matcher.Operation
+
+{- Adds a new token. -}
+addt :: String -> Annex ()
+addt = add . Utility.Matcher.Token
+
+{- Add a limit to skip files that do not match the glob. -}
+exclude :: String -> Annex ()
+exclude glob = addl $ return . notExcluded
+	where
+		notExcluded f = isNothing $ match cregex f []
+		cregex = compile regex []
+		regex = '^':wildToRegex glob

From b9aa944b09e60badb99c65a87f5689e0ab9010e3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 18:21:42 -0400
Subject: [PATCH 2222/2835] add --and --or --not -( and -)

I dislike -( and -), but without using a different option parser, can't
easily use bare parens.

--and and --or will become more useful once there are more interesting
limits than --exclude
---
 GitAnnex.hs |  6 +++---
 Limit.hs    | 12 ++++++------
 Options.hs  | 13 +++++++++++++
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index bb0f85119c..68c7a1805f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -98,8 +98,6 @@ options = commonOptions ++
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
-	, Option ['x'] ["exclude"] (ReqArg (Limit.exclude) paramGlob)
-		"skip files matching the glob pattern"
 	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
 	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)
@@ -110,7 +108,9 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	]
+	, Option ['x'] ["exclude"] (ReqArg (Limit.exclude) paramGlob)
+		"skip files matching the glob pattern"
+	] ++ matcherOptions
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
diff --git a/Limit.hs b/Limit.hs
index 324baee2e3..91ea3453cf 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -35,7 +35,7 @@ getMatcher = do
 			Annex.changeState $ \s -> s { Annex.limit = Right matcher }
 			return matcher
 
-{- Adds something to the limit list. -}
+{- Adds something to the limit list, which is built up reversed. -}
 add :: Limit -> Annex ()
 add l = Annex.changeState $ \s -> s { Annex.limit = append $ Annex.limit s }
 	where
@@ -43,16 +43,16 @@ add l = Annex.changeState $ \s -> s { Annex.limit = append $ Annex.limit s }
 		append _ = error "internal"
 
 {- Adds a new limit. -}
-addl :: (FilePath -> Annex Bool) -> Annex ()
-addl = add . Utility.Matcher.Operation
+addlimit :: (FilePath -> Annex Bool) -> Annex ()
+addlimit = add . Utility.Matcher.Operation
 
 {- Adds a new token. -}
-addt :: String -> Annex ()
-addt = add . Utility.Matcher.Token
+token :: String -> Annex ()
+token = add . Utility.Matcher.Token
 
 {- Add a limit to skip files that do not match the glob. -}
 exclude :: String -> Annex ()
-exclude glob = addl $ return . notExcluded
+exclude glob = addlimit $ return . notExcluded
 	where
 		notExcluded f = isNothing $ match cregex f []
 		cregex = compile regex []
diff --git a/Options.hs b/Options.hs
index ee3ce66203..44d5f3674c 100644
--- a/Options.hs
+++ b/Options.hs
@@ -14,6 +14,7 @@ import Control.Monad.State (liftIO)
 import qualified Annex
 import Types
 import Command
+import Limit
 
 {- Each dashed command-line option results in generation of an action
  - in the Annex monad that performs the necessary setting.
@@ -47,3 +48,15 @@ commonOptions =
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG
+	
+matcherOptions :: [Option]
+matcherOptions =
+	[ longopt "not" "negate next option"
+	, longopt "and" "both previous and next option must match"
+	, longopt "or" "either previous or next option must match"
+	, shortopt "(" "open group of options"
+	, shortopt ")" "close group of options"
+	]
+	where
+		longopt o d = Option [] [o] (NoArg (token o)) d
+		shortopt o d = Option o [] (NoArg (token o)) d

From d78b9f7d546bd4f13349e01777d5dd45fc01b0af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 18:24:10 -0400
Subject: [PATCH 2223/2835] update man page for file matching options

(--in is not yet implemented)
---
 doc/git-annex.mdwn | 55 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 48 insertions(+), 7 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d587f763ca..03277c6a0e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -379,13 +379,6 @@ subdirectories).
 
   It should be specified using the name of a configured remote.
 
-* --exclude=glob
-
-  Skips files matching the glob pattern. The glob is matched relative to
-  the current directory.
-
-  This option can be specified multiple times.
-
 * --numcopies=n
 
   Overrides the `annex.numcopies` setting, forcing git-annex to ensure the
@@ -415,6 +408,54 @@ subdirectories).
 
   Used to override git configuration settings. May be specified multiple times.
 
+# FILE MATCHING OPTIONS
+
+These options can all be specified multiple times, and can be combined to
+limit which files git-annex acts on.
+
+Arbitrarily complicated expressions can be built using these options.
+For example:
+
+	--exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+
+The above example prevents git-annex from working on mp3 files whose
+file contents are present at either of two repositories.
+
+* --exclude=glob
+
+  Skips files matching the glob pattern. The glob is matched relative to
+  the current directory. For example: --exclude='*.mp3' --exclude='subdir/*'
+
+* --in=repository
+
+  Matches only files that git-annex believes have their contents present
+  in a repository.
+
+  The repository should be specified using the name of a configured remote,
+  or the UUID or description of a repository.
+
+* --not
+
+  Inverts the next file matching option. For example, to only act on
+  mp3s, use: --not --exclude='*.mp3'
+
+* --and
+
+  Requires that both the previous and the next file matching option matches.
+  The default.
+
+* --or
+
+  Requires that either the previous, or the next file matching option matches.
+
+* -(
+
+  Opens a group of file matching options.
+
+* -)
+
+  Closes a group of file matching options.
+
 # CONFIGURATION
 
 Like other git commands, git-annex is configured via `.git/config`.

From dd463a3100f21d72c35ca1af5b0f63f6296cf322 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:11:39 -0400
Subject: [PATCH 2224/2835] rework annex-ignore handling

Only one place need to filter the list of remotes for ignored remotes:
keyPossibilities. Make the full list available to everything else.

This allows getting rid of the special case handing for --from and --to
to make ignored remotes not be ignored with those options.
---
 Annex.hs            |  4 ++--
 Config.hs           | 14 +++-----------
 Remote.hs           |  6 ++----
 Remote/Bup.hs       |  3 ++-
 Remote/Directory.hs |  3 ++-
 Remote/Git.hs       |  3 ++-
 Remote/Hook.hs      |  3 ++-
 Remote/Rsync.hs     |  3 ++-
 Remote/S3real.hs    |  3 ++-
 Remote/Web.hs       |  3 ++-
 Types/Remote.hs     |  4 +++-
 11 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index ad65e05ddf..1517a34708 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -26,7 +26,7 @@ import Control.Applicative hiding (empty)
 import qualified Git
 import Git.Queue
 import Types.Backend
-import Types.Remote
+import qualified Types.Remote
 import Types.Crypto
 import Types.BranchState
 import Types.TrustLevel
@@ -48,7 +48,7 @@ newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [Backend Annex]
-	, remotes :: [Remote Annex]
+	, remotes :: [Types.Remote.Remote Annex]
 	, repoqueue :: Queue
 	, output :: OutputType
 	, force :: Bool
diff --git a/Config.hs b/Config.hs
index b4f4c0b922..fe847fce1c 100644
--- a/Config.hs
+++ b/Config.hs
@@ -82,18 +82,10 @@ prop_cost_sane = False `notElem`
 {- Checks if a repo should be ignored, based either on annex-ignore
  - setting, or on command-line options. Allows command-line to override
  - annex-ignore. -}
-remoteNotIgnored :: Git.Repo -> Annex Bool
-remoteNotIgnored r = do
+repoNotIgnored :: Git.Repo -> Annex Bool
+repoNotIgnored r = do
 	ignored <- getConfig r "ignore" "false"
-	to <- match Annex.toremote
-	from <- match Annex.fromremote
-	if to || from
-		then return True
-		else return $ not $ Git.configTrue ignored
-	where
-		match a = do
-			n <- Annex.getState a
-			return $ n == Git.repoRemoteName r
+	return $ not $ Git.configTrue ignored
 
 {- If a value is specified, it is used; otherwise the default is looked up
  - in git config. forcenumcopies overrides everything. -}
diff --git a/Remote.hs b/Remote.hs
index 429f9058b6..0ce01872ac 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -16,7 +16,6 @@ module Remote (
 	hasKeyCheap,
 
 	remoteTypes,
-	genList,
 	byName,
 	prettyPrintUUIDs,
 	remotesWithUUID,
@@ -29,7 +28,7 @@ module Remote (
 	forceTrust
 ) where
 
-import Control.Monad (filterM)
+import Control.Monad.State (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
@@ -83,7 +82,6 @@ genList = do
 	where
 		process m t = 
 			enumerate t >>=
-			filterM remoteNotIgnored >>=
 			mapM (gen m t)
 		gen m t r = do
 			u <- getUUID r
@@ -184,7 +182,7 @@ keyPossibilities' withtrusted key = do
 	let validtrusteduuids = validuuids `intersect` trusted
 
 	-- remotes that match uuids that have the key
-	allremotes <- genList
+	allremotes <- filterM (repoNotIgnored . repo) =<< genList
 	let validremotes = remotesWithUUID allremotes validuuids
 
 	return (sort validremotes, validtrusteduuids)
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index ebb4b10a5b..29c7a0419f 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -66,7 +66,8 @@ gen r u c = do
 			removeKey = remove,
 			hasKey = checkPresent r bupr',
 			hasKeyCheap = bupLocal buprepo,
-			config = c
+			config = c,
+			repo = r
 		}
 
 bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 7ddb60462f..b183042ef6 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -57,7 +57,8 @@ gen r u c = do
 			removeKey = remove dir,
 			hasKey = checkPresent dir,
 			hasKeyCheap = True,
-			config = Nothing
+			config = Nothing,
+			repo = r
 		}
 
 directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Git.hs b/Remote/Git.hs
index fadd48a036..9789a06252 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -71,7 +71,8 @@ gen r u _ = do
 		removeKey = dropKey r',
 		hasKey = inAnnex r',
 		hasKeyCheap = cheap,
-		config = Nothing
+		config = Nothing,
+		repo = r'
 	}
 
 {- Tries to read the config for a specified remote, updates state, and
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 54b9806ffc..aaeb702c7b 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -58,7 +58,8 @@ gen r u c = do
 			removeKey = remove hooktype,
 			hasKey = checkPresent r hooktype,
 			hasKeyCheap = False,
-			config = Nothing
+			config = Nothing,
+			repo = r
 		}
 
 hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 3707966ad6..9d2d7ddcf7 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -66,7 +66,8 @@ gen r u c = do
 			removeKey = remove o,
 			hasKey = checkPresent r o,
 			hasKeyCheap = False,
-			config = Nothing
+			config = Nothing,
+			repo = r
 		}
 
 genRsyncOpts :: Git.Repo -> Annex RsyncOpts
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 456a77f0e4..77b6b6ca40 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -67,7 +67,8 @@ gen' r u c cst =
 			removeKey = remove this,
 			hasKey = checkPresent this,
 			hasKeyCheap = False,
-			config = c
+			config = c,
+			repo = r
 		}
 
 s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 3695bb1ab3..8fb29ec403 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -58,7 +58,8 @@ gen r _ _ =
 		removeKey = dropKey,
 		hasKey = checkKey,
 		hasKeyCheap = False,
-		config = Nothing
+		config = Nothing,
+		repo = r
 	}
 
 {- The urls for a key are stored in remote/web/hash/key.log 
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 8d9622c519..49f16bfdd7 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -51,7 +51,9 @@ data Remote a = Remote {
 	-- operation.
 	hasKeyCheap :: Bool,
 	-- a Remote can have a persistent configuration store
-	config :: Maybe RemoteConfig
+	config :: Maybe RemoteConfig,
+	-- git configuration for the remote
+	repo :: Git.Repo
 }
 
 instance Show (Remote a) where

From 1fc3ee24232059ae05ab18ed10bf151f86847ac1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:14:18 -0400
Subject: [PATCH 2225/2835] add --in limit

---
 GitAnnex.hs      |  4 +++-
 Limit.hs         | 20 ++++++++++++++++++--
 debian/changelog | 10 ++++++++++
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index 68c7a1805f..bcb30ff41f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -108,8 +108,10 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	, Option ['x'] ["exclude"] (ReqArg (Limit.exclude) paramGlob)
+	, Option ['x'] ["exclude"] (ReqArg (Limit.addExclude) paramGlob)
 		"skip files matching the glob pattern"
+	, Option ['i'] ["in"] (ReqArg (Limit.addIn) paramRemote)
+		"skip files not present in a remote"
 	] ++ matcherOptions
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
diff --git a/Limit.hs b/Limit.hs
index 91ea3453cf..13df5f6c27 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -14,6 +14,9 @@ import Data.Maybe
 
 import Annex
 import qualified Utility.Matcher
+import qualified Remote
+import qualified Backend
+import LocationLog
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
@@ -51,9 +54,22 @@ token :: String -> Annex ()
 token = add . Utility.Matcher.Token
 
 {- Add a limit to skip files that do not match the glob. -}
-exclude :: String -> Annex ()
-exclude glob = addlimit $ return . notExcluded
+addExclude :: String -> Annex ()
+addExclude glob = addlimit $ return . notExcluded
 	where
 		notExcluded f = isNothing $ match cregex f []
 		cregex = compile regex []
 		regex = '^':wildToRegex glob
+
+{- Adds a limit to skip files not believed to be present
+ - on a specfied remote. -}
+addIn :: String -> Annex ()
+addIn name = do
+	u <- Remote.nameToUUID name
+	addlimit $ check u
+	where
+		check u f = Backend.lookupFile f >>= handle u
+		handle _ Nothing = return False
+		handle u (Just (key, _)) = do
+			us <- keyLocations key
+			return $ u `elem` us
diff --git a/debian/changelog b/debian/changelog
index dff945c287..0b698bc66f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,13 @@
+git-annex (3.20110916) UNRELEASED; urgency=low
+
+  * --in can be used to make git-annex only operate on files
+    believed to be present in a given repository.
+  * Arbitrarily complex expressions can be built to limit the files git-annex
+    operates on, by combining the options --not --and --or -( and -)
+    Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+
+ -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
+
 git-annex (3.20110915) unstable; urgency=low
 
   * whereis: Show untrusted locations separately and do not include in

From 9da23dff78d80158ba01a271ac2a32830fd9bccc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:23:08 -0400
Subject: [PATCH 2226/2835] --copies=N can be used to make git-annex only
 operate on files with the specified number of copies.

(And --not --copies=N for the inverse.)
---
 GitAnnex.hs        |  2 ++
 Limit.hs           | 15 +++++++++++++++
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  5 +++++
 4 files changed, 24 insertions(+)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index bcb30ff41f..a284daad5c 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -112,6 +112,8 @@ options = commonOptions ++
 		"skip files matching the glob pattern"
 	, Option ['i'] ["in"] (ReqArg (Limit.addIn) paramRemote)
 		"skip files not present in a remote"
+	, Option ['C'] ["copies"] (ReqArg (Limit.addCopies) paramNumber)
+		"skip files with fewer copies"
 	] ++ matcherOptions
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
diff --git a/Limit.hs b/Limit.hs
index 13df5f6c27..4aeb8bafb4 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -17,6 +17,7 @@ import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
 import LocationLog
+import Utility
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
@@ -73,3 +74,17 @@ addIn name = do
 		handle u (Just (key, _)) = do
 			us <- keyLocations key
 			return $ u `elem` us
+
+{- Adds a limit to skip files not believed to have the specified number
+ - of copies. -}
+addCopies :: String -> Annex ()
+addCopies num = do
+	case readMaybe num :: Maybe Int of
+		Nothing -> error "bad number for --copies"
+		Just n -> addlimit $ check n
+	where
+		check n f = Backend.lookupFile f >>= handle n
+		handle _ Nothing = return False
+		handle n (Just (key, _)) = do
+			us <- keyLocations key
+			return $ length us >= n
diff --git a/debian/changelog b/debian/changelog
index 0b698bc66f..638661f735 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (3.20110916) UNRELEASED; urgency=low
   * Arbitrarily complex expressions can be built to limit the files git-annex
     operates on, by combining the options --not --and --or -( and -)
     Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+  * --copies=N can be used to make git-annex only operate on files with
+    the specified number of copies. (And --not --copies=N for the inverse.)
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 03277c6a0e..836e782cde 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -434,6 +434,11 @@ file contents are present at either of two repositories.
   The repository should be specified using the name of a configured remote,
   or the UUID or description of a repository.
 
+* --copies=number
+
+  Matches only files that git-annex believes to have the specified number
+  of copies.
+
 * --not
 
   Inverts the next file matching option. For example, to only act on

From 33cd1ffbfe200188a4fc61fd5715e7aa29556d7f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:41:51 -0400
Subject: [PATCH 2227/2835] make find show files meeting limits, even when not
 present

find: Rather than only showing files whose contents are present, when used
with --exclude --copies or --in, displays all files that match the
specified conditions.

Note that this is a behavior change for find --exclude! Old behavior
can be gotten with find --in . --exclude=...
---
 Command/Find.hs    |  9 ++++++---
 Limit.hs           |  5 +++++
 Utility/Matcher.hs | 10 +++++++++-
 debian/changelog   |  5 +++++
 doc/git-annex.mdwn |  5 ++++-
 5 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/Command/Find.hs b/Command/Find.hs
index 51849f6b85..effb331840 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,11 +7,12 @@
 
 module Command.Find where
 
-import Control.Monad.State (liftIO)
+import Control.Monad.State
 
 import Command
 import Content
 import Utility.Conditional
+import Limit
 
 command :: [Command]
 command = [repoCommand "find" paramPaths seek "lists available files"]
@@ -19,8 +20,10 @@ command = [repoCommand "find" paramPaths seek "lists available files"]
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
-{- Output a list of files. -}
 start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
-	whenM (inAnnex key) $ liftIO $ putStrLn file
+	-- only files inAnnex are shown, unless the user has requested
+	-- others via a limit
+	whenM (liftM2 (||) (inAnnex key) limited) $
+		liftIO $ putStrLn file
 	stop
diff --git a/Limit.hs b/Limit.hs
index 4aeb8bafb4..51f3fc9505 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -10,6 +10,7 @@ module Limit where
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
 import Control.Monad (filterM)
+import Control.Applicative
 import Data.Maybe
 
 import Annex
@@ -27,6 +28,10 @@ filterFiles l = do
 	matcher <- getMatcher
 	filterM (Utility.Matcher.matchM matcher) l
 
+{- Checks if there are user-specified limits. -}
+limited :: Annex Bool
+limited = (not . Utility.Matcher.matchesAny) <$> getMatcher
+
 {- Gets a matcher for the user-specified limits. The matcher is cached for
  - speed; once it's obtained the user-specified limits can't change. -}
 getMatcher :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 08534fc304..5cf000b1b6 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -20,7 +20,8 @@ module Utility.Matcher (
 	Matcher,
 	generate,
 	match,
-	matchM
+	matchM,
+	matchesAny
 ) where
 
 import Control.Monad
@@ -81,3 +82,10 @@ matchM m v = go m
 		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
 		go (Not m1) = liftM not (go m1)
 		go (Op o) = o v
+
+{- Checks is a matcher contains no limits, and so (presumably) matches
+ - anything. Note that this only checks the trivial case; it is possible
+ - to construct matchers that match anything but are more complicated. -}
+matchesAny :: Matcher a -> Bool
+matchesAny Any = True
+matchesAny _ = False
diff --git a/debian/changelog b/debian/changelog
index 638661f735..1f7decc1d8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,11 @@ git-annex (3.20110916) UNRELEASED; urgency=low
     Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
   * --copies=N can be used to make git-annex only operate on files with
     the specified number of copies. (And --not --copies=N for the inverse.)
+  * find: Rather than only showing files whose contents are present,
+    when used with --exclude --copies or --in, displays all files that
+    match the specified conditions.
+  * Note that this is a behavior change for find --exclude! Old behavior
+    can be gotten with: find --in . --exclude=...
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 836e782cde..4a8e28a6ee 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -222,6 +222,8 @@ subdirectories).
 * find [path ...]
 
   Outputs a list of annexed files whose content is currently present.
+  Or, if a file matching option is specified, outputs a list of all
+  matching files, whether or not their content is currently present.
 
   With no parameters, defaults to finding all files in the current directory
   and its subdirectories.
@@ -432,7 +434,8 @@ file contents are present at either of two repositories.
   in a repository.
 
   The repository should be specified using the name of a configured remote,
-  or the UUID or description of a repository.
+  or the UUID or description of a repository. For the current repository,
+  use "--in=."
 
 * --copies=number
 

From 8ea48c3e3976b16ee4588a24fe1524769ef42e56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:56:37 -0400
Subject: [PATCH 2228/2835] update docs

---
 doc/cheatsheet.mdwn                           |  1 +
 doc/todo/exclude_files_on_a_given_remote.mdwn |  2 ++
 doc/walkthrough/powerful_file_matching.mdwn   | 36 +++++++++++++++++++
 3 files changed, 39 insertions(+)
 create mode 100644 doc/walkthrough/powerful_file_matching.mdwn

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 9ccb22e3e2..65cbad187c 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -12,4 +12,5 @@ A supplement to the [[walkthrough]].
 	walkthrough/what_to_do_when_you_lose_a_repository
 	walkthrough/recover_data_from_lost+found
 	walkthrough/Internet_Archive_via_S3
+	walkthrough/powerful_file_matching
 """]]
diff --git a/doc/todo/exclude_files_on_a_given_remote.mdwn b/doc/todo/exclude_files_on_a_given_remote.mdwn
index fbb3f90bef..e8bb357d31 100644
--- a/doc/todo/exclude_files_on_a_given_remote.mdwn
+++ b/doc/todo/exclude_files_on_a_given_remote.mdwn
@@ -14,3 +14,5 @@ thought/known to be on A.
 	git annex drop --only-on A
 
 --[[Joey]] 
+
+[[done]]
diff --git a/doc/walkthrough/powerful_file_matching.mdwn b/doc/walkthrough/powerful_file_matching.mdwn
new file mode 100644
index 0000000000..c0af4615c4
--- /dev/null
+++ b/doc/walkthrough/powerful_file_matching.mdwn
@@ -0,0 +1,36 @@
+git-annex has a powerful syntax for making it act on only certian files.
+
+The simplest thing is to exclude some files, using wild cards:
+
+	git annex get --exclude '*.mp3' --exclude '*.ogg'
+
+But you can also exclude files that git-annex's [[location_tracking]]
+information indicates are present in a given repository. For example,
+if you want to populate newarchive with files, but not those already
+on oldarchive, you could do it like this:
+
+	git annex copy --not -in oldarchive --to newarchive
+
+Without the --not, --in makes it act on files that *are* in the specified
+repository. So, to remove files that are on oldarchive:
+
+	git annex drop --in oldarchive
+
+Or maybe you're curious which files have a lot of copies, and then
+also want to know which files have only one copy:
+
+	git annex find --copies 7
+	git annex find --not --copies 2
+
+The above are the simple examples of specifying what files git-annex
+should act on. But you can specify anything you can dream up by combining
+the things above, with --and --or -( and -). Those last two strange-looking
+options are parentheses, for grouping other options. You will probably
+have to escape them from your shell.
+
+Here are the mp3 files that are in either of two repositories, but have
+less than 3 copies:
+
+	git annex find --not --exclude '*.mp3' --and \
+		-\( --in usbdrive --or --in archive -\) --and \
+		--not --copies 3

From b516cecff2c315e495c7dc6ec51af6e5aefaf57c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:58:34 -0400
Subject: [PATCH 2229/2835] probably better to error on unknown token

---
 Utility/Matcher.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 5cf000b1b6..451c3f139e 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -57,7 +57,7 @@ consume m ((Token t):ts)
 	| t == "not" = cont $ m `And` (Not next)
 	| t == "(" = let (n, r) = consume next rest in (m `And` n, r)
 	| t == ")" = (m, ts)
-	| otherwise = (m, ts) -- ignore unknown token
+	| otherwise = error $ "unknown token " ++ t
 	where
 		(next, rest) = consume Any ts
 		cont v = (v, rest)

From 8d1e8c0760e0d9935223523f35f5b8ea954730ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 21:02:40 -0400
Subject: [PATCH 2230/2835] golfing with curry

---
 Utility/Matcher.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 451c3f139e..342775d68c 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -42,9 +42,7 @@ generate :: [Token op] -> Matcher op
 generate ts = generate' Any ts
 generate' :: Matcher op -> [Token op] -> Matcher op
 generate' m [] = m
-generate' m ts = generate' m' rest
-	where
-		(m', rest) = consume m ts
+generate' m ts = uncurry generate' $ consume m ts
 
 {- Consumes one or more Tokens, constructs a new Matcher,
  - and returns unconsumed Tokens. -}

From 4f1fea1a856a2c82ed200e805bb18e9f9aaaa67b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 22:40:31 -0400
Subject: [PATCH 2231/2835] fix memory leak

filterM is not a good idea if you were streaming in a large list of files.

Fixing this memory leak that I introduced earlier today was a PITA because
to avoid the filterM, it's necessary to do the filtering only after
building up the data structures like BackendFile, and that means each
separate data structure needs it own function to apply the filter,
at least in this naive implementation.

There is also a minor performance regression, when using copy/drop/get/fsck
with a filter, git is now asked to look up attributes for all files,
since that now comes before the filter is applied. This is only a very
minor thing, since getting the attributes is very fast and --exclude was
probably not typically used to speed it up.
---
 Command.hs | 53 +++++++++++++++++++++++++++++++++--------------------
 Limit.hs   | 15 +++++++--------
 2 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/Command.hs b/Command.hs
index fd58ca801a..3cfff268c8 100644
--- a/Command.hs
+++ b/Command.hs
@@ -107,17 +107,22 @@ notBareRepo a = do
 
 {- These functions find appropriate files or other things based on a
    user's parameters, and run a specified action on them. -}
-withFilesInGit :: (String -> CommandStart) -> CommandSeek
+withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	liftM (map a) $ filterFiles files
+	runFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	files' <- filterFiles files
-	liftM (map a) $ liftIO $ Git.checkAttr repo attr files'
+	run $ liftIO $ Git.checkAttr repo attr files
+	where
+		run fs = do
+			matcher <- Limit.getMatcher
+			liftM (map $ proc matcher) fs
+		proc matcher p@(f, _) = do
+			ok <- matcher f
+			if ok then a p else stop
 withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 	where
@@ -128,23 +133,17 @@ withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	files' <- filterFiles files
-	backendPairs a files'
+	backendPairs a files
 withFilesMissing :: (String -> CommandStart) -> CommandSeek
-withFilesMissing a params = do
-	files <- liftIO $ filterM missing params
-	liftM (map a) $ filterFiles files
+withFilesMissing a params = runFiltered a $ liftIO $ filterM missing params
 	where
-		missing f = do
-			e <- doesFileExist f
-			return $ not e
+		missing = liftM not . doesFileExist
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
 	force <- Annex.getState Annex.force
 	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
-	newfiles' <- filterFiles newfiles
-	backendPairs a newfiles'
+	backendPairs a newfiles
 withWords :: ([String] -> CommandStart) -> CommandSeek
 withWords a params = return [a params]
 withStrings :: (String -> CommandStart) -> CommandSeek
@@ -152,8 +151,8 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
-	liftM (map a) $ filterFiles tocommit
+	runFiltered a $
+		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
 withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
 withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
@@ -165,8 +164,7 @@ withFilesUnlocked' typechanged a params = do
 	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
-	unlockedfiles' <- filterFiles unlockedfiles
-	backendPairs a unlockedfiles'
+	backendPairs a unlockedfiles
 withKeys :: (Key -> CommandStart) -> CommandSeek
 withKeys a params = return $ map (a . parse) params
 	where
@@ -175,8 +173,23 @@ withNothing :: CommandStart -> CommandSeek
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
+runFiltered :: (FilePath -> Annex (Maybe a)) -> Annex [FilePath] -> Annex [Annex (Maybe a)]
+runFiltered a fs = do
+	matcher <- Limit.getMatcher
+	liftM (map $ proc matcher) fs
+	where
+		proc matcher f = do
+			ok <- matcher f
+			if ok then a f else stop
+
 backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
-backendPairs a files = map a <$> Backend.chooseBackends files
+backendPairs a fs = do
+	matcher <- Limit.getMatcher
+	liftM (map $ proc matcher) (Backend.chooseBackends fs)
+	where
+		proc matcher p@(_, f) = do
+			ok <- matcher f
+			if ok then a p else stop
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
diff --git a/Limit.hs b/Limit.hs
index 51f3fc9505..602da70013 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -22,20 +22,19 @@ import Utility
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
-{- Filter out files not matching user-specified limits. -}
-filterFiles :: [FilePath] -> Annex [FilePath]
-filterFiles l = do
-	matcher <- getMatcher
-	filterM (Utility.Matcher.matchM matcher) l
-
 {- Checks if there are user-specified limits. -}
 limited :: Annex Bool
-limited = (not . Utility.Matcher.matchesAny) <$> getMatcher
+limited = (not . Utility.Matcher.matchesAny) <$> getMatcher'
 
 {- Gets a matcher for the user-specified limits. The matcher is cached for
  - speed; once it's obtained the user-specified limits can't change. -}
-getMatcher :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
+getMatcher :: Annex (FilePath -> Annex Bool)
 getMatcher = do
+	m <- getMatcher'
+	return $ Utility.Matcher.matchM m
+
+getMatcher' :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
+getMatcher' = do
 	m <- Annex.getState Annex.limit
 	case m of
 		Right r -> return r

From c31a6a9e100d9d9a822fd5e93f2a59d1562db579 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 23:09:40 -0400
Subject: [PATCH 2232/2835] refactor

---
 Command.hs | 35 ++++++++++++-----------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/Command.hs b/Command.hs
index 3cfff268c8..a568da33b6 100644
--- a/Command.hs
+++ b/Command.hs
@@ -115,20 +115,11 @@ withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandS
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	run $ liftIO $ Git.checkAttr repo attr files
-	where
-		run fs = do
-			matcher <- Limit.getMatcher
-			liftM (map $ proc matcher) fs
-		proc matcher p@(f, _) = do
-			ok <- matcher f
-			if ok then a p else stop
+	runFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
 withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 	where
-		go (file, v) = do
-			let numcopies = readMaybe v
-			a file numcopies
+		go (file, v) = a file (readMaybe v)
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
@@ -174,22 +165,20 @@ withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
 runFiltered :: (FilePath -> Annex (Maybe a)) -> Annex [FilePath] -> Annex [Annex (Maybe a)]
-runFiltered a fs = do
+runFiltered a fs = runFilteredGen a id fs
+
+backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
+backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
+
+runFilteredGen :: (a1 -> Annex (Maybe a)) -> (a1 -> FilePath) -> Annex [a1] -> Annex [Annex (Maybe a)]
+runFilteredGen a d fs = do
 	matcher <- Limit.getMatcher
 	liftM (map $ proc matcher) fs
 	where
-		proc matcher f = do
+		proc matcher v = do
+			let f = d v
 			ok <- matcher f
-			if ok then a f else stop
-
-backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
-backendPairs a fs = do
-	matcher <- Limit.getMatcher
-	liftM (map $ proc matcher) (Backend.chooseBackends fs)
-	where
-		proc matcher p@(_, f) = do
-			ok <- matcher f
-			if ok then a p else stop
+			if ok then a v else stop
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool

From 6e80f195148ca689d85c6c8ed7f1a4f9720397a7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:03:16 -0400
Subject: [PATCH 2233/2835] golf

---
 Limit.hs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Limit.hs b/Limit.hs
index 602da70013..2482c8a656 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -9,7 +9,6 @@ module Limit where
 
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
-import Control.Monad (filterM)
 import Control.Applicative
 import Data.Maybe
 
@@ -29,9 +28,7 @@ limited = (not . Utility.Matcher.matchesAny) <$> getMatcher'
 {- Gets a matcher for the user-specified limits. The matcher is cached for
  - speed; once it's obtained the user-specified limits can't change. -}
 getMatcher :: Annex (FilePath -> Annex Bool)
-getMatcher = do
-	m <- getMatcher'
-	return $ Utility.Matcher.matchM m
+getMatcher = Utility.Matcher.matchM <$> getMatcher'
 
 getMatcher' :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
 getMatcher' = do

From dcded89129c4647bc71b474aac6d3e334b4321c1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:37:04 -0400
Subject: [PATCH 2234/2835] reorg

---
 Command.hs      | 52 +++++++++++--------------------------------------
 Utility/Path.hs | 27 +++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 41 deletions(-)

diff --git a/Command.hs b/Command.hs
index a568da33b6..cc9bcbf0c9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -1,6 +1,6 @@
 {- git-annex command infrastructure
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -10,9 +10,8 @@ module Command where
 import Control.Monad.State (liftIO)
 import System.Directory
 import System.Posix.Files
-import Control.Monad (filterM, liftM, when)
+import Control.Monad (filterM, liftM)
 import Control.Applicative
-import Data.List
 import Data.Maybe
 
 import Types
@@ -22,6 +21,8 @@ import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Utility
+import Utility.Conditional
+import Utility.Path
 import Types.Key
 import Trust
 import LocationLog
@@ -75,9 +76,8 @@ stop = return Nothing
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
 prepCommand :: Command -> [String] -> Annex [Annex Bool]
-prepCommand Command { cmdseek = seek } params = do
-	lists <- mapM (\s -> s params) seek
-	return $ map doCommand $ concat lists
+prepCommand Command { cmdseek = seek } params =
+	return . map doCommand . concat =<< mapM (\s -> s params) seek
 
 {- Runs a command through the start, perform and cleanup stages -}
 doCommand :: CommandStart -> CommandCleanup
@@ -86,11 +86,9 @@ doCommand = start
 		start   = stage $ maybe success perform
 		perform = stage $ maybe failure cleanup
 		cleanup = stage $ \r -> showEndResult r >> return r
-		stage a b = b >>= a
+		stage = (=<<)
 		success = return True
-		failure = do
-			showEndFail
-			return False
+		failure = showEndFail >> return False
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
@@ -100,13 +98,12 @@ isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
-	g <- Annex.gitRepo
-	when (Git.repoIsLocalBare g) $
+	whenM (Git.repoIsLocalBare <$> Annex.gitRepo) $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
 {- These functions find appropriate files or other things based on a
-   user's parameters, and run a specified action on them. -}
+   user's parameters, and prepare actions operating on them. -}
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
@@ -170,7 +167,7 @@ runFiltered a fs = runFilteredGen a id fs
 backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
 backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
 
-runFilteredGen :: (a1 -> Annex (Maybe a)) -> (a1 -> FilePath) -> Annex [a1] -> Annex [Annex (Maybe a)]
+runFilteredGen :: (b -> Annex (Maybe a)) -> (b -> FilePath) -> Annex [b] -> Annex [Annex (Maybe a)]
 runFilteredGen a d fs = do
 	matcher <- Limit.getMatcher
 	liftM (map $ proc matcher) fs
@@ -228,33 +225,6 @@ cmdlineKey  = do
 		nokey = error "please specify the key with --key"
 		badkey = error "bad key"
 
-{- Given an original list of files, and an expanded list derived from it,
- - ensures that the original list's ordering is preserved. 
- -
- - The input list may contain a directory, like "dir" or "dir/". Any
- - items in the expanded list that are contained in that directory will
- - appear at the same position as it did in the input list.
- -}
-preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]
--- optimisation, only one item in original list, so no reordering needed
-preserveOrder [_] new = new
-preserveOrder orig new = collect orig new
-	where
-		collect [] n = n
-		collect [_] n = n -- optimisation
-		collect (l:ls) n = found ++ collect ls rest
-			where (found, rest)=partition (l `dirContains`) n
-
-{- Runs an action that takes a list of FilePaths, and ensures that 
- - its return list preserves order.
- -
- - This assumes that it's cheaper to call preserveOrder on the result,
- - than it would be to run the action separately with each param. In the case
- - of git file list commands, that assumption tends to hold.
- -}
-runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
-runPreserveOrder a files = preserveOrder files <$> a files
-
 {- Used for commands that have an auto mode that checks the number of known
  - copies of a key.
  -
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 9b8041dad0..fe474ee825 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -90,3 +90,30 @@ prop_relPathDirToFile_basics from to
 	| otherwise = not (null r)
 	where
 		r = relPathDirToFile from to 
+
+{- Given an original list of files, and an expanded list derived from it,
+ - ensures that the original list's ordering is preserved. 
+ -
+ - The input list may contain a directory, like "dir" or "dir/". Any
+ - items in the expanded list that are contained in that directory will
+ - appear at the same position as it did in the input list.
+ -}
+preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]
+-- optimisation, only one item in original list, so no reordering needed
+preserveOrder [_] new = new
+preserveOrder orig new = collect orig new
+	where
+		collect [] n = n
+		collect [_] n = n -- optimisation
+		collect (l:ls) n = found ++ collect ls rest
+			where (found, rest)=partition (l `dirContains`) n
+
+{- Runs an action that takes a list of FilePaths, and ensures that 
+ - its return list preserves order.
+ -
+ - This assumes that it's cheaper to call preserveOrder on the result,
+ - than it would be to run the action separately with each param. In the case
+ - of git file list commands, that assumption tends to hold.
+ -}
+runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
+runPreserveOrder a files = preserveOrder files <$> a files

From 94ee28556fb0c42b774e0af9f69fbf9cb5f04d01 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:52:17 -0400
Subject: [PATCH 2235/2835] special case for --in .

Do not need to check the location log in this case, can just check inAnnex.
This is both an optimisation and perhaps a correctness measure
(fsck --in . should fsck files even if the location log is damaged.)
---
 Limit.hs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Limit.hs b/Limit.hs
index 2482c8a656..d8ec4484c8 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -18,6 +18,7 @@ import qualified Remote
 import qualified Backend
 import LocationLog
 import Utility
+import Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
@@ -64,17 +65,19 @@ addExclude glob = addlimit $ return . notExcluded
 		regex = '^':wildToRegex glob
 
 {- Adds a limit to skip files not believed to be present
- - on a specfied remote. -}
+ - on a specfied repository. -}
 addIn :: String -> Annex ()
 addIn name = do
 	u <- Remote.nameToUUID name
-	addlimit $ check u
+	addlimit $ if name == "." then check local else check (remote u)
 	where
-		check u f = Backend.lookupFile f >>= handle u
+		check a f = Backend.lookupFile f >>= handle a
 		handle _ Nothing = return False
-		handle u (Just (key, _)) = do
+		handle a (Just (key, _)) = a key
+		remote u key = do
 			us <- keyLocations key
 			return $ u `elem` us
+		local key = inAnnex key
 
 {- Adds a limit to skip files not believed to have the specified number
  - of copies. -}

From a4aef6f11508078b6d593ec9daea2c4225e630c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:54:20 -0400
Subject: [PATCH 2236/2835] clarify wording

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 1f7decc1d8..fc57fb361c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,8 +10,8 @@ git-annex (3.20110916) UNRELEASED; urgency=low
   * find: Rather than only showing files whose contents are present,
     when used with --exclude --copies or --in, displays all files that
     match the specified conditions.
-  * Note that this is a behavior change for find --exclude! Old behavior
-    can be gotten with: find --in . --exclude=...
+  * Note that this is a behavior change for git-annex find! Old behavior
+    can be gotten by using: git-annex find --in .
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 

From 3b2e4620184e2f7cc2dfd72b80e1e9eb8bd1dfef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:57:12 -0400
Subject: [PATCH 2237/2835] tweak

---
 Limit.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Limit.hs b/Limit.hs
index d8ec4484c8..f57895aeef 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -43,10 +43,10 @@ getMatcher' = do
 
 {- Adds something to the limit list, which is built up reversed. -}
 add :: Limit -> Annex ()
-add l = Annex.changeState $ \s -> s { Annex.limit = append $ Annex.limit s }
+add l = Annex.changeState $ \s -> s { Annex.limit = prepend $ Annex.limit s }
 	where
-		append (Left ls) = Left $ l:ls
-		append _ = error "internal"
+		prepend (Left ls) = Left $ l:ls
+		prepend _ = error "internal"
 
 {- Adds a new limit. -}
 addlimit :: (FilePath -> Annex Bool) -> Annex ()
@@ -65,7 +65,7 @@ addExclude glob = addlimit $ return . notExcluded
 		regex = '^':wildToRegex glob
 
 {- Adds a limit to skip files not believed to be present
- - on a specfied repository. -}
+ - in a specfied repository. -}
 addIn :: String -> Annex ()
 addIn name = do
 	u <- Remote.nameToUUID name

From 73f3a00c1cad9e77ba9517bdc25b8086abc9bc59 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 12:21:53 -0400
Subject: [PATCH 2238/2835] typo

---
 doc/walkthrough/powerful_file_matching.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/powerful_file_matching.mdwn b/doc/walkthrough/powerful_file_matching.mdwn
index c0af4615c4..d5d29377c4 100644
--- a/doc/walkthrough/powerful_file_matching.mdwn
+++ b/doc/walkthrough/powerful_file_matching.mdwn
@@ -9,7 +9,7 @@ information indicates are present in a given repository. For example,
 if you want to populate newarchive with files, but not those already
 on oldarchive, you could do it like this:
 
-	git annex copy --not -in oldarchive --to newarchive
+	git annex copy --not --in oldarchive --to newarchive
 
 Without the --not, --in makes it act on files that *are* in the specified
 repository. So, to remove files that are on oldarchive:

From 5c20ebcbf3df9e8641f5b08f6a2307a7cfb3eda3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Sep 2011 18:46:35 +0000
Subject: [PATCH 2239/2835] Added a comment

---
 .../comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment

diff --git a/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment
new file mode 100644
index 0000000000..787cf8f5d7
--- /dev/null
+++ b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-19T18:46:35Z"
+ content="""
+This is now almost completely implemented. See [[walkthrough/powerful_file_matching]].
+
+"""]]

From 1ddc207b585cf0dded363b5890010afff58aa478 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 14:50:16 -0400
Subject: [PATCH 2240/2835] some of these are now done

---
 doc/forum/new_microfeatures.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index 683cc69b8b..bfe44272a7 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -33,10 +33,16 @@ Come to think of it, the inject --bare thing is probably not a microfeature. Sho
 
 > I've thought about such things before; does not seem really micro and I'm unsure how well it would work, but it would be worth a [[todo]]. --[[Joey]]
 
+>> Update: Done as --auto. --[[Joey]] 
+
 ---
 
 Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.
 
+> Update: git annex get --auto basically does this; you can tune
+> --numcopies on the fly to make it get more files than needed by the
+> current numcopies setting. --[[Joey]] 
+
 ---
 
 If a remote could send on received files to another remote, I could use my own local bandwith efficiently while still having my git-annex repos replicate data. -- RichiH

From 10db73426aba8554c62298b59391cab42c677af8 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Sep 2011 18:54:46 +0000
Subject: [PATCH 2241/2835] Added a comment

---
 .../comment_4_63f24abf086d644dced8b01e1a9948c9._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_git-annex_replicate/comment_4_63f24abf086d644dced8b01e1a9948c9._comment

diff --git a/doc/forum/wishlist:_git-annex_replicate/comment_4_63f24abf086d644dced8b01e1a9948c9._comment b/doc/forum/wishlist:_git-annex_replicate/comment_4_63f24abf086d644dced8b01e1a9948c9._comment
new file mode 100644
index 0000000000..3805464a69
--- /dev/null
+++ b/doc/forum/wishlist:_git-annex_replicate/comment_4_63f24abf086d644dced8b01e1a9948c9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-09-19T18:54:46Z"
+ content="""
+git annex get/copy/drop all now support a --auto flag, which makes them only act on files that have not enough or too many copies. This allows for some crude replication; it doesn't take into account which repositories should be filled up more (beyond honoring annex.diskreserve), nor does it try to optimally use bandwidth (beyond honoring configured annex-cost). You have to run it in every repository that you want to participate in the replication, too. But it's probably a Good Enough solution. See [[walkthrough/automatically_managing_content]].
+"""]]

From a1578e33dce4a9e7dcbd700077fc73237c35d19a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Sep 2011 18:57:52 +0000
Subject: [PATCH 2242/2835] Added a comment

---
 .../comment_1_3480b0ec629ef29a151408d869186bf8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one/comment_1_3480b0ec629ef29a151408d869186bf8._comment

diff --git a/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one/comment_1_3480b0ec629ef29a151408d869186bf8._comment b/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one/comment_1_3480b0ec629ef29a151408d869186bf8._comment
new file mode 100644
index 0000000000..5d0edce2ea
--- /dev/null
+++ b/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one/comment_1_3480b0ec629ef29a151408d869186bf8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-09-19T18:57:52Z"
+ content="""
+JFTR, pushing now happens automatically from branchable.
+"""]]

From 52533799535ebe1c133e2687ff1c5612d5bdb51d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 00:49:40 -0400
Subject: [PATCH 2243/2835] convert Token to have separate constructors for
 each peice of syntax

---
 Limit.hs           | 18 ++++++-------
 Options.hs         |  4 +--
 Utility/Matcher.hs | 67 +++++++++++++++++++++++++++-------------------
 3 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/Limit.hs b/Limit.hs
index f57895aeef..b3b041396f 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -48,17 +48,17 @@ add l = Annex.changeState $ \s -> s { Annex.limit = prepend $ Annex.limit s }
 		prepend (Left ls) = Left $ l:ls
 		prepend _ = error "internal"
 
-{- Adds a new limit. -}
-addlimit :: (FilePath -> Annex Bool) -> Annex ()
-addlimit = add . Utility.Matcher.Operation
-
 {- Adds a new token. -}
-token :: String -> Annex ()
-token = add . Utility.Matcher.Token
+addToken :: String -> Annex ()
+addToken = add . Utility.Matcher.token
+
+{- Adds a new limit. -}
+addLimit :: (FilePath -> Annex Bool) -> Annex ()
+addLimit = add . Utility.Matcher.Operation
 
 {- Add a limit to skip files that do not match the glob. -}
 addExclude :: String -> Annex ()
-addExclude glob = addlimit $ return . notExcluded
+addExclude glob = addLimit $ return . notExcluded
 	where
 		notExcluded f = isNothing $ match cregex f []
 		cregex = compile regex []
@@ -69,7 +69,7 @@ addExclude glob = addlimit $ return . notExcluded
 addIn :: String -> Annex ()
 addIn name = do
 	u <- Remote.nameToUUID name
-	addlimit $ if name == "." then check local else check (remote u)
+	addLimit $ if name == "." then check local else check (remote u)
 	where
 		check a f = Backend.lookupFile f >>= handle a
 		handle _ Nothing = return False
@@ -85,7 +85,7 @@ addCopies :: String -> Annex ()
 addCopies num = do
 	case readMaybe num :: Maybe Int of
 		Nothing -> error "bad number for --copies"
-		Just n -> addlimit $ check n
+		Just n -> addLimit $ check n
 	where
 		check n f = Backend.lookupFile f >>= handle n
 		handle _ Nothing = return False
diff --git a/Options.hs b/Options.hs
index 44d5f3674c..eeb3639b4d 100644
--- a/Options.hs
+++ b/Options.hs
@@ -58,5 +58,5 @@ matcherOptions =
 	, shortopt ")" "close group of options"
 	]
 	where
-		longopt o d = Option [] [o] (NoArg (token o)) d
-		shortopt o d = Option o [] (NoArg (token o)) d
+		longopt o d = Option [] [o] (NoArg (addToken o)) d
+		shortopt o d = Option o [] (NoArg (addToken o)) d
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 342775d68c..323a84bfde 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -18,6 +18,7 @@
 module Utility.Matcher (
 	Token(..),
 	Matcher,
+	token,
 	generate,
 	match,
 	matchM,
@@ -26,20 +27,30 @@ module Utility.Matcher (
 
 import Control.Monad
 
-{- A Token can either be a single word, or an Operation of an arbitrary type. -}
-data Token op = Token String | Operation op
+{- A Token can be an Operation of an arbitrary type, or one of a few
+ - predefined peices of syntax. -}
+data Token op = Operation op | And | Or | Not | Open | Close
 	deriving (Show, Eq)
 
-data Matcher op = Any
-	| And (Matcher op) (Matcher op)
-	| Or (Matcher op) (Matcher op)
-	| Not (Matcher op)
-	| Op op
+data Matcher op = MAny
+	| MAnd (Matcher op) (Matcher op)
+	| MOr (Matcher op) (Matcher op)
+	| MNot (Matcher op)
+	| MOp op
 	deriving (Show, Eq)
 
+{- Converts a word of syntax into a token. Doesn't handle operations. -}
+token :: String -> Token op
+token "and" = And
+token "or" = Or
+token "not" = Not
+token "(" = Open
+token ")" = Close
+token t = error $ "unknown token " ++ t
+
 {- Converts a list of Tokens into a Matcher. -}
 generate :: [Token op] -> Matcher op
-generate ts = generate' Any ts
+generate ts = generate' MAny ts
 generate' :: Matcher op -> [Token op] -> Matcher op
 generate' m [] = m
 generate' m ts = uncurry generate' $ consume m ts
@@ -48,16 +59,16 @@ generate' m ts = uncurry generate' $ consume m ts
  - and returns unconsumed Tokens. -}
 consume :: Matcher op -> [Token op] -> (Matcher op, [Token op])
 consume m [] = (m, [])
-consume m ((Operation o):ts) = (m `And` Op o, ts)
-consume m ((Token t):ts)
-	| t == "and" = cont $ m `And` next
-	| t == "or" = cont $ m `Or` next
-	| t == "not" = cont $ m `And` (Not next)
-	| t == "(" = let (n, r) = consume next rest in (m `And` n, r)
-	| t == ")" = (m, ts)
-	| otherwise = error $ "unknown token " ++ t
+consume m (t:ts) = go t
 	where
-		(next, rest) = consume Any ts
+		go And = cont $ m `MAnd` next
+		go Or = cont $ m `MOr` next
+		go Not = cont $ m `MAnd` (MNot next)
+		go Open = let (n, r) = consume next rest in (m `MAnd` n, r)
+		go Close = (m, ts)
+		go (Operation o) = (m `MAnd` MOp o, ts)
+
+		(next, rest) = consume MAny ts
 		cont v = (v, rest)
 
 {- Checks if a Matcher matches, using a supplied function to check
@@ -65,25 +76,25 @@ consume m ((Token t):ts)
 match :: (op -> v -> Bool) -> Matcher op -> v -> Bool
 match a m v = go m
 	where
-		go Any = True
-		go (And m1 m2) = go m1 && go m2
-		go (Or m1 m2) = go m1 || go m2
-		go (Not m1) = not (go m1)
-		go (Op o) = a o v
+		go MAny = True
+		go (MAnd m1 m2) = go m1 && go m2
+		go (MOr m1 m2) = go m1 || go m2
+		go (MNot m1) = not (go m1)
+		go (MOp o) = a o v
 
 {- Runs a monadic Matcher, where Operations are actions in the monad. -}
 matchM :: Monad m => Matcher (v -> m Bool) -> v -> m Bool
 matchM m v = go m
 	where
-		go Any = return True
-		go (And m1 m2) = liftM2 (&&) (go m1) (go m2)
-		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
-		go (Not m1) = liftM not (go m1)
-		go (Op o) = o v
+		go MAny = return True
+		go (MAnd m1 m2) = liftM2 (&&) (go m1) (go m2)
+		go (MOr m1 m2) =  liftM2 (||) (go m1) (go m2)
+		go (MNot m1) = liftM not (go m1)
+		go (MOp o) = o v
 
 {- Checks is a matcher contains no limits, and so (presumably) matches
  - anything. Note that this only checks the trivial case; it is possible
  - to construct matchers that match anything but are more complicated. -}
 matchesAny :: Matcher a -> Bool
-matchesAny Any = True
+matchesAny MAny = True
 matchesAny _ = False

From b62123c378e7d134914f1479fbfa3409d94aa209 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 00:58:18 -0400
Subject: [PATCH 2244/2835] simplify

---
 Utility/Matcher.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 323a84bfde..69b78be4a0 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -50,10 +50,10 @@ token t = error $ "unknown token " ++ t
 
 {- Converts a list of Tokens into a Matcher. -}
 generate :: [Token op] -> Matcher op
-generate ts = generate' MAny ts
-generate' :: Matcher op -> [Token op] -> Matcher op
-generate' m [] = m
-generate' m ts = uncurry generate' $ consume m ts
+generate = go MAny
+	where
+		go m [] = m
+		go m ts = uncurry go $ consume m ts
 
 {- Consumes one or more Tokens, constructs a new Matcher,
  - and returns unconsumed Tokens. -}

From cabbefd9d2d16b52b28f69a8410a9eb84e506666 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 18:13:08 -0400
Subject: [PATCH 2245/2835] status: In --fast mode, all status info is
 displayed now; but some of it is only approximate, and is marked as such.

---
 Command/Status.hs  | 58 +++++++++++++++++++++++++---------------------
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  4 ++--
 3 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 76659a75e1..067128f620 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -23,13 +23,15 @@ import qualified Git
 import Command
 import Types
 import Utility.DataUnits
+import Utility.Conditional
 import Content
 import Types.Key
 import Locations
 import Backend
+import Messages
 
 -- a named computation that produces a statistic
-type Stat = StatState (Maybe (String, StatState String))
+type Stat = StatState (Maybe (String, Bool, StatState String))
 
 -- cached info that multiple Stats may need
 data StatInfo = StatInfo
@@ -58,16 +60,13 @@ seek = [withNothing start]
 {- Order is significant. Less expensive operations, and operations
  - that share data go together.
  -}
-faststats :: [Stat]
-faststats = 
+stats :: [Stat]
+stats = 
 	[ supported_backends
 	, supported_remote_types
 	, tmp_size
 	, bad_data_size
-	]
-slowstats :: [Stat]
-slowstats =
-	[ local_annex_keys
+	, local_annex_keys
 	, local_annex_size
 	, total_annex_keys
 	, total_annex_size
@@ -76,13 +75,16 @@ slowstats =
 
 start :: CommandStart
 start = do
-	fast <- Annex.getState Annex.fast
-	let todo = if fast then faststats else faststats ++ slowstats
-	evalStateT (mapM_ showStat todo) (StatInfo Nothing Nothing)
+	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
+	fastmode_note
 	stop
 
-stat :: String -> StatState String -> Stat
-stat desc a = return $ Just (desc, a)
+fastmode_note :: Annex ()
+fastmode_note = whenM (Annex.getState Annex.fast) $
+	showLongNote "(*) approximate due to fast mode"
+
+stat :: String -> Bool -> StatState String -> Stat
+stat desc approx a = return $ Just (desc, approx, a)
 
 nostat :: Stat
 nostat = return Nothing
@@ -90,33 +92,37 @@ nostat = return Nothing
 showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
-		calc (Just (desc, a)) = do
-			liftIO $ putStr $ desc ++ ": "
+		calc (Just (desc, approx, a)) = do
+			fast <- lift $ Annex.getState Annex.fast
+			let star = if fast && approx then "(*)" else ""
+			liftIO $ putStr $ desc ++ star ++ ": "
 			liftIO $ hFlush stdout
 			liftIO . putStrLn =<< a
 		calc Nothing = return ()
 
 supported_backends :: Stat
-supported_backends = stat "supported backends" $ 
+supported_backends = stat "supported backends" False $ 
 	return $ unwords $ map B.name Backend.list
 
 supported_remote_types :: Stat
-supported_remote_types = stat "supported remote types" $
+supported_remote_types = stat "supported remote types" False $
 	return $ unwords $ map R.typename Remote.remoteTypes
 
 local_annex_size :: Stat
-local_annex_size = stat "local annex size" $
+local_annex_size = stat "local annex size" False $
 	cachedKeysPresent >>= keySizeSum
 
 total_annex_size :: Stat
-total_annex_size = stat "total annex size" $
+total_annex_size = stat "total annex size" True $
 	cachedKeysReferenced >>= keySizeSum
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" $ show . snd <$> cachedKeysPresent
+local_annex_keys = stat "local annex keys" False $
+	show . snd <$> cachedKeysPresent
 
 total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" $ show . snd <$> cachedKeysReferenced
+total_annex_keys = stat "total annex keys" True $
+	show . snd <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -125,7 +131,7 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
+backend_usage = stat "backend usage" True $ usage <$> cachedKeysReferenced
 	where
 		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
 		splits :: [Key] -> [(String, Integer)]
@@ -135,7 +141,6 @@ backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 
-
 cachedKeysPresent :: StatState (SizeList Key)
 cachedKeysPresent = do
 	s <- get
@@ -153,10 +158,11 @@ cachedKeysReferenced = do
 	case keysReferencedCache s of
 		Just v -> return v
 		Nothing -> do
+			-- A given key may be referenced repeatedly,
+			-- so nub is needed for accuracy, but is slow.
 			keys <- lift Command.Unused.getKeysReferenced
-			-- A given key may be referenced repeatedly.
-			-- nub does not seem too slow (yet)..
-			let v = sizeList $ nub keys
+			fast <- lift $ Annex.getState Annex.fast
+			let v = sizeList $ if fast then keys else nub keys
 			put s { keysReferencedCache = Just v }
 			return v
 
@@ -175,7 +181,7 @@ staleSize label dirspec = do
 	keys <- lift (Command.Unused.staleKeys dirspec)
 	if null keys
 		then nostat
-		else stat label $ do
+		else stat label False $ do
 			s <- keySizeSum $ sizeList keys
 			return $ s ++ aside "clean up with git-annex unused"
 
diff --git a/debian/changelog b/debian/changelog
index fc57fb361c..938e2feaf3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,8 @@ git-annex (3.20110916) UNRELEASED; urgency=low
     match the specified conditions.
   * Note that this is a behavior change for git-annex find! Old behavior
     can be gotten by using: git-annex find --in .
+  * status: In --fast mode, all status info is displayed now; but some
+    of it is only approximate, and is marked as such.
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 4a8e28a6ee..26cfb2c613 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -240,8 +240,8 @@ subdirectories).
 
   Some of the statistics can take a while to generate, and those
   come last. You can ctrl-c this command once it's displayed the
-  information you wanted to see. Or, use --fast to only display
-  the first, fast(ish) statistics.
+  information you wanted to see. Or, use --fast to produce statistics
+  more quickly, but possibly less accurately.
 
 * map
 

From 9f5c7a246b786e350671551cafae0f9678d83648 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 18:57:05 -0400
Subject: [PATCH 2246/2835] status: Massively sped up; remove --fast mode.

Using Sets is the right thing; they have constant size lookup like my
SizeList, and logn insertation, which beats nub to death.

Runs faster than --fast mode did before, and gives accurate counts.

13 seconds total runtime with a warm cache in a repository with 40 thousand
keys.
---
 Command/Status.hs  | 96 ++++++++++++++++++----------------------------
 debian/changelog   |  3 +-
 doc/git-annex.mdwn |  5 ---
 3 files changed, 39 insertions(+), 65 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 067128f620..d06865b6aa 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -13,8 +13,9 @@ import Data.Maybe
 import System.IO
 import Data.List
 import qualified Data.Map as M
+import qualified Data.Set as S
+import Data.Set (Set)
 
-import qualified Annex
 import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
@@ -23,33 +24,23 @@ import qualified Git
 import Command
 import Types
 import Utility.DataUnits
-import Utility.Conditional
 import Content
 import Types.Key
 import Locations
 import Backend
-import Messages
 
 -- a named computation that produces a statistic
-type Stat = StatState (Maybe (String, Bool, StatState String))
+type Stat = StatState (Maybe (String, StatState String))
 
 -- cached info that multiple Stats may need
 data StatInfo = StatInfo
-	{ keysPresentCache :: Maybe (SizeList Key)
-	, keysReferencedCache :: Maybe (SizeList Key)
+	{ keysPresentCache :: Maybe (Set Key)
+	, keysReferencedCache :: Maybe (Set Key)
 	}
 
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
 
--- a list with a known length
--- (Integer is used for the length to avoid
--- blowing up if someone annexed billions of files..)
-type SizeList a = ([a], Integer)
-
-sizeList :: [a] -> SizeList a
-sizeList l = (l, genericLength l)
-
 command :: [Command]
 command = [repoCommand "status" paramNothing seek
 	"shows status information about the annex"]
@@ -76,15 +67,10 @@ stats =
 start :: CommandStart
 start = do
 	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
-	fastmode_note
 	stop
 
-fastmode_note :: Annex ()
-fastmode_note = whenM (Annex.getState Annex.fast) $
-	showLongNote "(*) approximate due to fast mode"
-
-stat :: String -> Bool -> StatState String -> Stat
-stat desc approx a = return $ Just (desc, approx, a)
+stat :: String -> StatState String -> Stat
+stat desc a = return $ Just (desc, a)
 
 nostat :: Stat
 nostat = return Nothing
@@ -92,37 +78,35 @@ nostat = return Nothing
 showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
-		calc (Just (desc, approx, a)) = do
-			fast <- lift $ Annex.getState Annex.fast
-			let star = if fast && approx then "(*)" else ""
-			liftIO $ putStr $ desc ++ star ++ ": "
+		calc (Just (desc, a)) = do
+			liftIO $ putStr $ desc ++ ": "
 			liftIO $ hFlush stdout
 			liftIO . putStrLn =<< a
 		calc Nothing = return ()
 
 supported_backends :: Stat
-supported_backends = stat "supported backends" False $ 
+supported_backends = stat "supported backends" $ 
 	return $ unwords $ map B.name Backend.list
 
 supported_remote_types :: Stat
-supported_remote_types = stat "supported remote types" False $
+supported_remote_types = stat "supported remote types" $
 	return $ unwords $ map R.typename Remote.remoteTypes
 
 local_annex_size :: Stat
-local_annex_size = stat "local annex size" False $
+local_annex_size = stat "local annex size" $
 	cachedKeysPresent >>= keySizeSum
 
 total_annex_size :: Stat
-total_annex_size = stat "total annex size" True $
+total_annex_size = stat "total annex size" $
 	cachedKeysReferenced >>= keySizeSum
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" False $
-	show . snd <$> cachedKeysPresent
+local_annex_keys = stat "local annex keys" $
+	show . S.size <$> cachedKeysPresent
 
 total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" True $
-	show . snd <$> cachedKeysReferenced
+total_annex_keys = stat "total annex keys" $
+	show . S.size <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -131,9 +115,9 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" True $ usage <$> cachedKeysReferenced
+backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 	where
-		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
+		usage ks = pp "" $ sort $ map swap $ splits $ S.toList ks
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
@@ -141,48 +125,44 @@ backend_usage = stat "backend usage" True $ usage <$> cachedKeysReferenced
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 
-cachedKeysPresent :: StatState (SizeList Key)
+cachedKeysPresent :: StatState (Set Key)
 cachedKeysPresent = do
 	s <- get
 	case keysPresentCache s of
 		Just v -> return v
 		Nothing -> do
-			keys <- lift getKeysPresent
-			let v = sizeList keys
-			put s { keysPresentCache = Just v }
-			return v
+			keys <- S.fromList <$> lift getKeysPresent
+			put s { keysPresentCache = Just keys }
+			return keys
 
-cachedKeysReferenced :: StatState (SizeList Key)
+cachedKeysReferenced :: StatState (Set Key)
 cachedKeysReferenced = do
 	s <- get
 	case keysReferencedCache s of
 		Just v -> return v
 		Nothing -> do
-			-- A given key may be referenced repeatedly,
-			-- so nub is needed for accuracy, but is slow.
-			keys <- lift Command.Unused.getKeysReferenced
-			fast <- lift $ Annex.getState Annex.fast
-			let v = sizeList $ if fast then keys else nub keys
-			put s { keysReferencedCache = Just v }
-			return v
+			keys <- S.fromList <$> lift Command.Unused.getKeysReferenced
+			put s { keysReferencedCache = Just keys }
+			return keys
 
-keySizeSum :: SizeList Key -> StatState String
-keySizeSum (keys, len) = do
-	let knownsizes = mapMaybe keySize keys
-	let total = roughSize storageUnits False $ sum knownsizes
-	let missing = len - genericLength knownsizes
+keySizeSum :: Set Key -> StatState String
+keySizeSum s = do
+	let (sizes, unknownsizes) = S.partition isJust $ S.map keySize s
+	let total = roughSize storageUnits False $
+		fromJust $ S.fold (liftM2 (+)) (Just 0) sizes
+	let num = S.size unknownsizes
 	return $ total ++
-		if missing > 0
-			then aside $ "but " ++ show missing ++ " keys have unknown size"
-			else ""
+		if num == 0
+			then ""
+			else aside $ "but " ++ show num ++ " keys have unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
 staleSize label dirspec = do
 	keys <- lift (Command.Unused.staleKeys dirspec)
 	if null keys
 		then nostat
-		else stat label False $ do
-			s <- keySizeSum $ sizeList keys
+		else stat label $ do
+			s <- keySizeSum $ S.fromList keys
 			return $ s ++ aside "clean up with git-annex unused"
 
 aside :: String -> String
diff --git a/debian/changelog b/debian/changelog
index 938e2feaf3..4807956ad9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,8 +12,7 @@ git-annex (3.20110916) UNRELEASED; urgency=low
     match the specified conditions.
   * Note that this is a behavior change for git-annex find! Old behavior
     can be gotten by using: git-annex find --in .
-  * status: In --fast mode, all status info is displayed now; but some
-    of it is only approximate, and is marked as such.
+  * status: Massively sped up; remove --fast mode.
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 26cfb2c613..59da9dd8cf 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -238,11 +238,6 @@ subdirectories).
   Displays some statistics and other information, including how much data
   is in the annex.
 
-  Some of the statistics can take a while to generate, and those
-  come last. You can ctrl-c this command once it's displayed the
-  information you wanted to see. Or, use --fast to produce statistics
-  more quickly, but possibly less accurately.
-
 * map
 
   Helps you keep track of your repositories, and the connections between them,

From 98fbeba0df112753e0c55b0e2f456cd89ed0ede7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 20:14:17 -0400
Subject: [PATCH 2247/2835] bugfix

Different keys can have the same size, so can't make a Set of the sizes.

This version actually runs faster yet, too..
---
 Command/Status.hs | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index d06865b6aa..26122f2694 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -147,14 +147,13 @@ cachedKeysReferenced = do
 
 keySizeSum :: Set Key -> StatState String
 keySizeSum s = do
-	let (sizes, unknownsizes) = S.partition isJust $ S.map keySize s
-	let total = roughSize storageUnits False $
-		fromJust $ S.fold (liftM2 (+)) (Just 0) sizes
-	let num = S.size unknownsizes
+	let knownsizes = mapMaybe keySize $ S.toList s
+	let total = roughSize storageUnits False $ sum knownsizes
+	let missing = S.size s - genericLength knownsizes
 	return $ total ++
-		if num == 0
+		if missing == 0
 			then ""
-			else aside $ "but " ++ show num ++ " keys have unknown size"
+			else aside $ "but " ++ show missing ++ " keys have unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
 staleSize label dirspec = do

From 9d26192350b9c7b0402720f6f29c99c24748f364 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 20:18:43 -0400
Subject: [PATCH 2248/2835] pull out pure code

---
 Command/Status.hs | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 26122f2694..fc306bbe5e 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -94,11 +94,11 @@ supported_remote_types = stat "supported remote types" $
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
-	cachedKeysPresent >>= keySizeSum
+	cachedKeysPresent >>= return . keySizeSum
 
 total_annex_size :: Stat
 total_annex_size = stat "total annex size" $
-	cachedKeysReferenced >>= keySizeSum
+	cachedKeysReferenced >>= return . keySizeSum
 
 local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $
@@ -145,15 +145,17 @@ cachedKeysReferenced = do
 			put s { keysReferencedCache = Just keys }
 			return keys
 
-keySizeSum :: Set Key -> StatState String
-keySizeSum s = do
-	let knownsizes = mapMaybe keySize $ S.toList s
-	let total = roughSize storageUnits False $ sum knownsizes
-	let missing = S.size s - genericLength knownsizes
-	return $ total ++
-		if missing == 0
-			then ""
-			else aside $ "but " ++ show missing ++ " keys have unknown size"
+keySizeSum :: Set Key -> String
+keySizeSum s = total ++ missingnote
+	where
+		knownsizes = mapMaybe keySize $ S.toList s
+		total = roughSize storageUnits False $ sum knownsizes
+		missing = S.size s - genericLength knownsizes
+		missingnote
+			| missing == 0 = ""
+			| otherwise = aside $
+				"but " ++ show missing ++
+				" keys have unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
 staleSize label dirspec = do
@@ -161,7 +163,7 @@ staleSize label dirspec = do
 	if null keys
 		then nostat
 		else stat label $ do
-			s <- keySizeSum $ S.fromList keys
+			let s = keySizeSum $ S.fromList keys
 			return $ s ++ aside "clean up with git-annex unused"
 
 aside :: String -> String

From 9f6b7935ddb3d5dcbe0b4b784dc8acd7288ddba6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 23:24:48 -0400
Subject: [PATCH 2249/2835] go go gadget hlint

---
 Backend.hs            |  2 +-
 Backend/SHA.hs        |  2 +-
 Command.hs            |  2 +-
 Command/AddUrl.hs     |  2 +-
 Command/InitRemote.hs |  2 +-
 Command/Migrate.hs    |  3 ++-
 Command/Status.hs     |  4 ++--
 Git.hs                | 13 ++++++-------
 GitAnnex.hs           |  6 +++---
 Init.hs               |  3 +--
 Limit.hs              |  5 ++---
 Options.hs            |  4 ++--
 Remote/Git.hs         |  4 ++--
 Remote/S3real.hs      |  4 ++--
 Upgrade/V1.hs         |  4 ++--
 Utility/JSONStream.hs |  2 +-
 Utility/Matcher.hs    |  2 +-
 Utility/Path.hs       |  2 +-
 Utility/Touch.hsc     |  2 +-
 configure.hs          |  2 +-
 git-union-merge.hs    |  2 +-
 21 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 0c9ea8d0b7..d129139850 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -111,7 +111,7 @@ chooseBackends :: [FilePath] -> Annex [BackendFile]
 chooseBackends fs = do
 	g <- Annex.gitRepo
 	forced <- Annex.getState Annex.forcebackend
-	if forced /= Nothing
+	if isJust forced
 		then do
 			l <- orderedList
 			return $ map (\f -> (Just $ head l, f)) fs
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index ed2a47db9b..15d3fa20d1 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -38,7 +38,7 @@ backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
 genBackend :: SHASize -> Maybe (Backend Annex)
 genBackend size
-	| shaCommand size == Nothing = Nothing
+	| isNothing (shaCommand size) = Nothing
 	| otherwise = Just b
 	where
 		b = Types.Backend.Backend
diff --git a/Command.hs b/Command.hs
index cc9bcbf0c9..c061c7c464 100644
--- a/Command.hs
+++ b/Command.hs
@@ -162,7 +162,7 @@ withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
 runFiltered :: (FilePath -> Annex (Maybe a)) -> Annex [FilePath] -> Annex [Annex (Maybe a)]
-runFiltered a fs = runFilteredGen a id fs
+runFiltered a = runFilteredGen a id
 
 backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
 backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index d9fcc17e2b..2e9e04fd3e 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -28,7 +28,7 @@ import Utility.Path
 import Utility.Conditional
 
 command :: [Command]
-command = [repoCommand "addurl" (paramRepeating $ paramUrl) seek
+command = [repoCommand "addurl" (paramRepeating paramUrl) seek
 	"add urls to annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 671f945d22..c6d9f52003 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -35,7 +35,7 @@ start ws = do
 	when (null ws) needname
 
 	(u, c) <- findByName name
-	let fullconfig = M.union config c	
+	let fullconfig = config `M.union` c	
 	t <- findType fullconfig
 
 	showStart "initremote" name
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 2be9108512..054db6e27b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -12,6 +12,7 @@ import Control.Applicative
 import System.Posix.Files
 import System.Directory
 import System.FilePath
+import Data.Maybe
 
 import Command
 import qualified Annex
@@ -48,7 +49,7 @@ start (b, file) = isAnnexed file $ \(key, oldbackend) -> do
 {- Checks if a key is upgradable to a newer representation. -}
 {- Ideally, all keys have file size metadata. Old keys may not. -}
 upgradableKey :: Key -> Bool
-upgradableKey key = Types.Key.keySize key == Nothing
+upgradableKey key = isNothing $ Types.Key.keySize key
 
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
diff --git a/Command/Status.hs b/Command/Status.hs
index fc306bbe5e..6da8064f85 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -94,11 +94,11 @@ supported_remote_types = stat "supported remote types" $
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
-	cachedKeysPresent >>= return . keySizeSum
+	keySizeSum <$> cachedKeysPresent
 
 total_annex_size :: Stat
 total_annex_size = stat "total annex size" $
-	cachedKeysReferenced >>= return . keySizeSum
+	keySizeSum <$> cachedKeysReferenced
 
 local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $
diff --git a/Git.hs b/Git.hs
index cd6cdfbfd0..86a8c7695c 100644
--- a/Git.hs
+++ b/Git.hs
@@ -62,7 +62,7 @@ module Git (
 	prop_idempotent_deencode
 ) where
 
-import Control.Monad (unless, when)
+import Control.Monad (unless, when, liftM2)
 import Control.Applicative
 import System.Directory
 import System.FilePath
@@ -425,7 +425,7 @@ getSha :: String -> IO String -> IO String
 getSha subcommand a = do
 	t <- a
 	let t' = if last t == '\n'
-		then take (length t - 1) t
+		then init t
 		else t
 	when (length t' /= shaSize) $
 		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
@@ -576,7 +576,7 @@ decodeGitFile f@(c:s)
 	| otherwise = f
 	where
 		e = '\\'
-		middle = take (length s - 1) s
+		middle = init s
 		unescape (b, []) = b
 		-- look for escapes starting with '\'
 		unescape (b, v) = b ++ beginning ++ unescape (decode rest)
@@ -702,7 +702,6 @@ isRepoTop dir = do
 	where
 		isRepo = gitSignature ".git" ".git/config"
 		isBareRepo = gitSignature "objects" "config"
-		gitSignature subdir file = do
-			s <- (doesDirectoryExist (dir ++ "/" ++ subdir))
-			f <- (doesFileExist (dir ++ "/" ++ file))
-			return (s && f)
+		gitSignature subdir file = liftM2 (&&)
+			(doesDirectoryExist (dir ++ "/" ++ subdir))
+			(doesFileExist (dir ++ "/" ++ file))
diff --git a/GitAnnex.hs b/GitAnnex.hs
index a284daad5c..a9d469b44e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -108,11 +108,11 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	, Option ['x'] ["exclude"] (ReqArg (Limit.addExclude) paramGlob)
+	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
-	, Option ['i'] ["in"] (ReqArg (Limit.addIn) paramRemote)
+	, Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote)
 		"skip files not present in a remote"
-	, Option ['C'] ["copies"] (ReqArg (Limit.addCopies) paramNumber)
+	, Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber)
 		"skip files with fewer copies"
 	] ++ matcherOptions
 	where
diff --git a/Init.hs b/Init.hs
index 2067c524cf..4df1599333 100644
--- a/Init.hs
+++ b/Init.hs
@@ -33,8 +33,7 @@ initialize = do
 	gitPreCommitHookWrite
 
 uninitialize :: Annex ()
-uninitialize = do
-	gitPreCommitHookUnWrite
+uninitialize = gitPreCommitHookUnWrite
 
 {- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
diff --git a/Limit.hs b/Limit.hs
index b3b041396f..10fc0ea6ca 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -69,7 +69,7 @@ addExclude glob = addLimit $ return . notExcluded
 addIn :: String -> Annex ()
 addIn name = do
 	u <- Remote.nameToUUID name
-	addLimit $ if name == "." then check local else check (remote u)
+	addLimit $ if name == "." then check inAnnex else check (remote u)
 	where
 		check a f = Backend.lookupFile f >>= handle a
 		handle _ Nothing = return False
@@ -77,12 +77,11 @@ addIn name = do
 		remote u key = do
 			us <- keyLocations key
 			return $ u `elem` us
-		local key = inAnnex key
 
 {- Adds a limit to skip files not believed to have the specified number
  - of copies. -}
 addCopies :: String -> Annex ()
-addCopies num = do
+addCopies num =
 	case readMaybe num :: Maybe Int of
 		Nothing -> error "bad number for --copies"
 		Just n -> addLimit $ check n
diff --git a/Options.hs b/Options.hs
index eeb3639b4d..b5eaf98cd8 100644
--- a/Options.hs
+++ b/Options.hs
@@ -58,5 +58,5 @@ matcherOptions =
 	, shortopt ")" "close group of options"
 	]
 	where
-		longopt o d = Option [] [o] (NoArg (addToken o)) d
-		shortopt o d = Option o [] (NoArg (addToken o)) d
+		longopt o = Option [] [o] $ NoArg $ addToken o
+		shortopt o = Option o [] $ NoArg $ addToken o
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 9789a06252..d50899c674 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -81,7 +81,7 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
 	| not $ M.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
-	| Git.repoIsHttp r = store $ safely $ geturlconfig
+	| Git.repoIsHttp r = store $ safely geturlconfig
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ do
 		onLocal r ensureInitialized
@@ -101,7 +101,7 @@ tryGitConfigRead r
 
 		geturlconfig = do
 			s <- Url.get (Git.repoLocation r ++ "/config")
-			withTempFile "git-annex.tmp" $ \tmpfile -> \h -> do
+			withTempFile "git-annex.tmp" $ \tmpfile h -> do
 				hPutStr h s
 				hClose h
 				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 77b6b6ca40..cafa4f15a8 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -95,7 +95,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 		defaulthost = do
 			c' <- encryptionSetup c
-			let fullconfig = M.union c' defaults
+			let fullconfig = c' `M.union` defaults
 			genBucket fullconfig
 			use fullconfig
 
@@ -209,7 +209,7 @@ s3Bool (Left e) = s3Warning e
 
 s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a
 s3Action r noconn action = do
-	when (config r == Nothing) $
+	when (isNothing $ config r) $
 		error $ "Missing configuration for special remote " ++ name r
 	let bucket = M.lookup "bucket" $ fromJust $ config r
 	conn <- s3Connection $ fromJust $ config r
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 9c3fd99595..78f7d3adbc 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -173,7 +173,7 @@ readKey1 v =
 			then Just (read (bits !! 2) :: Integer)
 			else Nothing
 		wormy = head bits == "WORM"
-		mixup = wormy && (isUpper $ head $ bits !! 1)
+		mixup = wormy && isUpper (head $ bits !! 1)
 
 showKey1 :: Key -> String
 showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } =
@@ -248,7 +248,7 @@ logFile' hasher repo key =
 	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
 
 stateDir :: FilePath
-stateDir = addTrailingPathSeparator $ ".git-annex"
+stateDir = addTrailingPathSeparator ".git-annex"
 
 gitStateDir :: Git.Repo -> FilePath
 gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
diff --git a/Utility/JSONStream.hs b/Utility/JSONStream.hs
index af3766948f..7910c11941 100644
--- a/Utility/JSONStream.hs
+++ b/Utility/JSONStream.hs
@@ -19,7 +19,7 @@ import Text.JSON
    later. -}
 start :: JSON a => [(String, a)] -> String
 start l
-	| last s == endchar = take (length s - 1)  s
+	| last s == endchar = init s
 	| otherwise = bad s
 	where
 		s = encodeStrict $ toJSObject l
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 69b78be4a0..01500a2111 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -63,7 +63,7 @@ consume m (t:ts) = go t
 	where
 		go And = cont $ m `MAnd` next
 		go Or = cont $ m `MOr` next
-		go Not = cont $ m `MAnd` (MNot next)
+		go Not = cont $ m `MAnd` MNot next
 		go Open = let (n, r) = consume next rest in (m `MAnd` n, r)
 		go Close = (m, ts)
 		go (Operation o) = (m `MAnd` MOp o, ts)
diff --git a/Utility/Path.hs b/Utility/Path.hs
index fe474ee825..ce54fb3695 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -19,7 +19,7 @@ import Control.Applicative
 parentDir :: FilePath -> FilePath
 parentDir dir =
 	if not $ null dirs
-	then slash ++ join s (take (length dirs - 1) dirs)
+	then slash ++ join s (init dirs)
 	else ""
 		where
 			dirs = filter (not . null) $ split s dir
diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc
index f27ac31360..fd3320cd1d 100644
--- a/Utility/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -24,7 +24,7 @@ newtype TimeSpec = TimeSpec CTime
 touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO ()
 
 touch :: FilePath -> TimeSpec -> Bool -> IO ()
-touch file mtime follow = touchBoth file mtime mtime follow
+touch file mtime = touchBoth file mtime mtime
 
 #include 
 #include 
diff --git a/configure.hs b/configure.hs
index 9f7179c539..b68fa12dbc 100644
--- a/configure.hs
+++ b/configure.hs
@@ -51,7 +51,7 @@ getVersionString = do
 	let verline = head $ lines changelog
 	return $ middle (words verline !! 1)
 	where
-		middle s = drop 1 $ take (length s - 1) s
+		middle = drop 1 . init
 
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 4e1a932b45..8b70e678c9 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -23,7 +23,7 @@ tmpIndex :: Git.Repo -> FilePath
 tmpIndex g = Git.gitDir g  "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
-setup g = cleanup g -- idempotency
+setup = cleanup -- idempotency
 
 cleanup :: Git.Repo -> IO ()
 cleanup g = do

From 6dc23b889e01d56a7aff241f9b7e347dd55cfc47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 23:26:35 -0400
Subject: [PATCH 2250/2835] one more hlint

---
 Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Branch.hs b/Branch.hs
index 5008b2e200..15681e6993 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -331,7 +331,7 @@ getJournalFilesRaw = do
 	g <- Annex.gitRepo
 	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
 		(const $ return [])
-	return $ filter (\f -> f /= "." && f /= "..") fs
+	return $ filter (`notElem` [".", ".."]) fs
 
 {- Stages all journal files into the index, and returns True if the index
  - was modified. -}

From d75da353b9905bb5757df08520e63607fbfd2073 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Sep 2011 18:04:38 -0400
Subject: [PATCH 2251/2835] documentation/warning message update for future
 feature

---
 Command/Unused.hs        | 24 +++++++++---------------
 debian/changelog         |  4 ++++
 doc/git-annex.mdwn       |  5 ++---
 doc/special_remotes.mdwn |  6 ------
 4 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 535b9b33e8..803debef82 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -112,21 +112,8 @@ staleBadMsg t = unlines $
 
 unusedMsg :: [(Int, Key)] -> String
 unusedMsg u = unusedMsg' u
-	["Some annexed data is no longer used by any files in the current branch:"]
-	[dropMsg Nothing,
-	"Please be cautious -- are you sure that another branch, or another",
-	"repository does not still use this data?"]
-
-remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String
-remoteUnusedMsg r u = unusedMsg' u
-	["Some annexed data on " ++ name ++ 
-	 " is not used by any files in the current branch:"]
-	[dropMsg $ Just r,
-	 "Please be cautious -- Are you sure that the remote repository",
-	 "does not use this data? Or that it's not used by another branch?"]
-	where
-		name = Remote.name r 
-
+	["Some annexed data is no longer used by any files:"]
+	[dropMsg Nothing]
 unusedMsg' :: [(Int, Key)] -> [String] -> [String] -> String
 unusedMsg' u header trailer = unlines $
 	header ++
@@ -134,6 +121,13 @@ unusedMsg' u header trailer = unlines $
 	["(To see where data was previously used, try: git log --stat -S'KEY')"] ++
 	trailer
 
+remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String
+remoteUnusedMsg r u = unusedMsg' u
+	["Some annexed data on " ++ name ++ " is not used by any files:"]
+	[dropMsg $ Just r]
+	where
+		name = Remote.name r 
+
 dropMsg :: Maybe (Remote.Remote Annex) -> String
 dropMsg Nothing = dropMsg' ""
 dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r
diff --git a/debian/changelog b/debian/changelog
index 4807956ad9..4bc442d511 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,10 @@ git-annex (3.20110916) UNRELEASED; urgency=low
   * Note that this is a behavior change for git-annex find! Old behavior
     can be gotten by using: git-annex find --in .
   * status: Massively sped up; remove --fast mode.
+  * unused: File contents used by branches and tags are no longer
+    considered unused, even when not used by the current branch. This is
+    the final piece of the puzzle needed for git-annex to to play nicely
+    with branches.
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 59da9dd8cf..714f23befa 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -182,12 +182,11 @@ subdirectories).
 * unused
 
   Checks the annex for data that does not correspond to any files present
-  in the currently checked out branch, and prints a numbered list of the data.
+  in any tar or branch, and prints a numbered list of the data.
 
   To only show unused temp and bad files, specify --fast
 
-  To check data on a remote that does not correspond to any files present
-  on the locally checked out branch, specify --from.
+  To check for annexed data on a remote, specify --from.
 
 * dropunused [number ...]
 
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 55cd1f1a0f..ddb2fd125d 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -29,11 +29,5 @@ on special remotes, instead use `git annex unused --from`. Example:
 	    1       WORM-s3-m1301674316--foo
 	  (To see where data was previously used, try: git log --stat -S'KEY')
 	  (To remove unwanted data: git-annex dropunused --from mys3 NUMBER)
-	  Please be cautious -- are you sure that the remote repository
-	  does not use this data?
 	$ git annex dropunused --from mys3 1
 	dropunused 12948 (from mys3...) ok
-
-Do be cautious when using this; it cannot detect if content in a remote
-is used by that remote, or is the last copy of data that is used by
-some *other* remote.

From 4bf1a5ef59026a095abf751ea60b586c299aa0b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Sep 2011 18:13:24 -0400
Subject: [PATCH 2252/2835] refactor

---
 Command/Fsck.hs     |  1 +
 Command/Unannex.hs  |  1 +
 Command/Unlock.hs   |  1 +
 Command/Unused.hs   |  2 +-
 Content.hs          | 16 +---------------
 Remote/Directory.hs |  2 +-
 Upgrade/V1.hs       |  1 +
 Utility.hs          | 11 -----------
 Utility/FileMode.hs | 32 ++++++++++++++++++++++++++++++++
 9 files changed, 39 insertions(+), 28 deletions(-)
 create mode 100644 Utility/FileMode.hs

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 142f755a7f..0c58add6a3 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -26,6 +26,7 @@ import Locations
 import Trust
 import Utility.DataUnits
 import Utility.Path
+import Utility.FileMode
 import Config
 
 command :: [Command]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 3dedd007ef..4d4281eb07 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -18,6 +18,7 @@ import qualified Annex
 import qualified AnnexQueue
 import Utility.SafeCommand
 import Utility.Path
+import Utility.FileMode
 import LocationLog
 import Types
 import Content
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 5817e8f221..44b92545c8 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -19,6 +19,7 @@ import Content
 import Utility.Conditional
 import Utility.CopyFile
 import Utility.Path
+import Utility.FileMode
 
 command :: [Command]
 command =
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 803debef82..f62e68c30f 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -184,7 +184,7 @@ getKeysReferenced = do
 staleKeysPrune :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key]
 staleKeysPrune dirspec present = do
 	contents <- staleKeys dirspec
-		
+	
 	let stale = contents `exclude` present
 	let dup = contents `exclude` stale
 
diff --git a/Content.hs b/Content.hs
index e4bbee5282..f963c48b47 100644
--- a/Content.hs
+++ b/Content.hs
@@ -13,8 +13,6 @@ module Content (
 	getViaTmpUnchecked,
 	withTmp,
 	checkDiskSpace,
-	preventWrite,
-	allowWrite,
 	moveAnnex,
 	removeAnnex,
 	fromAnnex,
@@ -43,6 +41,7 @@ import Utility
 import Utility.Conditional
 import Utility.StatFS
 import Utility.Path
+import Utility.FileMode
 import Types.Key
 import Utility.DataUnits
 import Config
@@ -152,19 +151,6 @@ checkDiskSpace' adjustment key = do
 				roughSize storageUnits True n ++
 				" more (use --force to override this check or adjust annex.diskreserve)"
 
-{- Removes the write bits from a file. -}
-preventWrite :: FilePath -> IO ()
-preventWrite f = unsetFileMode f writebits
-	where
-		writebits = foldl unionFileModes ownerWriteMode
-					[groupWriteMode, otherWriteMode]
-
-{- Turns a file's write bit back on. -}
-allowWrite :: FilePath -> IO ()
-allowWrite f = do
-	s <- getFileStatus f
-	setFileMode f $ fileMode s `unionFileModes` ownerWriteMode
-
 {- Moves a file into .git/annex/objects/
  -
  - What if the key there already has content? This could happen for
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index b183042ef6..18835c5de0 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -25,10 +25,10 @@ import UUID
 import Locations
 import Utility.CopyFile
 import Config
-import Content
 import Utility
 import Utility.Conditional
 import Utility.Path
+import Utility.FileMode
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 78f7d3adbc..329f90ed6c 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -32,6 +32,7 @@ import Backend
 import Messages
 import Version
 import Utility
+import Utility.FileMode
 import Utility.SafeCommand
 import Utility.Path
 import qualified Upgrade.V2
diff --git a/Utility.hs b/Utility.hs
index ce17363488..a3d461d286 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -8,7 +8,6 @@
 module Utility (
 	hGetContentsStrict,
 	readFileStrict,
-	unsetFileMode,
 	readMaybe,
 	viaTmp,
 	withTempFile,
@@ -24,12 +23,9 @@ module Utility (
 import IO (bracket)
 import System.IO
 import System.Posix.Process hiding (executeFile)
-import System.Posix.Files
-import System.Posix.Types
 import System.Posix.User
 import System.FilePath
 import System.Directory
-import Foreign (complement)
 import Utility.Path
 import Data.Maybe
 import Control.Monad (liftM)
@@ -43,13 +39,6 @@ hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict f = readFile f >>= \s -> length s `seq` return s
 
-{- Removes a FileMode from a file.
- - For example, call with otherWriteMode to chmod o-w -}
-unsetFileMode :: FilePath -> FileMode -> IO ()
-unsetFileMode f m = do
-	s <- getFileStatus f
-	setFileMode f $ fileMode s `intersectFileModes` complement m
-
 {- Attempts to read a value from a String. -}
 readMaybe :: (Read a) => String -> Maybe a
 readMaybe s = case reads s of
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
new file mode 100644
index 0000000000..f5b018c84a
--- /dev/null
+++ b/Utility/FileMode.hs
@@ -0,0 +1,32 @@
+{- File mode utilities.
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.FileMode where
+
+import System.Posix.Files
+import System.Posix.Types
+import Foreign (complement)
+
+{- Removes a FileMode from a file.
+ - For example, call with otherWriteMode to chmod o-w -}
+unsetFileMode :: FilePath -> FileMode -> IO ()
+unsetFileMode f m = do
+	s <- getFileStatus f
+	setFileMode f $ fileMode s `intersectFileModes` complement m
+
+{- Removes the write bits from a file. -}
+preventWrite :: FilePath -> IO ()
+preventWrite f = unsetFileMode f writebits
+	where
+		writebits = foldl unionFileModes ownerWriteMode
+					[groupWriteMode, otherWriteMode]
+
+{- Turns a file's write bit back on. -}
+allowWrite :: FilePath -> IO ()
+allowWrite f = do
+	s <- getFileStatus f
+	setFileMode f $ fileMode s `unionFileModes` ownerWriteMode

From b203a68cb75c585b7e8b0b70f46278b41fef2fad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 25 Sep 2011 14:26:05 -0400
Subject: [PATCH 2253/2835] include --bindir in all cabal install examples

---
 doc/install.mdwn        | 2 +-
 doc/install/Fedora.mdwn | 2 +-
 doc/install/OSX.mdwn    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index cd51b96d23..6f892e37a2 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -8,7 +8,7 @@
 
 ## Using cabal
 
-As a haskell package, git-annex can be built using cabal. For example:
+As a haskell package, git-annex can be installed using cabal. For example:
 
 	cabal install git-annex --bindir=$HOME/bin
 
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 068d5c111c..7e983597b2 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -3,5 +3,5 @@ Installation recipe for Fedora 14.
 
 sudo yum install ghc cabal-install
 sudo cabal update
-sudo cabal install git-annex
+cabal install git-annex --bindir=$HOME/bin
 

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 680c331ee6..f65e0bb4fa 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -9,7 +9,7 @@ sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that al
 export PATH=$PATH:/opt/local/libexec/gnubin
 
 sudo cabal update
-sudo cabal install git-annex
+cabal install git-annex --bindir=$HOME/bin
 

 
 Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]

From b57a4566d3468f713ad369fc5f41778dfd133f0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 25 Sep 2011 14:34:07 -0400
Subject: [PATCH 2254/2835] mention that add --force adds ignored files

---
 doc/git-annex.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 59da9dd8cf..9ea7dce970 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -64,6 +64,7 @@ subdirectories).
 
   Adds files in the path to the annex. Files that are already checked into
   git, or that git has been configured to ignore will be silently skipped.
+  (Use --force to add ignored files.)
 
 * get [path ...]
 

From 7724f895a810d5922fb0581403ff17169344f514 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 25 Sep 2011 14:37:13 -0400
Subject: [PATCH 2255/2835] tweak

---
 Git/LsFiles.hs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 1ecbb029b5..c778e5d69c 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -23,13 +23,16 @@ inRepo :: Repo -> [FilePath] -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo $
 	Params "ls-files --cached -z --" : map File l
 
-{- Scans for files at the specified locations that are not checked into
- - git. -}
+{- Scans for files at the specified locations that are not checked into git. -}
 notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
 notInRepo repo include_ignored l =
-	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l
+	pipeNullSplit repo $
+		[Params "ls-files --others"] ++ exclude ++
+		[Params "-z --"] ++ map File l
 	where
-		exclude = if include_ignored then [] else [Param "--exclude-standard"]
+		exclude
+			| include_ignored = []
+			| otherwise = [Param "--exclude-standard"]
 
 {- Returns a list of all files that are staged for commit. -}
 staged :: Repo -> [FilePath] -> IO [FilePath]

From 1c9c9a0cee33b63e9147863b95e570a8cc349304 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 02:35:23 -0400
Subject: [PATCH 2256/2835] golfing

---
 LocationLog.hs | 13 ++++++-------
 PresenceLog.hs | 20 ++++++++------------
 2 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index fa660c8b67..7e5e81d7af 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -23,7 +23,6 @@ module LocationLog (
 ) where
 
 import System.FilePath
-import Control.Monad (when)
 import Control.Applicative
 import Data.Maybe
 
@@ -36,16 +35,16 @@ import PresenceLog
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
-logChange repo key u s = do
-	when (null u) $
-		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
-			" (have you run git annex init there?)"
-	addLog (logFile key) =<< logNow s u
+logChange repo key u s
+	| null u = error $
+		"unknown UUID for " ++ Git.repoDescribe repo ++ 
+		" (have you run git annex init there?)"
+	| otherwise = addLog (logFile key) =<< logNow s u
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations key = currentLog $ logFile key
+keyLocations = currentLog . logFile
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/PresenceLog.hs b/PresenceLog.hs
index e0c8729979..5828f76af5 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -85,7 +85,7 @@ readLog :: FilePath -> Annex [LogLine]
 readLog file = parseLog <$> Branch.get file
 
 parseLog :: String -> [LogLine]
-parseLog s = filter parsable $ map read $ lines s
+parseLog = filter parsable . map read . lines
 	where
 		-- some lines may be unparseable, avoid them
 		parsable l = status l /= Undefined
@@ -102,23 +102,18 @@ logNow s i = do
 
 {- Reads a log and returns only the info that is still in effect. -}
 currentLog :: FilePath -> Annex [String]
-currentLog file = do
-	ls <- readLog file
-	return $ map info $ filterPresent ls
+currentLog file = map info . filterPresent <$> readLog file
 
 {- Returns the info from LogLines that are in effect. -}
 filterPresent :: [LogLine] -> [LogLine]
-filterPresent ls = filter (\l -> InfoPresent == status l) $ compactLog ls
-
-type LogMap = Map.Map String LogLine
+filterPresent = filter (\l -> InfoPresent == status l) . compactLog
 
 {- Compacts a set of logs, returning a subset that contains the current
  - status. -}
 compactLog :: [LogLine] -> [LogLine]
-compactLog = compactLog' Map.empty
-compactLog' :: LogMap -> [LogLine] -> [LogLine]
-compactLog' m [] = Map.elems m
-compactLog' m (l:ls) = compactLog' (mapLog m l) ls
+compactLog = Map.elems . foldl mapLog Map.empty
+
+type LogMap = Map.Map String LogLine
 
 {- Inserts a log into a map of logs, if the log has better (ie, newer)
  - information than the other logs in the map -}
@@ -128,5 +123,6 @@ mapLog m l =
 		then Map.insert i l m
 		else m
 	where
-		better = maybe True (\l' -> date l' <= date l) $ Map.lookup i m
+		better = maybe True newer $ Map.lookup i m
+		newer l' = date l' <= date l
 		i = info l

From d7d9e9aca08b752cc2e04809edc654be39413665 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 02:46:54 -0400
Subject: [PATCH 2257/2835] use a foldr

Should be faster here.
---
 PresenceLog.hs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/PresenceLog.hs b/PresenceLog.hs
index 5828f76af5..7742651b84 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -26,7 +26,7 @@ module PresenceLog (
 import Data.Time.Clock.POSIX
 import Data.Time
 import System.Locale
-import qualified Data.Map as Map
+import qualified Data.Map as M
 import Control.Monad.State (liftIO)
 import Control.Applicative
 
@@ -111,18 +111,18 @@ filterPresent = filter (\l -> InfoPresent == status l) . compactLog
 {- Compacts a set of logs, returning a subset that contains the current
  - status. -}
 compactLog :: [LogLine] -> [LogLine]
-compactLog = Map.elems . foldl mapLog Map.empty
+compactLog = M.elems . foldr mapLog M.empty
 
-type LogMap = Map.Map String LogLine
+type LogMap = M.Map String LogLine
 
 {- Inserts a log into a map of logs, if the log has better (ie, newer)
  - information than the other logs in the map -}
-mapLog :: LogMap -> LogLine -> LogMap
-mapLog m l = 
+mapLog :: LogLine -> LogMap -> LogMap
+mapLog l m = 
 	if better
-		then Map.insert i l m
+		then M.insert i l m
 		else m
 	where
-		better = maybe True newer $ Map.lookup i m
+		better = maybe True newer $ M.lookup i m
 		newer l' = date l' <= date l
 		i = info l

From 93807564d00b3f64ad2353731fffb5c45ef8c01a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 14:03:59 -0400
Subject: [PATCH 2258/2835] add ls-tree interface

This parser should be fast. I hope.
---
 Git/LsTree.hs | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 Git/LsTree.hs

diff --git a/Git/LsTree.hs b/Git/LsTree.hs
new file mode 100644
index 0000000000..8b530d2ad3
--- /dev/null
+++ b/Git/LsTree.hs
@@ -0,0 +1,48 @@
+{- git ls-tree interface
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.LsTree (
+	lsTree
+) where
+
+import Numeric
+import Control.Applicative
+import Data.Char
+
+import Git
+import Utility.SafeCommand
+
+type Treeish = String
+
+data TreeItem = TreeItem
+	{ mode :: Int
+	, objtype :: String
+	, sha :: String
+	, file :: FilePath
+	} deriving Show
+
+{- Lists the contents of a Treeish -}
+lsTree :: Repo -> Treeish -> IO [TreeItem]
+lsTree repo t = map parseLsTree <$>
+	pipeNullSplit repo [Params "ls-tree --full-tree -z -r --", File t]
+
+{- Parses a line of ls-tree output.
+ - (The --long format is not currently supported.) -}
+parseLsTree :: String -> TreeItem
+parseLsTree l = TreeItem m o s f
+	where
+		-- l =  SP  SP  TAB 
+		-- Since everything until the file is fixed-width,
+		-- do not need to split on words.
+		(m, past_m) = head $ readOct l
+		(o, past_o) = splitAt 4 $ space past_m
+		(s, past_s) = splitAt shaSize $ space past_o
+		f = decodeGitFile $ space past_s
+		space s@(sp:rest)
+			| isSpace sp = rest
+			| otherwise = error $
+				"ls-tree parse error at '" ++ s ++ "' in " ++ l

From a3cb5c47e5f4167c711ab57f4b06d6c9d56536c8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 14:14:52 -0400
Subject: [PATCH 2259/2835] use FileMode

---
 Git/LsTree.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 8b530d2ad3..2220cfc505 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -12,6 +12,7 @@ module Git.LsTree (
 import Numeric
 import Control.Applicative
 import Data.Char
+import System.Posix.Types
 
 import Git
 import Utility.SafeCommand
@@ -19,7 +20,7 @@ import Utility.SafeCommand
 type Treeish = String
 
 data TreeItem = TreeItem
-	{ mode :: Int
+	{ mode :: FileMode
 	, objtype :: String
 	, sha :: String
 	, file :: FilePath

From 4f4eaf387ab801157cb8986a9ca3542a977e9e03 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 14:47:56 -0400
Subject: [PATCH 2260/2835] golf

---
 Git.hs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/Git.hs b/Git.hs
index 86a8c7695c..b5464859e5 100644
--- a/Git.hs
+++ b/Git.hs
@@ -451,11 +451,8 @@ commit g message newref parentrefs = do
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it into a list of files/lines/whatever. -}
 pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
-pipeNullSplit repo params = do
-	fs0 <- pipeRead repo params
-	return $ split0 fs0
-	where
-		split0 s = filter (not . null) $ split "\0" s
+pipeNullSplit repo params = filter (not . null) . split "\0" <$>
+	pipeRead repo params
 
 {- Runs git config and populates a repo with its config. -}
 configRead :: Repo -> IO Repo

From ad245a6375b32a17a9aa18088ee006cad6b4c1ff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 15:15:42 -0400
Subject: [PATCH 2261/2835] refactor catfile code

split into generic IO code, and a thin Annex wrapper
---
 Annex.hs             |  3 +++
 Branch.hs            | 45 +++----------------------------
 CatFile.hs           | 26 ++++++++++++++++++
 Git/CatFile.hs       | 63 ++++++++++++++++++++++++++++++++++++++++++++
 Types/BranchState.hs |  7 +----
 5 files changed, 96 insertions(+), 48 deletions(-)
 create mode 100644 CatFile.hs
 create mode 100644 Git/CatFile.hs

diff --git a/Annex.hs b/Annex.hs
index 1517a34708..8a386a044b 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -24,6 +24,7 @@ import Control.Monad.IO.Control
 import Control.Applicative hiding (empty)
 
 import qualified Git
+import Git.CatFile
 import Git.Queue
 import Types.Backend
 import qualified Types.Remote
@@ -55,6 +56,7 @@ data AnnexState = AnnexState
 	, fast :: Bool
 	, auto :: Bool
 	, branchstate :: BranchState
+	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
@@ -79,6 +81,7 @@ newState gitrepo = AnnexState
 	, fast = False
 	, auto = False
 	, branchstate = startBranchState
+	, catfilehandle = Nothing
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, defaultkey = Nothing
diff --git a/Branch.hs b/Branch.hs
index 15681e6993..af3851635d 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -18,7 +18,7 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (when, unless, liftM)
+import Control.Monad (unless, liftM)
 import Control.Monad.State (liftIO)
 import Control.Applicative ((<$>))
 import System.FilePath
@@ -31,7 +31,6 @@ import System.IO
 import System.IO.Binary
 import System.Posix.Process
 import System.Exit
-import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
 import qualified Git
@@ -43,6 +42,7 @@ import Utility.SafeCommand
 import Types
 import Messages
 import Locations
+import CatFile
 
 type GitRef = String
 
@@ -244,49 +244,10 @@ get file = do
 					setCache file content
 					return content
 				Nothing -> withIndexUpdate $ do
-					content <- catFile file
+					content <- catFile fullname file
 					setCache file content
 					return content
 
-{- Uses git cat-file in batch mode to read the content of a file.
- -
- - Only one process is run, and it persists and is used for all accesses. -}
-catFile :: FilePath -> Annex String
-catFile file = do
-	state <- getState
-	maybe (startup state) ask (catFileHandles state)
-	where
-		startup state = do
-			g <- Annex.gitRepo
-			(_, from, to) <- liftIO $ hPipeBoth "git" $
-				toCommand $ Git.gitCommandLine g
-					[Param "cat-file", Param "--batch"]
-			setState state { catFileHandles = Just (from, to) }
-			ask (from, to)
-		ask (from, to) = liftIO $ do
-			let want = fullname ++ ":" ++ file
-			hPutStrLn to want
-			hFlush to
-			header <- hGetLine from
-			case words header of
-				[sha, blob, size]
-					| length sha == Git.shaSize &&
-					  blob == "blob" -> handle from size
-					| otherwise -> empty
-				_
-					| header == want ++ " missing" -> empty
-					| otherwise -> error $ "unknown response from git cat-file " ++ header
-		handle from size = case reads size of
-			[(bytes, "")] -> readcontent from bytes
-			_ -> empty
-		readcontent from bytes = do
-			content <- B.hGet from bytes
-			c <- hGetChar from
-			when (c /= '\n') $
-				error "missing newline from git cat-file"
-			return $ B.unpack content
-		empty = return ""
-
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
diff --git a/CatFile.hs b/CatFile.hs
new file mode 100644
index 0000000000..0eb1e74f6b
--- /dev/null
+++ b/CatFile.hs
@@ -0,0 +1,26 @@
+{- git cat-file interface, with handle automatically stored in the Annex monad
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module CatFile (
+	catFile
+) where
+
+import Control.Monad.State
+
+import qualified Git.CatFile
+import Types
+import qualified Annex
+
+catFile :: String -> FilePath -> Annex String
+catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
+	where
+		startup = do
+			g <- Annex.gitRepo
+			h <- liftIO $ Git.CatFile.catFileStart g
+			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
+			go h
+		go h = liftIO $ Git.CatFile.catFile h branch file
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
new file mode 100644
index 0000000000..64857c66a9
--- /dev/null
+++ b/Git/CatFile.hs
@@ -0,0 +1,63 @@
+{- git cat-file interface
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.CatFile (
+	CatFileHandle,
+	catFileStart,
+	catFileStop,
+	catFile
+) where
+
+import Control.Monad.State
+import System.Cmd.Utils
+import System.IO
+import qualified Data.ByteString.Char8 as B
+
+import Git
+import Utility.SafeCommand
+
+type CatFileHandle = (PipeHandle, Handle, Handle)
+
+{- Starts git cat-file running in batch mode in a repo and returns a handle. -}
+catFileStart :: Repo -> IO CatFileHandle
+catFileStart repo = hPipeBoth "git" $ toCommand $
+	Git.gitCommandLine repo [Param "cat-file", Param "--batch"]
+
+{- Stops git cat-file. -}
+catFileStop :: CatFileHandle -> IO ()
+catFileStop (pid, from, to) = do
+	hClose to
+	hClose from
+	forceSuccess pid
+
+{- Uses a running git cat-file read the content of a file from a branch.
+ - Files that do not exist on the branch will have "" returned. -}
+catFile :: CatFileHandle -> String -> FilePath -> IO String
+catFile (_, from, to) branch file = do
+	hPutStrLn to want
+	hFlush to
+	header <- hGetLine from
+	case words header of
+		[sha, blob, size]
+			| length sha == Git.shaSize &&
+			  blob == "blob" -> handle size
+			| otherwise -> empty
+		_
+			| header == want ++ " missing" -> empty
+			| otherwise -> error $ "unknown response from git cat-file " ++ header
+	where
+		want = branch ++ ":" ++ file
+		handle size = case reads size of
+			[(bytes, "")] -> readcontent bytes
+			_ -> empty
+		readcontent bytes = do
+			content <- B.hGet from bytes
+			c <- hGetChar from
+			when (c /= '\n') $
+				error "missing newline from git cat-file"
+			return $ B.unpack content
+		empty = return ""
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
index bc1d32e693..777edb32cb 100644
--- a/Types/BranchState.hs
+++ b/Types/BranchState.hs
@@ -7,18 +7,13 @@
 
 module Types.BranchState where
 
-import System.IO
-
 data BranchState = BranchState {
 	branchUpdated :: Bool, -- has the branch been updated this run?
 
-	-- (from, to) handles used to talk to a git-cat-file process
-	catFileHandles :: Maybe (Handle, Handle),
-
 	-- the content of one file is cached
 	cachedFile :: Maybe FilePath,
 	cachedContent :: String
 }
 
 startBranchState :: BranchState
-startBranchState = BranchState False Nothing Nothing ""
+startBranchState = BranchState False Nothing ""

From 5ae270001c978b306419e6799e67e5f14a1765a2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 15:17:45 -0400
Subject: [PATCH 2262/2835] fix

---
 test.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test.hs b/test.hs
index 4d751a707b..32278f0fcd 100644
--- a/test.hs
+++ b/test.hs
@@ -44,6 +44,7 @@ import qualified Types.Key
 import qualified Config
 import qualified Crypto
 import qualified Utility.Path
+import qualified Utility.FileMode
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -389,7 +390,7 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 
 		corrupt f = do
 			git_annex "get" ["-q", f] @? "get of file failed"
-			Content.allowWrite f
+			Utility.FileMode.allowWrite f
 			writeFile f (changedcontent f)
 			r <- git_annex "fsck" ["-q"]
 			not r @? "fsck failed to fail with corrupted file content"
@@ -558,7 +559,7 @@ cleanup dir = do
 		-- removed via directory permissions; undo
 		recurseDir SystemFS dir >>=
 			filterM doesDirectoryExist >>=
-			mapM_ Content.allowWrite
+			mapM_ Utility.FileMode.allowWrite
 		removeDirectoryRecursive dir
 	
 checklink :: FilePath -> Assertion

From 297bc648b9a3c1b950e65f23a0e974b7934dc4dd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 16:43:10 -0400
Subject: [PATCH 2263/2835] make unused check branches and tags too

needs time and space optimisation
---
 Backend.hs          |  5 ++---
 Command/Unused.hs   | 31 ++++++++++++++++++++++++++++++-
 Git/LsTree.hs       |  8 +++++---
 Utility/FileMode.hs |  4 ++++
 4 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index d129139850..ca822de5c1 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -17,6 +17,7 @@ module Backend (
 ) where
 
 import Control.Monad.State (liftIO, when)
+import Control.Applicative
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
@@ -86,9 +87,7 @@ lookupFile file = do
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = do
-			l <- readSymbolicLink file
-			return $ takeFileName l
+		getsymlink = takeFileName <$> readSymbolicLink file
 		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
 		makeret l k =
 			case maybeLookupBackendName bname of
diff --git a/Command/Unused.hs b/Command/Unused.hs
index f62e68c30f..e629f9fb9e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -15,6 +15,8 @@ import qualified Data.Set as S
 import Data.Maybe
 import System.FilePath
 import System.Directory
+import Data.List
+import Control.Applicative
 
 import Command
 import Types
@@ -22,12 +24,17 @@ import Content
 import Messages
 import Locations
 import Utility
+import Utility.FileMode
+import Utility.SafeCommand
 import LocationLog
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
+import qualified Git.LsTree as LsTree
 import qualified Backend
 import qualified Remote
+import qualified Branch
+import CatFile
 
 command :: [Command]
 command = [repoCommand "unused" paramNothing seek
@@ -173,7 +180,29 @@ getKeysReferenced = do
 	g <- Annex.gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
-	return $ map fst $ catMaybes keypairs
+	ingit <- getKeysReferencedInGit
+	return $ concat [ingit, map fst $ catMaybes keypairs]
+
+{- List of keys referenced by symlinks in all git branches and tags. -}
+getKeysReferencedInGit :: Annex [Key]
+getKeysReferencedInGit = do
+	g <- Annex.gitRepo
+	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
+	-- Skip the git-annex branches, and get all other unique refs.
+	let refs = nub $ map head $ filter ourbranches $ map words $ lines c
+	concat <$> mapM (\r -> findkeys r [] =<< liftIO (LsTree.lsTree g r)) refs
+	where
+		ourbranchend = "/" ++ Branch.name
+		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
+		findkeys _ c [] = return c
+		findkeys ref c (l:ls) = do
+			if isSymLink (LsTree.mode l)
+				then do
+					content <- catFile ref $ LsTree.file l
+					case fileKey (takeFileName content) of
+						Nothing -> findkeys ref c ls
+						Just k -> findkeys ref (k:c) ls
+				else findkeys ref c ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
  - of those that might still have value, or might be stale and removable. 
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 2220cfc505..4a6c509f94 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -6,6 +6,7 @@
  -}
 
 module Git.LsTree (
+	TreeItem(..),
 	lsTree
 ) where
 
@@ -43,7 +44,8 @@ parseLsTree l = TreeItem m o s f
 		(o, past_o) = splitAt 4 $ space past_m
 		(s, past_s) = splitAt shaSize $ space past_o
 		f = decodeGitFile $ space past_s
-		space s@(sp:rest)
+		space (sp:rest)
 			| isSpace sp = rest
-			| otherwise = error $
-				"ls-tree parse error at '" ++ s ++ "' in " ++ l
+			| otherwise = parseerr
+		space [] = parseerr
+		parseerr = "ls-tree parse error: " ++ l
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index f5b018c84a..6c1c06e82a 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -30,3 +30,7 @@ allowWrite :: FilePath -> IO ()
 allowWrite f = do
 	s <- getFileStatus f
 	setFileMode f $ fileMode s `unionFileModes` ownerWriteMode
+
+{- Checks if a file mode indicates it's a symlink. -}
+isSymLink :: FileMode -> Bool
+isSymLink mode = symbolicLinkMode `intersectFileModes` mode == symbolicLinkMode

From b4d5c10fb71a0aa938c7dde0b9aaf57d9e793874 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 17:35:47 -0400
Subject: [PATCH 2264/2835] refine new unused code

Fixed the laziness space leak, so it runs in 60 mb or so again. Slightly
faster due to using Data.Set.difference now, although this also makes it
use slightly more memory.

Also added display of the refs being checked, and made unused --from
also check all refs for things in the remote.
---
 Branch.hs         | 12 +--------
 Command/Unused.hs | 63 +++++++++++++++++++++++++++++++----------------
 Git.hs            |  9 +++++++
 3 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index af3851635d..92b1fe29e8 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -26,7 +26,6 @@ import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
-import Data.List
 import System.IO
 import System.IO.Binary
 import System.Posix.Process
@@ -58,15 +57,6 @@ fullname = "refs/heads/" ++ name
 originname :: GitRef
 originname = "origin/" ++ name
 
-{- Converts a fully qualified git ref into a short version for human
- - consumptiom. -}
-shortref :: GitRef -> String
-shortref = remove "refs/heads/" . remove "refs/remotes/"
-	where
-		remove prefix s
-			| prefix `isPrefixOf` s = drop (length prefix) s
-			| otherwise = s
-
 {- A separate index file for the branch. -}
 index :: Git.Repo -> FilePath
 index g = gitAnnexDir g  "index"
@@ -209,7 +199,7 @@ updateRef ref
 		if null diffs
 			then return Nothing
 			else do
-				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name
+				showSideAction $ "merging " ++ Git.refDescribe ref ++ " into " ++ name
 				-- By passing only one ref, it is actually
 				-- merged into the index, preserving any
 				-- changes that may already be staged.
diff --git a/Command/Unused.hs b/Command/Unused.hs
index e629f9fb9e..b15aa001ad 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,7 +16,6 @@ import Data.Maybe
 import System.FilePath
 import System.Directory
 import Data.List
-import Control.Applicative
 
 import Command
 import Types
@@ -76,9 +75,8 @@ checkRemoteUnused name = do
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
 	showAction "checking for unused data"
-	referenced <- getKeysReferenced
 	remotehas <- filterM isthere =<< loggedKeys
-	let remoteunused = remotehas `exclude` referenced
+	remoteunused <- excludeReferenced remotehas
 	let list = number 0 remoteunused
 	writeUnusedFile "" list
 	unless (null remoteunused) $ showLongNote $ remoteUnusedMsg r list
@@ -156,12 +154,40 @@ unusedKeys = do
 		else do
 			showAction "checking for unused data"
 			present <- getKeysPresent
-			referenced <- getKeysReferenced
-			let unused = present `exclude` referenced
+			unused <- excludeReferenced present
 			staletmp <- staleKeysPrune gitAnnexTmpDir present
 			stalebad <- staleKeysPrune gitAnnexBadDir present
 			return (unused, stalebad, staletmp)
 
+{- Finds keys in the list that are not referenced in the git repository. -}
+excludeReferenced :: [Key] -> Annex [Key]
+-- excludeReferenced [] = return [] -- optimisation
+excludeReferenced l = do
+	g <- Annex.gitRepo
+	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
+	excludeReferenced'
+		(getKeysReferenced : (map getKeysReferencedInGit $ refs c))
+		(S.fromList l)
+	where
+		-- Skip the git-annex branches, and get all other unique refs.
+		refs = map last .
+			nubBy cmpheads .
+			filter ourbranches .
+			map words . lines
+		cmpheads a b = head a == head b
+		ourbranchend = "/" ++ Branch.name
+		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
+excludeReferenced' :: ([Annex [Key]]) -> S.Set Key -> Annex [Key]
+excludeReferenced' [] s = return $ S.toList s
+excludeReferenced' (a:as) s
+	-- | s == S.empty = return [] -- optimisation
+	| otherwise = do
+		referenced <- a
+		let !s' = remove referenced
+		excludeReferenced' as s'
+	where
+		remove l = s `S.difference` S.fromList l
+
 {- Finds items in the first, smaller list, that are not
  - present in the second, larger list.
  - 
@@ -180,29 +206,24 @@ getKeysReferenced = do
 	g <- Annex.gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
-	ingit <- getKeysReferencedInGit
-	return $ concat [ingit, map fst $ catMaybes keypairs]
+	return $ map fst $ catMaybes keypairs
 
-{- List of keys referenced by symlinks in all git branches and tags. -}
-getKeysReferencedInGit :: Annex [Key]
-getKeysReferencedInGit = do
+{- List of keys referenced by symlinks in a git ref. -}
+getKeysReferencedInGit :: String -> Annex [Key]
+getKeysReferencedInGit ref = do
+	showAction $ "checking " ++ Git.refDescribe ref
 	g <- Annex.gitRepo
-	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
-	-- Skip the git-annex branches, and get all other unique refs.
-	let refs = nub $ map head $ filter ourbranches $ map words $ lines c
-	concat <$> mapM (\r -> findkeys r [] =<< liftIO (LsTree.lsTree g r)) refs
+	findkeys [] =<< liftIO (LsTree.lsTree g ref)
 	where
-		ourbranchend = "/" ++ Branch.name
-		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
-		findkeys _ c [] = return c
-		findkeys ref c (l:ls) = do
+		findkeys c [] = return c
+		findkeys c (l:ls) = do
 			if isSymLink (LsTree.mode l)
 				then do
 					content <- catFile ref $ LsTree.file l
 					case fileKey (takeFileName content) of
-						Nothing -> findkeys ref c ls
-						Just k -> findkeys ref (k:c) ls
-				else findkeys ref c ls
+						Nothing -> findkeys c ls
+						Just k -> findkeys (k:c) ls
+				else findkeys c ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
  - of those that might still have value, or might be stale and removable. 
diff --git a/Git.hs b/Git.hs
index b5464859e5..fe2afdcfe3 100644
--- a/Git.hs
+++ b/Git.hs
@@ -20,6 +20,7 @@ module Git (
 	repoIsHttp,
 	repoIsLocalBare,
 	repoDescribe,
+	refDescribe,
 	repoLocation,
 	workTree,
 	workTreeFile,
@@ -171,6 +172,14 @@ repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
 repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
+{- Converts a fully qualified git ref into a user-visible version -}
+refDescribe :: String -> String
+refDescribe = remove "refs/heads/" . remove "refs/remotes/"
+	where
+		remove prefix s
+			| prefix `isPrefixOf` s = drop (length prefix) s
+			| otherwise = s
+
 {- Location of the repo, either as a path or url. -}
 repoLocation :: Repo -> String
 repoLocation Repo { location = Url url } = show url

From 8e4bd621b9e8ecafc65e029f015a4c460ec95abc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 17:38:41 -0400
Subject: [PATCH 2265/2835] enable short-circuiting optimisatons

---
 Command/Unused.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index b15aa001ad..e65f0790ba 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -161,7 +161,7 @@ unusedKeys = do
 
 {- Finds keys in the list that are not referenced in the git repository. -}
 excludeReferenced :: [Key] -> Annex [Key]
--- excludeReferenced [] = return [] -- optimisation
+excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
 	g <- Annex.gitRepo
 	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
@@ -180,7 +180,7 @@ excludeReferenced l = do
 excludeReferenced' :: ([Annex [Key]]) -> S.Set Key -> Annex [Key]
 excludeReferenced' [] s = return $ S.toList s
 excludeReferenced' (a:as) s
-	-- | s == S.empty = return [] -- optimisation
+	| s == S.empty = return [] -- optimisation
 	| otherwise = do
 		referenced <- a
 		let !s' = remove referenced

From 26bb45d12a26e51d2e41af0ed66e5300cbe52fee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 17:48:11 -0400
Subject: [PATCH 2266/2835] update test suite for smarter unused

---
 test.hs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index 32278f0fcd..21e09e98e7 100644
--- a/test.hs
+++ b/test.hs
@@ -38,7 +38,6 @@ import qualified UUID
 import qualified Trust
 import qualified Remote
 import qualified RemoteLog
-import qualified Content
 import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Config
@@ -453,8 +452,14 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
 	checkunused []
 	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
+	checkunused []
+	boolSystem "git" [Params "commit -m foo"] @? "git commit failed"
+	checkunused []
+	-- unused checks origin/master; once it's gone it is really unused
+	boolSystem "git" [Params "remote rm origin"] @? "git remote rm origin failed"
 	checkunused [annexedfilekey]
 	boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "commit -m foo"] @? "git commit failed"
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also

From 7d0adfc5e8bcb318cfedcfc43cd2dca35d037631 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 17:48:45 -0400
Subject: [PATCH 2267/2835] typo

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3ba58f25af..f64f84e088 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -183,7 +183,7 @@ subdirectories).
 * unused
 
   Checks the annex for data that does not correspond to any files present
-  in any tar or branch, and prints a numbered list of the data.
+  in any tag or branch, and prints a numbered list of the data.
 
   To only show unused temp and bad files, specify --fast
 

From ed00bdb9954028e123bb25bfe1e1c1d06a5aaba7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 18:11:53 -0400
Subject: [PATCH 2268/2835] foo

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index f64f84e088..db19e9f7a2 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -185,7 +185,7 @@ subdirectories).
   Checks the annex for data that does not correspond to any files present
   in any tag or branch, and prints a numbered list of the data.
 
-  To only show unused temp and bad files, specify --fast
+  To only show unused temp and bad files, specify --fast.
 
   To check for annexed data on a remote, specify --from.
 

From 7dddb803a07f4eb77f271a9e954d1e07f74de6df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 19:17:12 -0400
Subject: [PATCH 2269/2835] releasing version 3.20110928

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4bc442d511..f337882714 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110916) UNRELEASED; urgency=low
+git-annex (3.20110928) unstable; urgency=low
 
   * --in can be used to make git-annex only operate on files
     believed to be present in a given repository.
@@ -18,7 +18,7 @@ git-annex (3.20110916) UNRELEASED; urgency=low
     the final piece of the puzzle needed for git-annex to to play nicely
     with branches.
 
- -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
+ -- Joey Hess   Wed, 28 Sep 2011 18:14:02 -0400
 
 git-annex (3.20110915) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index a518ca8cb9..3f31ee4dcc 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110915
+Version: 3.20110928
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 24a8b7f141af4874f7a0c71738614b53f671898d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 19:17:46 -0400
Subject: [PATCH 2270/2835] add news item for git-annex 3.20110928

---
 doc/news/version_3.20110817.mdwn |  6 ------
 doc/news/version_3.20110928.mdwn | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 6 deletions(-)
 delete mode 100644 doc/news/version_3.20110817.mdwn
 create mode 100644 doc/news/version_3.20110928.mdwn

diff --git a/doc/news/version_3.20110817.mdwn b/doc/news/version_3.20110817.mdwn
deleted file mode 100644
index 51388f3c78..0000000000
--- a/doc/news/version_3.20110817.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex 3.20110817 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix shell escaping in rsync special remote.
-   * addurl: --fast can be used to avoid immediately downloading the url.
-   * Added support for getting content from git remotes using http (and https).
-   * Added curl to Debian package dependencies."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110928.mdwn b/doc/news/version_3.20110928.mdwn
new file mode 100644
index 0000000000..3b11e18f5c
--- /dev/null
+++ b/doc/news/version_3.20110928.mdwn
@@ -0,0 +1,19 @@
+git-annex 3.20110928 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * --in can be used to make git-annex only operate on files
+     believed to be present in a given repository.
+   * Arbitrarily complex expressions can be built to limit the files git-annex
+     operates on, by combining the options --not --and --or -( and -)
+     Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+   * --copies=N can be used to make git-annex only operate on files with
+     the specified number of copies. (And --not --copies=N for the inverse.)
+   * find: Rather than only showing files whose contents are present,
+     when used with --exclude --copies or --in, displays all files that
+     match the specified conditions.
+   * Note that this is a behavior change for git-annex find! Old behavior
+     can be gotten by using: git-annex find --in .
+   * status: Massively sped up; remove --fast mode.
+   * unused: File contents used by branches and tags are no longer
+     considered unused, even when not used by the current branch. This is
+     the final piece of the puzzle needed for git-annex to to play nicely
+     with branches."""]]
\ No newline at end of file

From 7c2c17f706d990f2f61b6aa702795a31c040c88f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 20:12:11 -0400
Subject: [PATCH 2271/2835] golfing

---
 Command/Unused.hs | 39 +++++++++++++++++----------------------
 1 file changed, 17 insertions(+), 22 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index e65f0790ba..0c1ffe6039 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -165,8 +165,7 @@ excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
 	g <- Annex.gitRepo
 	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
-	excludeReferenced'
-		(getKeysReferenced : (map getKeysReferencedInGit $ refs c))
+	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
 		(S.fromList l)
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
@@ -175,18 +174,15 @@ excludeReferenced l = do
 			filter ourbranches .
 			map words . lines
 		cmpheads a b = head a == head b
-		ourbranchend = "/" ++ Branch.name
+		ourbranchend = '/' : Branch.name
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
-excludeReferenced' :: ([Annex [Key]]) -> S.Set Key -> Annex [Key]
-excludeReferenced' [] s = return $ S.toList s
-excludeReferenced' (a:as) s
-	| s == S.empty = return [] -- optimisation
-	| otherwise = do
-		referenced <- a
-		let !s' = remove referenced
-		excludeReferenced' as s'
-	where
-		remove l = s `S.difference` S.fromList l
+		removewith [] s = return $ S.toList s
+		removewith (a:as) s
+			| s == S.empty = return [] -- optimisation
+			| otherwise = do
+				referenced <- a
+				let !s' = s `S.difference` S.fromList referenced
+				removewith as s'
 
 {- Finds items in the first, smaller list, that are not
  - present in the second, larger list.
@@ -216,14 +212,13 @@ getKeysReferencedInGit ref = do
 	findkeys [] =<< liftIO (LsTree.lsTree g ref)
 	where
 		findkeys c [] = return c
-		findkeys c (l:ls) = do
-			if isSymLink (LsTree.mode l)
-				then do
-					content <- catFile ref $ LsTree.file l
-					case fileKey (takeFileName content) of
-						Nothing -> findkeys c ls
-						Just k -> findkeys (k:c) ls
-				else findkeys c ls
+		findkeys c (l:ls)
+			| isSymLink (LsTree.mode l) = do
+				content <- catFile ref $ LsTree.file l
+				case fileKey (takeFileName content) of
+					Nothing -> findkeys c ls
+					Just k -> findkeys (k:c) ls
+			| otherwise = findkeys c ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
  - of those that might still have value, or might be stale and removable. 

From 244ffef43f859152907f5202e85161f3e73cfe64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 16:43:00 -0400
Subject: [PATCH 2272/2835] add

---
 ...ex_processes_can_lead_to_locking_issues.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn

diff --git a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
new file mode 100644
index 0000000000..ef0a520a06
--- /dev/null
+++ b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
@@ -0,0 +1,17 @@
+When two git-annex processes are running and both modifying the git-annex
+branch, it's possible one will fail due to git's locking. When this
+happens, git-annex has already recorded its state in the journal (so no
+data is lost), but git-annex does crash, which can be surprising.
+
+I feel that, in general, multiple git-annex processes should be able to run
+concurrently. A big lock around all commands, or even all
+repository-modifying commands is a bad idea. Also, it's probably best to
+only worry about locking conflicts editing the git-annex branch. While `git
+annex add` and a few other commands make changes to the main git repo,
+and can have similar locking issues, so can any git commands that stage
+changes (I think.. check).
+
+Probably should KISS. Just add a lock file that is taken before changes to
+the git-annex branch, and if it's locked, wait. Changes to the git-annex
+branch tend to happen quickly (unless it's committing an enormous set of
+changes, and even that is relatively fast), so waiting seems ok. --[[Joey]] 

From a91c8a15d523791ea729976cd5c76bac1e7ec135 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 19:04:24 -0400
Subject: [PATCH 2273/2835] Sped up unused.

Added Git.ByteString which replaces Git IO methods with ones using lazy
ByteStrings. This can be more efficient when large quantities of data are
being read from git.

In Git.LsTree, parse git ls-tree output more efficiently, thanks
to ByteString. This benchmarks 25% faster, in a benchmark that includes
(probably predominately) the run time for git ls-tree itself.

In real world numbers, this makes git annex unused 2 seconds faster for
each branch it needs to check, in my usual large repo.
---
 Git.hs            |  5 ++--
 Git/ByteString.hs | 62 +++++++++++++++++++++++++++++++++++++++++++++++
 Git/LsTree.hs     | 34 +++++++++++++-------------
 debian/changelog  |  6 +++++
 4 files changed, 88 insertions(+), 19 deletions(-)
 create mode 100644 Git/ByteString.hs

diff --git a/Git.hs b/Git.hs
index fe2afdcfe3..f49cc21b54 100644
--- a/Git.hs
+++ b/Git.hs
@@ -59,6 +59,7 @@ module Git (
 	getSha,
 	shaSize,
 	commit,
+	assertLocal,
 
 	prop_idempotent_deencode
 ) where
@@ -458,8 +459,8 @@ commit g message newref parentrefs = do
 		ps = concatMap (\r -> ["-p", r]) parentrefs
 
 {- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it into a list of files/lines/whatever. -}
-pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
+ - parameter), and splits it. -}
+pipeNullSplit :: Repo -> [CommandParam] -> IO [String]
 pipeNullSplit repo params = filter (not . null) . split "\0" <$>
 	pipeRead repo params
 
diff --git a/Git/ByteString.hs b/Git/ByteString.hs
new file mode 100644
index 0000000000..4eb6a4876b
--- /dev/null
+++ b/Git/ByteString.hs
@@ -0,0 +1,62 @@
+{- module using Data.ByteString.Lazy.Char8 for git IO
+ -
+ - This can be imported instead of Git when more efficient ByteString IO
+ - is needed.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.ByteString (
+	module Git,
+	pipeRead,
+	pipeWrite,
+	pipeWriteRead,
+	pipeNullSplit
+) where
+
+import Control.Applicative
+import System.Cmd.Utils
+import System.IO
+import qualified Data.ByteString.Lazy.Char8 as L
+
+import Git hiding (pipeRead, pipeWrite, pipeWriteRead, pipeNullSplit)
+import Utility.SafeCommand
+
+{- Runs a git subcommand and returns its output, lazily. 
+ -
+ - Note that this leaves the git process running, and so zombies will
+ - result unless reap is called.
+ -}
+pipeRead :: Repo -> [CommandParam] -> IO L.ByteString
+pipeRead repo params = assertLocal repo $ do
+	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine repo params
+	hSetBinaryMode h True
+	L.hGetContents h
+
+{- Runs a git subcommand, feeding it input.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWrite :: Repo -> [CommandParam] -> L.ByteString -> IO PipeHandle
+pipeWrite repo params s = assertLocal repo $ do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine repo params)
+	L.hPut h s
+	hClose h
+	return p
+
+{- Runs a git subcommand, feeding it input, and returning its output.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWriteRead :: Repo -> [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString)
+pipeWriteRead repo params s = assertLocal repo $ do
+	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine repo params)
+	hSetBinaryMode from True
+	L.hPut to s
+	hClose to
+	c <- L.hGetContents from
+	return (p, c)
+
+{- Reads null terminated output of a git command (as enabled by the -z 
+ - parameter), and splits it. -}
+pipeNullSplit :: Repo -> [CommandParam] -> IO [L.ByteString]
+pipeNullSplit repo params = filter (not . L.null) . L.split '\0' <$>
+	pipeRead repo params
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 4a6c509f94..9d2fe7a373 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -7,22 +7,23 @@
 
 module Git.LsTree (
 	TreeItem(..),
-	lsTree
+	lsTree,
+	parseLsTree
 ) where
 
 import Numeric
 import Control.Applicative
-import Data.Char
 import System.Posix.Types
+import qualified Data.ByteString.Lazy.Char8 as L
 
-import Git
+import Git.ByteString
 import Utility.SafeCommand
 
 type Treeish = String
 
 data TreeItem = TreeItem
 	{ mode :: FileMode
-	, objtype :: String
+	, typeobj :: String
 	, sha :: String
 	, file :: FilePath
 	} deriving Show
@@ -34,18 +35,17 @@ lsTree repo t = map parseLsTree <$>
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
-parseLsTree :: String -> TreeItem
-parseLsTree l = TreeItem m o s f
+parseLsTree :: L.ByteString -> TreeItem
+parseLsTree l = TreeItem
+		(fst $ head $ readOct $ L.unpack m)
+		(L.unpack t)
+		(L.unpack s)
+		(decodeGitFile $ L.unpack f)
 	where
 		-- l =  SP  SP  TAB 
-		-- Since everything until the file is fixed-width,
-		-- do not need to split on words.
-		(m, past_m) = head $ readOct l
-		(o, past_o) = splitAt 4 $ space past_m
-		(s, past_s) = splitAt shaSize $ space past_o
-		f = decodeGitFile $ space past_s
-		space (sp:rest)
-			| isSpace sp = rest
-			| otherwise = parseerr
-		space [] = parseerr
-		parseerr = "ls-tree parse error: " ++ l
+		-- All fields are fixed, so we can pull them out of
+		-- specific positions in the line.
+		(m, past_m) = L.splitAt 7 l
+		(t, past_t) = L.splitAt 4 past_m
+		(s, past_s) = L.splitAt 40 $ L.tail past_t
+		f = L.tail past_s
diff --git a/debian/changelog b/debian/changelog
index f337882714..b793101e10 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110929) UNRELEASED; urgency=low
+
+  * Sped up unused.
+
+ -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
+
 git-annex (3.20110928) unstable; urgency=low
 
   * --in can be used to make git-annex only operate on files

From 67f2b7cb3eba19eb1ee55585f497e35172971e1a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 19:19:28 -0400
Subject: [PATCH 2274/2835] use ByteStrings when reading content of files

didn't bother to benchmark this
---
 Git.hs            | 13 -------------
 Git/UnionMerge.hs | 18 ++++++++++++++++--
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/Git.hs b/Git.hs
index f49cc21b54..4c4f00e8df 100644
--- a/Git.hs
+++ b/Git.hs
@@ -55,7 +55,6 @@ module Git (
 	repoAbsPath,
 	reap,
 	useIndex,
-	hashObject,
 	getSha,
 	shaSize,
 	commit,
@@ -417,18 +416,6 @@ useIndex index = do
 		reset (Right (Just v)) = setEnv var v True
 		reset _ = unsetEnv var
 
-{- Injects some content into git, returning its hash. -}
-hashObject :: Repo -> String -> IO String
-hashObject repo content = getSha subcmd $ do
-	(h, s) <- pipeWriteRead repo (map Param params) content
-	length s `seq` do
-		forceSuccess h
-		reap -- XXX unsure why this is needed
-		return s
-	where
-		subcmd = "hash-object"
-		params = [subcmd, "-w", "--stdin"]
-
 {- Runs an action that causes a git subcommand to emit a sha, and strips
    any trailing newline, returning the sha. -}
 getSha :: String -> IO String -> IO String
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a5bcbeac4a..a2b85dbdc5 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -16,8 +16,10 @@ import System.Cmd.Utils
 import Data.List
 import Data.Maybe
 import Data.String.Utils
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
+import qualified Git.ByteString as GitB
 import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
@@ -78,6 +80,18 @@ calc_merge g differ = do
 		pairs (_:[]) = error "calc_merge parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
 
+{- Injects some content into git, returning its hash. -}
+hashObject :: Repo -> L.ByteString -> IO String
+hashObject repo content = getSha subcmd $ do
+	(h, s) <- GitB.pipeWriteRead repo (map Param params) content
+	L.length s `seq` do
+		forceSuccess h
+		reap -- XXX unsure why this is needed
+		return $ L.unpack s
+	where
+		subcmd = "hash-object"
+		params = [subcmd, "-w", "--stdin"]
+
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
@@ -86,10 +100,10 @@ mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- pipeRead g $ map Param ("show":shas)
+		content <- GitB.pipeRead g $ map Param ("show":shas)
 		sha <- hashObject g $ unionmerge content
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
 		nullsha = replicate shaSize '0'
-		unionmerge = unlines . nub . lines
+		unionmerge = L.unlines . nub . L.lines

From 4050c24d7bc49345b914bc062c0a5aedb93cfc8d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 21:11:48 -0400
Subject: [PATCH 2275/2835] ssh

---
 test.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test.hs b/test.hs
index 21e09e98e7..f8701db669 100644
--- a/test.hs
+++ b/test.hs
@@ -453,13 +453,13 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	checkunused []
 	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
 	checkunused []
-	boolSystem "git" [Params "commit -m foo"] @? "git commit failed"
+	boolSystem "git" [Params "commit -q -m foo"] @? "git commit failed"
 	checkunused []
 	-- unused checks origin/master; once it's gone it is really unused
 	boolSystem "git" [Params "remote rm origin"] @? "git remote rm origin failed"
 	checkunused [annexedfilekey]
 	boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed"
-	boolSystem "git" [Params "commit -m foo"] @? "git commit failed"
+	boolSystem "git" [Params "commit -q -m foo"] @? "git commit failed"
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also

From 949ef94d5e5583e55d6ba9797cf71177b252173d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 22:31:20 -0400
Subject: [PATCH 2276/2835] layout

---
 Git/LsTree.hs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 9d2fe7a373..e0aa5a4431 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -36,11 +36,12 @@ lsTree repo t = map parseLsTree <$>
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
 parseLsTree :: L.ByteString -> TreeItem
-parseLsTree l = TreeItem
-		(fst $ head $ readOct $ L.unpack m)
-		(L.unpack t)
-		(L.unpack s)
-		(decodeGitFile $ L.unpack f)
+parseLsTree l = TreeItem 
+	{ mode = fst $ head $ readOct $ L.unpack m
+	, typeobj = L.unpack t
+	, sha = L.unpack s
+	, file = decodeGitFile $ L.unpack f
+	}
 	where
 		-- l =  SP  SP  TAB 
 		-- All fields are fixed, so we can pull them out of

From 7ff89ccfee13dcfe89cbdef83454e880dabd7186 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 23:43:42 -0400
Subject: [PATCH 2277/2835] convert all git read/write functions to use
 ByteStrings

This yields a second or so speedup in unused, find, etc. Seems that even
when the ByteString is immediately split and then converted to Strings,
it's faster.

I may try to push ByteStrings out into more of git-annex gradually,
although I suspect most of the time-critical parts are already covered
now, and many of the rest rely on libraries that only support Strings.
---
 Branch.hs         |  5 ++--
 Command/Unused.hs |  3 ++-
 Git.hs            | 54 +++++++++++++++++++++++++++--------------
 Git/ByteString.hs | 62 -----------------------------------------------
 Git/LsFiles.hs    |  6 ++---
 Git/LsTree.hs     |  4 +--
 Git/UnionMerge.hs |  7 +++---
 debian/changelog  |  2 +-
 8 files changed, 49 insertions(+), 94 deletions(-)
 delete mode 100644 Git/ByteString.hs

diff --git a/Branch.hs b/Branch.hs
index 92b1fe29e8..e4caeece77 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -30,6 +30,7 @@ import System.IO
 import System.IO.Binary
 import System.Posix.Process
 import System.Exit
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Types.BranchState
 import qualified Git
@@ -181,7 +182,7 @@ siblingBranches :: Annex [String]
 siblingBranches = do
 	g <- Annex.gitRepo
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-	return $ map (last . words) (lines r)
+	return $ map (last . words . L.unpack) (L.lines r)
 
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: GitRef -> Annex (Maybe String)
@@ -196,7 +197,7 @@ updateRef ref
 			Param (name++".."++ref),
 			Params "--oneline -n1"
 			]
-		if null diffs
+		if L.null diffs
 			then return Nothing
 			else do
 				showSideAction $ "merging " ++ Git.refDescribe ref ++ " into " ++ name
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 0c1ffe6039..987f367201 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,6 +16,7 @@ import Data.Maybe
 import System.FilePath
 import System.Directory
 import Data.List
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Command
 import Types
@@ -172,7 +173,7 @@ excludeReferenced l = do
 		refs = map last .
 			nubBy cmpheads .
 			filter ourbranches .
-			map words . lines
+			map words . lines . L.unpack
 		cmpheads a b = head a == head b
 		ourbranchend = '/' : Branch.name
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
diff --git a/Git.hs b/Git.hs
index 4c4f00e8df..d32aaaa562 100644
--- a/Git.hs
+++ b/Git.hs
@@ -44,6 +44,7 @@ module Git (
 	pipeWrite,
 	pipeWriteRead,
 	pipeNullSplit,
+	pipeNullSplitB,
 	attributes,
 	remotes,
 	remotesAdd,
@@ -85,6 +86,7 @@ import Text.Printf
 import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Utility
 import Utility.Path
@@ -379,22 +381,41 @@ run repo subcommand params = assertLocal repo $
  - Note that this leaves the git process running, and so zombies will
  - result unless reap is called.
  -}
-pipeRead :: Repo -> [CommandParam] -> IO String
+pipeRead :: Repo -> [CommandParam] -> IO L.ByteString
 pipeRead repo params = assertLocal repo $ do
-	(_, s) <- pipeFrom "git" $ toCommand $ gitCommandLine repo params
-	return s
+	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine repo params
+	hSetBinaryMode h True
+	L.hGetContents h
 
 {- Runs a git subcommand, feeding it input.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: Repo -> [CommandParam] -> String -> IO PipeHandle
-pipeWrite repo params s = assertLocal repo $
-	pipeTo "git" (toCommand $ gitCommandLine repo params) s
+pipeWrite :: Repo -> [CommandParam] -> L.ByteString -> IO PipeHandle
+pipeWrite repo params s = assertLocal repo $ do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine repo params)
+	L.hPut h s
+	hClose h
+	return p
 
 {- Runs a git subcommand, feeding it input, and returning its output.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: Repo -> [CommandParam] -> String -> IO (PipeHandle, String)
-pipeWriteRead repo params s = assertLocal repo $
-	pipeBoth "git" (toCommand $ gitCommandLine repo params) s
+pipeWriteRead :: Repo -> [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString)
+pipeWriteRead repo params s = assertLocal repo $ do
+	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine repo params)
+	hSetBinaryMode from True
+	L.hPut to s
+	hClose to
+	c <- L.hGetContents from
+	return (p, c)
+
+{- Reads null terminated output of a git command (as enabled by the -z 
+ - parameter), and splits it. -}
+pipeNullSplit :: Repo -> [CommandParam] -> IO [String]
+pipeNullSplit repo params = map L.unpack <$> pipeNullSplitB repo params
+
+{- For when Strings are not needed. -}
+pipeNullSplitB :: Repo -> [CommandParam] -> IO [L.ByteString]
+pipeNullSplitB repo params = filter (not . L.null) . L.split '\0' <$>
+	pipeRead repo params
 
 {- Reaps any zombie git processes. -}
 reap :: IO ()
@@ -436,21 +457,18 @@ shaSize = 40
  - with the specified parent refs. -}
 commit :: Repo -> String -> String -> [String] -> IO ()
 commit g message newref parentrefs = do
-	tree <- getSha "write-tree" $
+	tree <- getSha "write-tree" $ asString $
 		pipeRead g [Param "write-tree"]
-	sha <- getSha "commit-tree" $ ignorehandle $
-		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
+	sha <- getSha "commit-tree" $ asString $
+		ignorehandle $ pipeWriteRead g
+			(map Param $ ["commit-tree", tree] ++ ps)
+			(L.pack message)
 	run g "update-ref" [Param newref, Param sha]
 	where
 		ignorehandle a = snd <$> a
+		asString a = L.unpack <$> a
 		ps = concatMap (\r -> ["-p", r]) parentrefs
 
-{- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it. -}
-pipeNullSplit :: Repo -> [CommandParam] -> IO [String]
-pipeNullSplit repo params = filter (not . null) . split "\0" <$>
-	pipeRead repo params
-
 {- Runs git config and populates a repo with its config. -}
 configRead :: Repo -> IO Repo
 configRead repo@(Repo { location = Dir d }) = do
diff --git a/Git/ByteString.hs b/Git/ByteString.hs
deleted file mode 100644
index 4eb6a4876b..0000000000
--- a/Git/ByteString.hs
+++ /dev/null
@@ -1,62 +0,0 @@
-{- module using Data.ByteString.Lazy.Char8 for git IO
- -
- - This can be imported instead of Git when more efficient ByteString IO
- - is needed.
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Git.ByteString (
-	module Git,
-	pipeRead,
-	pipeWrite,
-	pipeWriteRead,
-	pipeNullSplit
-) where
-
-import Control.Applicative
-import System.Cmd.Utils
-import System.IO
-import qualified Data.ByteString.Lazy.Char8 as L
-
-import Git hiding (pipeRead, pipeWrite, pipeWriteRead, pipeNullSplit)
-import Utility.SafeCommand
-
-{- Runs a git subcommand and returns its output, lazily. 
- -
- - Note that this leaves the git process running, and so zombies will
- - result unless reap is called.
- -}
-pipeRead :: Repo -> [CommandParam] -> IO L.ByteString
-pipeRead repo params = assertLocal repo $ do
-	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine repo params
-	hSetBinaryMode h True
-	L.hGetContents h
-
-{- Runs a git subcommand, feeding it input.
- - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: Repo -> [CommandParam] -> L.ByteString -> IO PipeHandle
-pipeWrite repo params s = assertLocal repo $ do
-	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine repo params)
-	L.hPut h s
-	hClose h
-	return p
-
-{- Runs a git subcommand, feeding it input, and returning its output.
- - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: Repo -> [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString)
-pipeWriteRead repo params s = assertLocal repo $ do
-	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine repo params)
-	hSetBinaryMode from True
-	L.hPut to s
-	hClose to
-	c <- L.hGetContents from
-	return (p, c)
-
-{- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it. -}
-pipeNullSplit :: Repo -> [CommandParam] -> IO [L.ByteString]
-pipeNullSplit repo params = filter (not . L.null) . L.split '\0' <$>
-	pipeRead repo params
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index c778e5d69c..28e007a4dc 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -20,13 +20,11 @@ import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
-inRepo repo l = pipeNullSplit repo $
-	Params "ls-files --cached -z --" : map File l
+inRepo repo l = pipeNullSplit repo $ Params "ls-files --cached -z --" : map File l
 
 {- Scans for files at the specified locations that are not checked into git. -}
 notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
-notInRepo repo include_ignored l =
-	pipeNullSplit repo $
+notInRepo repo include_ignored l = pipeNullSplit repo $
 		[Params "ls-files --others"] ++ exclude ++
 		[Params "-z --"] ++ map File l
 	where
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index e0aa5a4431..c072ef5be9 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -16,7 +16,7 @@ import Control.Applicative
 import System.Posix.Types
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import Git.ByteString
+import Git
 import Utility.SafeCommand
 
 type Treeish = String
@@ -31,7 +31,7 @@ data TreeItem = TreeItem
 {- Lists the contents of a Treeish -}
 lsTree :: Repo -> Treeish -> IO [TreeItem]
 lsTree repo t = map parseLsTree <$>
-	pipeNullSplit repo [Params "ls-tree --full-tree -z -r --", File t]
+	pipeNullSplitB repo [Params "ls-tree --full-tree -z -r --", File t]
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a2b85dbdc5..ac002b374e 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -19,7 +19,6 @@ import Data.String.Utils
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
-import qualified Git.ByteString as GitB
 import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
@@ -44,7 +43,7 @@ merge _ _ = error "wrong number of branches to merge"
 update_index :: Repo -> [String] -> IO ()
 update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
-		togit ps content = pipeWrite g (map Param ps) content
+		togit ps content = pipeWrite g (map Param ps) (L.pack content)
 			>>= forceSuccess
 
 {- Generates a line suitable to be fed into update-index, to add
@@ -83,7 +82,7 @@ calc_merge g differ = do
 {- Injects some content into git, returning its hash. -}
 hashObject :: Repo -> L.ByteString -> IO String
 hashObject repo content = getSha subcmd $ do
-	(h, s) <- GitB.pipeWriteRead repo (map Param params) content
+	(h, s) <- pipeWriteRead repo (map Param params) content
 	L.length s `seq` do
 		forceSuccess h
 		reap -- XXX unsure why this is needed
@@ -100,7 +99,7 @@ mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- GitB.pipeRead g $ map Param ("show":shas)
+		content <- pipeRead g $ map Param ("show":shas)
 		sha <- hashObject g $ unionmerge content
 		return $ Just $ update_index_line sha file
 	where
diff --git a/debian/changelog b/debian/changelog
index b793101e10..62b65333d1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,6 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
-  * Sped up unused.
+  * Various speed improvements gained by using ByteStrings.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From c86a2f686aa447d1cf30cecaa2feedaf0e2c6ba2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
 
Date: Fri, 30 Sep 2011 04:32:24 +0000
Subject: [PATCH 2278/2835]

---
 doc/forum/location_tracking_cleanup.mdwn | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/location_tracking_cleanup.mdwn

diff --git a/doc/forum/location_tracking_cleanup.mdwn b/doc/forum/location_tracking_cleanup.mdwn
new file mode 100644
index 0000000000..7e2e230af7
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup.mdwn
@@ -0,0 +1,24 @@
+I recently started experimenting with git annex, adding files that I've had
+floating across several computers to repositories. During the testing I had
+a few occasions where I wrecked a repository somehow, and decided to wipe it
+and start anew (at this point there was no important files in them so I thought
+this is the easiest way). Well, as it turns out this interacts badly with location
+tracking, since now `git annex whereis` shows files residing in all those destroyed
+repositories, all having same names as some existing repositories. This makes it hard
+to follow whether a repo actually has a file, or was the file only seen in some dead
+repo with the same name.
+
+I planned on cleaning this up by looking up the UUIDs of the now stable, existing
+repos and untrusting all the dead copies (they should effectively disappear from
+git annex´s output then, right?), but I didn't find an easy way to look up the UUID
+of the current repository (maybe this could be included in `git annex status`?)
+I also noticed that untrust cannot remove the trust based on the UUID -- if I try
+it I simply get "there is no git remote named "11908472-...", so I guess untrust
+works with git remote names, which I find a bit confusing, since trust.log logs the
+trust levels based on the UUID. I could just write into trust.log manually, but I'm
+unsure how the changes would get propagated.
+
+What should I do? As a related wishlist item I would ask for some additional mechanisms
+for purging known-dead repositories from the location tracking database. And the ability
+to look up the UUID of the current repo, and to use the UUID to specify repositories when
+applicable (untrust and describe maybe).

From a7e7dda55a82cb5007c5eaa2f7752f5cefdcb1d2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 02:23:24 -0400
Subject: [PATCH 2279/2835] Fix referring to remotes by uuid.

I think that I broke this in some fairly recent refactoring.
---
 Remote.hs        | 12 +++++++++---
 debian/changelog |  1 +
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 0ce01872ac..05f70a5d7f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -87,7 +87,8 @@ genList = do
 			u <- getUUID r
 			generate t r u (M.lookup u m)
 
-{- Looks up a remote by name. (Or by UUID.) -}
+{- Looks up a remote by name. (Or by UUID.) Only finds currently configured
+ - git remotes. -}
 byName :: String -> Annex (Remote Annex)
 byName n = do
 	res <- byName' n
@@ -106,7 +107,8 @@ byName' n = do
 		matching r = n == name r || n == uuid r
 
 {- Looks up a remote by name (or by UUID, or even by description),
- - and returns its UUID. -}
+ - and returns its UUID. Finds even remotes that are not configured in
+ - .git/config. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = do
@@ -115,7 +117,11 @@ nameToUUID n = do
 		Left e -> fromMaybe (error e) <$> byDescription
 		Right r -> return $ uuid r
 	where
-		byDescription = M.lookup n . invertMap <$> uuidMap
+		byDescription = do
+			m <- uuidMap
+			case M.lookup n $ invertMap m of
+				Just u -> return $ Just u
+				Nothing -> return $ M.lookup n m
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
diff --git a/debian/changelog b/debian/changelog
index 62b65333d1..5354e710db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
   * Various speed improvements gained by using ByteStrings.
+  * Fix referring to remotes by uuid.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 17b29176b8350adf9f6d547d59b97c965f1aad7f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 02:50:34 -0400
Subject: [PATCH 2280/2835] fix handling of uuids with empty descriptions

---
 UUID.hs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/UUID.hs b/UUID.hs
index fa71bed391..a150dc333c 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -100,7 +100,4 @@ uuidMap = do
 	s <- Branch.get uuidLog
 	return $ M.fromList $ map pair $ lines s
 	where
-		pair l =
-			if 1 < length (words l)
-				then (head $ words l, unwords $ drop 1 $ words l)
-				else ("", "")
+		pair l = (head $ words l, unwords $ drop 1 $ words l)

From 03e54680ff1432915e2e02f87c95a9126af5a420 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 02:51:05 -0400
Subject: [PATCH 2281/2835] really fix referring to remotes by uuid

---
 Remote.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 05f70a5d7f..87fd8aab62 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -119,11 +119,12 @@ nameToUUID n = do
 	where
 		byDescription = do
 			m <- uuidMap
-			case M.lookup n $ invertMap m of
+			case M.lookup n $ transform swap m of
 				Just u -> return $ Just u
-				Nothing -> return $ M.lookup n m
-		invertMap = M.fromList . map swap . M.toList
+				Nothing -> return $ M.lookup n $ transform double m
+		transform a = M.fromList . map a . M.toList
 		swap (a, b) = (b, a)
+		double (a, _) = (a, a)
 
 {- Pretty-prints a list of UUIDs of remotes, for human display.
  -

From c68fc49184d9e59fe3d9cda1addca64b82d513d6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 30 Sep 2011 06:55:34 +0000
Subject: [PATCH 2282/2835] Added a comment

---
 ...comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment

diff --git a/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment b/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment
new file mode 100644
index 0000000000..8915ea3518
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-09-30T06:55:34Z"
+ content="""
+Specifying the UUID was supposed to work, I think I broke it a while ago. Fixed now in git.
+
+I'm not sure why you need to look up the UUID of the current repository. You can always refer to the current repository as \".\". Anyway, the UUID of the current repository is in `.git/config`, or use `git config annex.uuid`.
+"""]]

From a7102ca4d53f8a4ab7148ab3717765683b136cfc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 03:03:59 -0400
Subject: [PATCH 2283/2835] list backends with more keys first, not last

---
 Command/Status.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 6da8064f85..329d3f70bf 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -117,7 +117,7 @@ bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 backend_usage :: Stat
 backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 	where
-		usage ks = pp "" $ sort $ map swap $ splits $ S.toList ks
+		usage ks = pp "" $ reverse . sort $ map swap $ splits $ S.toList ks
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)

From 15eccdf124d13d8502ee5a23e73968e5208690f2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 03:05:10 -0400
Subject: [PATCH 2284/2835] better output layout

---
 Command/Status.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 329d3f70bf..ad490cae82 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -167,4 +167,4 @@ staleSize label dirspec = do
 			return $ s ++ aside "clean up with git-annex unused"
 
 aside :: String -> String
-aside s = "\t(" ++ s ++ ")"
+aside s = " (" ++ s ++ ")"

From 828f3f1b0c4cf8791c063c6a393797047084eee8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 03:20:24 -0400
Subject: [PATCH 2285/2835] status: List all known repositories.

---
 Command/Status.hs | 12 ++++++++++--
 debian/changelog  |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index ad490cae82..07c0958bbe 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -28,6 +28,8 @@ import Content
 import Types.Key
 import Locations
 import Backend
+import UUID
+import Remote
 
 -- a named computation that produces a statistic
 type Stat = StatState (Maybe (String, StatState String))
@@ -55,6 +57,7 @@ stats :: [Stat]
 stats = 
 	[ supported_backends
 	, supported_remote_types
+	, remote_list
 	, tmp_size
 	, bad_data_size
 	, local_annex_keys
@@ -92,6 +95,11 @@ supported_remote_types :: Stat
 supported_remote_types = stat "supported remote types" $
 	return $ unwords $ map R.typename Remote.remoteTypes
 
+remote_list :: Stat
+remote_list = stat "known repositories" $ lift $ do
+	s <- prettyPrintUUIDs "repos" =<< M.keys <$> uuidMap
+	return $ '\n':init s
+
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
 	keySizeSum <$> cachedKeysPresent
@@ -154,8 +162,8 @@ keySizeSum s = total ++ missingnote
 		missingnote
 			| missing == 0 = ""
 			| otherwise = aside $
-				"but " ++ show missing ++
-				" keys have unknown size"
+				"+ " ++ show missing ++
+				" keys of unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
 staleSize label dirspec = do
diff --git a/debian/changelog b/debian/changelog
index 5354e710db..3bdd044510 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
 
   * Various speed improvements gained by using ByteStrings.
   * Fix referring to remotes by uuid.
+  * status: List all known repositories.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 84801918f60481be20553da58ac5cd9bb9feae29 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
 
Date: Fri, 30 Sep 2011 11:55:36 +0000
Subject: [PATCH 2286/2835] Added a comment

---
 ...nt_2_e7395cb6e01f42da72adf71ea3ebcde4._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment

diff --git a/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment b/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment
new file mode 100644
index 0000000000..f612e53a4d
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
+ nickname="Perttu"
+ subject="comment 2"
+ date="2011-09-30T11:55:35Z"
+ content="""
+Thanks for the quick reply :)
+
+I wanted to look up the UUID of the current repo so that I can find out which repo is alive from the collection of repos with the same name.
+I could have looked for it in .git/config though, since it's pretty obvious. I just looked into the git-annex branch and didn't find it there.
+Thanks for the tip about using \".\". By the way, could there be some kind of warning about using non-unique names for repos? That would make this
+scenario less likely. Or maybe that is a bad idea given the decentralized nature of git.
+
+By the way, do the trust settings propagate to other repos? If I mark some UUID as untrusted on one computer does it become globally untrusted?
+"""]]

From d48ae1b8fd2d73746986bd40d22de450d647d223 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 30 Sep 2011 16:47:27 +0000
Subject: [PATCH 2287/2835] Added a comment

---
 ...comment_3_c15428cec90e969284a5e690fb4b2fde._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment

diff --git a/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment b/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment
new file mode 100644
index 0000000000..c676b9b615
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-09-30T16:47:27Z"
+ content="""
+`git annex status` now includes a list of all known repositories.
+
+Yes, trust setting propigate on git push/pull like any other git-annex information.
+"""]]

From f88738223ee5156baac8f0fa0bde6d701f1fdd07 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 14:59:35 -0400
Subject: [PATCH 2288/2835] don't crash on malformed lines in uuid.log

---
 UUID.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/UUID.hs b/UUID.hs
index a150dc333c..b1ccbb250c 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -100,4 +100,8 @@ uuidMap = do
 	s <- Branch.get uuidLog
 	return $ M.fromList $ map pair $ lines s
 	where
-		pair l = (head $ words l, unwords $ drop 1 $ words l)
+		pair l
+			| null ws = ("", "")
+			| otherwise = (head ws, unwords $ drop 1 ws)
+			where
+				ws = words l

From 29032cb70e66d18f25b9b942eb01dceeeb8aa300 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 15:02:08 -0400
Subject: [PATCH 2289/2835] When displaying a list of repositories, show git
 remote names in addition to their descriptions.

---
 Remote.hs                                     | 21 ++++++++++++-------
 debian/changelog                              |  2 ++
 ...95__remotes__47__hook_with_tahoe-lafs.mdwn |  2 +-
 .../automatically_managing_content.mdwn       |  4 ++--
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 87fd8aab62..83a593dd50 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -136,18 +136,25 @@ nameToUUID n = do
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
-	m <- M.union <$> uuidMap <*> availMap
+	m <- M.unionWith addname <$> uuidMap <*> remoteMap
 	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
-		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
+		addname d n
+			| d == n = d
+			| otherwise = n ++ " (" ++ d ++ ")"
+		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
-		prettify m here u = base ++ ishere
+		prettify m here u
+			| not (null d) = u ++ "  -- " ++ d
+			| otherwise = u
 			where
-				base = if not $ null $ findlog m u
-					then u ++ "  -- " ++ findlog m u
-					else u
-				ishere = if here == u then " <-- here" else ""
+				ishere = here == u
+				n = findlog m u
+				d
+					| null n && ishere = "here"
+					| ishere = addname n "here"
+					| otherwise = n
 		jsonify m here u = toJSObject
 			[ ("uuid", toJSON u)
 			, ("description", toJSON $ findlog m u)
diff --git a/debian/changelog b/debian/changelog
index 3bdd044510..bf41f553d5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Various speed improvements gained by using ByteStrings.
   * Fix referring to remotes by uuid.
   * status: List all known repositories.
+  * When displaying a list of repositories, show git remote names
+    in addition to their descriptions.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn
index 4f5f089a89..8981200d88 100644
--- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn
+++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn
@@ -12,7 +12,7 @@ The only quirk I've noticed is this...
 
 $ git annex whereis .
 whereis frink.jar (2 copies) 
-  	084603a8-7243-11e0-b1f5-83102bcd7953  -- testtest <-- here
+  	084603a8-7243-11e0-b1f5-83102bcd7953  -- here (testtest)
    	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a
 ok
 

diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
index ba0cad6092..ef883efef3 100644
--- a/doc/walkthrough/automatically_managing_content.mdwn
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -13,7 +13,7 @@ file.
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
 	whereis other_file (3 copies)
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
-		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive <-- here
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- here (usb drive)
 		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
 
 What would be handy is some automated versions of get and drop, that only
@@ -31,7 +31,7 @@ work toward having two copies of your files.
 	# git annex whereis
 	whereis my_cool_big_file (2 copies)
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
-		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive <-- here
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- here (usb drive)
 	whereis other_file (2 copies)
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
 		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive

From 1e6b7e901d47142a99ab4e6608f6307c01bc085e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 16:21:28 -0400
Subject: [PATCH 2290/2835] less space

---
 Remote.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote.hs b/Remote.hs
index 83a593dd50..efa7a5cc8e 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -146,7 +146,7 @@ prettyPrintUUIDs desc uuids = do
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
 		prettify m here u
-			| not (null d) = u ++ "  -- " ++ d
+			| not (null d) = u ++ " -- " ++ d
 			| otherwise = u
 			where
 				ishere = here == u

From 49f21dd9ba4819aedf04ebbc0c12fafdb3dc8ab5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 2 Oct 2011 11:16:34 -0400
Subject: [PATCH 2291/2835] Contain the zombie hordes.a

Specifically, when using gpg, a zombie is forked for each file, so waiting
until shutdown to reap won't do.
---
 CmdLine.hs       | 5 +++--
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 4ccd2c2c2f..0ee0c6adbb 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -85,7 +85,9 @@ tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
 tryRun' errnum state (a:as) = do
 	result <- try $ Annex.run state $ do
 		AnnexQueue.flushWhenFull
-		a
+		r <- a
+		liftIO Git.reap
+		return r
 	case result of
 		Left err -> do
 			Annex.eval state $ do
@@ -104,5 +106,4 @@ startup = return True
 shutdown :: Annex Bool
 shutdown = do
 	saveState
-	liftIO Git.reap
 	return True
diff --git a/debian/changelog b/debian/changelog
index bf41f553d5..834d2a5726 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * status: List all known repositories.
   * When displaying a list of repositories, show git remote names
     in addition to their descriptions.
+  * Contain the zombie hordes.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 61fbea992da0f2a1ed49e2e862ef937b25d8430b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 2 Oct 2011 11:42:34 -0400
Subject: [PATCH 2292/2835] when all you have is a zombie, everything looks
 like a shotgun

Actually, let's do a targeted fix of the actual forkProcess that was not
waited on. The global reap is moved back to the end, after the long-running
git processes actually exit.
---
 CmdLine.hs | 5 ++---
 Crypto.hs  | 3 ++-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 0ee0c6adbb..38d4754da9 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -85,9 +85,7 @@ tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
 tryRun' errnum state (a:as) = do
 	result <- try $ Annex.run state $ do
 		AnnexQueue.flushWhenFull
-		r <- a
-		liftIO Git.reap
-		return r
+		a
 	case result of
 		Left err -> do
 			Annex.eval state $ do
@@ -106,4 +104,5 @@ startup = return True
 shutdown :: Annex Bool
 shutdown = do
 	saveState
+	liftIO Git.reap -- zombies from long-running git processes
 	return True
diff --git a/Crypto.hs b/Crypto.hs
index d789b44556..9b2d73f28d 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -218,7 +218,7 @@ gpgCipherHandle params c a b = do
 
 	params' <- gpgParams $ passphrase ++ params
 	(pid, fromh, toh) <- hPipeBoth "gpg" params'
-	_ <- forkProcess $ do
+	pid2 <- forkProcess $ do
 		L.hPut toh =<< a
 		hClose toh
 		exitSuccess
@@ -227,6 +227,7 @@ gpgCipherHandle params c a b = do
 
 	-- cleanup
 	forceSuccess pid
+	_ <- getProcessStatus True False pid2
 	closeFd frompipe
 	return ret
 

From 6dfb94b2d783ef848c651ab20818b05c8a0504a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 14:48:04 -0400
Subject: [PATCH 2293/2835] update

---
 ..._processes_can_lead_to_locking_issues.mdwn | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
index ef0a520a06..87880df417 100644
--- a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
+++ b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
@@ -15,3 +15,37 @@ Probably should KISS. Just add a lock file that is taken before changes to
 the git-annex branch, and if it's locked, wait. Changes to the git-annex
 branch tend to happen quickly (unless it's committing an enormous set of
 changes, and even that is relatively fast), so waiting seems ok. --[[Joey]] 
+
+----
+
+Commit 7981eb4cb512fbe3c49a3dd165c31be14ae4bc49 is more pessimistic, 
+describes some other potential issues. 
+
+* The journal needs to be emptied (done) and kept locked (not done) during
+  a merge, since a merge operates at a level below the journal, and any
+  changes that are journaled during a merge can overwrite changes merged
+  in from another branch.
+
+* Two git-annex processes can be doing conflicting things and inconsistent
+  information be written to the journal.
+
+  - One example would be concurrent get and drop of the same key.
+    But could this really race? If the key was already present, the get
+    would do nothing, so record no changes. If the key was not yet present,
+    the drop would do nothing, and record no changes.
+
+  - Instead, consider two copys of a key to different locations. If the
+    slower copy starts first and ends last, it could cache the location 
+    info, add the  new location, and lose the other location it was copied to.
+    Tested it and the location is not cached during the whole copy (logChange
+    reads the current log immediatly before writing), so this
+    race's window is very small -- but does exist.
+
+---- 
+
+## Updated plan
+
+Make Branch.change transactional, so it takes a lock, reads a file,
+applies a function to it, and writes the changed file.
+
+Make Branch.update hold the same lock.

From f77979b8b5ef1dc59b45c03ba6febfacdf904491 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 15:41:25 -0400
Subject: [PATCH 2294/2835] improved git-annex branch changing

All changes to files in the branch are now made via pure functions that
transform the old file into the new. This will allow adding locking
to prevent read/write races. It also makes the code nicer, and purer.

I noticed a behavior change, really a sort of bug fix. Before,
'git annex untrust foo --trust bar' would change both trust levels
permanantly, now the --trust doesn't get stored.
---
 Branch.hs      | 16 +++++++++++++---
 LocationLog.hs |  1 -
 PresenceLog.hs | 12 +++++-------
 RemoteLog.hs   |  7 +++----
 Trust.hs       |  8 +++-----
 UUID.hs        | 16 ++++++++--------
 Upgrade/V2.hs  |  4 ++--
 7 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index e4caeece77..9340259c7d 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -213,9 +213,19 @@ updateRef ref
 				liftIO $ Git.UnionMerge.merge g [ref]
 				return $ Just ref
 
-{- Records changed content of a file into the journal. -}
-change :: FilePath -> String -> Annex ()
-change file content = do
+{- Applies a function to modifiy the content of a file. -}
+change :: FilePath -> (String -> String) -> Annex ()
+change file a = do
+	lock
+	get file >>= return . a >>= set file
+	unlock
+	where
+		lock = return ()
+		unlock = return ()
+
+{- Records new content of a file into the journal. -}
+set :: FilePath -> String -> Annex ()
+set file content = do
 	setJournalFile file content
 	setCache file content
 
diff --git a/LocationLog.hs b/LocationLog.hs
index 7e5e81d7af..0cdf88bc6b 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -15,7 +15,6 @@ module LocationLog (
 	LogStatus(..),
 	logChange,
 	readLog,
-	writeLog,
 	keyLocations,
 	loggedKeys,
 	logFile,
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 7742651b84..2db1ee59b4 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -16,7 +16,6 @@ module PresenceLog (
 	addLog,
 	readLog,
 	parseLog,
-	writeLog,
 	logNow,
 	compactLog,
 	currentLog,
@@ -75,9 +74,8 @@ instance Read LogLine where
 			ret v = [(v, "")]
 
 addLog :: FilePath -> LogLine -> Annex ()
-addLog file line = do
-	ls <- readLog file
-	writeLog file (compactLog $ line:ls)
+addLog file line = Branch.change file $ \s -> 
+	showLog $ compactLog (line : parseLog s)
 
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
@@ -90,9 +88,9 @@ parseLog = filter parsable . map read . lines
 		-- some lines may be unparseable, avoid them
 		parsable l = status l /= Undefined
 
-{- Stores a set of lines in a log file -}
-writeLog :: FilePath -> [LogLine] -> Annex ()
-writeLog file ls = Branch.change file (unlines $ map show ls)
+{- Generates a log file. -}
+showLog :: [LogLine] -> String
+showLog = unlines . map show
 
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> String -> Annex LogLine
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 620c0d8757..f9c7997e41 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -32,11 +32,10 @@ remoteLog = "remote.log"
 
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
-configSet u c = do
-	m <- readRemoteLog
-	Branch.change remoteLog $ unlines $ sort $
-		map toline $ M.toList $ M.insert u c m
+configSet u c = Branch.change remoteLog $
+	serialize . M.insert u c . remoteLogParse
 	where
+		serialize = unlines . sort . map toline . M.toList
 		toline (u', c') = u' ++ " " ++ unwords (configToKeyVal c')
 
 {- Map of remotes by uuid containing key/value config maps. -}
diff --git a/Trust.hs b/Trust.hs
index 232eea6a5d..0c8836c85a 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -64,11 +64,9 @@ trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
 	when (null uuid) $
 		error "unknown UUID; cannot modify trust level"
-        m <- trustMap
-	when (M.lookup uuid m /= Just level) $ do
-		let m' = M.insert uuid level m
-		Branch.change trustLog (serialize m')
-		Annex.changeState $ \s -> s { Annex.trustmap = Just m' }
+	Branch.change trustLog $
+		serialize . M.insert uuid level . M.fromList . trustMapParse
+	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
         where
                 serialize m = unlines $ map showpair $ M.toList m
 		showpair (u, t) = u ++ " " ++ show t
diff --git a/UUID.hs b/UUID.hs
index b1ccbb250c..eab6bd4dff 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -23,6 +23,7 @@ module UUID (
 ) where
 
 import Control.Monad.State
+import Control.Applicative
 import System.Cmd.Utils
 import System.IO
 import qualified Data.Map as M
@@ -87,18 +88,17 @@ prepUUID = do
 
 {- Records a description for a uuid in the uuidLog. -}
 describeUUID :: UUID -> String -> Annex ()
-describeUUID uuid desc = do
-	m <- uuidMap
-	let m' = M.insert uuid desc m
-	Branch.change uuidLog (serialize m')
+describeUUID uuid desc = Branch.change uuidLog $
+	serialize . M.insert uuid desc . parse
 	where
 		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
 
-{- Read and parse the uuidLog into a Map -}
+{- Read the uuidLog into a Map -}
 uuidMap :: Annex (M.Map UUID String)
-uuidMap = do
-	s <- Branch.get uuidLog
-	return $ M.fromList $ map pair $ lines s
+uuidMap = parse <$> Branch.get uuidLog
+
+parse :: String -> M.Map UUID String
+parse = M.fromList . map pair . lines
 	where
 		pair l
 			| null ws = ("", "")
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index e99a7cf817..4e686288fb 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -87,8 +87,8 @@ inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
 	g <- Annex.gitRepo
 	new <- liftIO (readFile $ olddir g  source)
-	prev <- Branch.get dest
-	Branch.change dest $ unlines $ nub $ lines prev ++ lines new
+	Branch.change dest $ \prev -> 
+		unlines $ nub $ lines prev ++ lines new
 	showProgress
 
 logFiles :: FilePath -> Annex [FilePath]

From d357556141b716a8c9d622cbfb44c38484065183 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 16:32:36 -0400
Subject: [PATCH 2295/2835] Add locking to avoid races when changing the
 git-annex branch.

---
 Branch.hs                                     | 33 ++++++++++++-------
 Locations.hs                                  |  5 +++
 debian/changelog                              |  1 +
 ..._processes_can_lead_to_locking_issues.mdwn |  2 ++
 4 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 9340259c7d..34486243e3 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -29,6 +29,8 @@ import Data.Maybe
 import System.IO
 import System.IO.Binary
 import System.Posix.Process
+import System.Posix.IO
+import System.Posix.Files
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -129,16 +131,17 @@ create = unlessM hasBranch $ do
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = whenM stageJournalFiles $ do
-	g <- Annex.gitRepo
-	withIndex $ liftIO $ Git.commit g message fullname [fullname]
+commit message = lockJournal $
+	whenM stageJournalFiles $ do
+		g <- Annex.gitRepo
+		withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
 	state <- getState
-	unless (branchUpdated state) $ withIndex $ do
+	unless (branchUpdated state) $ withIndex $ lockJournal $ do
 		{- Since branches get merged into the index, it's important to
 		 - first stage the journal into the index. Otherwise, any
 		 - changes in the journal would later get staged, and might
@@ -154,6 +157,7 @@ update = do
 		g <- Annex.gitRepo
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
+
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
@@ -215,13 +219,7 @@ updateRef ref
 
 {- Applies a function to modifiy the content of a file. -}
 change :: FilePath -> (String -> String) -> Annex ()
-change file a = do
-	lock
-	get file >>= return . a >>= set file
-	unlock
-	where
-		lock = return ()
-		unlock = return ()
+change file a = lockJournal $ get file >>= return . a >>= set file
 
 {- Records new content of a file into the journal. -}
 set :: FilePath -> String -> Annex ()
@@ -277,7 +275,7 @@ setJournalFile file content = do
 			writeBinaryFile tmpfile content
 			renameFile tmpfile jfile
 
-{- Gets journalled content for a file in the branch. -}
+{- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
 getJournalFile file = do
 	g <- Annex.gitRepo
@@ -346,3 +344,14 @@ journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
  - filename on the branch. -}
 fileJournal :: FilePath -> FilePath
 fileJournal = replace "//" "_" . replace "_" "/"
+
+{- Runs an action that modifies the journal, using locking to avoid
+ - contention with other git-annex processes. -}
+lockJournal :: Annex a -> Annex a
+lockJournal a = do
+	g <- Annex.gitRepo
+	h <- liftIO $ createFile (gitAnnexJournalLock g) stdFileMode
+	liftIO $ waitToSetLock h (WriteLock, AbsoluteSeek, 0, 0)
+	r <- a
+	liftIO $ closeFd h
+	return r
diff --git a/Locations.hs b/Locations.hs
index 942b687bbf..b18444e72b 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -18,6 +18,7 @@ module Locations (
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
 	gitAnnexJournalDir,
+	gitAnnexJournalLock,
 	isLinkToAnnex,
 	hashDirMixed,
 	hashDirLower,
@@ -109,6 +110,10 @@ gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 gitAnnexJournalDir :: Git.Repo -> FilePath
 gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r  "journal"
 
+{- Lock file for the journal. -}
+gitAnnexJournalLock :: Git.Repo -> FilePath
+gitAnnexJournalLock r = gitAnnexDir r  "journal.lck"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
diff --git a/debian/changelog b/debian/changelog
index 834d2a5726..7554cd5028 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * When displaying a list of repositories, show git remote names
     in addition to their descriptions.
   * Contain the zombie hordes.
+  * Add locking to avoid races when changing the git-annex branch.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
index 87880df417..2485e7b19e 100644
--- a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
+++ b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
@@ -49,3 +49,5 @@ Make Branch.change transactional, so it takes a lock, reads a file,
 applies a function to it, and writes the changed file.
 
 Make Branch.update hold the same lock.
+
+> [[Done]].

From 2636ea79c342f23f28a050bf8ad7f344a05210aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 17:27:48 -0400
Subject: [PATCH 2296/2835] avoid taking journal lock unnecessarily

---
 Branch.hs | 173 ++++++++++++++++++++++++++----------------------------
 1 file changed, 84 insertions(+), 89 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 34486243e3..82ae7029f2 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -18,14 +18,13 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (unless, liftM)
+import Control.Monad (unless, when, liftM, filterM)
 import Control.Monad.State (liftIO)
 import Control.Applicative ((<$>))
 import System.FilePath
 import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
-import Data.Maybe
 import System.IO
 import System.IO.Binary
 import System.Posix.Process
@@ -131,8 +130,10 @@ create = unlessM hasBranch $ do
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = lockJournal $
-	whenM stageJournalFiles $ do
+commit message = do
+	fs <- getJournalFiles
+	when (not $ null fs) $ lockJournal $ do
+		stageJournalFiles fs
 		g <- Annex.gitRepo
 		withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
@@ -141,25 +142,54 @@ commit message = lockJournal $
 update :: Annex ()
 update = do
 	state <- getState
-	unless (branchUpdated state) $ withIndex $ lockJournal $ do
-		{- Since branches get merged into the index, it's important to
-		 - first stage the journal into the index. Otherwise, any
-		 - changes in the journal would later get staged, and might
-		 - overwrite changes made during the merge.
-		 -
-		 - It would be cleaner to handle the merge by updating the
-		 - journal, not the index, with changes from the branches.
-		 -}
-		staged <- stageJournalFiles
-
-		refs <- siblingBranches
-		updated <- catMaybes <$> mapM updateRef refs
-		g <- Annex.gitRepo
-		unless (null updated && not staged) $ liftIO $
-			Git.commit g "update" fullname (fullname:updated)
-
-		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
-		invalidateCache
+	unless (branchUpdated state) $ do
+		-- check what needs updating before taking the lock
+		fs <- getJournalFiles
+		refs <- filterM checkref =<< siblingBranches
+		unless (null fs && null refs) $ withIndex $ lockJournal $ do
+			{- Before refs are merged into the index, it's
+			 - important to first stage the journal into the
+			 - index. Otherwise, any changes in the journal
+			 - would later get staged, and might overwrite
+			 - changes made during the merge.
+			 -
+			 - It would be cleaner to handle the merge by
+			 - updating the journal, not the index, with changes
+			 - from the branches.
+			 -}
+			unless (null fs) $ stageJournalFiles fs
+			mapM_ mergeref refs
+			g <- Annex.gitRepo
+			liftIO $ Git.commit g "update" fullname (fullname:refs)
+			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
+			invalidateCache
+	where
+		checkref ref = do
+			g <- Annex.gitRepo
+			-- checking with log to see if there have been changes
+			-- is less expensive than always merging
+			diffs <- liftIO $ Git.pipeRead g [
+				Param "log",
+				Param (name++".."++ref),
+				Params "--oneline -n1"
+				]
+			return $ not $ L.null diffs
+		mergeref ref = do
+			showSideAction $ "merging " ++
+				Git.refDescribe ref ++ " into " ++ name
+			{- By passing only one ref, it is actually
+			 - merged into the index, preserving any
+			 - changes that may already be staged.
+			 -
+			 - However, any changes in the git-annex
+			 - branch that are *not* reflected in the
+			 - index will be removed. So, documentation
+			 - advises users not to directly modify the
+			 - branch.
+			 -}
+			g <- Annex.gitRepo
+			liftIO $ Git.UnionMerge.merge g [ref]
+			return $ Just ref
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
@@ -188,35 +218,6 @@ siblingBranches = do
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 	return $ map (last . words . L.unpack) (L.lines r)
 
-{- Ensures that a given ref has been merged into the index. -}
-updateRef :: GitRef -> Annex (Maybe String)
-updateRef ref
-	| ref == fullname = return Nothing
-	| otherwise = do
-		g <- Annex.gitRepo
-		-- checking with log to see if there have been changes
-		-- is less expensive than always merging
-		diffs <- liftIO $ Git.pipeRead g [
-			Param "log",
-			Param (name++".."++ref),
-			Params "--oneline -n1"
-			]
-		if L.null diffs
-			then return Nothing
-			else do
-				showSideAction $ "merging " ++ Git.refDescribe ref ++ " into " ++ name
-				-- By passing only one ref, it is actually
-				-- merged into the index, preserving any
-				-- changes that may already be staged.
-				--
-				-- However, any changes in the git-annex
-				-- branch that are *not* reflected in the
-				-- index will be removed. So, documentation
-				-- advises users not to directly modify the
-				-- branch.
-				liftIO $ Git.UnionMerge.merge g [ref]
-				return $ Just ref
-
 {- Applies a function to modifiy the content of a file. -}
 change :: FilePath -> (String -> String) -> Annex ()
 change file a = lockJournal $ get file >>= return . a >>= set file
@@ -253,7 +254,7 @@ files = withIndexUpdate $ do
 	g <- Annex.gitRepo
 	bfiles <- liftIO $ Git.pipeNullSplit g
 		[Params "ls-tree --name-only -r -z", Param fullname]
-	jfiles <- getJournalFiles
+	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
 
 {- Records content for a file in the branch to the journal.
@@ -282,49 +283,43 @@ getJournalFile file = do
 	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)
 		(const $ return Nothing)
 
-{- List of journal files. -}
-getJournalFiles :: Annex [FilePath]
-getJournalFiles = map fileJournal <$> getJournalFilesRaw
+{- List of files that have updated content in the journal. -}
+getJournalledFiles :: Annex [FilePath]
+getJournalledFiles = map fileJournal <$> getJournalFiles
 
-getJournalFilesRaw :: Annex [FilePath]
-getJournalFilesRaw = do
+{- List of existing journal files. -}
+getJournalFiles :: Annex [FilePath]
+getJournalFiles = do
 	g <- Annex.gitRepo
 	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
 		(const $ return [])
 	return $ filter (`notElem` [".", ".."]) fs
 
-{- Stages all journal files into the index, and returns True if the index
- - was modified. -}
-stageJournalFiles :: Annex Bool
-stageJournalFiles = do
-	l <- getJournalFilesRaw
-	if null l
-		then return False
-		else do
-			g <- Annex.gitRepo
-			withIndex $ liftIO $ stage g l
-			return True
-	where
-		stage g fs = do
-			let dir = gitAnnexJournalDir g
-			let paths = map (dir ) fs
-			-- inject all the journal files directly into git
-			-- in one quick command
-			(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $
-				Git.gitCommandLine g [Param "hash-object", Param "-w", Param "--stdin-paths"]
-			_ <- forkProcess $ do
-				hPutStr toh $ unlines paths
-				hClose toh
-				exitSuccess
+{- Stages the specified journalfiles. -}
+stageJournalFiles :: [FilePath] -> Annex ()
+stageJournalFiles fs = do
+	g <- Annex.gitRepo
+	withIndex $ liftIO $ do
+		let dir = gitAnnexJournalDir g
+		let paths = map (dir ) fs
+		-- inject all the journal files directly into git
+		-- in one quick command
+		(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $
+			Git.gitCommandLine g [Param "hash-object", Param "-w", Param "--stdin-paths"]
+		_ <- forkProcess $ do
+			hPutStr toh $ unlines paths
 			hClose toh
-			s <- hGetContents fromh
-			-- update the index, also in just one command
-			Git.UnionMerge.update_index g $
-				index_lines (lines s) $ map fileJournal fs
-			hClose fromh
-			forceSuccess pid
-			mapM_ removeFile paths
-		index_lines shas fs = map genline $ zip shas fs
+			exitSuccess
+		hClose toh
+		s <- hGetContents fromh
+		-- update the index, also in just one command
+		Git.UnionMerge.update_index g $
+			index_lines (lines s) $ map fileJournal fs
+		hClose fromh
+		forceSuccess pid
+		mapM_ removeFile paths
+	where
+		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
 
 {- Produces a filename to use in the journal for a file on the branch.

From 003a604a6e48a8a0ffd1564e3399b54e8c673e92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 18:20:29 -0400
Subject: [PATCH 2297/2835] drop the lock on error

---
 Branch.hs | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 82ae7029f2..f1ba97c941 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -32,6 +32,8 @@ import System.Posix.IO
 import System.Posix.Files
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
+import Control.Monad.IO.Control (liftIOOp)
+import qualified Control.Exception.Base
 
 import Types.BranchState
 import qualified Git
@@ -345,8 +347,12 @@ fileJournal = replace "//" "_" . replace "_" "/"
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
 	g <- Annex.gitRepo
-	h <- liftIO $ createFile (gitAnnexJournalLock g) stdFileMode
-	liftIO $ waitToSetLock h (WriteLock, AbsoluteSeek, 0, 0)
-	r <- a
-	liftIO $ closeFd h
-	return r
+	let file = gitAnnexJournalLock g
+	liftIOOp (Control.Exception.Base.bracket (lock file) unlock) run
+	where
+		lock file = do
+			l <- createFile file stdFileMode
+			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
+			return l
+		unlock = closeFd
+		run _ = a

From 8ef2095fa00408ce6729596a42bc0abdc7778098 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 22:24:57 -0400
Subject: [PATCH 2298/2835] factor out common imports

no code changes
---
 Annex.hs                     |  6 ++---
 AnnexCommon.hs               | 13 ++++++++++
 AnnexQueue.hs                |  6 +----
 Backend.hs                   | 12 +++------
 Backend/SHA.hs               | 16 ++----------
 Backend/URL.hs               |  2 +-
 Backend/WORM.hs              |  6 +----
 Branch.hs                    | 50 ++++++++++++------------------------
 CatFile.hs                   |  6 ++---
 CmdLine.hs                   |  5 +---
 Command.hs                   | 27 ++++++-------------
 Command/Add.hs               | 13 ++--------
 Command/AddUrl.hs            | 11 ++------
 Command/ConfigList.hs        |  6 ++---
 Command/Describe.hs          |  2 +-
 Command/Drop.hs              |  8 +++---
 Command/DropKey.hs           |  3 +--
 Command/DropUnused.hs        | 12 +++------
 Command/Find.hs              |  4 +--
 Command/Fix.hs               |  8 +-----
 Command/FromKey.hs           |  9 +------
 Command/Fsck.hs              | 15 +++--------
 Command/Get.hs               |  3 +--
 Command/InAnnex.hs           |  5 +---
 Command/Init.hs              |  5 ++--
 Command/InitRemote.hs        |  7 +----
 Command/Lock.hs              |  6 +----
 Command/Map.hs               | 11 ++------
 Command/Merge.hs             |  2 +-
 Command/Migrate.hs           | 15 ++---------
 Command/Move.hs              | 12 +++------
 Command/RecvKey.hs           |  6 +----
 Command/Semitrust.hs         |  2 +-
 Command/SendKey.hs           | 11 ++------
 Command/SetKey.hs            |  5 +---
 Command/Status.hs            |  7 +----
 Command/Trust.hs             |  2 +-
 Command/Unannex.hs           | 15 +++--------
 Command/Uninit.hs            |  9 ++-----
 Command/Unlock.hs            | 14 +++-------
 Command/Untrust.hs           |  2 +-
 Command/Unused.hs            | 28 +++++++-------------
 Command/Upgrade.hs           |  2 +-
 Command/Version.hs           |  5 +---
 Command/Whereis.hs           |  5 +---
 Common.hs                    | 46 +++++++++++++++++++++++++++++++++
 Config.hs                    | 15 +++--------
 Content.hs                   | 34 ++++++++----------------
 Crypto.hs                    | 15 +++--------
 Git.hs                       | 16 +-----------
 GitAnnex.hs                  |  6 ++---
 Init.hs                      | 14 +++-------
 Limit.hs                     |  6 ++---
 LocationLog.hs               |  7 +----
 Locations.hs                 |  4 +--
 Messages.hs                  |  4 +--
 Options.hs                   |  3 +--
 PresenceLog.hs               |  4 +--
 Remote.hs                    | 16 ++++--------
 Remote/Bup.hs                | 21 +++------------
 Remote/Directory.hs          | 23 +++++------------
 Remote/Git.hs                | 31 ++++++++--------------
 Remote/Helper/Encryptable.hs |  4 +--
 Remote/Helper/Special.hs     | 11 +++-----
 Remote/Hook.hs               | 20 +++------------
 Remote/Rsync.hs              | 22 +++-------------
 Remote/S3real.hs             | 16 +++---------
 Remote/Web.hs                | 12 ++-------
 RemoteLog.hs                 |  5 +---
 Trust.hs                     |  4 +--
 Types/Key.hs                 |  3 ++-
 UUID.hs                      | 12 +++------
 Upgrade.hs                   |  2 +-
 Upgrade/V0.hs                | 12 ++-------
 Upgrade/V1.hs                | 26 +++++--------------
 Upgrade/V2.hs                | 20 +++------------
 Utility.hs                   |  1 -
 Utility/CopyFile.hs          |  6 ++---
 Version.hs                   |  5 ++--
 git-annex-shell.hs           |  4 +--
 git-annex.cabal              |  2 +-
 git-union-merge.hs           |  4 +--
 test.hs                      |  8 ++----
 83 files changed, 264 insertions(+), 619 deletions(-)
 create mode 100644 AnnexCommon.hs
 create mode 100644 Common.hs

diff --git a/Annex.hs b/Annex.hs
index 8a386a044b..6877537740 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -19,10 +19,10 @@ module Annex (
 	gitRepo
 ) where
 
-import Control.Monad.State
 import Control.Monad.IO.Control
-import Control.Applicative hiding (empty)
+import Control.Monad.State
 
+import Common
 import qualified Git
 import Git.CatFile
 import Git.Queue
@@ -75,7 +75,7 @@ newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
-	, repoqueue = empty
+	, repoqueue = Git.Queue.empty
 	, output = NormalOutput
 	, force = False
 	, fast = False
diff --git a/AnnexCommon.hs b/AnnexCommon.hs
new file mode 100644
index 0000000000..bcdc5e264e
--- /dev/null
+++ b/AnnexCommon.hs
@@ -0,0 +1,13 @@
+module AnnexCommon (
+	module Common,
+	module Types,
+	module Annex,
+	module Locations,
+	module Messages,
+) where
+
+import Common
+import Types
+import Annex (gitRepo)
+import Locations
+import Messages
diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index d155a7b81f..66843a75e3 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -11,13 +11,9 @@ module AnnexQueue (
 	flushWhenFull
 ) where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when, unless)
-
+import AnnexCommon
 import Annex
-import Messages
 import qualified Git.Queue
-import Utility.SafeCommand
 
 {- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
diff --git a/Backend.hs b/Backend.hs
index ca822de5c1..94fe29607e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -16,20 +16,14 @@ module Backend (
 	maybeLookupBackendName
 ) where
 
-import Control.Monad.State (liftIO, when)
-import Control.Applicative
 import System.IO.Error (try)
-import System.FilePath
 import System.Posix.Files
-import Data.Maybe
 
-import Locations
+import AnnexCommon
 import qualified Git
 import qualified Annex
-import Types
 import Types.Key
 import qualified Types.Backend as B
-import Messages
 
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
@@ -59,7 +53,7 @@ orderedList = do
 			Annex.changeState $ \state -> state { Annex.backends = l' }
 			return l'
 		getstandard = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			return $ parseBackendList $
 				Git.configGet g "annex.backends" ""
 
@@ -108,7 +102,7 @@ type BackendFile = (Maybe (Backend Annex), FilePath)
  -}
 chooseBackends :: [FilePath] -> Annex [BackendFile]
 chooseBackends fs = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	forced <- Annex.getState Annex.forcebackend
 	if isJust forced
 		then do
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 15d3fa20d1..0c36ef0dc7 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -7,23 +7,11 @@
 
 module Backend.SHA (backends) where
 
-import Control.Monad.State
-import Data.String.Utils
-import System.Cmd.Utils
-import System.IO
-import System.Directory
-import Data.Maybe
-import System.Posix.Files
-import System.FilePath
-
-import Messages
+import AnnexCommon
 import qualified Annex
-import Locations
 import Content
-import Types
 import Types.Backend
 import Types.Key
-import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
@@ -110,7 +98,7 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 {- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	fast <- Annex.getState Annex.fast
 	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
diff --git a/Backend/URL.hs b/Backend/URL.hs
index f20aa1f95e..0745de455d 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,9 +10,9 @@ module Backend.URL (
 	fromUrl
 ) where
 
+import AnnexCommon
 import Types.Backend
 import Types.Key
-import Types
 
 backends :: [Backend Annex]
 backends = [backend]
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 831c9e8ced..80c652558e 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -7,12 +7,8 @@
 
 module Backend.WORM (backends) where
 
-import Control.Monad.State
-import System.FilePath
-import System.Posix.Files
-
+import AnnexCommon
 import Types.Backend
-import Types
 import Types.Key
 
 backends :: [Backend Annex]
diff --git a/Branch.hs b/Branch.hs
index f1ba97c941..3f1153c098 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -18,33 +18,17 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (unless, when, liftM, filterM)
-import Control.Monad.State (liftIO)
-import Control.Applicative ((<$>))
-import System.FilePath
-import System.Directory
-import Data.String.Utils
-import System.Cmd.Utils
-import System.IO
 import System.IO.Binary
-import System.Posix.Process
-import System.Posix.IO
-import System.Posix.Files
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 import Control.Monad.IO.Control (liftIOOp)
-import qualified Control.Exception.Base
+import qualified Control.Exception
 
+import AnnexCommon
 import Types.BranchState
 import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
-import Utility
-import Utility.Conditional
-import Utility.SafeCommand
-import Types
-import Messages
-import Locations
 import CatFile
 
 type GitRef = String
@@ -79,7 +63,7 @@ withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = index g
 	reset <- liftIO $ Git.useIndex f
 
@@ -123,7 +107,7 @@ getCache file = getState >>= handle
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
 create = unlessM hasBranch $ do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	e <- hasOrigin
 	if e
 		then liftIO $ Git.run g "branch" [Param name, Param originname]
@@ -136,7 +120,7 @@ commit message = do
 	fs <- getJournalFiles
 	when (not $ null fs) $ lockJournal $ do
 		stageJournalFiles fs
-		g <- Annex.gitRepo
+		g <- gitRepo
 		withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
@@ -161,13 +145,13 @@ update = do
 			 -}
 			unless (null fs) $ stageJournalFiles fs
 			mapM_ mergeref refs
-			g <- Annex.gitRepo
+			g <- gitRepo
 			liftIO $ Git.commit g "update" fullname (fullname:refs)
 			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 			invalidateCache
 	where
 		checkref ref = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			-- checking with log to see if there have been changes
 			-- is less expensive than always merging
 			diffs <- liftIO $ Git.pipeRead g [
@@ -189,14 +173,14 @@ update = do
 			 - advises users not to directly modify the
 			 - branch.
 			 -}
-			g <- Annex.gitRepo
+			g <- gitRepo
 			liftIO $ Git.UnionMerge.merge g [ref]
 			return $ Just ref
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
 refExists ref = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ Git.runBool g "show-ref"
 		[Param "--verify", Param "-q", Param ref]
 
@@ -216,7 +200,7 @@ hasSomeBranch = not . null <$> siblingBranches
  - from remotes. -}
 siblingBranches :: Annex [String]
 siblingBranches = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 	return $ map (last . words . L.unpack) (L.lines r)
 
@@ -253,7 +237,7 @@ get file = do
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	bfiles <- liftIO $ Git.pipeNullSplit g
 		[Params "ls-tree --name-only -r -z", Param fullname]
 	jfiles <- getJournalledFiles
@@ -265,7 +249,7 @@ files = withIndexUpdate $ do
  - avoids git needing to rewrite the index after every change. -}
 setJournalFile :: FilePath -> String -> Annex ()
 setJournalFile file content = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ catch (write g) $ const $ do
 		createDirectoryIfMissing True $ gitAnnexJournalDir g
 		createDirectoryIfMissing True $ gitAnnexTmpDir g
@@ -281,7 +265,7 @@ setJournalFile file content = do
 {- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
 getJournalFile file = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)
 		(const $ return Nothing)
 
@@ -292,7 +276,7 @@ getJournalledFiles = map fileJournal <$> getJournalFiles
 {- List of existing journal files. -}
 getJournalFiles :: Annex [FilePath]
 getJournalFiles = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
 		(const $ return [])
 	return $ filter (`notElem` [".", ".."]) fs
@@ -300,7 +284,7 @@ getJournalFiles = do
 {- Stages the specified journalfiles. -}
 stageJournalFiles :: [FilePath] -> Annex ()
 stageJournalFiles fs = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	withIndex $ liftIO $ do
 		let dir = gitAnnexJournalDir g
 		let paths = map (dir ) fs
@@ -346,9 +330,9 @@ fileJournal = replace "//" "_" . replace "_" "/"
  - contention with other git-annex processes. -}
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let file = gitAnnexJournalLock g
-	liftIOOp (Control.Exception.Base.bracket (lock file) unlock) run
+	liftIOOp (Control.Exception.bracket (lock file) unlock) run
 	where
 		lock file = do
 			l <- createFile file stdFileMode
diff --git a/CatFile.hs b/CatFile.hs
index 0eb1e74f6b..8762109e78 100644
--- a/CatFile.hs
+++ b/CatFile.hs
@@ -9,17 +9,15 @@ module CatFile (
 	catFile
 ) where
 
-import Control.Monad.State
-
+import AnnexCommon
 import qualified Git.CatFile
-import Types
 import qualified Annex
 
 catFile :: String -> FilePath -> Annex String
 catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
 	where
 		startup = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			h <- liftIO $ Git.CatFile.catFileStart g
 			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
 			go h
diff --git a/CmdLine.hs b/CmdLine.hs
index 38d4754da9..34adb25569 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -13,17 +13,14 @@ module CmdLine (
 
 import System.IO.Error (try)
 import System.Console.GetOpt
-import Control.Monad.State (liftIO)
-import Control.Monad (when)
 
+import AnnexCommon
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
 import Content
-import Types
 import Command
 import Options
-import Messages
 import Init
 
 {- Runs the passed command line. -}
diff --git a/Command.hs b/Command.hs
index c061c7c464..20f3d79b62 100644
--- a/Command.hs
+++ b/Command.hs
@@ -7,22 +7,11 @@
 
 module Command where
 
-import Control.Monad.State (liftIO)
-import System.Directory
-import System.Posix.Files
-import Control.Monad (filterM, liftM)
-import Control.Applicative
-import Data.Maybe
-
-import Types
+import AnnexCommon
 import qualified Backend
-import Messages
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
-import Utility
-import Utility.Conditional
-import Utility.Path
 import Types.Key
 import Trust
 import LocationLog
@@ -98,7 +87,7 @@ isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
-	whenM (Git.repoIsLocalBare <$> Annex.gitRepo) $
+	whenM (Git.repoIsLocalBare <$> gitRepo) $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
@@ -106,11 +95,11 @@ notBareRepo a = do
    user's parameters, and prepare actions operating on them. -}
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	runFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	runFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
 withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
@@ -119,7 +108,7 @@ withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 		go (file, v) = a file (readMaybe v)
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	backendPairs a files
 withFilesMissing :: (String -> CommandStart) -> CommandSeek
@@ -128,7 +117,7 @@ withFilesMissing a params = runFiltered a $ liftIO $ filterM missing params
 		missing = liftM not . doesFileExist
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	force <- Annex.getState Annex.force
 	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
 	backendPairs a newfiles
@@ -138,7 +127,7 @@ withStrings :: (String -> CommandStart) -> CommandSeek
 withStrings a params = return $ map a params
 withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
 withFilesToBeCommitted a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	runFiltered a $
 		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
 withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
@@ -148,7 +137,7 @@ withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
diff --git a/Command/Add.hs b/Command/Add.hs
index 4b2ef24cd9..c66c381311 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -7,26 +7,17 @@
 
 module Command.Add where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when)
-import System.Posix.Files
-import System.Directory
 import Control.Exception.Control (handle)
 import Control.Exception.Base (throwIO)
-import Control.Exception.Extensible (IOException)
 
+import AnnexCommon
 import Command
 import qualified Annex
 import qualified AnnexQueue
 import qualified Backend
 import LocationLog
-import Types
 import Content
-import Messages
-import Utility.Conditional
 import Utility.Touch
-import Utility.SafeCommand
-import Locations
 import Backend
 
 command :: [Command]
@@ -72,7 +63,7 @@ undo file key e = do
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			liftIO $ renameFile (gitAnnexLocation g key) file
 
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 2e9e04fd3e..ce6e70699c 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -7,12 +7,9 @@
 
 module Command.AddUrl where
 
-import Control.Monad.State
 import Network.URI
-import Data.String.Utils
-import Data.Maybe
-import System.Directory
 
+import AnnexCommon
 import Command
 import qualified Backend
 import qualified Utility.Url as Url
@@ -20,12 +17,8 @@ import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
-import Messages
 import Content
 import PresenceLog
-import Locations
-import Utility.Path
-import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "addurl" (paramRepeating paramUrl) seek
@@ -51,7 +44,7 @@ perform url file = do
 
 download :: String -> FilePath -> CommandPerform
 download url file = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	showAction $ "downloading " ++ url ++ " "
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 3de26c8925..c38539fa0b 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -7,9 +7,7 @@
 
 module Command.ConfigList where
 
-import Control.Monad.State (liftIO)
-
-import Annex
+import AnnexCommon
 import Command
 import UUID
 
@@ -22,7 +20,7 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	liftIO $ putStrLn $ "annex.uuid=" ++ u
 	stop
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 8d2f9071b0..b1c144872b 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -7,10 +7,10 @@
 
 module Command.Describe where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import UUID
-import Messages
 
 command :: [Command]
 command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 4a7596921f..7210184f88 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -7,14 +7,12 @@
 
 module Command.Drop where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import qualified Annex
 import LocationLog
-import Types
 import Content
-import Messages
-import Utility.Conditional
 import Trust
 import Config
 
@@ -71,9 +69,9 @@ dropKey key numcopiesM = do
 			| length have >= need = return True
 			| otherwise = do
 				let u = Remote.uuid r
-				let dup = u `elem` have
+				let duplicate = u `elem` have
 				haskey <- Remote.hasKey r key
-				case (dup, haskey) of
+				case (duplicate, haskey) of
 					(False, Right True)	-> findcopies need (u:have) rs bad
 					(False, Left _)		-> findcopies need have rs (r:bad)
 					_			-> findcopies need have rs bad
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index b9938585e9..7ead1c4bca 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -7,12 +7,11 @@
 
 module Command.DropKey where
 
+import AnnexCommon
 import Command
 import qualified Annex
 import LocationLog
-import Types
 import Content
-import Messages
 
 command :: [Command]
 command = [repoCommand "dropkey" (paramRepeating paramKey) seek
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 90fea050e8..ed4f71e7e4 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -7,22 +7,16 @@
 
 module Command.DropUnused where
 
-import Control.Monad.State (liftIO)
 import qualified Data.Map as M
-import System.Directory
-import Data.Maybe
 
+import AnnexCommon
 import Command
-import Types
-import Messages
-import Locations
 import qualified Annex
 import qualified Command.Drop
 import qualified Command.Move
 import qualified Remote
 import qualified Git
 import Types.Key
-import Utility.Conditional
 
 type UnusedMap = M.Map String Key
 
@@ -67,14 +61,14 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 
 performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = filespec g key
 	liftIO $ whenM (doesFileExist f) $ removeFile f
 	next $ return True
 
 readUnusedLog :: FilePath -> Annex UnusedMap
 readUnusedLog prefix = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = gitAnnexUnusedLog prefix g
 	e <- liftIO $ doesFileExist f
 	if e
diff --git a/Command/Find.hs b/Command/Find.hs
index effb331840..8d80659d0d 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,11 +7,9 @@
 
 module Command.Find where
 
-import Control.Monad.State
-
+import AnnexCommon
 import Command
 import Content
-import Utility.Conditional
 import Limit
 
 command :: [Command]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 481da52f2b..a66a1c44a7 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -7,16 +7,10 @@
 
 module Command.Fix where
 
-import Control.Monad.State (liftIO)
-import System.Posix.Files
-import System.Directory
-
+import AnnexCommon
 import Command
 import qualified AnnexQueue
-import Utility.Path
-import Utility.SafeCommand
 import Content
-import Messages
 
 command :: [Command]
 command = [repoCommand "fix" paramPaths seek
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 9ff126a45e..e60025bf7e 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -7,18 +7,11 @@
 
 module Command.FromKey where
 
-import Control.Monad.State (liftIO)
-import System.Posix.Files
-import System.Directory
-import Control.Monad (unless)
-
+import AnnexCommon
 import Command
 import qualified AnnexQueue
-import Utility.SafeCommand
 import Content
-import Messages
 import Types.Key
-import Utility.Path
 
 command :: [Command]
 command = [repoCommand "fromkey" paramPath seek
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0c58add6a3..33a8405a69 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,25 +7,16 @@
 
 module Command.Fsck where
 
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
-import System.Directory
-import System.Posix.Files
-
+import AnnexCommon
 import Command
-import qualified Annex
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
 import UUID
-import Types
-import Messages
 import Content
 import LocationLog
-import Locations
 import Trust
 import Utility.DataUnits
-import Utility.Path
 import Utility.FileMode
 import Config
 
@@ -54,7 +45,7 @@ perform key file backend numcopies = do
    in this repository only. -}
 verifyLocationLog :: Key -> FilePath -> Annex Bool
 verifyLocationLog key file = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	present <- inAnnex key
 	
 	-- Since we're checking that a key's file is present, throw
@@ -98,7 +89,7 @@ fsckKey backend key file numcopies = do
  - the key's metadata, if available. -}
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
 	case (present, Types.Key.keySize key) of
diff --git a/Command/Get.hs b/Command/Get.hs
index 4fd654f63e..34f56aa2d5 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -7,12 +7,11 @@
 
 module Command.Get where
 
+import AnnexCommon
 import Command
 import qualified Annex
 import qualified Remote
-import Types
 import Content
-import Messages
 import qualified Command.Move
 
 command :: [Command]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 713492c2fe..36b6d40e6e 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -7,12 +7,9 @@
 
 module Command.InAnnex where
 
-import Control.Monad.State (liftIO)
-import System.Exit
-
+import AnnexCommon
 import Command
 import Content
-import Types
 
 command :: [Command]
 command = [repoCommand "inannex" (paramRepeating paramKey) seek
diff --git a/Command/Init.hs b/Command/Init.hs
index 2351763a95..f3d8834ba1 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,10 +7,9 @@
 
 module Command.Init where
 
+import AnnexCommon
 import Command
-import qualified Annex
 import UUID
-import Messages
 import Init
 	
 command :: [Command]
@@ -30,7 +29,7 @@ start ws = do
 perform :: String -> CommandPerform
 perform description = do
 	initialize
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	describeUUID u description
 	next $ return True
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index c6d9f52003..2ce86e9c6b 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -8,18 +8,13 @@
 module Command.InitRemote where
 
 import qualified Data.Map as M
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
-import Data.Maybe
-import Data.String.Utils
 
+import AnnexCommon
 import Command
 import qualified Remote
 import qualified RemoteLog
 import qualified Types.Remote as R
-import Types
 import UUID
-import Messages
 
 command :: [Command]
 command = [repoCommand "initremote"
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 04d1bb94d6..af7b92ad68 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -7,13 +7,9 @@
 
 module Command.Lock where
 
-import Control.Monad.State (liftIO)
-import System.Directory
-
+import AnnexCommon
 import Command
-import Messages
 import qualified AnnexQueue
-import Utility.SafeCommand
 import Backend
 	
 command :: [Command]
diff --git a/Command/Map.hs b/Command/Map.hs
index 7e23da774d..8e63f6dd6a 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -7,19 +7,12 @@
 
 module Command.Map where
 
-import Control.Monad.State (liftIO)
 import Control.Exception.Extensible
-import System.Cmd.Utils
 import qualified Data.Map as M
-import Data.List.Utils
-import Data.Maybe
 
+import AnnexCommon
 import Command
-import qualified Annex
 import qualified Git
-import Messages
-import Types
-import Utility.SafeCommand
 import UUID
 import Trust
 import Utility.Ssh
@@ -36,7 +29,7 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	rs <- spider g
 
 	umap <- uuidMap
diff --git a/Command/Merge.hs b/Command/Merge.hs
index 832cde5127..b365e0e0cd 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -7,9 +7,9 @@
 
 module Command.Merge where
 
+import AnnexCommon
 import Command
 import qualified Branch
-import Messages
 
 command :: [Command]
 command = [repoCommand "merge" paramNothing seek
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 054db6e27b..24f23baf5c 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -7,22 +7,11 @@
 
 module Command.Migrate where
 
-import Control.Monad.State (liftIO)
-import Control.Applicative
-import System.Posix.Files
-import System.Directory
-import System.FilePath
-import Data.Maybe
-
+import AnnexCommon
 import Command
-import qualified Annex
 import qualified Backend
 import qualified Types.Key
-import Locations
-import Types
 import Content
-import Messages
-import Utility.Conditional
 import qualified Command.Add
 import Backend
 
@@ -53,7 +42,7 @@ upgradableKey key = isNothing $ Types.Key.keySize key
 
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 
 	-- Store the old backend's cached key in the new backend
 	-- (the file can't be stored as usual, because it's already a symlink).
diff --git a/Command/Move.hs b/Command/Move.hs
index 15dae39385..d2870b1e42 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -7,18 +7,14 @@
 
 module Command.Move where
 
-import Control.Monad (when)
-
+import AnnexCommon
 import Command
 import qualified Command.Drop
 import qualified Annex
 import LocationLog
-import Types
 import Content
 import qualified Remote
 import UUID
-import Messages
-import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "move" paramPaths seek
@@ -60,7 +56,7 @@ showMoveAction False file = showStart "copy" file
 remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()
 remoteHasKey remote key present	= do
 	let remoteuuid = Remote.uuid remote
-	g <- Annex.gitRepo
+	g <- gitRepo
 	logChange g key remoteuuid status
 	where
 		status = if present then InfoPresent else InfoMissing
@@ -76,7 +72,7 @@ remoteHasKey remote key present	= do
  -}
 toStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 toStart dest move file = isAnnexed file $ \(key, _) -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	ishere <- inAnnex key
 	if not ishere || u == Remote.uuid dest
@@ -126,7 +122,7 @@ toCleanup dest move key = do
  -}
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 fromStart src move file = isAnnexed file $ \(key, _) -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	remotes <- Remote.keyPossibilities key
 	if u == Remote.uuid src || not (any (== src) remotes)
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 33792e5b6e..400e81102b 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -7,15 +7,11 @@
 
 module Command.RecvKey where
 
-import Control.Monad.State (liftIO)
-import System.Exit
-
+import AnnexCommon
 import Command
 import CmdLine
 import Content
 import Utility.RsyncFile
-import Utility.Conditional
-import Types
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 3b12bb747d..236ba28794 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -7,11 +7,11 @@
 
 module Command.Semitrust where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import UUID
 import Trust
-import Messages
 
 command :: [Command]
 command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 98d2573380..f397d9ae6a 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -7,17 +7,10 @@
 
 module Command.SendKey where
 
-import Control.Monad.State (liftIO)
-import System.Exit
-
-import Locations
-import qualified Annex
+import AnnexCommon
 import Command
 import Content
 import Utility.RsyncFile
-import Utility.Conditional
-import Messages
-import Types
 
 command :: [Command]
 command = [repoCommand "sendkey" paramKey seek
@@ -28,7 +21,7 @@ seek = [withKeys start]
 
 start :: Key -> CommandStart
 start key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let file = gitAnnexLocation g key
 	whenM (inAnnex key) $
 		liftIO $ rsyncServerSend file -- does not return
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index c03c5d0445..12ef5b74a5 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -7,13 +7,10 @@
 
 module Command.SetKey where
 
-import Control.Monad.State (liftIO)
-
+import AnnexCommon
 import Command
-import Utility.SafeCommand
 import LocationLog
 import Content
-import Messages
 
 command :: [Command]
 command = [repoCommand "setkey" paramPath seek
diff --git a/Command/Status.hs b/Command/Status.hs
index 07c0958bbe..de49f84d5a 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -8,25 +8,20 @@
 module Command.Status where
 
 import Control.Monad.State
-import Control.Applicative
-import Data.Maybe
-import System.IO
-import Data.List
 import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
 
+import AnnexCommon
 import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
 import qualified Command.Unused
 import qualified Git
 import Command
-import Types
 import Utility.DataUnits
 import Content
 import Types.Key
-import Locations
 import Backend
 import UUID
 import Remote
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 5e25b519bc..04c68a5d32 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -7,11 +7,11 @@
 
 module Command.Trust where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import Trust
 import UUID
-import Messages
 
 command :: [Command]
 command = [repoCommand "trust" (paramRepeating paramRemote) seek
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 4d4281eb07..c5c5e90a67 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -7,25 +7,16 @@
 
 module Command.Unannex where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (unless)
-import System.Directory
-import System.Posix.Files
-
+import AnnexCommon
 import Command
 import qualified Command.Drop
 import qualified Annex
 import qualified AnnexQueue
-import Utility.SafeCommand
-import Utility.Path
 import Utility.FileMode
 import LocationLog
-import Types
 import Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
-import Messages
-import Locations
 
 command :: [Command]
 command = [repoCommand "unannex" paramPaths seek "undo accidential add command"]
@@ -41,7 +32,7 @@ start file = isAnnexed file $ \(key, _) -> do
 		then do
 			force <- Annex.getState Annex.force
 			unless force $ do
-				g <- Annex.gitRepo
+				g <- gitRepo
 				staged <- liftIO $ LsFiles.staged g [Git.workTree g]
 				unless (null staged) $
 					error "This command cannot be run when there are already files staged for commit."
@@ -60,7 +51,7 @@ perform file key = do
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 
 	liftIO $ removeFile file
 	liftIO $ Git.run g "rm" [Params "--quiet --", File file]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index ce12665424..3ba7a7cf33 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,19 +7,14 @@
 
 module Command.Uninit where
 
-import Control.Monad.State (liftIO)
-import System.Directory
-import System.Exit
-
+import AnnexCommon
 import Command
-import Utility.SafeCommand
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
 import Init
 import qualified Branch
 import Content
-import Locations
 
 command :: [Command]
 command = [repoCommand "uninit" paramPaths seek 
@@ -44,7 +39,7 @@ perform = next cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 44b92545c8..220d57829b 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -7,18 +7,10 @@
 
 module Command.Unlock where
 
-import Control.Monad.State (liftIO)
-import System.Directory hiding (copyFile)
-
+import AnnexCommon
 import Command
-import qualified Annex
-import Types
-import Messages
-import Locations
 import Content
-import Utility.Conditional
 import Utility.CopyFile
-import Utility.Path
 import Utility.FileMode
 
 command :: [Command]
@@ -43,12 +35,12 @@ perform dest key = do
 	
 	checkDiskSpace key
 
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g key
 	let tmpdest = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showAction "copying"
-	ok <- liftIO $ copyFile src tmpdest
+	ok <- liftIO $ copyFileExternal src tmpdest
         if ok
                 then do
 			liftIO $ do
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 9f7e521987..30ade85ce0 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -7,11 +7,11 @@
 
 module Command.Untrust where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import UUID
 import Trust
-import Messages
 
 command :: [Command]
 command = [repoCommand "untrust" (paramRepeating paramRemote) seek
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 987f367201..1ba4f53019 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -9,23 +9,13 @@
 
 module Command.Unused where
 
-import Control.Monad (filterM, unless, forM_)
-import Control.Monad.State (liftIO)
 import qualified Data.Set as S
-import Data.Maybe
-import System.FilePath
-import System.Directory
-import Data.List
 import qualified Data.ByteString.Lazy.Char8 as L
 
+import AnnexCommon
 import Command
-import Types
 import Content
-import Messages
-import Locations
-import Utility
 import Utility.FileMode
-import Utility.SafeCommand
 import LocationLog
 import qualified Annex
 import qualified Git
@@ -92,7 +82,7 @@ checkRemoteUnused' r = do
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ viaTmp writeFile (gitAnnexUnusedLog prefix g) $
 		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
 
@@ -164,7 +154,7 @@ unusedKeys = do
 excludeReferenced :: [Key] -> Annex [Key]
 excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
 	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
 		(S.fromList l)
@@ -200,7 +190,7 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 {- List of keys referenced by symlinks in the git repo. -}
 getKeysReferenced :: Annex [Key]
 getKeysReferenced = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
@@ -209,7 +199,7 @@ getKeysReferenced = do
 getKeysReferencedInGit :: String -> Annex [Key]
 getKeysReferencedInGit ref = do
 	showAction $ "checking " ++ Git.refDescribe ref
-	g <- Annex.gitRepo
+	g <- gitRepo
 	findkeys [] =<< liftIO (LsTree.lsTree g ref)
 	where
 		findkeys c [] = return c
@@ -232,17 +222,17 @@ staleKeysPrune dirspec present = do
 	contents <- staleKeys dirspec
 	
 	let stale = contents `exclude` present
-	let dup = contents `exclude` stale
+	let dups = contents `exclude` stale
 
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let dir = dirspec g
-	liftIO $ forM_ dup $ \t -> removeFile $ dir  keyFile t
+	liftIO $ forM_ dups $ \t -> removeFile $ dir  keyFile t
 
 	return stale
 
 staleKeys :: (Git.Repo -> FilePath) -> Annex [Key]
 staleKeys dirspec = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let dir = dirspec g
 	exists <- liftIO $ doesDirectoryExist dir
 	if not exists
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 5d9ed92fae..d79f895d8d 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -7,10 +7,10 @@
 
 module Command.Upgrade where
 
+import AnnexCommon
 import Command
 import Upgrade
 import Version
-import Messages
 
 command :: [Command]
 command = [standaloneCommand "upgrade" paramNothing seek
diff --git a/Command/Version.hs b/Command/Version.hs
index af547949c6..1e44fbb0b1 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -7,10 +7,7 @@
 
 module Command.Version where
 
-import Control.Monad.State (liftIO)
-import Data.String.Utils
-import Data.Maybe
-
+import AnnexCommon
 import Command
 import qualified Build.SysConfig as SysConfig
 import Version
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index a414428f72..3fb636c04c 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,13 +7,10 @@
 
 module Command.Whereis where
 
-import Control.Monad
-
+import AnnexCommon
 import LocationLog
 import Command
-import Messages
 import Remote
-import Types
 import Trust
 
 command :: [Command]
diff --git a/Common.hs b/Common.hs
new file mode 100644
index 0000000000..e88342ae4a
--- /dev/null
+++ b/Common.hs
@@ -0,0 +1,46 @@
+module Common (
+	module Control.Monad,
+	module Control.Applicative,
+	module Control.Monad.State,
+	module Control.Exception.Extensible,
+	module Data.Maybe,
+	module Data.List,
+	module Data.String.Utils,
+	module System.Path,
+	module System.FilePath,
+	module System.Directory,
+	module System.Cmd.Utils,
+	module System.IO,
+	module System.Posix.Files,
+	module System.Posix.IO,
+	module System.Posix.Process,
+	module System.Exit,
+	module Utility,
+	module Utility.Conditional,
+	module Utility.SafeCommand,
+	module Utility.Path,
+) where
+
+import Control.Monad hiding (join)
+import Control.Applicative
+import Control.Monad.State (liftIO)
+import Control.Exception.Extensible (IOException)
+
+import Data.Maybe
+import Data.List
+import Data.String.Utils
+
+import System.Path
+import System.FilePath
+import System.Directory
+import System.Cmd.Utils
+import System.IO hiding (FilePath)
+import System.Posix.Files
+import System.Posix.IO
+import System.Posix.Process hiding (executeFile)
+import System.Exit
+
+import Utility
+import Utility.Conditional
+import Utility.SafeCommand
+import Utility.Path
diff --git a/Config.hs b/Config.hs
index fe847fce1c..c0328794e1 100644
--- a/Config.hs
+++ b/Config.hs
@@ -7,23 +7,16 @@
 
 module Config where
 
-import Data.Maybe
-import Control.Monad.State (liftIO)
-import Control.Applicative
-import System.Cmd.Utils
-
+import AnnexCommon
 import qualified Git
 import qualified Annex
-import Types
-import Utility
-import Utility.SafeCommand
 
 type ConfigKey = String
 
 {- Changes a git config setting in both internal state and .git/config -}
 setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ Git.run g "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
 	g' <- liftIO $ Git.configRead g
@@ -33,7 +26,7 @@ setConfig k value = do
  - Failing that, tries looking for a global config option. -}
 getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
 getConfig r key def = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let def' = Git.configGet g ("annex." ++ key) def
 	return $ Git.configGet g (remoteConfig r key) def'
 
@@ -95,7 +88,7 @@ getNumCopies v =
 	where
 		use (Just n) = return n
 		use Nothing = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			return $ read $ Git.configGet g config "1"
 		config = "annex.numcopies"
 
diff --git a/Content.hs b/Content.hs
index f963c48b47..567e4caa5d 100644
--- a/Content.hs
+++ b/Content.hs
@@ -21,26 +21,14 @@ module Content (
 	saveState
 ) where
 
-import System.Directory
-import Control.Monad.State (liftIO)
-import System.Path
-import Control.Monad
-import System.Posix.Files
-import System.FilePath
-import Data.Maybe
-
-import Types
-import Locations
+import AnnexCommon
 import LocationLog
 import UUID
 import qualified Git
 import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
-import Utility
-import Utility.Conditional
 import Utility.StatFS
-import Utility.Path
 import Utility.FileMode
 import Types.Key
 import Utility.DataUnits
@@ -49,14 +37,14 @@ import Config
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
 inAnnex key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo"
 	liftIO $ doesFileExist $ gitAnnexLocation g key
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	cwd <- liftIO getCurrentDirectory
 	let absfile = fromMaybe whoops $ absNormPath cwd file
 	return $ relPathDirToFile (parentDir absfile) 
@@ -68,7 +56,7 @@ calcGitLink file key = do
  - repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	logChange g key u status
 
@@ -77,7 +65,7 @@ logStatus key status = do
  - the annex as a key's content. -}
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let tmp = gitAnnexTmpLocation g key
 
 	-- Check that there is enough free disk space.
@@ -96,7 +84,7 @@ getViaTmp key action = do
 
 prepTmp :: Key -> Annex FilePath
 prepTmp key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let tmp = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	return tmp
@@ -133,7 +121,7 @@ checkDiskSpace = checkDiskSpace' 0
 
 checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	r <- getConfig g "diskreserve" ""
 	let reserve = fromMaybe megabyte $ readSize dataUnits r
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
@@ -174,7 +162,7 @@ checkDiskSpace' adjustment key = do
  -}
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let dest = gitAnnexLocation g key
 	let dir = parentDir dest
 	e <- liftIO $ doesFileExist dest
@@ -189,7 +177,7 @@ moveAnnex key src = do
 
 withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
 withObjectLoc key a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let file = gitAnnexLocation g key
 	let dir = parentDir file
 	a (dir, file)
@@ -213,7 +201,7 @@ fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
  - returns the file it was moved to. -}
 moveBad :: Key -> Annex FilePath
 moveBad key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g key
 	let dest = gitAnnexBadDir g  takeFileName src
 	liftIO $ do
@@ -227,7 +215,7 @@ moveBad key = do
 {- List of keys whose content exists in .git/annex/objects/ -}
 getKeysPresent :: Annex [Key]
 getKeysPresent = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	getKeysPresent' $ gitAnnexObjectDir g
 getKeysPresent' :: FilePath -> Annex [Key]
 getKeysPresent' dir = do
diff --git a/Crypto.hs b/Crypto.hs
index 9b2d73f28d..4cc16b4242 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -30,26 +30,17 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.ByteString.Lazy.UTF8 (fromString)
 import Data.Digest.Pure.SHA
-import System.Cmd.Utils
-import Data.String.Utils
-import Data.List
-import Data.Maybe
-import System.IO
-import System.Posix.IO
 import System.Posix.Types
-import System.Posix.Process
 import Control.Applicative
 import Control.Concurrent
 import Control.Exception (finally)
 import System.Exit
 import System.Environment
 
-import Types
+import AnnexCommon
 import Types.Key
 import Types.Remote
-import Utility
 import Utility.Base64
-import Utility.SafeCommand
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
@@ -97,9 +88,9 @@ updateCipher :: RemoteConfig -> EncryptedCipher -> IO EncryptedCipher
 updateCipher c encipher@(EncryptedCipher _ ks) = do
 	ks' <- configKeyIds c
 	cipher <- decryptCipher c encipher
-	encryptCipher cipher (combine ks ks')
+	encryptCipher cipher (merge ks ks')
 	where
-		combine (KeyIds a) (KeyIds b) = KeyIds $ a ++ b
+		merge (KeyIds a) (KeyIds b) = KeyIds $ a ++ b
 
 describeCipher :: EncryptedCipher -> String
 describeCipher (EncryptedCipher _ (KeyIds ks)) =
diff --git a/Git.hs b/Git.hs
index d32aaaa562..7a3d55deaa 100644
--- a/Git.hs
+++ b/Git.hs
@@ -64,34 +64,20 @@ module Git (
 	prop_idempotent_deencode
 ) where
 
-import Control.Monad (unless, when, liftM2)
-import Control.Applicative
-import System.Directory
-import System.FilePath
 import System.Posix.Directory
 import System.Posix.User
-import System.Posix.Process
-import System.Path
-import System.Cmd.Utils
 import IO (bracket_, try)
-import Data.String.Utils
-import System.IO
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
-import Data.Maybe
 import Data.Char
 import Data.Word (Word8)
 import Codec.Binary.UTF8.String (encode)
 import Text.Printf
-import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import Utility
-import Utility.Path
-import Utility.Conditional
-import Utility.SafeCommand
+import Common
 
 {- There are two types of repositories; those on local disk and those
  - accessed via an URL. -}
diff --git a/GitAnnex.hs b/GitAnnex.hs
index a9d469b44e..fcfbf44b4b 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -8,14 +8,12 @@
 module GitAnnex where
 
 import System.Console.GetOpt
-import Control.Monad.State (liftIO)
 
+import AnnexCommon
 import qualified Git
 import CmdLine
 import Command
 import Options
-import Utility
-import Types
 import Types.TrustLevel
 import qualified Annex
 import qualified Remote
@@ -122,7 +120,7 @@ options = commonOptions ++
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			g' <- liftIO $ Git.configStore g v
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 
diff --git a/Init.hs b/Init.hs
index 4df1599333..57eaf39d20 100644
--- a/Init.hs
+++ b/Init.hs
@@ -11,18 +11,10 @@ module Init (
 	uninitialize
 ) where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (unless)
-import System.Directory
-
-import qualified Annex
+import AnnexCommon
 import qualified Git
 import qualified Branch
 import Version
-import Messages
-import Types
-import Utility
-import Utility.Conditional
 import UUID
 
 initialize :: Annex ()
@@ -73,12 +65,12 @@ gitPreCommitHookUnWrite = unlessBare $ do
 
 unlessBare :: Annex () -> Annex ()
 unlessBare a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	unless (Git.repoIsLocalBare g) a
 
 preCommitHook :: Annex FilePath
 preCommitHook = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	return $ Git.gitDir g ++ "/hooks/pre-commit"
 
 preCommitScript :: String
diff --git a/Limit.hs b/Limit.hs
index 10fc0ea6ca..334ae325d6 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -9,15 +9,13 @@ module Limit where
 
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
-import Control.Applicative
-import Data.Maybe
 
-import Annex
+import AnnexCommon
+import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
 import LocationLog
-import Utility
 import Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
diff --git a/LocationLog.hs b/LocationLog.hs
index 0cdf88bc6b..759bee830e 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -21,15 +21,10 @@ module LocationLog (
 	logFileKey
 ) where
 
-import System.FilePath
-import Control.Applicative
-import Data.Maybe
-
+import AnnexCommon
 import qualified Git
 import qualified Branch
 import UUID
-import Types
-import Locations
 import PresenceLog
 
 {- Log a change in the presence of a key's value in a repository. -}
diff --git a/Locations.hs b/Locations.hs
index b18444e72b..4579fe05b7 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -26,13 +26,11 @@ module Locations (
 	prop_idempotent_fileKey
 ) where
 
-import System.FilePath
-import Data.String.Utils
-import Data.List
 import Bits
 import Word
 import Data.Hash.MD5
 
+import Common
 import Types
 import Types.Key
 import qualified Git
diff --git a/Messages.hs b/Messages.hs
index c663c17c20..e029c50722 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -23,11 +23,9 @@ module Messages (
 	setupConsole
 ) where
 
-import Control.Monad.State (liftIO)
-import System.IO
-import Data.String.Utils
 import Text.JSON
 
+import Common
 import Types
 import qualified Annex
 import qualified Messages.JSON as JSON
diff --git a/Options.hs b/Options.hs
index b5eaf98cd8..9d60292002 100644
--- a/Options.hs
+++ b/Options.hs
@@ -9,10 +9,9 @@ module Options where
 
 import System.Console.GetOpt
 import System.Log.Logger
-import Control.Monad.State (liftIO)
 
+import AnnexCommon
 import qualified Annex
-import Types
 import Command
 import Limit
 
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 2db1ee59b4..23c2882571 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -26,11 +26,9 @@ import Data.Time.Clock.POSIX
 import Data.Time
 import System.Locale
 import qualified Data.Map as M
-import Control.Monad.State (liftIO)
-import Control.Applicative
 
+import AnnexCommon
 import qualified Branch
-import Types
 
 data LogLine = LogLine {
 	date :: POSIXTime,
diff --git a/Remote.hs b/Remote.hs
index efa7a5cc8e..2371b7bf2d 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -28,23 +28,17 @@ module Remote (
 	forceTrust
 ) where
 
-import Control.Monad.State (filterM)
-import Data.List
 import qualified Data.Map as M
-import Data.String.Utils
-import Data.Maybe
-import Control.Applicative
 import Text.JSON
 import Text.JSON.Generic
 
-import Types
+import AnnexCommon
 import Types.Remote
 import UUID
 import qualified Annex
 import Config
 import Trust
 import LocationLog
-import Messages
 import RemoteLog
 
 import qualified Remote.Git
@@ -110,7 +104,7 @@ byName' n = do
  - and returns its UUID. Finds even remotes that are not configured in
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
-nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
+nameToUUID "." = getUUID =<< gitRepo -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
@@ -135,7 +129,7 @@ nameToUUID n = do
  - of the UUIDs. -}
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
-	here <- getUUID =<< Annex.gitRepo
+	here <- getUUID =<< gitRepo
 	m <- M.unionWith addname <$> uuidMap <*> remoteMap
 	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
@@ -184,7 +178,7 @@ keyPossibilitiesTrusted = keyPossibilities' True
 
 keyPossibilities' :: Bool -> Key -> Annex ([Remote Annex], [UUID])
 keyPossibilities' withtrusted key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	trusted <- if withtrusted then trustGet Trusted else return []
 
@@ -204,7 +198,7 @@ keyPossibilities' withtrusted key = do
 {- Displays known locations of a key. -}
 showLocations :: Key -> [UUID] -> Annex ()
 showLocations key exclude = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	uuids <- keyLocations key
 	untrusteduuids <- trustGet UnTrusted
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 29c7a0419f..9588310194 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -8,30 +8,15 @@
 module Remote.Bup (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import System.IO
 import System.IO.Error
-import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
 import System.Process
-import System.Exit
-import System.FilePath
-import Data.Maybe
-import Data.List.Utils
-import System.Cmd.Utils
 
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Locations
 import Config
-import Utility
-import Utility.Conditional
-import Utility.SafeCommand
-import Messages
 import Utility.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
@@ -118,14 +103,14 @@ bupSplitParams r buprepo k src = do
 
 store :: Git.Repo -> BupRepo -> Key -> Annex Bool
 store r buprepo k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g k
 	params <- bupSplitParams r buprepo k (File src)
 	liftIO $ boolSystem "bup" params
 
 storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r buprepo (cipher, enck) k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g k
 	params <- bupSplitParams r buprepo enck (Param "-")
 	liftIO $ catchBool $
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 18835c5de0..664f8ca5fe 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -9,25 +9,14 @@ module Remote.Directory (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
 import System.IO.Error
-import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
-import System.Directory hiding (copyFile)
-import System.FilePath
-import Data.Maybe
 
-import Types
+import AnnexCommon
+import Utility.CopyFile
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Locations
-import Utility.CopyFile
 import Config
-import Utility
-import Utility.Conditional
-import Utility.Path
 import Utility.FileMode
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
@@ -82,14 +71,14 @@ dirKey d k = d  hashDirMixed k  f  f
 
 store :: FilePath -> Key -> Annex Bool
 store d k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g k
 	let dest = dirKey d k
-	liftIO $ catchBool $ storeHelper dest $ copyFile src dest
+	liftIO $ catchBool $ storeHelper dest $ copyFileExternal src dest
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g k
 	let dest = dirKey d enck
 	liftIO $ catchBool $ storeHelper dest $ encrypt src dest
@@ -110,7 +99,7 @@ storeHelper dest a = do
 	return ok
 
 retrieve :: FilePath -> Key -> FilePath -> Annex Bool
-retrieve d k f = liftIO $ copyFile (dirKey d k) f
+retrieve d k f = liftIO $ copyFileExternal (dirKey d k) f
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d50899c674..a457c5905d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -8,26 +8,17 @@
 module Remote.Git (remote) where
 
 import Control.Exception.Extensible
-import Control.Monad.State (liftIO)
 import qualified Data.Map as M
-import System.Cmd.Utils
-import System.Posix.Files
-import System.IO
 
-import Types
-import Types.Remote
-import qualified Git
-import qualified Annex
-import Locations
-import UUID
-import Utility
-import qualified Content
-import Messages
+import AnnexCommon
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
-import Utility.SafeCommand
-import Utility.Path
+import Types.Remote
+import qualified Git
+import qualified Annex
+import UUID
+import qualified Content
 import qualified Utility.Url as Url
 import Config
 import Init
@@ -42,7 +33,7 @@ remote = RemoteType {
 
 list :: Annex [Git.Repo]
 list = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	return $ Git.remotes g
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
@@ -109,7 +100,7 @@ tryGitConfigRead r
 
 		store a = do
 			r' <- a
-			g <- Annex.gitRepo
+			g <- gitRepo
 			let l = Git.remotes g
 			let g' = Git.remotesAdd g $ exchange l r'
 			Annex.changeState $ \s -> s { Annex.repo = g' }
@@ -169,7 +160,7 @@ copyFromRemote r key file
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
 	| not $ Git.repoIsUrl r = do
-		g <- Annex.gitRepo
+		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
@@ -178,7 +169,7 @@ copyToRemote r key
 			Content.saveState
 			return ok
 	| Git.repoIsSsh r = do
-		g <- Annex.gitRepo
+		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
@@ -200,7 +191,7 @@ rsyncOrCopyFile r src dest = do
 	ss <- liftIO $ getFileStatus $ parentDir src
 	ds <- liftIO $ getFileStatus $ parentDir dest
 	if deviceID ss == deviceID ds
-		then liftIO $ copyFile src dest
+		then liftIO $ copyFileExternal src dest
 		else do
 			params <- rsyncParams r
 			rsyncHelper $ params ++ [Param src, Param dest]
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 04041c6553..42503e4d41 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -8,13 +8,11 @@
 module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
-import Control.Monad.State (liftIO)
 
-import Types
+import AnnexCommon
 import Types.Remote
 import Crypto
 import qualified Annex
-import Messages
 import Config
 
 {- Encryption setup for a remote. The user must specify whether to use
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index b842588c0c..905db04c55 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -8,16 +8,11 @@
 module Remote.Helper.Special where
 
 import qualified Data.Map as M
-import Data.Maybe
-import Data.String.Utils
-import Control.Monad.State (liftIO)
 
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Utility.SafeCommand
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
@@ -25,7 +20,7 @@ import Utility.SafeCommand
  -}
 findSpecialRemotes :: String -> Annex [Git.Repo]
 findSpecialRemotes s = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	return $ map construct $ remotepairs g
 	where
 		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
@@ -35,7 +30,7 @@ findSpecialRemotes s = do
 {- Sets up configuration for a special remote in .git/config. -}
 gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex ()
 gitConfigSpecialRemote u c k v = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
 		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index aaeb702c7b..3bbda19240 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -8,31 +8,19 @@
 module Remote.Hook (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad.State (liftIO)
-import System.FilePath
-import System.Posix.Process hiding (executeFile)
-import System.Posix.IO
-import System.IO
 import System.IO.Error (try)
 import System.Exit
-import Data.Maybe
 
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Locations
 import Config
 import Content
-import Utility
-import Utility.SafeCommand
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
-import Messages
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -86,7 +74,7 @@ hookEnv k f = Just $ fileenv f ++ keyenv
 
 lookupHook :: String -> String -> Annex (Maybe String)
 lookupHook hooktype hook =do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	command <- getConfig g hookname ""
 	if null command
 		then do
@@ -111,12 +99,12 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h
 
 store :: String -> Key -> Annex Bool
 store h k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	runHook h "store" k (Just $ gitAnnexLocation g k) $ return True
 
 storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = gitAnnexLocation g k
 	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 	runHook h "store" enck (Just tmp) $ return True
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9d2d7ddcf7..6a1c297c5d 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -8,32 +8,18 @@
 module Remote.Rsync (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad.State (liftIO)
-import System.FilePath
-import System.Directory
-import System.Posix.Files
-import System.Posix.Process
-import Data.Maybe
 
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Locations
 import Config
 import Content
-import Utility
-import Utility.Conditional
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
-import Messages
 import Utility.RsyncFile
-import Utility.SafeCommand
-import Utility.Path
 
 type RsyncUrl = String
 
@@ -106,12 +92,12 @@ rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  shellEscape (keyFile k)
 
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	rsyncSend o k (gitAnnexLocation g k)
 
 storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = gitAnnexLocation g k
 	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 	rsyncSend o enck tmp
@@ -166,7 +152,7 @@ partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial"
  - up trees for rsync. -}
 withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool
 withRsyncScratchDir a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	pid <- liftIO getProcessID
 	let tmp = gitAnnexTmpDir g  "rsynctmp"  show pid
 	nuke tmp
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index cafa4f15a8..b2ea4b0c86 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -7,31 +7,21 @@
 
 module Remote.S3 (remote) where
 
-import Control.Exception.Extensible (IOException)
 import Network.AWS.AWSConnection
 import Network.AWS.S3Object
 import Network.AWS.S3Bucket hiding (size)
 import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
-import Data.Maybe
-import Data.List
 import Data.Char
-import Data.String.Utils
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
 import System.Environment
-import System.Posix.Files
 import System.Posix.Env (setEnv)
 
-import Types
+import AnnexCommon
 import Types.Remote
 import Types.Key
 import qualified Git
-import qualified Annex
 import UUID
-import Messages
-import Locations
 import Config
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
@@ -123,7 +113,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 store :: Remote Annex -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	res <- liftIO $ storeHelper (conn, bucket) r k $ gitAnnexLocation g k
 	s3Bool res
 
@@ -132,7 +122,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) ->
 	-- To get file size of the encrypted content, have to use a temp file.
 	-- (An alternative would be chunking to to a constant size.)
 	withTmp enck $ \tmp -> do
-		g <- Annex.gitRepo
+		g <- gitRepo
 		let f = gitAnnexLocation g k
 		liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 		res <- liftIO $ storeHelper (conn, bucket) r enck tmp
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 8fb29ec403..732f4d46c0 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -10,21 +10,13 @@ module Remote.Web (
 	setUrl
 ) where
 
-import Control.Monad.State (liftIO)
-import Control.Exception
-import System.FilePath
-
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
-import Messages
 import UUID
 import Config
 import PresenceLog
 import LocationLog
-import Locations
-import Utility
 import qualified Utility.Url as Url
 
 type URLString = String
@@ -80,7 +72,7 @@ getUrls key = do
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()
 setUrl key url status = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	addLog (urlLog key) =<< logNow status url
 
 	-- update location log to indicate that the web has the key, or not
diff --git a/RemoteLog.hs b/RemoteLog.hs
index f9c7997e41..2e43265f5c 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -15,14 +15,11 @@ module RemoteLog (
 	prop_idempotent_configEscape
 ) where
 
-import Data.List
 import qualified Data.Map as M
-import Data.Maybe
 import Data.Char
-import Control.Applicative
 
+import AnnexCommon
 import qualified Branch
-import Types
 import Types.Remote
 import UUID
 
diff --git a/Trust.hs b/Trust.hs
index 0c8836c85a..13f0354bd3 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -13,13 +13,11 @@ module Trust (
 	trustPartition
 ) where
 
-import Control.Monad.State
 import qualified Data.Map as M
-import Data.List
 
+import AnnexCommon
 import Types.TrustLevel
 import qualified Branch
-import Types
 import UUID
 import qualified Annex
 
diff --git a/Types/Key.hs b/Types/Key.hs
index b26bb89896..165f814d4b 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -15,9 +15,10 @@ module Types.Key (
 	prop_idempotent_key_read_show
 ) where
 
-import Utility
 import System.Posix.Types
 
+import Common
+
 {- A Key has a unique name, is associated with a key/value backend,
  - and may contain other optional metadata. -}
 data Key = Key {
diff --git a/UUID.hs b/UUID.hs
index eab6bd4dff..633938be48 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -22,18 +22,12 @@ module UUID (
 	uuidLog
 ) where
 
-import Control.Monad.State
-import Control.Applicative
-import System.Cmd.Utils
-import System.IO
 import qualified Data.Map as M
-import Data.Maybe
 
+import AnnexCommon
 import qualified Git
 import qualified Branch
-import Types
 import Types.UUID
-import qualified Annex
 import qualified Build.SysConfig as SysConfig
 import Config
 
@@ -60,7 +54,7 @@ genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
  -}
 getUUID :: Git.Repo -> Annex UUID
 getUUID r = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 
 	let c = cached g
 	let u = getUncachedUUID r
@@ -81,7 +75,7 @@ getUncachedUUID r = Git.configGet r configkey ""
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
 prepUUID = do
-	u <- getUUID =<< Annex.gitRepo
+	u <- getUUID =<< gitRepo
 	when ("" == u) $ do
 		uuid <- liftIO genUUID
 		setConfig configkey uuid
diff --git a/Upgrade.hs b/Upgrade.hs
index a724ecce31..666f8d08a7 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -7,7 +7,7 @@
 
 module Upgrade where
 
-import Types
+import AnnexCommon
 import Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index 3aabe07700..f8e6cda56f 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -8,23 +8,15 @@
 module Upgrade.V0 where
 
 import System.IO.Error (try)
-import System.Directory
-import Control.Monad.State (liftIO)
-import Control.Monad (filterM, forM_)
-import System.Posix.Files
-import System.FilePath
 
+import AnnexCommon
 import Content
-import Types
-import Locations
-import qualified Annex
-import Messages
 import qualified Upgrade.V1
 
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v0 to v1"
-	g <- Annex.gitRepo
+	g <- gitRepo
 
 	-- do the reorganisation of the key files
 	let olddir = gitAnnexDir g
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 329f90ed6c..bc50b857c6 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -8,33 +8,19 @@
 module Upgrade.V1 where
 
 import System.IO.Error (try)
-import System.Directory
-import Control.Monad.State (liftIO)
-import Control.Monad (filterM, forM_, unless)
-import Control.Applicative
-import System.Posix.Files
-import System.FilePath
-import Data.String.Utils
 import System.Posix.Types
-import Data.Maybe
 import Data.Char
 
+import AnnexCommon
 import Types.Key
 import Content
-import Types
-import Locations
 import PresenceLog
-import qualified Annex
 import qualified AnnexQueue
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Backend
-import Messages
 import Version
-import Utility
 import Utility.FileMode
-import Utility.SafeCommand
-import Utility.Path
 import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
@@ -64,7 +50,7 @@ upgrade :: Annex Bool
 upgrade = do
 	showAction "v1 to v2"
 
-	g <- Annex.gitRepo
+	g <- gitRepo
 	if Git.repoIsLocalBare g
 		then do
 			moveContent
@@ -96,7 +82,7 @@ moveContent = do
 updateSymlinks :: Annex ()
 updateSymlinks = do
 	showAction "updating symlinks"
-	g <- Annex.gitRepo
+	g <- gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	forM_ files fixlink
 	where
@@ -117,7 +103,7 @@ moveLocationLogs = do
 	forM_ logkeys move
 		where
 			oldlocationlogs = do
-				g <- Annex.gitRepo
+				g <- gitRepo
 				let dir = Upgrade.V2.gitStateDir g
 				exists <- liftIO $ doesDirectoryExist dir
 				if exists
@@ -126,7 +112,7 @@ moveLocationLogs = do
 						return $ mapMaybe oldlog2key contents
 					else return []
 			move (l, k) = do
-				g <- Annex.gitRepo
+				g <- gitRepo
 				let dest = logFile2 g k
 				let dir = Upgrade.V2.gitStateDir g
 				let f = dir  l
@@ -220,7 +206,7 @@ lookupFile1 file = do
 
 getKeyFilesPresent1 :: Annex [FilePath]
 getKeyFilesPresent1  = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	getKeyFilesPresent1' $ gitAnnexObjectDir g
 getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
 getKeyFilesPresent1' dir = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 4e686288fb..922dfff28f 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -7,21 +7,9 @@
 
 module Upgrade.V2 where
 
-import System.Directory
-import System.FilePath
-import Control.Monad.State (unless, when, liftIO)
-import Data.List
-import Data.Maybe
-
-import Types.Key
-import Types
-import qualified Annex
+import AnnexCommon
 import qualified Git
 import qualified Branch
-import Messages
-import Utility
-import Utility.Conditional
-import Utility.SafeCommand
 import LocationLog
 import Content
 
@@ -48,7 +36,7 @@ olddir g
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v2 to v3"
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let bare = Git.repoIsLocalBare g
 
 	Branch.create
@@ -85,7 +73,7 @@ locationLogs repo = liftIO $ do
 
 inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	new <- liftIO (readFile $ olddir g  source)
 	Branch.change dest $ \prev -> 
 		unlines $ nub $ lines prev ++ lines new
@@ -114,7 +102,7 @@ push = do
 			Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
-			g <- Annex.gitRepo
+			g <- gitRepo
 			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
diff --git a/Utility.hs b/Utility.hs
index a3d461d286..4e82e63c9d 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -11,7 +11,6 @@ module Utility (
 	readMaybe,
 	viaTmp,
 	withTempFile,
-	dirContains,
 	dirContents,
 	myHomeDir,
 	catchBool,
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 9019357196..5d6855bf01 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.CopyFile (copyFile) where
+module Utility.CopyFile (copyFileExternal) where
 
 import System.Directory (doesFileExist, removeFile)
 
@@ -15,8 +15,8 @@ import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
  - and because this allows easy access to features like cp --reflink. -}
-copyFile :: FilePath -> FilePath -> IO Bool
-copyFile src dest = do
+copyFileExternal :: FilePath -> FilePath -> IO Bool
+copyFileExternal src dest = do
 	whenM (doesFileExist dest) $
 		removeFile dest
 	boolSystem "cp" [params, File src, File dest]
diff --git a/Version.hs b/Version.hs
index fcf6bc4d1d..304e9f0e18 100644
--- a/Version.hs
+++ b/Version.hs
@@ -7,8 +7,7 @@
 
 module Version where
 
-import Types
-import qualified Annex
+import AnnexCommon
 import qualified Git
 import Config
 
@@ -28,7 +27,7 @@ versionField = "annex.version"
 
 getVersion :: Annex (Maybe Version)
 getVersion = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let v = Git.configGet g versionField ""
 	if not $ null v
 		then return $ Just v
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 1fb928f9da..6147545ab6 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -6,13 +6,11 @@
  -}
 
 import System.Environment
-import Data.List
 
+import AnnexCommon
 import qualified Git
 import CmdLine
 import Command
-import Utility.Conditional
-import Utility.SafeCommand
 import Options
 
 import qualified Command.ConfigList
diff --git a/git-annex.cabal b/git-annex.cabal
index 3f31ee4dcc..5645eb043a 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110928
+Version: 3.20110929
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 8b70e678c9..34f22d06fe 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -6,10 +6,8 @@
  -}
 
 import System.Environment
-import System.FilePath
-import System.Directory
-import Control.Monad (when)
 
+import Common
 import qualified Git.UnionMerge
 import qualified Git
 
diff --git a/test.hs b/test.hs
index f8701db669..654af5713f 100644
--- a/test.hs
+++ b/test.hs
@@ -9,23 +9,19 @@ import Test.HUnit
 import Test.HUnit.Tools
 import Test.QuickCheck
 
-import System.Directory
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
 import IO (bracket_, bracket)
-import Control.Monad (unless, when, filterM)
-import Data.List
 import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
 import Control.Exception (throw)
-import Data.Maybe
 import qualified Data.Map as M
-import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
 
-import Utility.SafeCommand
+import Common
 
+import qualified Utility.SafeCommand
 import qualified Annex
 import qualified Backend
 import qualified Git

From 1a96d4ab35ed5c2af95a1598620cbbd13bc295b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 4 Oct 2011 00:19:42 -0400
Subject: [PATCH 2299/2835] use bracket to reset environment

In case the exception is caught higher up, don't leave the environment
dirty.
---
 Branch.hs | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 3f1153c098..554f168481 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -65,17 +65,13 @@ withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
 	g <- gitRepo
 	let f = index g
-	reset <- liftIO $ Git.useIndex f
 
-	e <- liftIO $ doesFileExist f
-	unless e $ do
-		unless bootstrapping create
-		liftIO $ createDirectoryIfMissing True $ takeDirectory f
-		liftIO $ unless bootstrapping $ genIndex g
-
-	r <- a
-	liftIO reset
-	return r
+	bracket (Git.useIndex f) id $ do
+		unlessM (liftIO $ doesFileExist f) $ do
+			unless bootstrapping create
+			liftIO $ createDirectoryIfMissing True $ takeDirectory f
+			unless bootstrapping $ liftIO $ genIndex g
+		a
 
 withIndexUpdate :: Annex a -> Annex a
 withIndexUpdate a = update >> withIndex a
@@ -332,11 +328,14 @@ lockJournal :: Annex a -> Annex a
 lockJournal a = do
 	g <- gitRepo
 	let file = gitAnnexJournalLock g
-	liftIOOp (Control.Exception.bracket (lock file) unlock) run
+	bracket (lock file) unlock a
 	where
 		lock file = do
 			l <- createFile file stdFileMode
 			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
 			return l
 		unlock = closeFd
-		run _ = a
+
+bracket :: IO c -> (c -> IO b) -> Annex a -> Annex a
+bracket start cleanup go =
+	liftIOOp (Control.Exception.bracket start cleanup) (const go)

From ff21fd4a652cc6516d0e06ab885adf1c93eddced Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 4 Oct 2011 00:34:04 -0400
Subject: [PATCH 2300/2835] factor out Annex exception handling module

---
 Annex/Exception.hs | 27 +++++++++++++++++++++++++++
 Branch.hs          | 15 +++++----------
 Command/Add.hs     |  6 ++----
 3 files changed, 34 insertions(+), 14 deletions(-)
 create mode 100644 Annex/Exception.hs

diff --git a/Annex/Exception.hs b/Annex/Exception.hs
new file mode 100644
index 0000000000..549ef4fd50
--- /dev/null
+++ b/Annex/Exception.hs
@@ -0,0 +1,27 @@
+{- exception handling in the git-annex monad
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.Exception (
+	bracketIO,
+	handle,
+	throw,
+) where
+
+import Control.Exception.Control (handle)
+import Control.Monad.IO.Control (liftIOOp)
+import Control.Exception hiding (handle, throw)
+
+import AnnexCommon
+
+{- Runs an Annex action, with setup and cleanup both in the IO monad. -}
+bracketIO :: IO c -> (c -> IO b) -> Annex a -> Annex a
+bracketIO setup cleanup go =
+	liftIOOp (Control.Exception.bracket setup cleanup) (const go)
+
+{- Throws an exception in the Annex monad. -}
+throw :: Control.Exception.Exception e => e -> Annex a
+throw = liftIO . throwIO
diff --git a/Branch.hs b/Branch.hs
index 554f168481..a2ddc70ac7 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -21,10 +21,9 @@ module Branch (
 import System.IO.Binary
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
-import Control.Monad.IO.Control (liftIOOp)
-import qualified Control.Exception
 
 import AnnexCommon
+import Annex.Exception
 import Types.BranchState
 import qualified Git
 import qualified Git.UnionMerge
@@ -66,7 +65,7 @@ withIndex' bootstrapping a = do
 	g <- gitRepo
 	let f = index g
 
-	bracket (Git.useIndex f) id $ do
+	bracketIO (Git.useIndex f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
 			liftIO $ createDirectoryIfMissing True $ takeDirectory f
@@ -93,9 +92,9 @@ invalidateCache = do
 	setState state { cachedFile = Nothing, cachedContent = "" }
 
 getCache :: FilePath -> Annex (Maybe String)
-getCache file = getState >>= handle
+getCache file = getState >>= go
 	where
-		handle state
+		go state
 			| cachedFile state == Just file =
 				return $ Just $ cachedContent state
 			| otherwise = return Nothing
@@ -328,14 +327,10 @@ lockJournal :: Annex a -> Annex a
 lockJournal a = do
 	g <- gitRepo
 	let file = gitAnnexJournalLock g
-	bracket (lock file) unlock a
+	bracketIO (lock file) unlock a
 	where
 		lock file = do
 			l <- createFile file stdFileMode
 			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
 			return l
 		unlock = closeFd
-
-bracket :: IO c -> (c -> IO b) -> Annex a -> Annex a
-bracket start cleanup go =
-	liftIOOp (Control.Exception.bracket start cleanup) (const go)
diff --git a/Command/Add.hs b/Command/Add.hs
index c66c381311..299b5f36e0 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -7,10 +7,8 @@
 
 module Command.Add where
 
-import Control.Exception.Control (handle)
-import Control.Exception.Base (throwIO)
-
 import AnnexCommon
+import Annex.Exception
 import Command
 import qualified Annex
 import qualified AnnexQueue
@@ -58,7 +56,7 @@ undo file key e = do
 	logStatus key InfoMissing
 	rethrow
 	where
-		rethrow = liftIO $ throwIO e
+		rethrow = throw e
 
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()

From cfe21e85e7fba61ac588e210f2a9b75f8d081f42 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 4 Oct 2011 00:40:47 -0400
Subject: [PATCH 2301/2835] rename

---
 Branch.hs => Annex/Branch.hs      |  6 +++---
 CatFile.hs => Annex/CatFile.hs    |  4 ++--
 AnnexCommon.hs => Annex/Common.hs |  2 +-
 Content.hs => Annex/Content.hs    | 12 ++++++------
 Annex/Exception.hs                |  2 +-
 AnnexQueue.hs => Annex/Queue.hs   |  4 ++--
 Version.hs => Annex/Version.hs    |  4 ++--
 Backend.hs                        |  2 +-
 Backend/SHA.hs                    |  4 ++--
 Backend/URL.hs                    |  2 +-
 Backend/WORM.hs                   |  2 +-
 CmdLine.hs                        |  8 ++++----
 Command.hs                        |  2 +-
 Command/Add.hs                    | 10 +++++-----
 Command/AddUrl.hs                 |  4 ++--
 Command/ConfigList.hs             |  2 +-
 Command/Describe.hs               |  2 +-
 Command/Drop.hs                   |  4 ++--
 Command/DropKey.hs                |  4 ++--
 Command/DropUnused.hs             |  2 +-
 Command/Find.hs                   |  4 ++--
 Command/Fix.hs                    |  8 ++++----
 Command/FromKey.hs                |  8 ++++----
 Command/Fsck.hs                   |  4 ++--
 Command/Get.hs                    |  4 ++--
 Command/InAnnex.hs                |  4 ++--
 Command/Init.hs                   |  2 +-
 Command/InitRemote.hs             |  2 +-
 Command/Lock.hs                   |  6 +++---
 Command/Map.hs                    |  2 +-
 Command/Merge.hs                  |  6 +++---
 Command/Migrate.hs                |  4 ++--
 Command/Move.hs                   |  4 ++--
 Command/RecvKey.hs                |  4 ++--
 Command/Semitrust.hs              |  2 +-
 Command/SendKey.hs                |  4 ++--
 Command/SetKey.hs                 |  4 ++--
 Command/Status.hs                 |  4 ++--
 Command/Trust.hs                  |  2 +-
 Command/Unannex.hs                |  8 ++++----
 Command/Uninit.hs                 |  8 ++++----
 Command/Unlock.hs                 |  4 ++--
 Command/Untrust.hs                |  2 +-
 Command/Unused.hs                 | 10 +++++-----
 Command/Upgrade.hs                |  4 ++--
 Command/Version.hs                |  4 ++--
 Command/Whereis.hs                |  2 +-
 Config.hs                         |  2 +-
 Crypto.hs                         |  2 +-
 GitAnnex.hs                       |  2 +-
 Init.hs                           | 10 +++++-----
 Limit.hs                          |  4 ++--
 LocationLog.hs                    |  6 +++---
 Options.hs                        |  2 +-
 PresenceLog.hs                    |  8 ++++----
 Remote.hs                         |  2 +-
 Remote/Bup.hs                     |  2 +-
 Remote/Directory.hs               |  2 +-
 Remote/Git.hs                     | 10 +++++-----
 Remote/Helper/Encryptable.hs      |  2 +-
 Remote/Helper/Special.hs          |  2 +-
 Remote/Hook.hs                    |  4 ++--
 Remote/Rsync.hs                   |  4 ++--
 Remote/S3real.hs                  |  4 ++--
 Remote/Web.hs                     |  2 +-
 RemoteLog.hs                      |  8 ++++----
 Trust.hs                          |  8 ++++----
 UUID.hs                           |  8 ++++----
 Upgrade.hs                        |  4 ++--
 Upgrade/V0.hs                     |  4 ++--
 Upgrade/V1.hs                     | 18 +++++++++---------
 Upgrade/V2.hs                     | 22 +++++++++++-----------
 git-annex-shell.hs                |  2 +-
 73 files changed, 173 insertions(+), 173 deletions(-)
 rename Branch.hs => Annex/Branch.hs (99%)
 rename CatFile.hs => Annex/CatFile.hs (93%)
 rename AnnexCommon.hs => Annex/Common.hs (88%)
 rename Content.hs => Annex/Content.hs (97%)
 rename AnnexQueue.hs => Annex/Queue.hs (95%)
 rename Version.hs => Annex/Version.hs (95%)

diff --git a/Branch.hs b/Annex/Branch.hs
similarity index 99%
rename from Branch.hs
rename to Annex/Branch.hs
index a2ddc70ac7..c6db9decaf 100644
--- a/Branch.hs
+++ b/Annex/Branch.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Branch (
+module Annex.Branch (
 	create,
 	update,
 	get,
@@ -22,13 +22,13 @@ import System.IO.Binary
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import AnnexCommon
+import Annex.Common
 import Annex.Exception
 import Types.BranchState
 import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
-import CatFile
+import Annex.CatFile
 
 type GitRef = String
 
diff --git a/CatFile.hs b/Annex/CatFile.hs
similarity index 93%
rename from CatFile.hs
rename to Annex/CatFile.hs
index 8762109e78..4f98815f8f 100644
--- a/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -5,11 +5,11 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module CatFile (
+module Annex.CatFile (
 	catFile
 ) where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git.CatFile
 import qualified Annex
 
diff --git a/AnnexCommon.hs b/Annex/Common.hs
similarity index 88%
rename from AnnexCommon.hs
rename to Annex/Common.hs
index bcdc5e264e..ca7b1bff7e 100644
--- a/AnnexCommon.hs
+++ b/Annex/Common.hs
@@ -1,4 +1,4 @@
-module AnnexCommon (
+module Annex.Common (
 	module Common,
 	module Types,
 	module Annex,
diff --git a/Content.hs b/Annex/Content.hs
similarity index 97%
rename from Content.hs
rename to Annex/Content.hs
index 567e4caa5d..a3fa79da85 100644
--- a/Content.hs
+++ b/Annex/Content.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Content (
+module Annex.Content (
 	inAnnex,
 	calcGitLink,
 	logStatus,
@@ -21,13 +21,13 @@ module Content (
 	saveState
 ) where
 
-import AnnexCommon
+import Annex.Common
 import LocationLog
 import UUID
 import qualified Git
 import qualified Annex
-import qualified AnnexQueue
-import qualified Branch
+import qualified Annex.Queue
+import qualified Annex.Branch
 import Utility.StatFS
 import Utility.FileMode
 import Types.Key
@@ -233,5 +233,5 @@ getKeysPresent' dir = do
 {- Things to do to record changes to content. -}
 saveState :: Annex ()
 saveState = do
-	AnnexQueue.flush False
-	Branch.commit "update"
+	Annex.Queue.flush False
+	Annex.Branch.commit "update"
diff --git a/Annex/Exception.hs b/Annex/Exception.hs
index 549ef4fd50..7ea8fb89a2 100644
--- a/Annex/Exception.hs
+++ b/Annex/Exception.hs
@@ -15,7 +15,7 @@ import Control.Exception.Control (handle)
 import Control.Monad.IO.Control (liftIOOp)
 import Control.Exception hiding (handle, throw)
 
-import AnnexCommon
+import Annex.Common
 
 {- Runs an Annex action, with setup and cleanup both in the IO monad. -}
 bracketIO :: IO c -> (c -> IO b) -> Annex a -> Annex a
diff --git a/AnnexQueue.hs b/Annex/Queue.hs
similarity index 95%
rename from AnnexQueue.hs
rename to Annex/Queue.hs
index 66843a75e3..8d0a32bec9 100644
--- a/AnnexQueue.hs
+++ b/Annex/Queue.hs
@@ -5,13 +5,13 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module AnnexQueue (
+module Annex.Queue (
 	add,
 	flush,
 	flushWhenFull
 ) where
 
-import AnnexCommon
+import Annex.Common
 import Annex
 import qualified Git.Queue
 
diff --git a/Version.hs b/Annex/Version.hs
similarity index 95%
rename from Version.hs
rename to Annex/Version.hs
index 304e9f0e18..e501dbf2ec 100644
--- a/Version.hs
+++ b/Annex/Version.hs
@@ -5,9 +5,9 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Version where
+module Annex.Version where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import Config
 
diff --git a/Backend.hs b/Backend.hs
index 94fe29607e..9a7df692cd 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -19,7 +19,7 @@ module Backend (
 import System.IO.Error (try)
 import System.Posix.Files
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import qualified Annex
 import Types.Key
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 0c36ef0dc7..2be02c9f62 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -7,9 +7,9 @@
 
 module Backend.SHA (backends) where
 
-import AnnexCommon
+import Annex.Common
 import qualified Annex
-import Content
+import Annex.Content
 import Types.Backend
 import Types.Key
 import qualified Build.SysConfig as SysConfig
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 0745de455d..555e0617cf 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,7 +10,7 @@ module Backend.URL (
 	fromUrl
 ) where
 
-import AnnexCommon
+import Annex.Common
 import Types.Backend
 import Types.Key
 
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 80c652558e..b45ec7b0c9 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -7,7 +7,7 @@
 
 module Backend.WORM (backends) where
 
-import AnnexCommon
+import Annex.Common
 import Types.Backend
 import Types.Key
 
diff --git a/CmdLine.hs b/CmdLine.hs
index 34adb25569..faf5222a2a 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -14,11 +14,11 @@ module CmdLine (
 import System.IO.Error (try)
 import System.Console.GetOpt
 
-import AnnexCommon
+import Annex.Common
 import qualified Annex
-import qualified AnnexQueue
+import qualified Annex.Queue
 import qualified Git
-import Content
+import Annex.Content
 import Command
 import Options
 import Init
@@ -81,7 +81,7 @@ tryRun = tryRun' 0
 tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
 tryRun' errnum state (a:as) = do
 	result <- try $ Annex.run state $ do
-		AnnexQueue.flushWhenFull
+		Annex.Queue.flushWhenFull
 		a
 	case result of
 		Left err -> do
diff --git a/Command.hs b/Command.hs
index 20f3d79b62..1f418b870b 100644
--- a/Command.hs
+++ b/Command.hs
@@ -7,7 +7,7 @@
 
 module Command where
 
-import AnnexCommon
+import Annex.Common
 import qualified Backend
 import qualified Annex
 import qualified Git
diff --git a/Command/Add.hs b/Command/Add.hs
index 299b5f36e0..70b38e809a 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -7,14 +7,14 @@
 
 module Command.Add where
 
-import AnnexCommon
+import Annex.Common
 import Annex.Exception
 import Command
 import qualified Annex
-import qualified AnnexQueue
+import qualified Annex.Queue
 import qualified Backend
 import LocationLog
-import Content
+import Annex.Content
 import Utility.Touch
 import Backend
 
@@ -81,6 +81,6 @@ cleanup file key hascontent = do
 
 	force <- Annex.getState Annex.force
 	if force
-		then AnnexQueue.add "add" [Param "-f", Param "--"] [file]
-		else AnnexQueue.add "add" [Param "--"] [file]
+		then Annex.Queue.add "add" [Param "-f", Param "--"] [file]
+		else Annex.Queue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index ce6e70699c..35f85ca26f 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -9,7 +9,7 @@ module Command.AddUrl where
 
 import Network.URI
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Backend
 import qualified Utility.Url as Url
@@ -17,7 +17,7 @@ import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
-import Content
+import Annex.Content
 import PresenceLog
 
 command :: [Command]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index c38539fa0b..443656f174 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -7,7 +7,7 @@
 
 module Command.ConfigList where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import UUID
 
diff --git a/Command/Describe.hs b/Command/Describe.hs
index b1c144872b..48f74dcd87 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -7,7 +7,7 @@
 
 module Command.Describe where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 7210184f88..45feab2f32 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -7,12 +7,12 @@
 
 module Command.Drop where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import qualified Annex
 import LocationLog
-import Content
+import Annex.Content
 import Trust
 import Config
 
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 7ead1c4bca..185041ad09 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -7,11 +7,11 @@
 
 module Command.DropKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Annex
 import LocationLog
-import Content
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "dropkey" (paramRepeating paramKey) seek
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index ed4f71e7e4..c4d9b765e3 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -9,7 +9,7 @@ module Command.DropUnused where
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Annex
 import qualified Command.Drop
diff --git a/Command/Find.hs b/Command/Find.hs
index 8d80659d0d..b8c9eeec27 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,9 +7,9 @@
 
 module Command.Find where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 import Limit
 
 command :: [Command]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index a66a1c44a7..44e36009b6 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -7,10 +7,10 @@
 
 module Command.Fix where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import qualified AnnexQueue
-import Content
+import qualified Annex.Queue
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "fix" paramPaths seek
@@ -39,5 +39,5 @@ perform file link = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	AnnexQueue.add "add" [Param "--"] [file]
+	Annex.Queue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index e60025bf7e..f4dceb3315 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -7,10 +7,10 @@
 
 module Command.FromKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import qualified AnnexQueue
-import Content
+import qualified Annex.Queue
+import Annex.Content
 import Types.Key
 
 command :: [Command]
@@ -39,5 +39,5 @@ perform file = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	AnnexQueue.add "add" [Param "--"] [file]
+	Annex.Queue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 33a8405a69..0a75003207 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,13 +7,13 @@
 
 module Command.Fsck where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
 import UUID
-import Content
+import Annex.Content
 import LocationLog
 import Trust
 import Utility.DataUnits
diff --git a/Command/Get.hs b/Command/Get.hs
index 34f56aa2d5..c9fdf56530 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -7,11 +7,11 @@
 
 module Command.Get where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Annex
 import qualified Remote
-import Content
+import Annex.Content
 import qualified Command.Move
 
 command :: [Command]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 36b6d40e6e..05544366ba 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -7,9 +7,9 @@
 
 module Command.InAnnex where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "inannex" (paramRepeating paramKey) seek
diff --git a/Command/Init.hs b/Command/Init.hs
index f3d8834ba1..1a306ae963 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,7 +7,7 @@
 
 module Command.Init where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import UUID
 import Init
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 2ce86e9c6b..5080b7b2b8 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -9,7 +9,7 @@ module Command.InitRemote where
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import qualified RemoteLog
diff --git a/Command/Lock.hs b/Command/Lock.hs
index af7b92ad68..9acc5fe4aa 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -7,9 +7,9 @@
 
 module Command.Lock where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import qualified AnnexQueue
+import qualified Annex.Queue
 import Backend
 	
 command :: [Command]
@@ -30,5 +30,5 @@ perform file = do
 	-- Checkout from HEAD to get rid of any changes that might be 
 	-- staged in the index, and get back to the previous symlink to
 	-- the content.
-	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] [file]
+	Annex.Queue.add "checkout" [Param "HEAD", Param "--"] [file]
 	next $ return True -- no cleanup needed
diff --git a/Command/Map.hs b/Command/Map.hs
index 8e63f6dd6a..39737289ca 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -10,7 +10,7 @@ module Command.Map where
 import Control.Exception.Extensible
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Git
 import UUID
diff --git a/Command/Merge.hs b/Command/Merge.hs
index b365e0e0cd..1de1eb6eeb 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -7,9 +7,9 @@
 
 module Command.Merge where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import qualified Branch
+import qualified Annex.Branch
 
 command :: [Command]
 command = [repoCommand "merge" paramNothing seek
@@ -25,5 +25,5 @@ start = do
 
 perform :: CommandPerform
 perform = do
-	Branch.update
+	Annex.Branch.update
 	next $ return True
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 24f23baf5c..a9591a81a9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -7,11 +7,11 @@
 
 module Command.Migrate where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Backend
 import qualified Types.Key
-import Content
+import Annex.Content
 import qualified Command.Add
 import Backend
 
diff --git a/Command/Move.hs b/Command/Move.hs
index d2870b1e42..06d58d602e 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -7,12 +7,12 @@
 
 module Command.Move where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Command.Drop
 import qualified Annex
 import LocationLog
-import Content
+import Annex.Content
 import qualified Remote
 import UUID
 
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 400e81102b..babe04cd01 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -7,10 +7,10 @@
 
 module Command.RecvKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import CmdLine
-import Content
+import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 236ba28794..31b44bc59d 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -7,7 +7,7 @@
 
 module Command.Semitrust where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index f397d9ae6a..f3db996637 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -7,9 +7,9 @@
 
 module Command.SendKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 12ef5b74a5..dde1ec12ad 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -7,10 +7,10 @@
 
 module Command.SetKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import LocationLog
-import Content
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "setkey" paramPath seek
diff --git a/Command/Status.hs b/Command/Status.hs
index de49f84d5a..edb74166d0 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -12,7 +12,7 @@ import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
 
-import AnnexCommon
+import Annex.Common
 import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
@@ -20,7 +20,7 @@ import qualified Command.Unused
 import qualified Git
 import Command
 import Utility.DataUnits
-import Content
+import Annex.Content
 import Types.Key
 import Backend
 import UUID
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 04c68a5d32..3c3473e219 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -7,7 +7,7 @@
 
 module Command.Trust where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import Trust
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index c5c5e90a67..9413cb88c2 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -7,14 +7,14 @@
 
 module Command.Unannex where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Command.Drop
 import qualified Annex
-import qualified AnnexQueue
+import qualified Annex.Queue
 import Utility.FileMode
 import LocationLog
-import Content
+import Annex.Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 
@@ -71,6 +71,6 @@ cleanup file key = do
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
-	AnnexQueue.add "commit" [Param "-m", Param "content removed from git annex"] []
+	Annex.Queue.add "commit" [Param "-m", Param "content removed from git annex"] []
 	
 	return True
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 3ba7a7cf33..c5afe30f23 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,14 +7,14 @@
 
 module Command.Uninit where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
 import Init
-import qualified Branch
-import Content
+import qualified Annex.Branch
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "uninit" paramPaths seek 
@@ -46,5 +46,5 @@ cleanup = do
 	-- avoid normal shutdown
 	saveState
 	liftIO $ do
-		Git.run g "branch" [Param "-D", Param Branch.name]
+		Git.run g "branch" [Param "-D", Param Annex.Branch.name]
 		exitSuccess
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 220d57829b..b2e4aa09a5 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -7,9 +7,9 @@
 
 module Command.Unlock where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 import Utility.CopyFile
 import Utility.FileMode
 
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 30ade85ce0..b8a107d901 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -7,7 +7,7 @@
 
 module Command.Untrust where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 1ba4f53019..6681fca3c7 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -12,9 +12,9 @@ module Command.Unused where
 import qualified Data.Set as S
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 import Utility.FileMode
 import LocationLog
 import qualified Annex
@@ -23,8 +23,8 @@ import qualified Git.LsFiles as LsFiles
 import qualified Git.LsTree as LsTree
 import qualified Backend
 import qualified Remote
-import qualified Branch
-import CatFile
+import qualified Annex.Branch
+import Annex.CatFile
 
 command :: [Command]
 command = [repoCommand "unused" paramNothing seek
@@ -165,7 +165,7 @@ excludeReferenced l = do
 			filter ourbranches .
 			map words . lines . L.unpack
 		cmpheads a b = head a == head b
-		ourbranchend = '/' : Branch.name
+		ourbranchend = '/' : Annex.Branch.name
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
 		removewith [] s = return $ S.toList s
 		removewith (a:as) s
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index d79f895d8d..7b6b127a5e 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -7,10 +7,10 @@
 
 module Command.Upgrade where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import Upgrade
-import Version
+import Annex.Version
 
 command :: [Command]
 command = [standaloneCommand "upgrade" paramNothing seek
diff --git a/Command/Version.hs b/Command/Version.hs
index 1e44fbb0b1..bc895b1945 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -7,10 +7,10 @@
 
 module Command.Version where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Build.SysConfig as SysConfig
-import Version
+import Annex.Version
 
 command :: [Command]
 command = [standaloneCommand "version" paramNothing seek "show version info"]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 3fb636c04c..fec41f410c 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,7 +7,7 @@
 
 module Command.Whereis where
 
-import AnnexCommon
+import Annex.Common
 import LocationLog
 import Command
 import Remote
diff --git a/Config.hs b/Config.hs
index c0328794e1..80637c3933 100644
--- a/Config.hs
+++ b/Config.hs
@@ -7,7 +7,7 @@
 
 module Config where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import qualified Annex
 
diff --git a/Crypto.hs b/Crypto.hs
index 4cc16b4242..af0a216d7d 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -37,7 +37,7 @@ import Control.Exception (finally)
 import System.Exit
 import System.Environment
 
-import AnnexCommon
+import Annex.Common
 import Types.Key
 import Types.Remote
 import Utility.Base64
diff --git a/GitAnnex.hs b/GitAnnex.hs
index fcfbf44b4b..9814880792 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -9,7 +9,7 @@ module GitAnnex where
 
 import System.Console.GetOpt
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import CmdLine
 import Command
diff --git a/Init.hs b/Init.hs
index 57eaf39d20..fa4598966b 100644
--- a/Init.hs
+++ b/Init.hs
@@ -11,16 +11,16 @@ module Init (
 	uninitialize
 ) where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
-import qualified Branch
-import Version
+import qualified Annex.Branch
+import Annex.Version
 import UUID
 
 initialize :: Annex ()
 initialize = do
 	prepUUID
-	Branch.create
+	Annex.Branch.create
 	setVersion
 	gitPreCommitHookWrite
 
@@ -35,7 +35,7 @@ ensureInitialized :: Annex ()
 ensureInitialized = getVersion >>= maybe needsinit checkVersion
 	where
 		needsinit = do
-			annexed <- Branch.hasSomeBranch
+			annexed <- Annex.Branch.hasSomeBranch
 			if annexed
 				then initialize
 				else error "First run: git-annex init"
diff --git a/Limit.hs b/Limit.hs
index 334ae325d6..91857c687d 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -10,13 +10,13 @@ module Limit where
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
 
-import AnnexCommon
+import Annex.Common
 import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
 import LocationLog
-import Content
+import Annex.Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
diff --git a/LocationLog.hs b/LocationLog.hs
index 759bee830e..a633f26374 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -21,9 +21,9 @@ module LocationLog (
 	logFileKey
 ) where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
-import qualified Branch
+import qualified Annex.Branch
 import UUID
 import PresenceLog
 
@@ -43,7 +43,7 @@ keyLocations = currentLog . logFile
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Annex [Key]
-loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Branch.files
+loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Annex.Branch.files
 
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
diff --git a/Options.hs b/Options.hs
index 9d60292002..9f573ef5dc 100644
--- a/Options.hs
+++ b/Options.hs
@@ -10,7 +10,7 @@ module Options where
 import System.Console.GetOpt
 import System.Log.Logger
 
-import AnnexCommon
+import Annex.Common
 import qualified Annex
 import Command
 import Limit
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 23c2882571..01c1ad0943 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -27,8 +27,8 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as M
 
-import AnnexCommon
-import qualified Branch
+import Annex.Common
+import qualified Annex.Branch
 
 data LogLine = LogLine {
 	date :: POSIXTime,
@@ -72,13 +72,13 @@ instance Read LogLine where
 			ret v = [(v, "")]
 
 addLog :: FilePath -> LogLine -> Annex ()
-addLog file line = Branch.change file $ \s -> 
+addLog file line = Annex.Branch.change file $ \s -> 
 	showLog $ compactLog (line : parseLog s)
 
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
-readLog file = parseLog <$> Branch.get file
+readLog file = parseLog <$> Annex.Branch.get file
 
 parseLog :: String -> [LogLine]
 parseLog = filter parsable . map read . lines
diff --git a/Remote.hs b/Remote.hs
index 2371b7bf2d..86fda270e1 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -32,7 +32,7 @@ import qualified Data.Map as M
 import Text.JSON
 import Text.JSON.Generic
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import UUID
 import qualified Annex
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 9588310194..2d0ba47423 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -12,7 +12,7 @@ import System.IO.Error
 import qualified Data.Map as M
 import System.Process
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 664f8ca5fe..bfea850e52 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -11,7 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import System.IO.Error
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Utility.CopyFile
 import Types.Remote
 import qualified Git
diff --git a/Remote/Git.hs b/Remote/Git.hs
index a457c5905d..15e8991f54 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -10,7 +10,7 @@ module Remote.Git (remote) where
 import Control.Exception.Extensible
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
@@ -18,7 +18,7 @@ import Types.Remote
 import qualified Git
 import qualified Annex
 import UUID
-import qualified Content
+import qualified Annex.Content
 import qualified Utility.Url as Url
 import Config
 import Init
@@ -121,7 +121,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = safely checklocal
 	where
-		checklocal = onLocal r (Content.inAnnex key)
+		checklocal = onLocal r (Annex.Content.inAnnex key)
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
@@ -164,9 +164,9 @@ copyToRemote r key
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
-			ok <- Content.getViaTmp key $
+			ok <- Annex.Content.getViaTmp key $
 				rsyncOrCopyFile r keysrc
-			Content.saveState
+			Annex.Content.saveState
 			return ok
 	| Git.repoIsSsh r = do
 		g <- gitRepo
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 42503e4d41..6f43b4367c 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -9,7 +9,7 @@ module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import Crypto
 import qualified Annex
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 905db04c55..c4e0b7f07f 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -9,7 +9,7 @@ module Remote.Helper.Special where
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 3bbda19240..a18bc51e66 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -12,12 +12,12 @@ import qualified Data.Map as M
 import System.IO.Error (try)
 import System.Exit
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
 import Config
-import Content
+import Annex.Content
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 6a1c297c5d..9b870f2b10 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -10,12 +10,12 @@ module Remote.Rsync (remote) where
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
 import Config
-import Content
+import Annex.Content
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index b2ea4b0c86..c846390b8e 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -17,7 +17,7 @@ import Data.Char
 import System.Environment
 import System.Posix.Env (setEnv)
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import Types.Key
 import qualified Git
@@ -26,7 +26,7 @@ import Config
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
-import Content
+import Annex.Content
 import Utility.Base64
 
 remote :: RemoteType Annex
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 732f4d46c0..9132967c7e 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -10,7 +10,7 @@ module Remote.Web (
 	setUrl
 ) where
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 2e43265f5c..4ffe712050 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -18,8 +18,8 @@ module RemoteLog (
 import qualified Data.Map as M
 import Data.Char
 
-import AnnexCommon
-import qualified Branch
+import Annex.Common
+import qualified Annex.Branch
 import Types.Remote
 import UUID
 
@@ -29,7 +29,7 @@ remoteLog = "remote.log"
 
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
-configSet u c = Branch.change remoteLog $
+configSet u c = Annex.Branch.change remoteLog $
 	serialize . M.insert u c . remoteLogParse
 	where
 		serialize = unlines . sort . map toline . M.toList
@@ -37,7 +37,7 @@ configSet u c = Branch.change remoteLog $
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = remoteLogParse <$> Branch.get remoteLog
+readRemoteLog = remoteLogParse <$> Annex.Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
diff --git a/Trust.hs b/Trust.hs
index 13f0354bd3..5f76003813 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -15,9 +15,9 @@ module Trust (
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Types.TrustLevel
-import qualified Branch
+import qualified Annex.Branch
 import UUID
 import qualified Annex
 
@@ -40,7 +40,7 @@ trustMap = do
 		Just m -> return m
 		Nothing -> do
 			overrides <- Annex.getState Annex.forcetrust
-			l <- Branch.get trustLog
+			l <- Annex.Branch.get trustLog
 			let m = M.fromList $ trustMapParse l ++ overrides
 			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
 			return m
@@ -62,7 +62,7 @@ trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
 	when (null uuid) $
 		error "unknown UUID; cannot modify trust level"
-	Branch.change trustLog $
+	Annex.Branch.change trustLog $
 		serialize . M.insert uuid level . M.fromList . trustMapParse
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
         where
diff --git a/UUID.hs b/UUID.hs
index 633938be48..208866ad72 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -24,9 +24,9 @@ module UUID (
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
-import qualified Branch
+import qualified Annex.Branch
 import Types.UUID
 import qualified Build.SysConfig as SysConfig
 import Config
@@ -82,14 +82,14 @@ prepUUID = do
 
 {- Records a description for a uuid in the uuidLog. -}
 describeUUID :: UUID -> String -> Annex ()
-describeUUID uuid desc = Branch.change uuidLog $
+describeUUID uuid desc = Annex.Branch.change uuidLog $
 	serialize . M.insert uuid desc . parse
 	where
 		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
 
 {- Read the uuidLog into a Map -}
 uuidMap :: Annex (M.Map UUID String)
-uuidMap = parse <$> Branch.get uuidLog
+uuidMap = parse <$> Annex.Branch.get uuidLog
 
 parse :: String -> M.Map UUID String
 parse = M.fromList . map pair . lines
diff --git a/Upgrade.hs b/Upgrade.hs
index 666f8d08a7..6f1da773dc 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -7,8 +7,8 @@
 
 module Upgrade where
 
-import AnnexCommon
-import Version
+import Annex.Common
+import Annex.Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
 import qualified Upgrade.V2
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index f8e6cda56f..af91741a0c 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -9,8 +9,8 @@ module Upgrade.V0 where
 
 import System.IO.Error (try)
 
-import AnnexCommon
-import Content
+import Annex.Common
+import Annex.Content
 import qualified Upgrade.V1
 
 upgrade :: Annex Bool
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index bc50b857c6..f4e44acdcb 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -11,15 +11,15 @@ import System.IO.Error (try)
 import System.Posix.Types
 import Data.Char
 
-import AnnexCommon
+import Annex.Common
 import Types.Key
-import Content
+import Annex.Content
 import PresenceLog
-import qualified AnnexQueue
+import qualified Annex.Queue
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Backend
-import Version
+import Annex.Version
 import Utility.FileMode
 import qualified Upgrade.V2
 
@@ -60,7 +60,7 @@ upgrade = do
 			updateSymlinks
 			moveLocationLogs
 	
-			AnnexQueue.flush True
+			Annex.Queue.flush True
 			setVersion
 	
 	Upgrade.V2.upgrade
@@ -94,7 +94,7 @@ updateSymlinks = do
 					link <- calcGitLink f k
 					liftIO $ removeFile f
 					liftIO $ createSymbolicLink link f
-					AnnexQueue.add "add" [Param "--"] [f]
+					Annex.Queue.add "add" [Param "--"] [f]
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
@@ -124,9 +124,9 @@ moveLocationLogs = do
 				old <- liftIO $ readLog1 f
 				new <- liftIO $ readLog1 dest
 				liftIO $ writeLog1 dest (old++new)
-				AnnexQueue.add "add" [Param "--"] [dest]
-				AnnexQueue.add "add" [Param "--"] [f]
-				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
+				Annex.Queue.add "add" [Param "--"] [dest]
+				Annex.Queue.add "add" [Param "--"] [f]
+				Annex.Queue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
 		
 oldlog2key :: FilePath -> Maybe (FilePath, Key)
 oldlog2key l = 
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 922dfff28f..8ac26dc525 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -7,11 +7,11 @@
 
 module Upgrade.V2 where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
-import qualified Branch
+import qualified Annex.Branch
 import LocationLog
-import Content
+import Annex.Content
 
 olddir :: Git.Repo -> FilePath
 olddir g
@@ -39,7 +39,7 @@ upgrade = do
 	g <- gitRepo
 	let bare = Git.repoIsLocalBare g
 
-	Branch.create
+	Annex.Branch.create
 	showProgress
 
 	e <- liftIO $ doesDirectoryExist (olddir g)
@@ -75,7 +75,7 @@ inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
 	g <- gitRepo
 	new <- liftIO (readFile $ olddir g  source)
-	Branch.change dest $ \prev -> 
+	Annex.Branch.change dest $ \prev -> 
 		unlines $ nub $ lines prev ++ lines new
 	showProgress
 
@@ -85,8 +85,8 @@ logFiles dir = return . filter (".log" `isSuffixOf`)
 
 push :: Annex ()
 push = do
-	origin_master <- Branch.refExists "origin/master"
-	origin_gitannex <- Branch.hasOrigin
+	origin_master <- Annex.Branch.refExists "origin/master"
+	origin_gitannex <- Annex.Branch.hasOrigin
 	case (origin_master, origin_gitannex) of
 		(_, True) -> do
 			-- Merge in the origin's git-annex branch,
@@ -94,20 +94,20 @@ push = do
 			-- will immediately work. Not pushed here,
 			-- because it's less obnoxious to let the user
 			-- push.
-			Branch.update
+			Annex.Branch.update
 		(True, False) -> do
 			-- push git-annex to origin, so that
 			-- "git push" will from then on
 			-- automatically push it
-			Branch.update -- just in case
+			Annex.Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
 			g <- gitRepo
-			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]
+			liftIO $ Git.run g "push" [Param "origin", Param Annex.Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
 			-- know about the new branch
-			Branch.update
+			Annex.Branch.update
 			showLongNote $
 				"git-annex branch created\n" ++
 				"Be sure to push this branch when pushing to remotes.\n"
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 6147545ab6..d03b0910ea 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -7,7 +7,7 @@
 
 import System.Environment
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import CmdLine
 import Command

From c199a01dd13e3c25bebcf6e67b9f870dc9be5ce8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 5 Oct 2011 15:59:49 -0400
Subject: [PATCH 2302/2835] bug

---
 ...rust.log_and_remote.log_merge_wackiness.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn

diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
new file mode 100644
index 0000000000..af140403ff
--- /dev/null
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -0,0 +1,17 @@
+Since uuid.log, trust.log and remote.log are union merged, it's possible
+for any given item in them to have multiple values after a merge.
+This would happen, for example, if the value was changed in different ways
+in two repos which were then merged. git-annex will use an arbitrary
+one of the multiple values.
+
+A workaround if this should happen to you is to use `git annex describe`
+or other commands to re-set the value you want. The process of setting
+the value will remove the multiple lines.
+
+To fix this the file format needs to be changed to include a timestamp
+as is done with the other log files, then git-annex can consistently
+pick the newest value -- which is as close to the "right" value as can be
+determined in this situation.
+
+(For backwards compatability, git-annex should
+treat lines with no timestamp as being timestamped with 0.)

From 6a6ea06cee8ce69f391f7ce78b98b8f26a599e66 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 5 Oct 2011 16:02:51 -0400
Subject: [PATCH 2303/2835] rename

---
 Annex/Branch.hs                    | 2 +-
 Annex/CatFile.hs                   | 2 +-
 Annex/Content.hs                   | 2 +-
 Annex/Exception.hs                 | 2 +-
 Annex/Queue.hs                     | 2 +-
 Annex/Version.hs                   | 2 +-
 Backend.hs                         | 2 +-
 Backend/SHA.hs                     | 2 +-
 Backend/URL.hs                     | 2 +-
 Backend/WORM.hs                    | 2 +-
 CmdLine.hs                         | 2 +-
 Command.hs                         | 2 +-
 Command/Add.hs                     | 2 +-
 Command/AddUrl.hs                  | 2 +-
 Command/ConfigList.hs              | 2 +-
 Command/Describe.hs                | 2 +-
 Command/Drop.hs                    | 2 +-
 Command/DropKey.hs                 | 2 +-
 Command/DropUnused.hs              | 2 +-
 Command/Find.hs                    | 2 +-
 Command/Fix.hs                     | 2 +-
 Command/FromKey.hs                 | 2 +-
 Command/Fsck.hs                    | 2 +-
 Command/Get.hs                     | 2 +-
 Command/InAnnex.hs                 | 2 +-
 Command/Init.hs                    | 2 +-
 Command/InitRemote.hs              | 2 +-
 Command/Lock.hs                    | 2 +-
 Command/Map.hs                     | 2 +-
 Command/Merge.hs                   | 2 +-
 Command/Migrate.hs                 | 2 +-
 Command/Move.hs                    | 2 +-
 Command/RecvKey.hs                 | 2 +-
 Command/Semitrust.hs               | 2 +-
 Command/SendKey.hs                 | 2 +-
 Command/SetKey.hs                  | 2 +-
 Command/Status.hs                  | 2 +-
 Command/Trust.hs                   | 2 +-
 Command/Unannex.hs                 | 2 +-
 Command/Uninit.hs                  | 2 +-
 Command/Unlock.hs                  | 2 +-
 Command/Untrust.hs                 | 2 +-
 Command/Unused.hs                  | 2 +-
 Command/Upgrade.hs                 | 2 +-
 Command/Version.hs                 | 2 +-
 Command/Whereis.hs                 | 2 +-
 Annex/Common.hs => Common/Annex.hs | 2 +-
 Config.hs                          | 2 +-
 Crypto.hs                          | 2 +-
 GitAnnex.hs                        | 2 +-
 Init.hs                            | 2 +-
 Limit.hs                           | 2 +-
 LocationLog.hs                     | 2 +-
 Options.hs                         | 2 +-
 PresenceLog.hs                     | 2 +-
 Remote.hs                          | 2 +-
 Remote/Bup.hs                      | 2 +-
 Remote/Directory.hs                | 2 +-
 Remote/Git.hs                      | 2 +-
 Remote/Helper/Encryptable.hs       | 2 +-
 Remote/Helper/Special.hs           | 2 +-
 Remote/Hook.hs                     | 2 +-
 Remote/Rsync.hs                    | 2 +-
 Remote/S3real.hs                   | 2 +-
 Remote/Web.hs                      | 2 +-
 RemoteLog.hs                       | 2 +-
 Trust.hs                           | 2 +-
 UUID.hs                            | 2 +-
 Upgrade.hs                         | 2 +-
 Upgrade/V0.hs                      | 2 +-
 Upgrade/V1.hs                      | 2 +-
 Upgrade/V2.hs                      | 2 +-
 git-annex-shell.hs                 | 2 +-
 73 files changed, 73 insertions(+), 73 deletions(-)
 rename Annex/Common.hs => Common/Annex.hs (88%)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index c6db9decaf..0b4bea051d 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -22,7 +22,7 @@ import System.IO.Binary
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import Annex.Common
+import Common.Annex
 import Annex.Exception
 import Types.BranchState
 import qualified Git
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 4f98815f8f..2707ed3ea3 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -9,7 +9,7 @@ module Annex.CatFile (
 	catFile
 ) where
 
-import Annex.Common
+import Common.Annex
 import qualified Git.CatFile
 import qualified Annex
 
diff --git a/Annex/Content.hs b/Annex/Content.hs
index a3fa79da85..21403954ae 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -21,7 +21,7 @@ module Annex.Content (
 	saveState
 ) where
 
-import Annex.Common
+import Common.Annex
 import LocationLog
 import UUID
 import qualified Git
diff --git a/Annex/Exception.hs b/Annex/Exception.hs
index 7ea8fb89a2..c147439a1c 100644
--- a/Annex/Exception.hs
+++ b/Annex/Exception.hs
@@ -15,7 +15,7 @@ import Control.Exception.Control (handle)
 import Control.Monad.IO.Control (liftIOOp)
 import Control.Exception hiding (handle, throw)
 
-import Annex.Common
+import Common.Annex
 
 {- Runs an Annex action, with setup and cleanup both in the IO monad. -}
 bracketIO :: IO c -> (c -> IO b) -> Annex a -> Annex a
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index 8d0a32bec9..4c1182750a 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -11,7 +11,7 @@ module Annex.Queue (
 	flushWhenFull
 ) where
 
-import Annex.Common
+import Common.Annex
 import Annex
 import qualified Git.Queue
 
diff --git a/Annex/Version.hs b/Annex/Version.hs
index e501dbf2ec..935f777abf 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -7,7 +7,7 @@
 
 module Annex.Version where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import Config
 
diff --git a/Backend.hs b/Backend.hs
index 9a7df692cd..d1ff114059 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -19,7 +19,7 @@ module Backend (
 import System.IO.Error (try)
 import System.Posix.Files
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex
 import Types.Key
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 2be02c9f62..4b5b14cc36 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -7,7 +7,7 @@
 
 module Backend.SHA (backends) where
 
-import Annex.Common
+import Common.Annex
 import qualified Annex
 import Annex.Content
 import Types.Backend
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 555e0617cf..32a72335a5 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,7 +10,7 @@ module Backend.URL (
 	fromUrl
 ) where
 
-import Annex.Common
+import Common.Annex
 import Types.Backend
 import Types.Key
 
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index b45ec7b0c9..5a3e2d694c 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -7,7 +7,7 @@
 
 module Backend.WORM (backends) where
 
-import Annex.Common
+import Common.Annex
 import Types.Backend
 import Types.Key
 
diff --git a/CmdLine.hs b/CmdLine.hs
index faf5222a2a..b1c9c17285 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -14,7 +14,7 @@ module CmdLine (
 import System.IO.Error (try)
 import System.Console.GetOpt
 
-import Annex.Common
+import Common.Annex
 import qualified Annex
 import qualified Annex.Queue
 import qualified Git
diff --git a/Command.hs b/Command.hs
index 1f418b870b..6f8684e4af 100644
--- a/Command.hs
+++ b/Command.hs
@@ -7,7 +7,7 @@
 
 module Command where
 
-import Annex.Common
+import Common.Annex
 import qualified Backend
 import qualified Annex
 import qualified Git
diff --git a/Command/Add.hs b/Command/Add.hs
index 70b38e809a..e6e7a7c770 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -7,7 +7,7 @@
 
 module Command.Add where
 
-import Annex.Common
+import Common.Annex
 import Annex.Exception
 import Command
 import qualified Annex
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 35f85ca26f..c5417bf5bc 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -9,7 +9,7 @@ module Command.AddUrl where
 
 import Network.URI
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Backend
 import qualified Utility.Url as Url
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 443656f174..d52c33f3b9 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -7,7 +7,7 @@
 
 module Command.ConfigList where
 
-import Annex.Common
+import Common.Annex
 import Command
 import UUID
 
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 48f74dcd87..3368046393 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -7,7 +7,7 @@
 
 module Command.Describe where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 45feab2f32..fabacbb724 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -7,7 +7,7 @@
 
 module Command.Drop where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import qualified Annex
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 185041ad09..35ebfc219b 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -7,7 +7,7 @@
 
 module Command.DropKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex
 import LocationLog
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index c4d9b765e3..0050685565 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -9,7 +9,7 @@ module Command.DropUnused where
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex
 import qualified Command.Drop
diff --git a/Command/Find.hs b/Command/Find.hs
index b8c9eeec27..98501078e8 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,7 +7,7 @@
 
 module Command.Find where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 import Limit
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 44e36009b6..5b6f1f7a47 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -7,7 +7,7 @@
 
 module Command.Fix where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex.Queue
 import Annex.Content
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index f4dceb3315..1b05d71fb8 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -7,7 +7,7 @@
 
 module Command.FromKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex.Queue
 import Annex.Content
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0a75003207..689271dd7e 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,7 +7,7 @@
 
 module Command.Fsck where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import qualified Types.Backend
diff --git a/Command/Get.hs b/Command/Get.hs
index c9fdf56530..acf7e07228 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -7,7 +7,7 @@
 
 module Command.Get where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex
 import qualified Remote
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 05544366ba..773693b65f 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -7,7 +7,7 @@
 
 module Command.InAnnex where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 
diff --git a/Command/Init.hs b/Command/Init.hs
index 1a306ae963..ace06c2c34 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,7 +7,7 @@
 
 module Command.Init where
 
-import Annex.Common
+import Common.Annex
 import Command
 import UUID
 import Init
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 5080b7b2b8..918604f852 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -9,7 +9,7 @@ module Command.InitRemote where
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import qualified RemoteLog
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 9acc5fe4aa..c6c66a1585 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -7,7 +7,7 @@
 
 module Command.Lock where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex.Queue
 import Backend
diff --git a/Command/Map.hs b/Command/Map.hs
index 39737289ca..1155c4a6ef 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -10,7 +10,7 @@ module Command.Map where
 import Control.Exception.Extensible
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Git
 import UUID
diff --git a/Command/Merge.hs b/Command/Merge.hs
index 1de1eb6eeb..eef2f3857a 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -7,7 +7,7 @@
 
 module Command.Merge where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex.Branch
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index a9591a81a9..23ed6fd162 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -7,7 +7,7 @@
 
 module Command.Migrate where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Backend
 import qualified Types.Key
diff --git a/Command/Move.hs b/Command/Move.hs
index 06d58d602e..d650c52510 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -7,7 +7,7 @@
 
 module Command.Move where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Command.Drop
 import qualified Annex
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index babe04cd01..d3b77d8ac1 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -7,7 +7,7 @@
 
 module Command.RecvKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import CmdLine
 import Annex.Content
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 31b44bc59d..53b29c98ac 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -7,7 +7,7 @@
 
 module Command.Semitrust where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index f3db996637..ad47cd009f 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -7,7 +7,7 @@
 
 module Command.SendKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 import Utility.RsyncFile
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index dde1ec12ad..28bc9a48d6 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -7,7 +7,7 @@
 
 module Command.SetKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import LocationLog
 import Annex.Content
diff --git a/Command/Status.hs b/Command/Status.hs
index edb74166d0..37e13f0d8e 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -12,7 +12,7 @@ import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
 
-import Annex.Common
+import Common.Annex
 import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 3c3473e219..bc655e3f61 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -7,7 +7,7 @@
 
 module Command.Trust where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import Trust
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 9413cb88c2..d0cef76781 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -7,7 +7,7 @@
 
 module Command.Unannex where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Command.Drop
 import qualified Annex
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index c5afe30f23..7b3b4fe329 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,7 +7,7 @@
 
 module Command.Uninit where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Git
 import qualified Annex
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index b2e4aa09a5..9b568b5a6b 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -7,7 +7,7 @@
 
 module Command.Unlock where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 import Utility.CopyFile
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index b8a107d901..bcde0e0a06 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -7,7 +7,7 @@
 
 module Command.Untrust where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 6681fca3c7..abf5a5361a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -12,7 +12,7 @@ module Command.Unused where
 import qualified Data.Set as S
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 import Utility.FileMode
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 7b6b127a5e..90d3a4e95b 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -7,7 +7,7 @@
 
 module Command.Upgrade where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Upgrade
 import Annex.Version
diff --git a/Command/Version.hs b/Command/Version.hs
index bc895b1945..5ac87099bb 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -7,7 +7,7 @@
 
 module Command.Version where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Build.SysConfig as SysConfig
 import Annex.Version
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index fec41f410c..0eeb174142 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,7 +7,7 @@
 
 module Command.Whereis where
 
-import Annex.Common
+import Common.Annex
 import LocationLog
 import Command
 import Remote
diff --git a/Annex/Common.hs b/Common/Annex.hs
similarity index 88%
rename from Annex/Common.hs
rename to Common/Annex.hs
index ca7b1bff7e..43f1ea0af3 100644
--- a/Annex/Common.hs
+++ b/Common/Annex.hs
@@ -1,4 +1,4 @@
-module Annex.Common (
+module Common.Annex (
 	module Common,
 	module Types,
 	module Annex,
diff --git a/Config.hs b/Config.hs
index 80637c3933..f4c3843af8 100644
--- a/Config.hs
+++ b/Config.hs
@@ -7,7 +7,7 @@
 
 module Config where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex
 
diff --git a/Crypto.hs b/Crypto.hs
index af0a216d7d..21e4d75605 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -37,7 +37,7 @@ import Control.Exception (finally)
 import System.Exit
 import System.Environment
 
-import Annex.Common
+import Common.Annex
 import Types.Key
 import Types.Remote
 import Utility.Base64
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 9814880792..3cd7207d3e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -9,7 +9,7 @@ module GitAnnex where
 
 import System.Console.GetOpt
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import CmdLine
 import Command
diff --git a/Init.hs b/Init.hs
index fa4598966b..145413e8da 100644
--- a/Init.hs
+++ b/Init.hs
@@ -11,7 +11,7 @@ module Init (
 	uninitialize
 ) where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import Annex.Version
diff --git a/Limit.hs b/Limit.hs
index 91857c687d..3812ceea4b 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -10,7 +10,7 @@ module Limit where
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
 
-import Annex.Common
+import Common.Annex
 import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
diff --git a/LocationLog.hs b/LocationLog.hs
index a633f26374..5cbdbb28a1 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -21,7 +21,7 @@ module LocationLog (
 	logFileKey
 ) where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import UUID
diff --git a/Options.hs b/Options.hs
index 9f573ef5dc..0c7b4d5f41 100644
--- a/Options.hs
+++ b/Options.hs
@@ -10,7 +10,7 @@ module Options where
 import System.Console.GetOpt
 import System.Log.Logger
 
-import Annex.Common
+import Common.Annex
 import qualified Annex
 import Command
 import Limit
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 01c1ad0943..4e4960f0a6 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -27,7 +27,7 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import qualified Annex.Branch
 
 data LogLine = LogLine {
diff --git a/Remote.hs b/Remote.hs
index 86fda270e1..27ebd724ac 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -32,7 +32,7 @@ import qualified Data.Map as M
 import Text.JSON
 import Text.JSON.Generic
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import UUID
 import qualified Annex
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 2d0ba47423..986be7fc40 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -12,7 +12,7 @@ import System.IO.Error
 import qualified Data.Map as M
 import System.Process
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index bfea850e52..03b17456a4 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -11,7 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import System.IO.Error
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Utility.CopyFile
 import Types.Remote
 import qualified Git
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 15e8991f54..704bdc04d3 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -10,7 +10,7 @@ module Remote.Git (remote) where
 import Control.Exception.Extensible
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 6f43b4367c..004b70408b 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -9,7 +9,7 @@ module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import Crypto
 import qualified Annex
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index c4e0b7f07f..1fbd7b19ed 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -9,7 +9,7 @@ module Remote.Helper.Special where
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index a18bc51e66..cc8ed69ab5 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -12,7 +12,7 @@ import qualified Data.Map as M
 import System.IO.Error (try)
 import System.Exit
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9b870f2b10..3474f8b251 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -10,7 +10,7 @@ module Remote.Rsync (remote) where
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index c846390b8e..a754731281 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -17,7 +17,7 @@ import Data.Char
 import System.Environment
 import System.Posix.Env (setEnv)
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import Types.Key
 import qualified Git
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 9132967c7e..30a1ff0086 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -10,7 +10,7 @@ module Remote.Web (
 	setUrl
 ) where
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 4ffe712050..149104fbac 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -18,7 +18,7 @@ module RemoteLog (
 import qualified Data.Map as M
 import Data.Char
 
-import Annex.Common
+import Common.Annex
 import qualified Annex.Branch
 import Types.Remote
 import UUID
diff --git a/Trust.hs b/Trust.hs
index 5f76003813..1920bc6170 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -15,7 +15,7 @@ module Trust (
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Types.TrustLevel
 import qualified Annex.Branch
 import UUID
diff --git a/UUID.hs b/UUID.hs
index 208866ad72..bd7b6b1d01 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -24,7 +24,7 @@ module UUID (
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import Types.UUID
diff --git a/Upgrade.hs b/Upgrade.hs
index 6f1da773dc..8b2e939dde 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -7,7 +7,7 @@
 
 module Upgrade where
 
-import Annex.Common
+import Common.Annex
 import Annex.Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index af91741a0c..b1443fa460 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -9,7 +9,7 @@ module Upgrade.V0 where
 
 import System.IO.Error (try)
 
-import Annex.Common
+import Common.Annex
 import Annex.Content
 import qualified Upgrade.V1
 
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index f4e44acdcb..132e694ebc 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -11,7 +11,7 @@ import System.IO.Error (try)
 import System.Posix.Types
 import Data.Char
 
-import Annex.Common
+import Common.Annex
 import Types.Key
 import Annex.Content
 import PresenceLog
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 8ac26dc525..39c7b15611 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -7,7 +7,7 @@
 
 module Upgrade.V2 where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import LocationLog
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index d03b0910ea..2e0d309794 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -7,7 +7,7 @@
 
 import System.Environment
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import CmdLine
 import Command

From e139a99aa0528af28de8544564e99562d2333d64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 5 Oct 2011 16:35:01 -0400
Subject: [PATCH 2304/2835] investigated file formats; appending timestamp
 should preserve back-compat

---
 ...ust.log_and_remote.log_merge_wackiness.mdwn | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
index af140403ff..2724611fbf 100644
--- a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -13,5 +13,19 @@ as is done with the other log files, then git-annex can consistently
 pick the newest value -- which is as close to the "right" value as can be
 determined in this situation.
 
-(For backwards compatability, git-annex should
-treat lines with no timestamp as being timestamped with 0.)
+----
+
+File format backwards-compatability is the issue. Ideally, old git-annex
+would keep working, ignoring the timestamp.
+
+- uuid.log: "uuid description timestamp" would work; old git-annex 
+  would just treat the timestamp as part of the description which would be
+  ok
+- trust.log: "uuid trustlevel timestamp" would work; old git-annex
+  ignores trailing words
+- remote.log: "uuid key=value ... timestamp" is on the edge but does work
+  (old git-annex will include the timestamp in the key/value map it builds,
+  but that should not break anything really)
+
+Appending "timestamp=xxxxx" would be good for clarity, and make
+it easier to parse the timestamp out from lines that have it.

From dab5bddc64ab4ad479a1104748c15d194e138847 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 11:12:03 -0400
Subject: [PATCH 2305/2835] propigate test suite failure

(but not test suite build failure)
---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index ff58362766..73e4260008 100644
--- a/Makefile
+++ b/Makefile
@@ -57,6 +57,7 @@ test: $(bins)
 	else \
 		if ! ./test; then \
 			echo "** test suite failed!" >&2; \
+			exit 1; \
 		fi; \
 	fi
 

From 52fa4096480ba74c355dffcddbda766994f4d5b7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 15:23:26 -0400
Subject: [PATCH 2306/2835] add UUIDLog, a generic module for mergable
 uuid-based logs

---
 UUIDLog.hs | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 UUIDLog.hs

diff --git a/UUIDLog.hs b/UUIDLog.hs
new file mode 100644
index 0000000000..d6eb8fbbbe
--- /dev/null
+++ b/UUIDLog.hs
@@ -0,0 +1,110 @@
+{- git-annex uuid-based logs
+ -
+ - This is used to store information about a UUID in a way that can
+ - be union merged.
+ -
+ - A line of the log will look like: "UUID[ INFO[ timestamp=foo]]"
+ - The timestamp is last for backwards compatability reasons,
+ - and may not be present on old log lines.
+ - 
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module UUIDLog (
+	Log,
+	LogEntry(..),
+	parseLog,
+	showLog,
+	changeLog,
+	addLog,
+	simpleMap,
+
+	prop_TimeStamp_sane,
+	prop_addLog_sane,
+) where
+
+import qualified Data.Map as M
+import Data.Time.Clock.POSIX
+import Data.Time
+import System.Locale
+
+import Common
+import Types.UUID
+
+data TimeStamp = Unknown | Date POSIXTime
+	deriving (Eq, Ord, Show)
+
+data LogEntry a = LogEntry
+	{ changed :: TimeStamp
+	, value :: a
+	} deriving (Eq, Show)
+
+type Log a = M.Map UUID (LogEntry a)
+
+tskey :: String
+tskey = "timestamp="
+
+showLog :: (a -> String) -> Log a -> String
+showLog shower = unlines . map showpair . M.toList
+	where
+		showpair (k, LogEntry (Date p) v) =
+			unwords [k, shower v, tskey ++ show p]
+		showpair (k, LogEntry Unknown v) =
+			unwords [k, shower v]
+
+parseLog :: (String -> Maybe a) -> String -> Log a
+parseLog parser = M.fromListWith best . catMaybes . map pair . lines
+	where
+		pair line
+			| null ws = Nothing
+			| otherwise = case parser $ unwords info of
+				Nothing -> Nothing
+				Just v -> Just (u, LogEntry c v)
+			where
+				ws = words line
+				u = head ws
+				end = last ws
+				c
+					| tskey `isPrefixOf` end =
+						pdate $ tail $ dropWhile (/= '=') end
+					| otherwise = Unknown
+				info
+					| c == Unknown = drop 1 ws
+					| otherwise = drop 1 $ init ws
+				pdate s = case parseTime defaultTimeLocale "%s%Qs" s of
+					Nothing -> Unknown
+					Just d -> Date $ utcTimeToPOSIXSeconds d
+
+changeLog :: POSIXTime -> UUID -> a -> Log a -> Log a
+changeLog t u v = M.insert u $ LogEntry (Date t) v
+
+{- Only add an LogEntry if it's newer (or at least as new as) than any
+ - existing LogEntry for a UUID. -}
+addLog :: UUID -> LogEntry a -> Log a -> Log a
+addLog = M.insertWith best
+
+{- Converts a Log into a simple Map without the timestamp information.
+ - This is a one-way trip, but useful for code that never needs to change
+ - the log. -}
+simpleMap :: Log a -> M.Map UUID a
+simpleMap = M.map value
+
+best :: LogEntry a -> LogEntry a -> LogEntry a
+best new old
+	| changed old > changed new = old
+	| otherwise = new
+
+-- Unknown is oldest.
+prop_TimeStamp_sane :: Bool
+prop_TimeStamp_sane = Unknown < Date 1
+
+prop_addLog_sane :: Bool
+prop_addLog_sane = newWins && newestWins
+	where
+		newWins = addLog "foo" (LogEntry (Date 1) "new") l == l2
+		newestWins = addLog "foo" (LogEntry (Date 1) "newest") l2 /= l2
+
+		l = M.fromList [("foo", LogEntry (Date 0) "old")]
+		l2 = M.fromList [("foo", LogEntry (Date 1) "new")]

From 3e0d2a080333f3566312da0e1982739873603457 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 15:31:25 -0400
Subject: [PATCH 2307/2835] add timestamp to uuid.log

* New or changed repository descriptions in uuid.log now have a timestamp,
  which is used to ensure the newest description is used when the uuid.log
  has been merged.
* Note that older versions of git-annex will display the timestamp as part
  of the repository description, which is ugly but otherwise harmless.
---
 UUID.hs                                       | 41 ++++++++-----------
 debian/changelog                              |  5 +++
 ...st.log_and_remote.log_merge_wackiness.mdwn |  1 +
 doc/internals.mdwn                            |  6 +--
 test.hs                                       |  3 ++
 5 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/UUID.hs b/UUID.hs
index bd7b6b1d01..c4b870f39f 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -6,7 +6,9 @@
  - UUIDs of remotes are cached in git config, using keys named
  - remote..annex-uuid
  -
- - Copyright 2010 Joey Hess 
+ - uuid.log stores a list of known uuids, and their descriptions.
+ -
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -18,11 +20,11 @@ module UUID (
 	prepUUID,
 	genUUID,
 	describeUUID,
-	uuidMap,
-	uuidLog
+	uuidMap
 ) where
 
 import qualified Data.Map as M
+import Data.Time.Clock.POSIX
 
 import Common.Annex
 import qualified Git
@@ -30,13 +32,14 @@ import qualified Annex.Branch
 import Types.UUID
 import qualified Build.SysConfig as SysConfig
 import Config
+import UUIDLog
 
 configkey :: String
 configkey = "annex.uuid"
 
 {- Filename of uuid.log. -}
-uuidLog :: FilePath
-uuidLog = "uuid.log"
+logfile :: FilePath
+logfile = "uuid.log"
 
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
@@ -50,8 +53,7 @@ genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
 			-- uuidgen generates random uuid by default
 			else []
 
-{- Looks up a repo's UUID. May return "" if none is known.
- -}
+{- Looks up a repo's UUID. May return "" if none is known. -}
 getUUID :: Git.Repo -> Annex UUID
 getUUID r = do
 	g <- gitRepo
@@ -76,26 +78,17 @@ getUncachedUUID r = Git.configGet r configkey ""
 prepUUID :: Annex ()
 prepUUID = do
 	u <- getUUID =<< gitRepo
-	when ("" == u) $ do
+	when (null u) $ do
 		uuid <- liftIO genUUID
 		setConfig configkey uuid
 
-{- Records a description for a uuid in the uuidLog. -}
+{- Records a description for a uuid in the log. -}
 describeUUID :: UUID -> String -> Annex ()
-describeUUID uuid desc = Annex.Branch.change uuidLog $
-	serialize . M.insert uuid desc . parse
-	where
-		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
+describeUUID uuid desc = do
+	ts <- liftIO $ getPOSIXTime
+	Annex.Branch.change logfile $
+		showLog id . changeLog ts uuid desc . parseLog Just
 
-{- Read the uuidLog into a Map -}
+{- Read the uuidLog into a simple Map -}
 uuidMap :: Annex (M.Map UUID String)
-uuidMap = parse <$> Annex.Branch.get uuidLog
-
-parse :: String -> M.Map UUID String
-parse = M.fromList . map pair . lines
-	where
-		pair l
-			| null ws = ("", "")
-			| otherwise = (head ws, unwords $ drop 1 ws)
-			where
-				ws = words l
+uuidMap = (simpleMap . parseLog Just) <$> Annex.Branch.get logfile
diff --git a/debian/changelog b/debian/changelog
index 7554cd5028..5eb7870ef5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,11 @@ git-annex (3.20110929) UNRELEASED; urgency=low
     in addition to their descriptions.
   * Contain the zombie hordes.
   * Add locking to avoid races when changing the git-annex branch.
+  * New or changed repository descriptions in uuid.log now have a timestamp,
+    which is used to ensure the newest description is used when the uuid.log
+    has been merged.
+  * Note that older versions of git-annex will display the timestamp as part
+    of the repository description, which is ugly but otherwise harmless.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
index 2724611fbf..5a703b2cbd 100644
--- a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -21,6 +21,7 @@ would keep working, ignoring the timestamp.
 - uuid.log: "uuid description timestamp" would work; old git-annex 
   would just treat the timestamp as part of the description which would be
   ok
+  > update: converted! --[[Joey]] 
 - trust.log: "uuid trustlevel timestamp" would work; old git-annex
   ignores trailing words
 - remote.log: "uuid key=value ... timestamp" is on the edge but does work
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index e80ecbac0d..4881588ca6 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -42,10 +42,10 @@ more useful than a UUID when it refers to a repository that does not have
 a configured git remote pointing at it.
 
 The file format is simply one line per repository, with the uuid followed by a
-space and then the description through to the end of the line. Example:
+space and then the description, followed by a timestamp. Example:
 
-	e605dca6-446a-11e0-8b2a-002170d25c55 laptop
-	26339d22-446b-11e0-9101-002170d25c55 usb disk
+	e605dca6-446a-11e0-8b2a-002170d25c55 laptop timestamp=1317929189.157237s
+	26339d22-446b-11e0-9101-002170d25c55 usb disk timestamp=1317929330.769997s
 
 ## `remotes.log`
 
diff --git a/test.hs b/test.hs
index 654af5713f..16f2a2bdf6 100644
--- a/test.hs
+++ b/test.hs
@@ -31,6 +31,7 @@ import qualified Types
 import qualified GitAnnex
 import qualified LocationLog
 import qualified UUID
+import qualified UUIDLog
 import qualified Trust
 import qualified Remote
 import qualified RemoteLog
@@ -78,6 +79,8 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
+	, qctest "prop_TimeStamp_sane" UUIDLog.prop_TimeStamp_sane
+	, qctest "prop_addLog_sane" UUIDLog.prop_addLog_sane
 	]
 
 blackbox :: Test

From f929d0229c05ebf0fe2c26d443fe6f843f270983 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 15:55:50 -0400
Subject: [PATCH 2308/2835] Add timestamps to trust.log.

---
 Trust.hs                                      | 38 ++++++++-----------
 debian/changelog                              |  1 +
 ...st.log_and_remote.log_merge_wackiness.mdwn |  1 +
 3 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/Trust.hs b/Trust.hs
index 1920bc6170..2971256a45 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -7,29 +7,29 @@
 
 module Trust (
 	TrustLevel(..),
-	trustLog,
 	trustGet,
 	trustSet,
 	trustPartition
 ) where
 
 import qualified Data.Map as M
+import Data.Time.Clock.POSIX
 
 import Common.Annex
 import Types.TrustLevel
 import qualified Annex.Branch
-import UUID
 import qualified Annex
 
+import UUID
+import UUIDLog
+
 {- Filename of trust.log. -}
 trustLog :: FilePath
 trustLog = "trust.log"
 
 {- Returns a list of UUIDs at the specified trust level. -}
 trustGet :: TrustLevel -> Annex [UUID]
-trustGet level = do
-	m <- trustMap
-	return $ M.keys $ M.filter (== level) m
+trustGet level = M.keys . M.filter (== level) <$> trustMap
 
 {- Read the trustLog into a map, overriding with any
  - values from forcetrust. The map is cached for speed. -}
@@ -39,35 +39,29 @@ trustMap = do
 	case cached of
 		Just m -> return m
 		Nothing -> do
-			overrides <- Annex.getState Annex.forcetrust
-			l <- Annex.Branch.get trustLog
-			let m = M.fromList $ trustMapParse l ++ overrides
+			overrides <- M.fromList <$> Annex.getState Annex.forcetrust
+			m <- (M.union overrides . simpleMap . parseLog parseTrust) <$>
+				Annex.Branch.get trustLog
 			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
 			return m
 
-{- Trust map parser. -}
-trustMapParse :: String -> [(UUID, TrustLevel)]
-trustMapParse s = map pair $ filter (not . null) $ lines s
+parseTrust :: String -> Maybe TrustLevel
+parseTrust s
+	| length w > 0 = readMaybe $ head w
+	-- back-compat; the trust.log used to only list trusted repos
+	| otherwise = Just Trusted
 	where
-		pair l
-			| length w > 1 = (w !! 0, read (w !! 1) :: TrustLevel)
-			-- for back-compat; the trust log used to only
-			-- list trusted uuids
-			| otherwise = (w !! 0, Trusted)
-			where
-				w = words l
+		w = words s
 
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
 	when (null uuid) $
 		error "unknown UUID; cannot modify trust level"
+	ts <- liftIO $ getPOSIXTime
 	Annex.Branch.change trustLog $
-		serialize . M.insert uuid level . M.fromList . trustMapParse
+		showLog show . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
-        where
-                serialize m = unlines $ map showpair $ M.toList m
-		showpair (u, t) = u ++ " " ++ show t
 
 {- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
 trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
diff --git a/debian/changelog b/debian/changelog
index 5eb7870ef5..e5f74bec5b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
     has been merged.
   * Note that older versions of git-annex will display the timestamp as part
     of the repository description, which is ugly but otherwise harmless.
+  * Add timestamps to trust.log.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
index 5a703b2cbd..f300d8708c 100644
--- a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -24,6 +24,7 @@ would keep working, ignoring the timestamp.
   > update: converted! --[[Joey]] 
 - trust.log: "uuid trustlevel timestamp" would work; old git-annex
   ignores trailing words
+  > update: converted! --[[Joey]] 
 - remote.log: "uuid key=value ... timestamp" is on the edge but does work
   (old git-annex will include the timestamp in the key/value map it builds,
   but that should not break anything really)

From f011033869bbeeb7941c1c6e16a2a138b11c92e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 16:07:51 -0400
Subject: [PATCH 2309/2835] add timestamps to remote.log

---
 RemoteLog.hs                                  | 30 ++++++++-----------
 debian/changelog                              |  2 +-
 ...st.log_and_remote.log_merge_wackiness.mdwn |  3 ++
 doc/internals.mdwn                            | 12 ++++----
 4 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/RemoteLog.hs b/RemoteLog.hs
index 149104fbac..d49635b930 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -6,7 +6,6 @@
  -}
 
 module RemoteLog (
-	remoteLog,
 	readRemoteLog,
 	configSet,
 	keyValToConfig,
@@ -16,12 +15,14 @@ module RemoteLog (
 ) where
 
 import qualified Data.Map as M
+import Data.Time.Clock.POSIX
 import Data.Char
 
 import Common.Annex
 import qualified Annex.Branch
 import Types.Remote
 import UUID
+import UUIDLog
 
 {- Filename of remote.log. -}
 remoteLog :: FilePath
@@ -29,27 +30,20 @@ remoteLog = "remote.log"
 
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
-configSet u c = Annex.Branch.change remoteLog $
-	serialize . M.insert u c . remoteLogParse
-	where
-		serialize = unlines . sort . map toline . M.toList
-		toline (u', c') = u' ++ " " ++ unwords (configToKeyVal c')
+configSet u c = do
+	ts <- liftIO $ getPOSIXTime
+	Annex.Branch.change remoteLog $
+		showLog showConfig . changeLog ts u c . parseLog parseConfig
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = remoteLogParse <$> Annex.Branch.get remoteLog
+readRemoteLog = (simpleMap . parseLog parseConfig) <$> Annex.Branch.get remoteLog
 
-remoteLogParse :: String -> M.Map UUID RemoteConfig
-remoteLogParse s =
-	M.fromList $ mapMaybe parseline $ filter (not . null) $ lines s
-	where
-		parseline l
-			| length w > 2 = Just (u, c)
-			| otherwise = Nothing
-			where
-				w = words l
-				u = head w
-				c = keyValToConfig $ tail w
+parseConfig :: String -> Maybe RemoteConfig
+parseConfig = Just . keyValToConfig . words
+
+showConfig :: RemoteConfig -> String
+showConfig = unwords . configToKeyVal
 
 {- Given Strings like "key=value", generates a RemoteConfig. -}
 keyValToConfig :: [String] -> RemoteConfig
diff --git a/debian/changelog b/debian/changelog
index e5f74bec5b..31455c9f0c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,7 +12,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
     has been merged.
   * Note that older versions of git-annex will display the timestamp as part
     of the repository description, which is ugly but otherwise harmless.
-  * Add timestamps to trust.log.
+  * Add timestamps to trust.log and remote.log too.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
index f300d8708c..a84d8cb568 100644
--- a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -28,6 +28,9 @@ would keep working, ignoring the timestamp.
 - remote.log: "uuid key=value ... timestamp" is on the edge but does work
   (old git-annex will include the timestamp in the key/value map it builds,
   but that should not break anything really)
+  > update: converted! --[[Joey]] 
 
 Appending "timestamp=xxxxx" would be good for clarity, and make
 it easier to parse the timestamp out from lines that have it.
+
+> [[done]] --[[Joey]] 
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 4881588ca6..5559d122b9 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -54,7 +54,7 @@ Amazon S3.
 
 The file format is one line per remote, starting with the uuid of the
 remote, followed by a space, and then a series of key=value pairs,
-each separated by whitespace.
+each separated by whitespace, and finally a timestamp.
 
 ## `trust.log`
 
@@ -62,13 +62,15 @@ Records the [[trust]] information for repositories. Does not exist unless
 [[trust]] values are configured.
 
 The file format is one line per repository, with the uuid followed by a
-space, and then either 1 (trusted), 0 (untrusted), or ? (semi-trusted).
-Repositories not listed are semi-trusted.
+space, and then either 1 (trusted), 0 (untrusted), or ? (semi-trusted),
+and finally a timestamp.
 
 Example:
 
-	e605dca6-446a-11e0-8b2a-002170d25c55 1
-	26339d22-446b-11e0-9101-002170d25c55 ?
+	e605dca6-446a-11e0-8b2a-002170d25c55 1 timestamp=1317929189.157237s
+	26339d22-446b-11e0-9101-002170d25c55 ? timestamp=1317929330.769997s
+
+Repositories not listed are semi-trusted.
 
 ## `aaa/bbb/*.log`
 

From 5414bbce58041aa92f2a50a8e721507879000f77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 19:11:30 -0400
Subject: [PATCH 2310/2835] git-annex-shell uuid verification

* git-annex now asks git-annex-shell to verify that it's operating in
  the expected repository.
* Note that this git-annex will not interoperate with remotes using
  older versions of git-annex-shell.

The reason for this check is to avoid git-annex getting confused about
what remote repository actually contains a value. It's a prerequisite for
supporting git insteadOf aliases.
---
 Command.hs         |  2 ++
 Utility/Ssh.hs     | 12 +++++++++---
 debian/changelog   |  5 +++++
 git-annex-shell.hs | 16 ++++++++++++++--
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index 6f8684e4af..54dc9603fa 100644
--- a/Command.hs
+++ b/Command.hs
@@ -189,6 +189,8 @@ paramGlob :: String
 paramGlob = "GLOB"
 paramName :: String
 paramName = "NAME"
+paramUUID :: String
+paramUUID = "UUID"
 paramType :: String
 paramType = "TYPE"
 paramKeyValue :: String
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 05269552c7..4d17a47ba8 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -13,6 +13,7 @@ import qualified Git
 import Utility.SafeCommand
 import Types
 import Config
+import UUID
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
@@ -33,15 +34,20 @@ git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePat
 git_annex_shell r command params
 	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)
 	| Git.repoIsSsh r = do
-		sshparams <- sshToRepo r [Param sshcmd]
+		uuid <- getUUID r
+		sshparams <- sshToRepo r [Param $ sshcmd uuid ]
 		return $ Just ("ssh", sshparams)
 	| otherwise = return Nothing
 	where
 		dir = Git.workTree r
 		shellcmd = "git-annex-shell"
 		shellopts = Param command : File dir : params
-		sshcmd = shellcmd ++ " " ++ 
-			unwords (map shellEscape $ toCommand shellopts)
+		sshcmd uuid = unwords $
+			shellcmd : (map shellEscape $ toCommand shellopts) ++
+			uuidcheck uuid
+		uuidcheck uuid
+			| null uuid = []
+			| otherwise = ["--uuid", uuid]
 
 {- Uses a supplied function (such as boolSystem) to run a git-annex-shell
  - command on a remote.
diff --git a/debian/changelog b/debian/changelog
index 31455c9f0c..6a08d62e89 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,11 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Note that older versions of git-annex will display the timestamp as part
     of the repository description, which is ugly but otherwise harmless.
   * Add timestamps to trust.log and remote.log too.
+  * git-annex-shell: Added the --uuid option.
+  * git-annex now asks git-annex-shell to verify that it's operating in 
+    the expected repository.
+  * Note that this git-annex will not interoperate with remotes using 
+    older versions of git-annex-shell.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 2e0d309794..79b5da69a1 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -6,12 +6,14 @@
  -}
 
 import System.Environment
+import System.Console.GetOpt
 
 import Common.Annex
 import qualified Git
 import CmdLine
 import Command
 import Options
+import UUID
 
 import qualified Command.ConfigList
 import qualified Command.InAnnex
@@ -30,6 +32,16 @@ cmds = map adddirparam $ concat
 	where
 		adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c }
 
+options :: [OptDescr (Annex ())]
+options = uuid : commonOptions
+	where
+		uuid = Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
+		check expected = do
+			u <- getUUID =<< gitRepo
+			when (u /= expected) $ error $
+				"expected repository UUID " ++ expected
+					++ " but found UUID " ++ u
+
 header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
 
@@ -57,7 +69,7 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params =
 	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
-		dispatch (cmd : filterparams params) cmds commonOptions header
+		dispatch (cmd : filterparams params) cmds options header
 
 external :: [String] -> IO ()
 external params =
@@ -72,4 +84,4 @@ filterparams ("--":_) = []
 filterparams (a:as) = a:filterparams as
 
 failure :: IO ()
-failure = error $ "bad parameters\n\n" ++ usage header cmds commonOptions
+failure = error $ "bad parameters\n\n" ++ usage header cmds options

From 3acdba3995941907028905a7c18362309af924b5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 7 Oct 2011 13:17:54 -0400
Subject: [PATCH 2311/2835] faster union merge of multiple branches into index

only write index once
---
 Annex/Branch.hs    | 28 +++++++++++-----------------
 Git/UnionMerge.hs  | 17 +++++++++--------
 git-union-merge.hs |  2 +-
 3 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 0b4bea051d..db4379243f 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -139,8 +139,18 @@ update = do
 			 - from the branches.
 			 -}
 			unless (null fs) $ stageJournalFiles fs
-			mapM_ mergeref refs
 			g <- gitRepo
+			unless (null refs) $ do
+				showSideAction $ "merging " ++
+					(unwords $ map Git.refDescribe refs) ++
+					" into " ++ name
+				{- Note: This merges the branches into the index.
+				 - Any unstaged changes in the git-annex branch
+				 - (if it's checked out) will be removed. So,
+				 - documentation advises users not to directly
+				 - modify the branch.
+				 -}
+				liftIO $ Git.UnionMerge.merge_index g refs
 			liftIO $ Git.commit g "update" fullname (fullname:refs)
 			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 			invalidateCache
@@ -155,22 +165,6 @@ update = do
 				Params "--oneline -n1"
 				]
 			return $ not $ L.null diffs
-		mergeref ref = do
-			showSideAction $ "merging " ++
-				Git.refDescribe ref ++ " into " ++ name
-			{- By passing only one ref, it is actually
-			 - merged into the index, preserving any
-			 - changes that may already be staged.
-			 -
-			 - However, any changes in the git-annex
-			 - branch that are *not* reflected in the
-			 - index will be removed. So, documentation
-			 - advises users not to directly modify the
-			 - branch.
-			 -}
-			g <- gitRepo
-			liftIO $ Git.UnionMerge.merge g [ref]
-			return $ Just ref
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index ac002b374e..859a66ca07 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -7,6 +7,7 @@
 
 module Git.UnionMerge (
 	merge,
+	merge_index,
 	update_index,
 	update_index_line,
 	ls_tree
@@ -18,24 +19,24 @@ import Data.Maybe
 import Data.String.Utils
 import qualified Data.ByteString.Lazy.Char8 as L
 
+import Common
 import Git
-import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
  -
- - When only one branch is specified, it is merged into the index.
- - In this case, previously staged changes in the index are preserved.
- -
  - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: Repo -> [String] -> IO ()
-merge g (x:y:[]) = do
+merge :: Repo -> String -> String -> IO ()
+merge g x y = do
 	a <- ls_tree g x
 	b <- merge_trees g x y
 	update_index g (a++b)
-merge g [x] = merge_tree_index g x >>= update_index g
-merge _ _ = error "wrong number of branches to merge"
+
+{- Merges a list of branches into the index. Previously staged changed in
+ - the index are preserved (and participate in the merge). -}
+merge_index :: Repo -> [String] -> IO ()
+merge_index g bs = update_index g =<< concat <$> mapM (merge_tree_index g) bs
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 34f22d06fe..0d1d0819d2 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -41,6 +41,6 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
-	Git.UnionMerge.merge g [aref, bref]
+	Git.UnionMerge.merge g aref bref
 	Git.commit g "union merge" newref [aref, bref]
 	cleanup g

From 44fc358885f2d6ae20afb21d9526a14b2966901c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 7 Oct 2011 13:37:01 -0400
Subject: [PATCH 2312/2835] avoid merging multiple branches that point to the
 same tree

avoids git warning "error: duplicate parent xxx ignored"
---
 Annex/Branch.hs | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index db4379243f..ea8ac2cec0 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -126,7 +126,8 @@ update = do
 	unless (branchUpdated state) $ do
 		-- check what needs updating before taking the lock
 		fs <- getJournalFiles
-		refs <- filterM checkref =<< siblingBranches
+		c <- filterM changedbranch =<< siblingBranches
+		let (refs, branches) = unzip c
 		unless (null fs && null refs) $ withIndex $ lockJournal $ do
 			{- Before refs are merged into the index, it's
 			 - important to first stage the journal into the
@@ -140,9 +141,9 @@ update = do
 			 -}
 			unless (null fs) $ stageJournalFiles fs
 			g <- gitRepo
-			unless (null refs) $ do
+			unless (null branches) $ do
 				showSideAction $ "merging " ++
-					(unwords $ map Git.refDescribe refs) ++
+					(unwords $ map Git.refDescribe branches) ++
 					" into " ++ name
 				{- Note: This merges the branches into the index.
 				 - Any unstaged changes in the git-annex branch
@@ -150,18 +151,18 @@ update = do
 				 - documentation advises users not to directly
 				 - modify the branch.
 				 -}
-				liftIO $ Git.UnionMerge.merge_index g refs
-			liftIO $ Git.commit g "update" fullname (fullname:refs)
+				liftIO $ Git.UnionMerge.merge_index g branches
+			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
 			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 			invalidateCache
 	where
-		checkref ref = do
+		changedbranch (_, branch) = do
 			g <- gitRepo
 			-- checking with log to see if there have been changes
 			-- is less expensive than always merging
 			diffs <- liftIO $ Git.pipeRead g [
 				Param "log",
-				Param (name++".."++ref),
+				Param (name ++ ".." ++ branch),
 				Params "--oneline -n1"
 				]
 			return $ not $ L.null diffs
@@ -185,13 +186,15 @@ hasOrigin = refExists originname
 hasSomeBranch :: Annex Bool
 hasSomeBranch = not . null <$> siblingBranches
 
-{- List of all git-annex branches, including the main one and any
+{- List of all git-annex (refs, branches), including the main one and any
  - from remotes. -}
-siblingBranches :: Annex [String]
+siblingBranches :: Annex [(String, String)]
 siblingBranches = do
 	g <- gitRepo
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-	return $ map (last . words . L.unpack) (L.lines r)
+	return $ map (pair . words . L.unpack) (L.lines r)
+	where
+		pair l = (head l, last l)
 
 {- Applies a function to modifiy the content of a file. -}
 change :: FilePath -> (String -> String) -> Annex ()

From 82e655efd060ce3c674681b0c51f2efc1efa069d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 7 Oct 2011 13:38:56 -0400
Subject: [PATCH 2313/2835] performance fix

It was checking if it needed to merge on every branch access, fix it to
only check once.
---
 Annex/Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index ea8ac2cec0..29e3a3956f 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -153,8 +153,8 @@ update = do
 				 -}
 				liftIO $ Git.UnionMerge.merge_index g branches
 			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
-			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 			invalidateCache
+		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 	where
 		changedbranch (_, branch) = do
 			g <- gitRepo

From dfee6e1ed65e6df3fe66df6e8351d41a7572903f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 7 Oct 2011 13:59:34 -0400
Subject: [PATCH 2314/2835] better layout

And a theoretical fix to branchstate cache invalidation, but not a bug
that could actually happen.
---
 Annex/Branch.hs | 71 ++++++++++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 33 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 29e3a3956f..a548798d5d 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -121,40 +121,37 @@ commit message = do
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
-update = do
-	state <- getState
-	unless (branchUpdated state) $ do
-		-- check what needs updating before taking the lock
-		fs <- getJournalFiles
-		c <- filterM changedbranch =<< siblingBranches
-		let (refs, branches) = unzip c
-		unless (null fs && null refs) $ withIndex $ lockJournal $ do
-			{- Before refs are merged into the index, it's
-			 - important to first stage the journal into the
-			 - index. Otherwise, any changes in the journal
-			 - would later get staged, and might overwrite
-			 - changes made during the merge.
-			 -
-			 - It would be cleaner to handle the merge by
-			 - updating the journal, not the index, with changes
-			 - from the branches.
+update = onceonly $ do
+	-- check what needs updating before taking the lock
+	fs <- getJournalFiles
+	c <- filterM changedbranch =<< siblingBranches
+	let (refs, branches) = unzip c
+	unless (null fs && null refs) $ withIndex $ lockJournal $ do
+		{- Before refs are merged into the index, it's
+		 - important to first stage the journal into the
+		 - index. Otherwise, any changes in the journal
+		 - would later get staged, and might overwrite
+		 - changes made during the merge.
+		 -
+		 - It would be cleaner to handle the merge by
+		 - updating the journal, not the index, with changes
+		 - from the branches.
+		 -}
+		unless (null fs) $ stageJournalFiles fs
+		g <- gitRepo
+		unless (null branches) $ do
+			showSideAction $ "merging " ++
+				(unwords $ map Git.refDescribe branches) ++
+				" into " ++ name
+			{- Note: This merges the branches into the index.
+			 - Any unstaged changes in the git-annex branch
+			 - (if it's checked out) will be removed. So,
+			 - documentation advises users not to directly
+			 - modify the branch.
 			 -}
-			unless (null fs) $ stageJournalFiles fs
-			g <- gitRepo
-			unless (null branches) $ do
-				showSideAction $ "merging " ++
-					(unwords $ map Git.refDescribe branches) ++
-					" into " ++ name
-				{- Note: This merges the branches into the index.
-				 - Any unstaged changes in the git-annex branch
-				 - (if it's checked out) will be removed. So,
-				 - documentation advises users not to directly
-				 - modify the branch.
-				 -}
-				liftIO $ Git.UnionMerge.merge_index g branches
-			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
-			invalidateCache
-		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
+			liftIO $ Git.UnionMerge.merge_index g branches
+		liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
+		invalidateCache
 	where
 		changedbranch (_, branch) = do
 			g <- gitRepo
@@ -166,6 +163,14 @@ update = do
 				Params "--oneline -n1"
 				]
 			return $ not $ L.null diffs
+		onceonly a = unlessM (branchUpdated <$> getState) $ do
+			r <- a
+			Annex.changeState setupdated
+			return r
+		setupdated s = s { Annex.branchstate = new }
+			where
+				new = old { branchUpdated = True }
+				old = Annex.branchstate s
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool

From 81ed7b203d1c1af7a9c7a52c939eb0797fa16ab7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 9 Oct 2011 14:58:32 -0400
Subject: [PATCH 2315/2835] Now supports git's insteadOf configuration, to
 modify the url used to access a remote. Note that pushInsteadOf is not used;
 that and pushurl are reserved for actual git pushes. Closes: #644278

---
 Git.hs           | 22 ++++++++++++++++++----
 debian/changelog |  3 +++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/Git.hs b/Git.hs
index 7a3d55deaa..b87f2a2659 100644
--- a/Git.hs
+++ b/Git.hs
@@ -496,15 +496,29 @@ configParse s = M.fromList $ map pair $ lines s
 configRemotes :: Repo -> IO [Repo]
 configRemotes repo = mapM construct remotepairs
 	where
-		remotepairs = M.toList $ filterremotes $ config repo
-		filterremotes = M.filterWithKey (\k _ -> isremote k)
+		filterconfig f = filter f $ M.toList $ config repo
+		filterkeys f = filterconfig (\(k,_) -> f k)
+		remotepairs = filterkeys isremote
 		isremote k = startswith "remote." k && endswith ".url" k
 		construct (k,v) = do
-			r <- gen v
+			r <- gen $ calcloc v
 			return $ repoRemoteNameSet r k
-		gen v	| scpstyle v = repoFromUrl $ scptourl v
+		gen v
+			| scpstyle v = repoFromUrl $ scptourl v
 			| isURI v = repoFromUrl v
 			| otherwise = repoFromRemotePath v repo
+		-- insteadof config can rewrite remote location
+		calcloc l
+			| null insteadofs = l
+			| otherwise = replacement ++ drop (length replacement) l
+			where
+				replacement = take (length bestkey - length prefix) bestkey
+				bestkey = fst $ maximumBy longestvalue insteadofs
+				longestvalue (_, a) (_, b) = compare b a
+				insteadofs = filterconfig $ \(k, v) -> 
+					endswith prefix k &&
+					startswith v l
+				prefix = ".insteadof"
 		-- git remotes can be written scp style -- [user@]host:dir
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
diff --git a/debian/changelog b/debian/changelog
index 6a08d62e89..a72a85a8a7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -18,6 +18,9 @@ git-annex (3.20110929) UNRELEASED; urgency=low
     the expected repository.
   * Note that this git-annex will not interoperate with remotes using 
     older versions of git-annex-shell.
+  * Now supports git's insteadOf configuration, to modify the url
+    used to access a remote. Note that pushInsteadOf is not used;
+    that and pushurl are reserved for actual git pushes. Closes: #644278
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From f0153f9fd7c753b12d612d1e57833abb2a7aa059 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 9 Oct 2011 16:19:09 -0400
Subject: [PATCH 2316/2835] fix a race

Another process may stage journalled files before the lock is
taken, so need to get the list of journalled files afterwards.

It's unfortunate this means getting the directory contents twice,
but it seems better to do that than sometimes take the lock
unnecessarily.
---
 Annex/Branch.hs | 48 ++++++++++++++++++++++++++----------------------
 1 file changed, 26 insertions(+), 22 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a548798d5d..0002befecd 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -111,33 +111,32 @@ create = unlessM hasBranch $ do
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = do
-	fs <- getJournalFiles
-	when (not $ null fs) $ lockJournal $ do
-		stageJournalFiles fs
-		g <- gitRepo
-		withIndex $ liftIO $ Git.commit g message fullname [fullname]
+commit message = whenM journalDirty $ lockJournal $ do
+	stageJournalFiles
+	g <- gitRepo
+	withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
- - data is read from it. Runs only once per git-annex run. -}
+ - data is read from it. Runs only once per git-annex run.
+ -
+ - Before refs are merged into the index, it's
+ - important to first stage the journal into the
+ - index. Otherwise, any changes in the journal
+ - would later get staged, and might overwrite
+ - changes made during the merge.
+ -
+ - It would be cleaner to handle the merge by
+ - updating the journal, not the index, with changes
+ - from the branches.
+ -}
 update :: Annex ()
 update = onceonly $ do
 	-- check what needs updating before taking the lock
-	fs <- getJournalFiles
+	dirty <- journalDirty
 	c <- filterM changedbranch =<< siblingBranches
 	let (refs, branches) = unzip c
-	unless (null fs && null refs) $ withIndex $ lockJournal $ do
-		{- Before refs are merged into the index, it's
-		 - important to first stage the journal into the
-		 - index. Otherwise, any changes in the journal
-		 - would later get staged, and might overwrite
-		 - changes made during the merge.
-		 -
-		 - It would be cleaner to handle the merge by
-		 - updating the journal, not the index, with changes
-		 - from the branches.
-		 -}
-		unless (null fs) $ stageJournalFiles fs
+	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
+		when dirty $ stageJournalFiles
 		g <- gitRepo
 		unless (null branches) $ do
 			showSideAction $ "merging " ++
@@ -279,8 +278,9 @@ getJournalFiles = do
 	return $ filter (`notElem` [".", ".."]) fs
 
 {- Stages the specified journalfiles. -}
-stageJournalFiles :: [FilePath] -> Annex ()
-stageJournalFiles fs = do
+stageJournalFiles :: Annex ()
+stageJournalFiles = do
+	fs <- getJournalFiles
 	g <- gitRepo
 	withIndex $ liftIO $ do
 		let dir = gitAnnexJournalDir g
@@ -305,6 +305,10 @@ stageJournalFiles fs = do
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
 
+{- Checks if there are changes in the journal. -}
+journalDirty :: Annex Bool
+journalDirty = not . null <$> getJournalFiles
+
 {- Produces a filename to use in the journal for a file on the branch.
  -
  - The journal typically won't have a lot of files in it, so the hashing

From 10edaf6dc9c5a8f091487e8ba82ac8f54697bf0c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 10 Oct 2011 16:03:32 -0400
Subject: [PATCH 2317/2835] reorder

---
 debian/changelog | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index a72a85a8a7..9bcaa40466 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,12 +1,6 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
-  * Various speed improvements gained by using ByteStrings.
   * Fix referring to remotes by uuid.
-  * status: List all known repositories.
-  * When displaying a list of repositories, show git remote names
-    in addition to their descriptions.
-  * Contain the zombie hordes.
-  * Add locking to avoid races when changing the git-annex branch.
   * New or changed repository descriptions in uuid.log now have a timestamp,
     which is used to ensure the newest description is used when the uuid.log
     has been merged.
@@ -21,6 +15,12 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Now supports git's insteadOf configuration, to modify the url
     used to access a remote. Note that pushInsteadOf is not used;
     that and pushurl are reserved for actual git pushes. Closes: #644278
+  * status: List all known repositories.
+  * When displaying a list of repositories, show git remote names
+    in addition to their descriptions.
+  * Add locking to avoid races when changing the git-annex branch.
+  * Various speed improvements gained by using ByteStrings.
+  * Contain the zombie hordes.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 025ded4a2dfb58a6ec0cb47b9d625d593a4e1977 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 10 Oct 2011 17:37:44 -0400
Subject: [PATCH 2318/2835] tweaks

---
 Annex/Branch.hs  | 30 ++++++++++++++----------------
 Backend.hs       | 42 +++++++++++++++++++-----------------------
 Types/Backend.hs |  2 +-
 3 files changed, 34 insertions(+), 40 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 0002befecd..aea0d2bffe 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -215,20 +215,16 @@ set file content = do
  -
  - Returns an empty string if the file doesn't exist yet. -}
 get :: FilePath -> Annex String
-get file = do
-	cached <- getCache file
-	case cached of
-		Just content -> return content
-		Nothing -> do
-			j <- getJournalFile file
-			case j of
-				Just content -> do
-					setCache file content
-					return content
-				Nothing -> withIndexUpdate $ do
-					content <- catFile fullname file
-					setCache file content
-					return content
+get file = fromcache =<< getCache file
+	where
+		fromcache (Just content) = return content
+		fromcache Nothing = fromjournal =<< getJournalFile file
+		fromjournal (Just content) = cache content
+		fromjournal Nothing = withIndexUpdate $
+			cache =<< catFile fullname file
+		cache content = do
+			setCache file content
+			return content
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
@@ -287,8 +283,7 @@ stageJournalFiles = do
 		let paths = map (dir ) fs
 		-- inject all the journal files directly into git
 		-- in one quick command
-		(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $
-			Git.gitCommandLine g [Param "hash-object", Param "-w", Param "--stdin-paths"]
+		(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object g
 		_ <- forkProcess $ do
 			hPutStr toh $ unlines paths
 			hClose toh
@@ -304,6 +299,9 @@ stageJournalFiles = do
 	where
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
+		git_hash_object g = Git.gitCommandLine g
+			[Param "hash-object", Param "-w", Param "--stdin-paths"]
+
 
 {- Checks if there are changes in the journal. -}
 journalDirty :: Annex Bool
diff --git a/Backend.hs b/Backend.hs
index d1ff114059..a09fc0e990 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -39,23 +39,20 @@ orderedList = do
 	l <- Annex.getState Annex.backends -- list is cached here
 	if not $ null l
 		then return l
-		else do
-			s <- getstandard
-			d <- Annex.getState Annex.forcebackend
-			handle d s
+		else handle =<< Annex.getState Annex.forcebackend
 	where
-		parseBackendList [] = list
-		parseBackendList s = map lookupBackendName $ words s
-		handle Nothing s = return s
-		handle (Just "") s = return s
-		handle (Just name) s = do
-			let l' = lookupBackendName name : s
-			Annex.changeState $ \state -> state { Annex.backends = l' }
+		handle Nothing = standard
+		handle (Just "") = standard
+		handle (Just name) = do
+			l' <- (lookupBackendName name :) <$> standard
+			Annex.changeState $ \s -> s { Annex.backends = l' }
 			return l'
-		getstandard = do
+		standard = do
 			g <- gitRepo
 			return $ parseBackendList $
 				Git.configGet g "annex.backends" ""
+		parseBackendList [] = list
+		parseBackendList s = map lookupBackendName $ words s
 
 {- Generates a key for a file, trying each backend in turn until one
  - accepts it. -}
@@ -83,17 +80,15 @@ lookupFile file = do
 	where
 		getsymlink = takeFileName <$> readSymbolicLink file
 		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
-		makeret l k =
+		makeret l k = let bname = keyBackendName k in
 			case maybeLookupBackendName bname of
-					Just backend -> return $ Just (k, backend)
-					Nothing -> do
-						when (isLinkToAnnex l) $
-							warning skip
-						return Nothing
-			where
-				bname = keyBackendName k
-				skip = "skipping " ++ file ++ 
-					" (unknown backend " ++ bname ++ ")"
+				Just backend -> return $ Just (k, backend)
+				Nothing -> do
+					when (isLinkToAnnex l) $ warning $
+						"skipping " ++ file ++
+						" (unknown backend " ++
+						bname ++ ")"
+					return Nothing
 
 type BackendFile = (Maybe (Backend Annex), FilePath)
 
@@ -121,4 +116,5 @@ maybeLookupBackendName :: String -> Maybe (Backend Annex)
 maybeLookupBackendName s
 	| length matches == 1 = Just $ head matches
 	| otherwise = Nothing
-	where matches = filter (\b -> s == B.name b) list
+	where
+		matches = filter (\b -> s == B.name b) list
diff --git a/Types/Backend.hs b/Types/Backend.hs
index f86d0845cd..4f82267045 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -1,6 +1,6 @@
 {- git-annex key/value backend data type
  -
- - Most things should not need this, using Types instead
+ - Most things should not need this, using Remotes instead
  -
  - Copyright 2010 Joey Hess 
  -

From b505ba83e8b62a9ed0ec2fb96448c5fc801184d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 14:43:45 -0400
Subject: [PATCH 2319/2835] minor syntax changes

---
 Annex/Content.hs             |  2 +-
 Backend/SHA.hs               | 16 ++++++++--------
 Command/ConfigList.hs        |  3 +--
 Command/Fsck.hs              |  2 +-
 Command/Init.hs              |  3 +--
 Command/Move.hs              |  6 ++----
 Crypto.hs                    | 25 ++++++++++++-------------
 Init.hs                      |  6 +++---
 Limit.hs                     |  7 +++----
 Messages.hs                  | 27 +++++++++++++++------------
 Remote.hs                    | 12 +++++-------
 Remote/Git.hs                |  2 +-
 Remote/Helper/Encryptable.hs |  6 ++----
 UUID.hs                      | 17 +++++++++--------
 Utility.hs                   | 10 +++-------
 Utility/Path.hs              | 21 ++++++++-------------
 Utility/Ssh.hs               |  2 +-
 git-annex-shell.hs           |  2 +-
 test.hs                      |  4 +---
 19 files changed, 78 insertions(+), 95 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 21403954ae..3827154a67 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -57,7 +57,7 @@ calcGitLink file key = do
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
 	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	logChange g key u status
 
 {- Runs an action, passing it a temporary filename to download,
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 4b5b14cc36..3a54a8871b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -104,11 +104,11 @@ checkKeyChecksum size key = do
 	present <- liftIO $ doesFileExist file
 	if not present || fast
 		then return True
-		else do
-			s <- shaN size file
-			if s == dropExtension (keyName key)
-				then return True
-				else do
-					dest <- moveBad key
-					warning $ "Bad file content; moved to " ++ dest
-					return False
+		else check =<< shaN size file
+	where
+		check s
+			| s == dropExtension (keyName key) = return True
+			| otherwise = do
+				dest <- moveBad key
+				warning $ "Bad file content; moved to " ++ dest
+				return False
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index d52c33f3b9..60f71eee6d 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -20,7 +20,6 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	liftIO $ putStrLn $ "annex.uuid=" ++ u
 	stop
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 689271dd7e..16e834fbf1 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -55,7 +55,7 @@ verifyLocationLog key file = do
 		preventWrite f
 		preventWrite (parentDir f)
 
-	u <- getUUID g
+	u <- getUUID
         uuids <- keyLocations key
 
 	case (present, u `elem` uuids) of
diff --git a/Command/Init.hs b/Command/Init.hs
index ace06c2c34..b9dffb5cd9 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -29,7 +29,6 @@ start ws = do
 perform :: String -> CommandPerform
 perform description = do
 	initialize
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	describeUUID u description
 	next $ return True
diff --git a/Command/Move.hs b/Command/Move.hs
index d650c52510..52eb49da14 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -72,8 +72,7 @@ remoteHasKey remote key present	= do
  -}
 toStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 toStart dest move file = isAnnexed file $ \(key, _) -> do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	ishere <- inAnnex key
 	if not ishere || u == Remote.uuid dest
 		then stop -- not here, so nothing to do
@@ -122,8 +121,7 @@ toCleanup dest move key = do
  -}
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 fromStart src move file = isAnnexed file $ \(key, _) -> do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	remotes <- Remote.keyPossibilities key
 	if u == Remote.uuid src || not (any (== src) remotes)
 		then stop
diff --git a/Crypto.hs b/Crypto.hs
index 21e4d75605..ced7c144c4 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -135,13 +135,12 @@ decryptCipher _ (EncryptedCipher encipher _) =
 {- Generates an encrypted form of a Key. The encryption does not need to be
  - reversable, nor does it need to be the same type of encryption used
  - on content. It does need to be repeatable. -}
-encryptKey :: Cipher -> Key -> IO Key
-encryptKey c k =
-	return Key {
-		keyName = hmacWithCipher c (show k),
-		keyBackendName = "GPGHMACSHA1",
-		keySize = Nothing, -- size and mtime omitted
-		keyMtime = Nothing -- to avoid leaking data
+encryptKey :: Cipher -> Key -> Key
+encryptKey c k = Key
+	{ keyName = hmacWithCipher c (show k)
+	, keyBackendName = "GPGHMACSHA1"
+	, keySize = Nothing -- size and mtime omitted
+	, keyMtime = Nothing -- to avoid leaking data
 	}
 
 {- Runs an action, passing it a handle from which it can 
@@ -223,18 +222,18 @@ gpgCipherHandle params c a b = do
 	return ret
 
 configKeyIds :: RemoteConfig -> IO KeyIds
-configKeyIds c = do
-	let k = configGet c "encryption"
-	s <- gpgRead [Params "--with-colons --list-public-keys", Param k]
-	return $ KeyIds $ parseWithColons s
+configKeyIds c = parse <$> gpgRead params
 	where
-		parseWithColons s = map keyIdField $ filter pubKey $ lines s
+		params = [Params "--with-colons --list-public-keys",
+			Param $ configGet c "encryption"]
+		parse = KeyIds . map keyIdField . filter pubKey . lines
 		pubKey = isPrefixOf "pub:"
 		keyIdField s = split ":" s !! 4
 
 configGet :: RemoteConfig -> String -> String
 configGet c key = fromMaybe missing $ M.lookup key c
-	where missing = error $ "missing " ++ key ++ " in remote config"
+	where
+		missing = error $ "missing " ++ key ++ " in remote config"
 
 hmacWithCipher :: Cipher -> String -> String
 hmacWithCipher c = hmacWithCipher' (cipherHmac c) 
diff --git a/Init.hs b/Init.hs
index 145413e8da..509cbca15d 100644
--- a/Init.hs
+++ b/Init.hs
@@ -75,6 +75,6 @@ preCommitHook = do
 
 preCommitScript :: String
 preCommitScript = 
-		"#!/bin/sh\n" ++
-		"# automatically configured by git-annex\n" ++ 
-		"git annex pre-commit .\n"
+	"#!/bin/sh\n" ++
+	"# automatically configured by git-annex\n" ++ 
+	"git annex pre-commit .\n"
diff --git a/Limit.hs b/Limit.hs
index 3812ceea4b..8dd88e72b8 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -65,14 +65,13 @@ addExclude glob = addLimit $ return . notExcluded
 {- Adds a limit to skip files not believed to be present
  - in a specfied repository. -}
 addIn :: String -> Annex ()
-addIn name = do
-	u <- Remote.nameToUUID name
-	addLimit $ if name == "." then check inAnnex else check (remote u)
+addIn name = addLimit $ check $ if name == "." then inAnnex else inremote
 	where
 		check a f = Backend.lookupFile f >>= handle a
 		handle _ Nothing = return False
 		handle a (Just (key, _)) = a key
-		remote u key = do
+		inremote key = do
+			u <- Remote.nameToUUID name
 			us <- keyLocations key
 			return $ u `elem` us
 
diff --git a/Messages.hs b/Messages.hs
index e029c50722..6f4880e2d5 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -31,31 +31,31 @@ import qualified Annex
 import qualified Messages.JSON as JSON
 
 showStart :: String -> String -> Annex ()
-showStart command file = handle (JSON.start command file) $ do
-	putStr $ command ++ " " ++ file ++ " "
-	hFlush stdout
+showStart command file = handle (JSON.start command file) $
+	flushed $ putStr $ command ++ " " ++ file ++ " "
 
 showNote :: String -> Annex ()
-showNote s = handle (JSON.note s) $ do
-	putStr $ "(" ++ s ++ ") "
-	hFlush stdout
+showNote s = handle (JSON.note s) $
+	flushed $ putStr $ "(" ++ s ++ ") "
 
 showAction :: String -> Annex ()
 showAction s = showNote $ s ++ "..."
 
 showProgress :: Annex ()
-showProgress = handle q $ do
-	putStr "."
-	hFlush stdout
+showProgress = handle q $
+	flushed $ putStr "."
 
 showSideAction :: String -> Annex ()
-showSideAction s = handle q $ putStrLn $ "(" ++ s ++ "...)"
+showSideAction s = handle q $
+	putStrLn $ "(" ++ s ++ "...)"
 
 showOutput :: Annex ()
-showOutput = handle q $ putStr "\n"
+showOutput = handle q $
+	putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = handle (JSON.note s) $ putStrLn $ '\n' : indent s
+showLongNote s = handle (JSON.note s) $
+	putStrLn $ '\n' : indent s
 
 showEndOk :: Annex ()
 showEndOk = showEndResult True
@@ -113,3 +113,6 @@ maybeShowJSON v = handle (JSON.add v) q
 
 q :: Monad m => m ()
 q = return ()
+
+flushed :: IO () -> IO ()
+flushed a = a >> hFlush stdout
diff --git a/Remote.hs b/Remote.hs
index 27ebd724ac..b1305b9e0f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -78,7 +78,7 @@ genList = do
 			enumerate t >>=
 			mapM (gen m t)
 		gen m t r = do
-			u <- getUUID r
+			u <- getRepoUUID r
 			generate t r u (M.lookup u m)
 
 {- Looks up a remote by name. (Or by UUID.) Only finds currently configured
@@ -104,7 +104,7 @@ byName' n = do
  - and returns its UUID. Finds even remotes that are not configured in
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
-nameToUUID "." = getUUID =<< gitRepo -- special case for current repo
+nameToUUID "." = getUUID -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
@@ -129,7 +129,7 @@ nameToUUID n = do
  - of the UUIDs. -}
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
-	here <- getUUID =<< gitRepo
+	here <- getUUID
 	m <- M.unionWith addname <$> uuidMap <*> remoteMap
 	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
@@ -178,8 +178,7 @@ keyPossibilitiesTrusted = keyPossibilities' True
 
 keyPossibilities' :: Bool -> Key -> Annex ([Remote Annex], [UUID])
 keyPossibilities' withtrusted key = do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	trusted <- if withtrusted then trustGet Trusted else return []
 
 	-- get uuids of all remotes that are recorded to have the key
@@ -198,8 +197,7 @@ keyPossibilities' withtrusted key = do
 {- Displays known locations of a key. -}
 showLocations :: Key -> [UUID] -> Annex ()
 showLocations key exclude = do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	uuids <- keyLocations key
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 704bdc04d3..183fcd8548 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -48,7 +48,7 @@ gen r u _ = do
 		(False, "") -> tryGitConfigRead r
 		_ -> return r
 
-	u' <- getUUID r'
+	u' <- getRepoUUID r'
 
 	let defcst = if cheap then cheapRemoteCost else expensiveRemoteCost
 	cst <- remoteCost r' defcst
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 004b70408b..85d269a213 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -78,8 +78,6 @@ remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher
 {- Gets encryption Cipher, and encrypted version of Key. -}
 cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))
 cipherKey Nothing _ = return Nothing
-cipherKey (Just c) k = remoteCipher c >>= maybe (return Nothing) encrypt
+cipherKey (Just c) k = maybe Nothing encrypt <$> remoteCipher c
 	where
-		encrypt ciphertext = do
-			k' <- liftIO $ encryptKey ciphertext k
-			return $ Just (ciphertext, k')
+		encrypt ciphertext = Just (ciphertext, encryptKey ciphertext k)
diff --git a/UUID.hs b/UUID.hs
index c4b870f39f..63ce87f03e 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -16,6 +16,7 @@
 module UUID (
 	UUID,
 	getUUID,
+	getRepoUUID,
 	getUncachedUUID,
 	prepUUID,
 	genUUID,
@@ -44,7 +45,7 @@ logfile = "uuid.log"
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
 genUUID :: IO UUID
-genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
+genUUID = pOpen ReadFromPipe command params hGetLine
 	where
 		command = SysConfig.uuid
 		params = if command == "uuid"
@@ -53,9 +54,12 @@ genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
 			-- uuidgen generates random uuid by default
 			else []
 
+getUUID :: Annex UUID
+getUUID = getRepoUUID =<< gitRepo
+
 {- Looks up a repo's UUID. May return "" if none is known. -}
-getUUID :: Git.Repo -> Annex UUID
-getUUID r = do
+getRepoUUID :: Git.Repo -> Annex UUID
+getRepoUUID r = do
 	g <- gitRepo
 
 	let c = cached g
@@ -76,11 +80,8 @@ getUncachedUUID r = Git.configGet r configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
-prepUUID = do
-	u <- getUUID =<< gitRepo
-	when (null u) $ do
-		uuid <- liftIO genUUID
-		setConfig configkey uuid
+prepUUID = whenM (null <$> getUUID) $
+	setConfig configkey =<< liftIO genUUID
 
 {- Records a description for a uuid in the log. -}
 describeUUID :: UUID -> String -> Annex ()
diff --git a/Utility.hs b/Utility.hs
index 4e82e63c9d..8ef60a0814 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -19,6 +19,7 @@ module Utility (
 	anyM
 ) where
 
+import Control.Applicative
 import IO (bracket)
 import System.IO
 import System.Posix.Process hiding (executeFile)
@@ -69,9 +70,7 @@ withTempFile template a = bracket create remove use
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]
-dirContents d = do
-	c <- getDirectoryContents d
-	return $ map (d ) $ filter notcruft c
+dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
 	where
 		notcruft "." = False
 		notcruft ".." = False
@@ -79,10 +78,7 @@ dirContents d = do
 
 {- Current user's home directory. -}
 myHomeDir :: IO FilePath
-myHomeDir = do
-	uid <- getEffectiveUserID
-	u <- getUserEntryForID uid
-	return $ homeDirectory u
+myHomeDir = homeDirectory <$> (getUserEntryForID =<< getEffectiveUserID)
 
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
diff --git a/Utility/Path.hs b/Utility/Path.hs
index ce54fb3695..1c68b87bbc 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -17,10 +17,9 @@ import Control.Applicative
 
 {- Returns the parent directory of a path. Parent of / is "" -}
 parentDir :: FilePath -> FilePath
-parentDir dir =
-	if not $ null dirs
-	then slash ++ join s (init dirs)
-	else ""
+parentDir dir
+	| not $ null dirs = slash ++ join s (init dirs)
+	| otherwise = ""
 		where
 			dirs = filter (not . null) $ split s dir
 			slash = if isAbsolute dir then s else ""
@@ -72,7 +71,7 @@ relPathCwdToFile f = relPathDirToFile <$> getCurrentDirectory <*> absPath f
  - Both must be absolute, and normalized (eg with absNormpath).
  -}
 relPathDirToFile :: FilePath -> FilePath -> FilePath
-relPathDirToFile from to = path
+relPathDirToFile from to = join s $ dotdots ++ uncommon
 	where
 		s = [pathSeparator]
 		pfrom = split s from
@@ -82,7 +81,6 @@ relPathDirToFile from to = path
 		uncommon = drop numcommon pto
 		dotdots = replicate (length pfrom - numcommon) ".."
 		numcommon = length common
-		path = join s $ dotdots ++ uncommon
 
 prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
 prop_relPathDirToFile_basics from to
@@ -99,14 +97,11 @@ prop_relPathDirToFile_basics from to
  - appear at the same position as it did in the input list.
  -}
 preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]
--- optimisation, only one item in original list, so no reordering needed
-preserveOrder [_] new = new
-preserveOrder orig new = collect orig new
+preserveOrder [] new = new
+preserveOrder [_] new = new -- optimisation
+preserveOrder (l:ls) new = found ++ preserveOrder ls rest
 	where
-		collect [] n = n
-		collect [_] n = n -- optimisation
-		collect (l:ls) n = found ++ collect ls rest
-			where (found, rest)=partition (l `dirContains`) n
+		(found, rest)=partition (l `dirContains`) new
 
 {- Runs an action that takes a list of FilePaths, and ensures that 
  - its return list preserves order.
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 4d17a47ba8..a0e52507d6 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -34,7 +34,7 @@ git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePat
 git_annex_shell r command params
 	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)
 	| Git.repoIsSsh r = do
-		uuid <- getUUID r
+		uuid <- getRepoUUID r
 		sshparams <- sshToRepo r [Param $ sshcmd uuid ]
 		return $ Just ("ssh", sshparams)
 	| otherwise = return Nothing
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 79b5da69a1..a4d8dd65b6 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -37,7 +37,7 @@ options = uuid : commonOptions
 	where
 		uuid = Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
 		check expected = do
-			u <- getUUID =<< gitRepo
+			u <- getUUID
 			when (u /= expected) $ error $
 				"expected repository UUID " ++ expected
 					++ " but found UUID " ++ u
diff --git a/test.hs b/test.hs
index 16f2a2bdf6..1c511b4d4b 100644
--- a/test.hs
+++ b/test.hs
@@ -609,9 +609,7 @@ checkdangling f = do
 
 checklocationlog :: FilePath -> Bool -> Assertion
 checklocationlog f expected = do
-	thisuuid <- annexeval $ do
-		g <- Annex.gitRepo
-		UUID.getUUID g
+	thisuuid <- annexeval UUID.getUUID
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do

From 9c04d1e523dc513c64adf251046e13c4e7f7b5dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 22:52:55 -0400
Subject: [PATCH 2320/2835] fix git 1.7.7 breakage

* This version of git-annex only works with git 1.7.7 and newer.
  The breakage with old versions is subtle, and affects
  annex.numcopies .gitattributes settings, so be sure to upgrade git
  to 1.7.7. (Debian package now depends on that version.)
* Don't pass absolute paths to git show-attr, as it started following
  symlinks when that's done in 1.7.7. Instead, use relative paths,
  which show-attr only handles 100% correctly in 1.7.7. Closes: #645046

Unfortunatly I can find no way to work with the old and new gits, as
the old had bugs that require absolute paths, while the new doesn't like
them at all. And the behavior of git show-attr in 1.7.7. is the same as
eg, git add of an absolute path to a symlink, so seems entirely
intentional and not likely to change.
---
 Git.hs           | 19 +++----------------
 debian/changelog | 31 +++++++++++++++++++++++++++++++
 debian/control   |  2 +-
 3 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/Git.hs b/Git.hs
index b87f2a2659..173deba6e8 100644
--- a/Git.hs
+++ b/Git.hs
@@ -547,36 +547,23 @@ configMap = config
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
 checkAttr repo attr files = do
-	-- git check-attr wants files that are absolute (or relative to the
-	-- top of the repo). But we're passed files relative to the current
-	-- directory. Convert to absolute, and then convert the filenames
-	-- in its output back to relative.
-	cwd <- getCurrentDirectory
-	let top = workTree repo
-	let absfiles = map (absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
         _ <- forkProcess $ do
 		hClose fromh
-                hPutStr toh $ join "\0" absfiles
+                hPutStr toh $ join "\0" files
                 hClose toh
                 exitSuccess
         hClose toh
-	s <- hGetContents fromh
-	return $ map (topair cwd top) $ lines s
+	(map topair . lines) <$> hGetContents fromh
 	where
 		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]
-		topair cwd top l = (relfile, value)
+		topair l = (file, value)
 			where 
-				relfile
-					| startswith cwd' file = drop (length cwd') file
-					| otherwise = relPathDirToFile top' file
 				file = decodeGitFile $ join sep $ take end bits
 				value = bits !! end
 				end = length bits - 1
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
-				cwd' = cwd ++ "/"
-				top' = top ++ "/"
 
 {- Some git commands output encoded filenames. Decode that (annoyingly
  - complex) encoding. -}
diff --git a/debian/changelog b/debian/changelog
index 9bcaa40466..85506655fd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,6 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
+<<<<<<<
   * Fix referring to remotes by uuid.
   * New or changed repository descriptions in uuid.log now have a timestamp,
     which is used to ensure the newest description is used when the uuid.log
@@ -21,6 +22,36 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Add locking to avoid races when changing the git-annex branch.
   * Various speed improvements gained by using ByteStrings.
   * Contain the zombie hordes.
+=======
+  * This version of git-annex only works with git 1.7.7 and newer.
+    The breakage with old versions is subtle, and affects
+    annex.numcopies .gitattributes settings, so be sure to upgrade git
+    to 1.7.7. (Debian package now depends on that version.)
+  * Don't pass absolute paths to git show-attr, as it started following
+    symlinks when that's done in 1.7.7. Instead, use relative paths,
+    which show-attr only handles 100% correctly in 1.7.7. Closes: #645046
+  * Various speed improvements gained by using ByteStrings.
+  * Fix referring to remotes by uuid.
+  * status: List all known repositories.
+  * When displaying a list of repositories, show git remote names
+    in addition to their descriptions.
+  * Contain the zombie hordes.
+  * Add locking to avoid races when changing the git-annex branch.
+  * New or changed repository descriptions in uuid.log now have a timestamp,
+    which is used to ensure the newest description is used when the uuid.log
+    has been merged.
+  * Note that older versions of git-annex will display the timestamp as part
+    of the repository description, which is ugly but otherwise harmless.
+  * Add timestamps to trust.log and remote.log too.
+  * git-annex-shell: Added the --uuid option.
+  * git-annex now asks git-annex-shell to verify that it's operating in 
+    the expected repository.
+  * Note that this git-annex will not interoperate with remotes using 
+    older versions of git-annex-shell.
+  * Now supports git's insteadOf configuration, to modify the url
+    used to access a remote. Note that pushInsteadOf is not used;
+    that and pushurl are reserved for actual git pushes. Closes: #644278
+>>>>>>>
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/debian/control b/debian/control
index cb5a8212ab..49f564fdb9 100644
--- a/debian/control
+++ b/debian/control
@@ -29,7 +29,7 @@ Package: git-annex
 Architecture: any
 Section: utils
 Depends: ${misc:Depends}, ${shlibs:Depends},
-	git | git-core,
+	git (>= 1:1.7.7),
 	uuid,
 	rsync,
 	wget | curl,

From 402d9c7c5fa79d78fe09e5ed0b5a3b60e16a7f0b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 22:54:38 -0400
Subject: [PATCH 2321/2835] oops

---
 debian/changelog | 36 ++++++------------------------------
 1 file changed, 6 insertions(+), 30 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 85506655fd..e985166dcd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,28 +1,5 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
-<<<<<<<
-  * Fix referring to remotes by uuid.
-  * New or changed repository descriptions in uuid.log now have a timestamp,
-    which is used to ensure the newest description is used when the uuid.log
-    has been merged.
-  * Note that older versions of git-annex will display the timestamp as part
-    of the repository description, which is ugly but otherwise harmless.
-  * Add timestamps to trust.log and remote.log too.
-  * git-annex-shell: Added the --uuid option.
-  * git-annex now asks git-annex-shell to verify that it's operating in 
-    the expected repository.
-  * Note that this git-annex will not interoperate with remotes using 
-    older versions of git-annex-shell.
-  * Now supports git's insteadOf configuration, to modify the url
-    used to access a remote. Note that pushInsteadOf is not used;
-    that and pushurl are reserved for actual git pushes. Closes: #644278
-  * status: List all known repositories.
-  * When displaying a list of repositories, show git remote names
-    in addition to their descriptions.
-  * Add locking to avoid races when changing the git-annex branch.
-  * Various speed improvements gained by using ByteStrings.
-  * Contain the zombie hordes.
-=======
   * This version of git-annex only works with git 1.7.7 and newer.
     The breakage with old versions is subtle, and affects
     annex.numcopies .gitattributes settings, so be sure to upgrade git
@@ -30,13 +7,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Don't pass absolute paths to git show-attr, as it started following
     symlinks when that's done in 1.7.7. Instead, use relative paths,
     which show-attr only handles 100% correctly in 1.7.7. Closes: #645046
-  * Various speed improvements gained by using ByteStrings.
   * Fix referring to remotes by uuid.
-  * status: List all known repositories.
-  * When displaying a list of repositories, show git remote names
-    in addition to their descriptions.
-  * Contain the zombie hordes.
-  * Add locking to avoid races when changing the git-annex branch.
   * New or changed repository descriptions in uuid.log now have a timestamp,
     which is used to ensure the newest description is used when the uuid.log
     has been merged.
@@ -51,7 +22,12 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Now supports git's insteadOf configuration, to modify the url
     used to access a remote. Note that pushInsteadOf is not used;
     that and pushurl are reserved for actual git pushes. Closes: #644278
->>>>>>>
+  * status: List all known repositories.
+  * When displaying a list of repositories, show git remote names
+    in addition to their descriptions.
+  * Add locking to avoid races when changing the git-annex branch.
+  * Various speed improvements gained by using ByteStrings.
+  * Contain the zombie hordes.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 11b154e811a9dcea42ceaaf9740f811ac310a615 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:03:19 -0400
Subject: [PATCH 2322/2835] prep release

---
 debian/changelog | 8 ++++----
 debian/control   | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e985166dcd..c4e53e1795 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,8 @@
-git-annex (3.20110929) UNRELEASED; urgency=low
+git-annex (3.20111011) unstable; urgency=low
 
   * This version of git-annex only works with git 1.7.7 and newer.
-    The breakage with old versions is subtle, and affects
-    annex.numcopies .gitattributes settings, so be sure to upgrade git
+    The breakage with old versions is subtle, and affects the
+    annex.numcopies settings in .gitattributes, so be sure to upgrade git
     to 1.7.7. (Debian package now depends on that version.)
   * Don't pass absolute paths to git show-attr, as it started following
     symlinks when that's done in 1.7.7. Instead, use relative paths,
@@ -29,7 +29,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Various speed improvements gained by using ByteStrings.
   * Contain the zombie hordes.
 
- -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
+ -- Joey Hess   Tue, 11 Oct 2011 23:00:02 -0400
 
 git-annex (3.20110928) unstable; urgency=low
 
diff --git a/debian/control b/debian/control
index 49f564fdb9..6f59ada5b8 100644
--- a/debian/control
+++ b/debian/control
@@ -17,7 +17,7 @@ Build-Depends:
 	libghc-json-dev,
 	ikiwiki,
 	perlmagick,
-	git | git-core,
+	git,
 	uuid,
 	rsync,
 Maintainer: Joey Hess 

From 5a9e3f73772daff3ffadd2359d99914f0567b78c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:39:29 -0400
Subject: [PATCH 2323/2835] force files relative

Other code is currently depending on checkAttr forcing absolute filenames
to relative, so keep it doing so.

This is a quick fix, and should sometime be moved elsewhere.
---
 Git.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Git.hs b/Git.hs
index 173deba6e8..fb998b1b0e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -547,10 +547,12 @@ configMap = config
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
 checkAttr repo attr files = do
+	cwd <- getCurrentDirectory
+	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
         _ <- forkProcess $ do
 		hClose fromh
-                hPutStr toh $ join "\0" files
+                hPutStr toh $ join "\0" relfiles
                 hClose toh
                 exitSuccess
         hClose toh

From bcec418495f1330ee502b59ba485a69a9b5e3088 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:49:19 -0400
Subject: [PATCH 2324/2835] releasing version 3.20111011

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 5645eb043a..450adea75b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110929
+Version: 3.20111011
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 2b26b95226566773f8019826671d4955600dbffe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:49:25 -0400
Subject: [PATCH 2325/2835] add news item for git-annex 3.20111011

---
 doc/news/version_3.20110819.mdwn |  9 ---------
 doc/news/version_3.20111011.mdwn | 30 ++++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_3.20110819.mdwn
 create mode 100644 doc/news/version_3.20111011.mdwn

diff --git a/doc/news/version_3.20110819.mdwn b/doc/news/version_3.20110819.mdwn
deleted file mode 100644
index bbc6abdbc7..0000000000
--- a/doc/news/version_3.20110819.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 3.20110819 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Now "git annex init" only has to be run once, when a git repository
-     is first being created. Clones will automatically notice that git-annex
-     is in use and automatically perform a basic initalization. It's
-     still recommended to run "git annex init" in any clones, to describe them.
-   * Added annex-cost-command configuration, which can be used to vary the
-     cost of a remote based on the output of a shell command.
-   * Fix broken upgrade from V1 repository. Closes: #[638584](http://bugs.debian.org/638584)"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111011.mdwn b/doc/news/version_3.20111011.mdwn
new file mode 100644
index 0000000000..0346cbca82
--- /dev/null
+++ b/doc/news/version_3.20111011.mdwn
@@ -0,0 +1,30 @@
+git-annex 3.20111011 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * This version of git-annex only works with git 1.7.7 and newer.
+     The breakage with old versions is subtle, and affects the
+     annex.numcopies settings in .gitattributes, so be sure to upgrade git
+     to 1.7.7. (Debian package now depends on that version.)
+   * Don't pass absolute paths to git show-attr, as it started following
+     symlinks when that's done in 1.7.7. Instead, use relative paths,
+     which show-attr only handles 100% correctly in 1.7.7. Closes: #[645046](http://bugs.debian.org/645046)
+   * Fix referring to remotes by uuid.
+   * New or changed repository descriptions in uuid.log now have a timestamp,
+     which is used to ensure the newest description is used when the uuid.log
+     has been merged.
+   * Note that older versions of git-annex will display the timestamp as part
+     of the repository description, which is ugly but otherwise harmless.
+   * Add timestamps to trust.log and remote.log too.
+   * git-annex-shell: Added the --uuid option.
+   * git-annex now asks git-annex-shell to verify that it's operating in
+     the expected repository.
+   * Note that this git-annex will not interoperate with remotes using
+     older versions of git-annex-shell.
+   * Now supports git's insteadOf configuration, to modify the url
+     used to access a remote. Note that pushInsteadOf is not used;
+     that and pushurl are reserved for actual git pushes. Closes: #[644278](http://bugs.debian.org/644278)
+   * status: List all known repositories.
+   * When displaying a list of repositories, show git remote names
+     in addition to their descriptions.
+   * Add locking to avoid races when changing the git-annex branch.
+   * Various speed improvements gained by using ByteStrings.
+   * Contain the zombie hordes."""]]
\ No newline at end of file

From fa5c016585b61e84c9ef7e7f6e6814693599a66a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:49:58 -0400
Subject: [PATCH 2326/2835] add comment about relative/absolute filenames

---
 Git.hs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Git.hs b/Git.hs
index fb998b1b0e..044fc08aea 100644
--- a/Git.hs
+++ b/Git.hs
@@ -547,6 +547,9 @@ configMap = config
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
 checkAttr repo attr files = do
+	-- git check-attr needs relative filenames input; it will choke
+	-- on some absolute filenames. This also means it will output
+	-- all relative filenames.
 	cwd <- getCurrentDirectory
 	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)

From 82d127de99767a8a6387cb923eb964fa51474aa6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 12 Oct 2011 00:25:05 -0400
Subject: [PATCH 2327/2835] add git version check to configure

Rather ugly dotted version comparison method, but it does work.
---
 configure.hs    | 24 ++++++++++++++++++++++++
 git-annex.cabal |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/configure.hs b/configure.hs
index b68fa12dbc..1d1c023356 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,12 +2,16 @@
 
 import System.Directory
 import Data.List
+import Data.String.Utils
+import System.Cmd.Utils
 
 import Build.TestConfig
 
 tests :: [TestCase]
 tests =
 	[ TestCase "version" getVersion
+	, TestCase "git" $ requireCmd "git" "git --version >/dev/null"
+	, TestCase "git version" checkGitVersion
 	, testCp "cp_a" "-a"
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
@@ -53,6 +57,26 @@ getVersionString = do
 	where
 		middle = drop 1 . init
 
+{- Checks for a new enough version of git. -}
+checkGitVersion :: Test
+checkGitVersion = do
+	(_, s) <- pipeFrom "git" ["--version"]
+	let version = last $ words $ head $ lines s
+	if dotted version < dotted need
+		then error $ "git version " ++ version ++ " too old; need " ++ need
+		else return $ Config "gitversion" (StringConfig version)
+	where
+		-- for git-check-attr behavior change
+		need = "1.7.7"
+		dotted = sum . mult 1 . reverse . extend 10 . map readi . split "." 
+		extend n l = l ++ take (n - length l) (repeat 0)
+		mult _ [] = []
+		mult n (x:xs) = (n*x) : (mult (n*100) xs)
+		readi :: String -> Integer
+		readi s = case reads s of
+			((x,_):_) -> x
+			_ -> 0
+
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
 cabalSetup = do
diff --git a/git-annex.cabal b/git-annex.cabal
index 5645eb043a..450adea75b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110929
+Version: 3.20111011
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 205a5b2aaa76fac31d23f9b5a47180400a3008bf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 12 Oct 2011 00:29:49 -0400
Subject: [PATCH 2328/2835] typo

---
 debian/NEWS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/NEWS b/debian/NEWS
index ad4e946e6c..f807d05255 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -4,7 +4,7 @@ git-annex (3.20110702) unstable; urgency=low
 
  -- Joey Hess   Fri, 01 Jul 2011 15:40:51 -0400
 
-git-annex (3.20110624) exerimental; urgency=low
+git-annex (3.20110624) experimental; urgency=low
 
   There has been another change to the git-annex data store.
   Use `git annex upgrade` to migrate your repositories to the new

From 88e9bba97ef5e321407519f39bd81d992e19e708 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
 
Date: Thu, 13 Oct 2011 14:52:43 +0000
Subject: [PATCH 2329/2835]

---
 doc/forum/git_tag_missing_for_3.20111011.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/git_tag_missing_for_3.20111011.mdwn

diff --git a/doc/forum/git_tag_missing_for_3.20111011.mdwn b/doc/forum/git_tag_missing_for_3.20111011.mdwn
new file mode 100644
index 0000000000..781d0c91a0
--- /dev/null
+++ b/doc/forum/git_tag_missing_for_3.20111011.mdwn
@@ -0,0 +1 @@
+Well, the subject pretty much says it all :)

From 3e07780bf881988eed491e1451b95cab86b7f552 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 13 Oct 2011 15:36:59 +0000
Subject: [PATCH 2330/2835] Added a comment: fixed that

---
 .../comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment

diff --git a/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment b/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment
new file mode 100644
index 0000000000..87cda998bf
--- /dev/null
+++ b/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="fixed that"
+ date="2011-10-13T15:36:59Z"
+ content="""
+:)
+"""]]

From 9fa92141064a7682e1559bfa91a360c1ad5cb3dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 14 Oct 2011 18:17:46 -0400
Subject: [PATCH 2331/2835] A remote can have a annexUrl configured, that is
 used by git-annex instead of its usual url. (Similar to pushUrl.)

---
 Git.hs                   | 22 +++++++++++++++++-----
 Remote/Git.hs            | 13 ++++++++++++-
 Remote/Helper/Special.hs |  2 +-
 Remote/Web.hs            |  2 +-
 debian/changelog         |  7 +++++++
 doc/git-annex.mdwn       |  6 ++++++
 6 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/Git.hs b/Git.hs
index 044fc08aea..836e063c2a 100644
--- a/Git.hs
+++ b/Git.hs
@@ -48,8 +48,10 @@ module Git (
 	attributes,
 	remotes,
 	remotesAdd,
+	genRemote,
 	repoRemoteName,
 	repoRemoteNameSet,
+	repoRemoteNameFromKey,
 	checkAttr,
 	decodeGitFile,
 	encodeGitFile,
@@ -185,10 +187,14 @@ repoRemoteName :: Repo -> Maybe String
 repoRemoteName Repo { remoteName = Just name } = Just name
 repoRemoteName _ = Nothing
 
+{- Sets the name of a remote. -}
+repoRemoteNameSet :: Repo -> String -> Repo
+repoRemoteNameSet r n = r { remoteName = Just n }
+
 {- Sets the name of a remote based on the git config key, such as
    "remote.foo.url". -}
-repoRemoteNameSet :: Repo -> String -> Repo
-repoRemoteNameSet r k = r { remoteName = Just basename }
+repoRemoteNameFromKey :: Repo -> String -> Repo
+repoRemoteNameFromKey r k = repoRemoteNameSet r basename
 	where
 		basename = join "." $ reverse $ drop 1 $
 				reverse $ drop 1 $ split "." k
@@ -501,9 +507,15 @@ configRemotes repo = mapM construct remotepairs
 		remotepairs = filterkeys isremote
 		isremote k = startswith "remote." k && endswith ".url" k
 		construct (k,v) = do
-			r <- gen $ calcloc v
-			return $ repoRemoteNameSet r k
-		gen v
+			r <- genRemote repo v
+			return $ repoRemoteNameFromKey r k
+
+{- Generates one of a repo's remotes using a given location (ie, an url). -}
+genRemote :: Repo -> String -> IO Repo
+genRemote repo = gen . calcloc
+	where
+		filterconfig f = filter f $ M.toList $ config repo
+		gen v	
 			| scpstyle v = repoFromUrl $ scptourl v
 			| isURI v = repoFromUrl v
 			| otherwise = repoFromRemotePath v repo
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 183fcd8548..e9919e636c 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -34,7 +34,18 @@ remote = RemoteType {
 list :: Annex [Git.Repo]
 list = do
 	g <- gitRepo
-	return $ Git.remotes g
+	let c = Git.configMap g
+	mapM (tweakurl c) $ Git.remotes g
+	where
+		annexurl n = "remote." ++ n ++ ".annexurl"
+		tweakurl c r = do
+			let n = fromJust $ Git.repoRemoteName r
+			case M.lookup (annexurl n) c of
+				Nothing -> return r
+				Just url -> do
+					g <- gitRepo
+					r' <- liftIO $ Git.genRemote g url
+					return $ Git.repoRemoteNameSet r' n
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r u _ = do
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 1fbd7b19ed..5f66e8cd9f 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -24,7 +24,7 @@ findSpecialRemotes s = do
 	return $ map construct $ remotepairs g
 	where
 		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
-		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
+		construct (k,_) = Git.repoRemoteNameFromKey Git.repoFromUnknown k
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 30a1ff0086..ed9c94909e 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -33,7 +33,7 @@ remote = RemoteType {
 -- (If the web should cease to exist, remove this module and redistribute
 -- a new release to the survivors by carrier pigeon.)
 list :: Annex [Git.Repo]
-list = return [Git.repoRemoteNameSet Git.repoFromUnknown "remote.web.dummy"]
+list = return [Git.repoRemoteNameSet Git.repoFromUnknown "web"]
 
 -- Dummy uuid for the whole web. Do not alter.
 webUUID :: UUID
diff --git a/debian/changelog b/debian/changelog
index c4e53e1795..6e3450a926 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20111012) UNRELEASED; urgency=low
+
+  * A remote can have a annexUrl configured, that is used by git-annex
+    instead of its usual url. (Similar to pushUrl.)
+
+ -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
+
 git-annex (3.20111011) unstable; urgency=low
 
   * This version of git-annex only works with git 1.7.7 and newer.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index db19e9f7a2..eefedbfbf9 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -501,6 +501,12 @@ Here are all the supported configuration settings.
   Or, it could be used if the network connection between two
   repositories is too slow to be used normally.
 
+* `remote..annexUrl`
+
+  Can be used to specify a different url than the regular `remote..url`
+  for git-annex to use when talking with the remote. Similar to the `pushUrl`
+  used by git-push.
+
 * `remote..annex-uuid`
 
   git-annex caches UUIDs of remote repositories here.

From ae2b1308a62981e14a03275e5d6ea65a2bc7d098 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 14 Oct 2011 18:23:17 -0400
Subject: [PATCH 2332/2835] reorg

---
 doc/git-annex.mdwn | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index eefedbfbf9..1b8464b314 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -477,6 +477,20 @@ Here are all the supported configuration settings.
   Space-separated list of names of the key-value backends to use. 
   The first listed is used to store new files by default. 
 
+* `annex.diskreserve`
+
+  Amount of disk space to reserve. Disk space is checked when transferring
+  content to avoid running out, and additional free space can be reserved
+  via this option, to make space for more important content (such as git
+  commit logs). Can be specified with any commonly used units, for example,
+  "0.5 gb" or "100 KiloBytes"
+
+  The default reserve is 1 megabyte.
+
+* `annex.version`
+
+  Automatically maintained, and used to automate upgrades between versions.
+
 * `remote..annex-cost`
 
   When determining which repository to
@@ -532,20 +546,6 @@ Here are all the supported configuration settings.
   Default ssh, rsync, and bup options to use if a remote does not have
   specific options.
 
-* `annex.diskreserve`
-
-  Amount of disk space to reserve. Disk space is checked when transferring
-  content to avoid running out, and additional free space can be reserved
-  via this option, to make space for more important content (such as git
-  commit logs). Can be specified with any commonly used units, for example,
-  "0.5 gb" or "100 KiloBytes"
-
-  The default reserve is 1 megabyte.
-
-* `annex.version`
-
-  Automatically maintained, and used to automate upgrades between versions.
-
 * `remote..buprepo`
 
   Used by bup special remotes, this configures

From 872057303e14dd6623772de2d8d85b511e21214c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 01:37:55 -0400
Subject: [PATCH 2333/2835] tweak

---
 Git.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Git.hs b/Git.hs
index 836e063c2a..b426d5c5fe 100644
--- a/Git.hs
+++ b/Git.hs
@@ -338,8 +338,9 @@ urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r
 
 {- The full authority portion an URL repo. (ie, "user@host:port") -}
 urlAuthority :: Repo -> String
-urlAuthority r = flip urlAuthPart r $ \a ->
-	uriUserInfo a ++ uriRegName' a ++ uriPort a
+urlAuthority = urlAuthPart combine
+	where
+		combine a = uriUserInfo a ++ uriRegName' a ++ uriPort a
 
 {- Applies a function to extract part of the uriAuthority of an URL repo. -}
 urlAuthPart :: (URIAuth -> a) -> Repo -> a

From c867ae842ae15e03eb7d36704a33ce62f2a8fe16 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 13:13:57 -0400
Subject: [PATCH 2334/2835] add

---
 doc/todo/gitolite_and_gitosis_support.mdwn | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/todo/gitolite_and_gitosis_support.mdwn

diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn
new file mode 100644
index 0000000000..0131cdc076
--- /dev/null
+++ b/doc/todo/gitolite_and_gitosis_support.mdwn
@@ -0,0 +1,14 @@
+gitosis and gitolite should support git-annex being used to send/receive
+files from the repositories they manage. Users with read-only access
+could only get files, while users with write access could also put and drop
+files.
+
+Doing this right requires modifying both programs, to add [[git-annex-shell]]
+to the list of things they can run, and only allow through appropriate
+git-annex-shell subcommands to read-only users.
+
+I have posted an RFC for modifying gitolite to the
+[gitolite mailing list](http://groups.google.com/group/gitolite?lnk=srg).
+
+As I don't write python, someone else is needed to work on gitosis.
+--[[Joey]] 

From bae3008d17d492b174db552f64c0cdfaf1804bdd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 13:17:00 -0400
Subject: [PATCH 2335/2835] add a copy of my mailing list post

---
 doc/todo/gitolite_and_gitosis_support.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn
index 0131cdc076..daa54854d3 100644
--- a/doc/todo/gitolite_and_gitosis_support.mdwn
+++ b/doc/todo/gitolite_and_gitosis_support.mdwn
@@ -10,5 +10,25 @@ git-annex-shell subcommands to read-only users.
 I have posted an RFC for modifying gitolite to the
 [gitolite mailing list](http://groups.google.com/group/gitolite?lnk=srg).
 
+> I have not developed a patch yet, but all that git-annex needs is a way
+> to ssh to the server and run the git-annex-shell command there.
+> git-annex-shell is very similar to git-shell. So, one way to enable
+> it is simply to set GL_ADC_PATH to a directory containing git-annex-shell.
+> 
+> But, that's not optimal, since git-annex-shell will send off receive-pack
+> commands to git, which would bypass gitolite's permissions checking.
+> Also, it makes sense to limit readonly users to only download, not
+> upload/delete files from git-annex. Instead, I suggest adding something
+> like this to gitolite's config:
+ 
+	# If set, users with W access can write file contents into the git-annex,
+	# and users with R access can read file contents from the git-annex.
+	$GL_GIT_ANNEX = 0;
+
+> If this makes sense, I'm sure I can put a patch together for your
+> review. It would involve modifying gl-auth-command so it knows how
+> to run git-annex-shell, and how to parse out the "verb" from a
+> git-annex-shell command line, and modifying R_COMMANDS and W_COMMANDS.
+
 As I don't write python, someone else is needed to work on gitosis.
 --[[Joey]] 

From 279150ccd5ad937a44cbff798ab7bb118ad1dbee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 13:30:04 -0400
Subject: [PATCH 2336/2835] update

---
 doc/todo/gitolite_and_gitosis_support.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn
index daa54854d3..12e26243e4 100644
--- a/doc/todo/gitolite_and_gitosis_support.mdwn
+++ b/doc/todo/gitolite_and_gitosis_support.mdwn
@@ -32,3 +32,19 @@ I have posted an RFC for modifying gitolite to the
 
 As I don't write python, someone else is needed to work on gitosis.
 --[[Joey]] 
+
+## readonly commands
+
+* git-annex-shell configlist $directory
+* git-annex-shell inannex $directory [$key ...]
+* git-annex-shell sendkey $directory $key
+
+## read-write commands
+
+* git-annex-shell dropkey $directory [$key ...]
+* git-annex-shell recvkey $directory $key
+
+## other git-annex-shell parameters
+
+All parameters like --uuid=foo and --force are safe and need to be allowed
+through.

From 1a29b5b52eec641a5456d7c8dc24356c90107bc0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 16:21:08 -0400
Subject: [PATCH 2337/2835] reorganize log modules

no code changes
---
 Annex/Content.hs                   |  6 +++---
 Command.hs                         |  4 ++--
 Command/Add.hs                     |  2 +-
 Command/AddUrl.hs                  |  2 +-
 Command/ConfigList.hs              |  2 +-
 Command/Describe.hs                |  2 +-
 Command/Drop.hs                    |  4 ++--
 Command/DropKey.hs                 |  2 +-
 Command/Fsck.hs                    |  6 +++---
 Command/Init.hs                    |  2 +-
 Command/InitRemote.hs              | 12 +++++------
 Command/Map.hs                     |  4 ++--
 Command/Move.hs                    |  4 ++--
 Command/Semitrust.hs               |  4 ++--
 Command/SetKey.hs                  |  2 +-
 Command/Status.hs                  |  2 +-
 Command/Trust.hs                   |  4 ++--
 Command/Unannex.hs                 |  2 +-
 Command/Untrust.hs                 |  4 ++--
 Command/Unused.hs                  |  2 +-
 Command/Whereis.hs                 |  4 ++--
 Git.hs                             |  4 ++--
 Init.hs                            |  2 +-
 Limit.hs                           |  2 +-
 LocationLog.hs => Logs/Location.hs |  6 +++---
 PresenceLog.hs => Logs/Presence.hs |  2 +-
 RemoteLog.hs => Logs/Remote.hs     |  6 +++---
 Trust.hs => Logs/Trust.hs          |  6 +++---
 UUID.hs => Logs/UUID.hs            |  4 ++--
 UUIDLog.hs => Logs/UUIDBased.hs    |  2 +-
 Remote.hs                          | 12 +++++------
 Remote/Bup.hs                      |  2 +-
 Remote/Directory.hs                |  2 +-
 Remote/Git.hs                      |  2 +-
 Remote/Helper/Special.hs           |  2 +-
 Remote/Hook.hs                     |  2 +-
 Remote/Rsync.hs                    |  2 +-
 Remote/Web.hs                      |  6 +++---
 Upgrade/V1.hs                      |  2 +-
 Upgrade/V2.hs                      |  2 +-
 Utility/Ssh.hs                     |  2 +-
 git-annex-shell.hs                 |  2 +-
 git-annex.cabal                    |  2 +-
 test.hs                            | 34 +++++++++++++++---------------
 44 files changed, 92 insertions(+), 92 deletions(-)
 rename LocationLog.hs => Logs/Location.hs (96%)
 rename PresenceLog.hs => Logs/Presence.hs (99%)
 rename RemoteLog.hs => Logs/Remote.hs (97%)
 rename Trust.hs => Logs/Trust.hs (96%)
 rename UUID.hs => Logs/UUID.hs (98%)
 rename UUIDLog.hs => Logs/UUIDBased.hs (99%)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 3827154a67..9cf7ea8f21 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -22,8 +22,8 @@ module Annex.Content (
 ) where
 
 import Common.Annex
-import LocationLog
-import UUID
+import Logs.Location
+import Logs.UUID
 import qualified Git
 import qualified Annex
 import qualified Annex.Queue
@@ -52,7 +52,7 @@ calcGitLink file key = do
 	where
 		whoops = error $ "unable to normalize " ++ file
 
-{- Updates the LocationLog when a key's presence changes in the current
+{- Updates the Logs.Location when a key's presence changes in the current
  - repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
diff --git a/Command.hs b/Command.hs
index 54dc9603fa..f282791fb3 100644
--- a/Command.hs
+++ b/Command.hs
@@ -13,8 +13,8 @@ import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Types.Key
-import Trust
-import LocationLog
+import Logs.Trust
+import Logs.Location
 import Config
 import Backend
 import Limit
diff --git a/Command/Add.hs b/Command/Add.hs
index e6e7a7c770..bfddd72ee7 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -13,7 +13,7 @@ import Command
 import qualified Annex
 import qualified Annex.Queue
 import qualified Backend
-import LocationLog
+import Logs.Location
 import Annex.Content
 import Utility.Touch
 import Backend
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index c5417bf5bc..4447dee812 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -18,7 +18,7 @@ import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
 import Annex.Content
-import PresenceLog
+import Logs.Presence
 
 command :: [Command]
 command = [repoCommand "addurl" (paramRepeating paramUrl) seek
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 60f71eee6d..b50c759eee 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -9,7 +9,7 @@ module Command.ConfigList where
 
 import Common.Annex
 import Command
-import UUID
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "configlist" paramNothing seek
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 3368046393..65cd8d0bf5 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -10,7 +10,7 @@ module Command.Describe where
 import Common.Annex
 import Command
 import qualified Remote
-import UUID
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
diff --git a/Command/Drop.hs b/Command/Drop.hs
index fabacbb724..dc858fb29b 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -11,9 +11,9 @@ import Common.Annex
 import Command
 import qualified Remote
 import qualified Annex
-import LocationLog
+import Logs.Location
+import Logs.Trust
 import Annex.Content
-import Trust
 import Config
 
 command :: [Command]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 35ebfc219b..fde6ce02ea 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -10,7 +10,7 @@ module Command.DropKey where
 import Common.Annex
 import Command
 import qualified Annex
-import LocationLog
+import Logs.Location
 import Annex.Content
 
 command :: [Command]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 16e834fbf1..632570b110 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -12,10 +12,10 @@ import Command
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
-import UUID
 import Annex.Content
-import LocationLog
-import Trust
+import Logs.Location
+import Logs.Trust
+import Logs.UUID
 import Utility.DataUnits
 import Utility.FileMode
 import Config
diff --git a/Command/Init.hs b/Command/Init.hs
index b9dffb5cd9..dcc6bfe6b2 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -9,7 +9,7 @@ module Command.Init where
 
 import Common.Annex
 import Command
-import UUID
+import Logs.UUID
 import Init
 	
 command :: [Command]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 918604f852..240528b879 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -12,9 +12,9 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Remote
-import qualified RemoteLog
+import qualified Logs.Remote
 import qualified Types.Remote as R
-import UUID
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "initremote"
@@ -38,7 +38,7 @@ start ws = do
 
 	where
 		name = head ws
-		config = RemoteLog.keyValToConfig $ tail ws
+		config = Logs.Remote.keyValToConfig $ tail ws
 		needname = do
 			let err s = error $ "Specify a name for the remote. " ++ s
 			names <- remoteNames
@@ -54,13 +54,13 @@ perform t u c = do
 
 cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
 cleanup u c = do
-	RemoteLog.configSet u c
+	Logs.Remote.configSet u c
         return True
 
 {- Look up existing remote's UUID and config by name, or generate a new one -}
 findByName :: String -> Annex (UUID, R.RemoteConfig)
 findByName name = do
-	m <- RemoteLog.readRemoteLog
+	m <- Logs.Remote.readRemoteLog
 	maybe generate return $ findByName' name m
 	where
 		generate = do
@@ -79,7 +79,7 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 
 remoteNames :: Annex [String]
 remoteNames = do
-	m <- RemoteLog.readRemoteLog
+	m <- Logs.Remote.readRemoteLog
 	return $ mapMaybe (M.lookup nameKey . snd) $ M.toList m
 
 {- find the specified remote type -}
diff --git a/Command/Map.hs b/Command/Map.hs
index 1155c4a6ef..5cbf51b27f 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -13,8 +13,8 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Git
-import UUID
-import Trust
+import Logs.UUID
+import Logs.Trust
 import Utility.Ssh
 import qualified Utility.Dot as Dot
 
diff --git a/Command/Move.hs b/Command/Move.hs
index 52eb49da14..62f38224ca 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -11,10 +11,10 @@ import Common.Annex
 import Command
 import qualified Command.Drop
 import qualified Annex
-import LocationLog
+import Logs.Location
 import Annex.Content
 import qualified Remote
-import UUID
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "move" paramPaths seek
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 53b29c98ac..e13785a38f 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -10,8 +10,8 @@ module Command.Semitrust where
 import Common.Annex
 import Command
 import qualified Remote
-import UUID
-import Trust
+import Logs.UUID
+import Logs.Trust
 
 command :: [Command]
 command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 28bc9a48d6..b707e0b918 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -9,7 +9,7 @@ module Command.SetKey where
 
 import Common.Annex
 import Command
-import LocationLog
+import Logs.Location
 import Annex.Content
 
 command :: [Command]
diff --git a/Command/Status.hs b/Command/Status.hs
index 37e13f0d8e..70282b79ee 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -23,7 +23,7 @@ import Utility.DataUnits
 import Annex.Content
 import Types.Key
 import Backend
-import UUID
+import Logs.UUID
 import Remote
 
 -- a named computation that produces a statistic
diff --git a/Command/Trust.hs b/Command/Trust.hs
index bc655e3f61..fb7f47ec0e 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -10,8 +10,8 @@ module Command.Trust where
 import Common.Annex
 import Command
 import qualified Remote
-import Trust
-import UUID
+import Logs.Trust
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "trust" (paramRepeating paramRemote) seek
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index d0cef76781..083984d0c5 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -13,7 +13,7 @@ import qualified Command.Drop
 import qualified Annex
 import qualified Annex.Queue
 import Utility.FileMode
-import LocationLog
+import Logs.Location
 import Annex.Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index bcde0e0a06..6f2b602038 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -10,8 +10,8 @@ module Command.Untrust where
 import Common.Annex
 import Command
 import qualified Remote
-import UUID
-import Trust
+import Logs.UUID
+import Logs.Trust
 
 command :: [Command]
 command = [repoCommand "untrust" (paramRepeating paramRemote) seek
diff --git a/Command/Unused.hs b/Command/Unused.hs
index abf5a5361a..874b0ca061 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,7 +16,7 @@ import Common.Annex
 import Command
 import Annex.Content
 import Utility.FileMode
-import LocationLog
+import Logs.Location
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 0eeb174142..b1646ae69a 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -8,10 +8,10 @@
 module Command.Whereis where
 
 import Common.Annex
-import LocationLog
+import Logs.Location
 import Command
 import Remote
-import Trust
+import Logs.Trust
 
 command :: [Command]
 command = [repoCommand "whereis" paramPaths seek
diff --git a/Git.hs b/Git.hs
index b426d5c5fe..b05c3b2d5e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -338,9 +338,9 @@ urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r
 
 {- The full authority portion an URL repo. (ie, "user@host:port") -}
 urlAuthority :: Repo -> String
-urlAuthority = urlAuthPart combine
+urlAuthority = urlAuthPart assemble
 	where
-		combine a = uriUserInfo a ++ uriRegName' a ++ uriPort a
+		assemble a = uriUserInfo a ++ uriRegName' a ++ uriPort a
 
 {- Applies a function to extract part of the uriAuthority of an URL repo. -}
 urlAuthPart :: (URIAuth -> a) -> Repo -> a
diff --git a/Init.hs b/Init.hs
index 509cbca15d..00a3d12fab 100644
--- a/Init.hs
+++ b/Init.hs
@@ -15,7 +15,7 @@ import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import Annex.Version
-import UUID
+import Logs.UUID
 
 initialize :: Annex ()
 initialize = do
diff --git a/Limit.hs b/Limit.hs
index 8dd88e72b8..490577e80f 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -15,7 +15,7 @@ import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
-import LocationLog
+import Logs.Location
 import Annex.Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
diff --git a/LocationLog.hs b/Logs/Location.hs
similarity index 96%
rename from LocationLog.hs
rename to Logs/Location.hs
index 5cbdbb28a1..4e8b2b5357 100644
--- a/LocationLog.hs
+++ b/Logs/Location.hs
@@ -11,7 +11,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module LocationLog (
+module Logs.Location (
 	LogStatus(..),
 	logChange,
 	readLog,
@@ -24,8 +24,8 @@ module LocationLog (
 import Common.Annex
 import qualified Git
 import qualified Annex.Branch
-import UUID
-import PresenceLog
+import Logs.UUID
+import Logs.Presence
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
diff --git a/PresenceLog.hs b/Logs/Presence.hs
similarity index 99%
rename from PresenceLog.hs
rename to Logs/Presence.hs
index 4e4960f0a6..7211eba039 100644
--- a/PresenceLog.hs
+++ b/Logs/Presence.hs
@@ -11,7 +11,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module PresenceLog (
+module Logs.Presence (
 	LogStatus(..),
 	addLog,
 	readLog,
diff --git a/RemoteLog.hs b/Logs/Remote.hs
similarity index 97%
rename from RemoteLog.hs
rename to Logs/Remote.hs
index d49635b930..47c2d7472b 100644
--- a/RemoteLog.hs
+++ b/Logs/Remote.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module RemoteLog (
+module Logs.Remote (
 	readRemoteLog,
 	configSet,
 	keyValToConfig,
@@ -21,8 +21,8 @@ import Data.Char
 import Common.Annex
 import qualified Annex.Branch
 import Types.Remote
-import UUID
-import UUIDLog
+import Logs.UUID
+import Logs.UUIDBased
 
 {- Filename of remote.log. -}
 remoteLog :: FilePath
diff --git a/Trust.hs b/Logs/Trust.hs
similarity index 96%
rename from Trust.hs
rename to Logs/Trust.hs
index 2971256a45..6966ffdd6e 100644
--- a/Trust.hs
+++ b/Logs/Trust.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Trust (
+module Logs.Trust (
 	TrustLevel(..),
 	trustGet,
 	trustSet,
@@ -20,8 +20,8 @@ import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
 
-import UUID
-import UUIDLog
+import Logs.UUID
+import Logs.UUIDBased
 
 {- Filename of trust.log. -}
 trustLog :: FilePath
diff --git a/UUID.hs b/Logs/UUID.hs
similarity index 98%
rename from UUID.hs
rename to Logs/UUID.hs
index 63ce87f03e..baf6650015 100644
--- a/UUID.hs
+++ b/Logs/UUID.hs
@@ -13,7 +13,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module UUID (
+module Logs.UUID (
 	UUID,
 	getUUID,
 	getRepoUUID,
@@ -33,7 +33,7 @@ import qualified Annex.Branch
 import Types.UUID
 import qualified Build.SysConfig as SysConfig
 import Config
-import UUIDLog
+import Logs.UUIDBased
 
 configkey :: String
 configkey = "annex.uuid"
diff --git a/UUIDLog.hs b/Logs/UUIDBased.hs
similarity index 99%
rename from UUIDLog.hs
rename to Logs/UUIDBased.hs
index d6eb8fbbbe..46fa80be0d 100644
--- a/UUIDLog.hs
+++ b/Logs/UUIDBased.hs
@@ -12,7 +12,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module UUIDLog (
+module Logs.UUIDBased (
 	Log,
 	LogEntry(..),
 	parseLog,
diff --git a/Remote.hs b/Remote.hs
index b1305b9e0f..87c4e23b26 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -34,12 +34,12 @@ import Text.JSON.Generic
 
 import Common.Annex
 import Types.Remote
-import UUID
 import qualified Annex
 import Config
-import Trust
-import LocationLog
-import RemoteLog
+import Logs.UUID
+import Logs.Trust
+import Logs.Location
+import Logs.Remote
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -163,12 +163,12 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+{- Cost ordered lists of remotes that the Logs.Location indicate may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote Annex]
 keyPossibilities key = fst <$> keyPossibilities' False key
 
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+{- Cost ordered lists of remotes that the Logs.Location indicate may have a key.
  -
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 986be7fc40..dfc9116882 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -15,7 +15,7 @@ import System.Process
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Utility.Ssh
 import Remote.Helper.Special
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 03b17456a4..270c78f838 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -15,7 +15,7 @@ import Common.Annex
 import Utility.CopyFile
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Utility.FileMode
 import Remote.Helper.Special
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e9919e636c..42d1b5858d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -17,7 +17,7 @@ import Utility.Ssh
 import Types.Remote
 import qualified Git
 import qualified Annex
-import UUID
+import Logs.UUID
 import qualified Annex.Content
 import qualified Utility.Url as Url
 import Config
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 5f66e8cd9f..5603d13aa2 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -12,7 +12,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index cc8ed69ab5..2c6b50c7da 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -15,7 +15,7 @@ import System.Exit
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Annex.Content
 import Remote.Helper.Special
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 3474f8b251..3216567479 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -13,7 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Annex.Content
 import Remote.Helper.Special
diff --git a/Remote/Web.hs b/Remote/Web.hs
index ed9c94909e..51373a49c3 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -13,10 +13,10 @@ module Remote.Web (
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
 import Config
-import PresenceLog
-import LocationLog
+import Logs.Presence
+import Logs.Location
+import Logs.UUID
 import qualified Utility.Url as Url
 
 type URLString = String
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 132e694ebc..6c6531acea 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -14,7 +14,7 @@ import Data.Char
 import Common.Annex
 import Types.Key
 import Annex.Content
-import PresenceLog
+import Logs.Presence
 import qualified Annex.Queue
 import qualified Git
 import qualified Git.LsFiles as LsFiles
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 39c7b15611..d6334ed654 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -10,7 +10,7 @@ module Upgrade.V2 where
 import Common.Annex
 import qualified Git
 import qualified Annex.Branch
-import LocationLog
+import Logs.Location
 import Annex.Content
 
 olddir :: Git.Repo -> FilePath
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index a0e52507d6..1847ff2440 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -13,7 +13,7 @@ import qualified Git
 import Utility.SafeCommand
 import Types
 import Config
-import UUID
+import Logs.UUID
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index a4d8dd65b6..f19abe6c3f 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -13,7 +13,7 @@ import qualified Git
 import CmdLine
 import Command
 import Options
-import UUID
+import Logs.UUID
 
 import qualified Command.ConfigList
 import qualified Command.InAnnex
diff --git a/git-annex.cabal b/git-annex.cabal
index 450adea75b..e9ec089677 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111011
+Version: 3.20111012
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
diff --git a/test.hs b/test.hs
index 1c511b4d4b..6a6ad441e5 100644
--- a/test.hs
+++ b/test.hs
@@ -29,12 +29,12 @@ import qualified Locations
 import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
-import qualified LocationLog
-import qualified UUID
-import qualified UUIDLog
-import qualified Trust
+import qualified Logs.Location
+import qualified Logs.UUID
+import qualified Logs.UUIDBased
+import qualified Logs.Trust
+import qualified Logs.Remote
 import qualified Remote
-import qualified RemoteLog
 import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Config
@@ -73,14 +73,14 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape
 	, qctest "prop_idempotent_shellEscape_multiword" Utility.SafeCommand.prop_idempotent_shellEscape_multiword
-	, qctest "prop_idempotent_configEscape" RemoteLog.prop_idempotent_configEscape
+	, qctest "prop_idempotent_configEscape" Logs.Remote.prop_idempotent_configEscape
 	, qctest "prop_parentDir_basics" Utility.Path.prop_parentDir_basics
 
 	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
-	, qctest "prop_TimeStamp_sane" UUIDLog.prop_TimeStamp_sane
-	, qctest "prop_addLog_sane" UUIDLog.prop_addLog_sane
+	, qctest "prop_TimeStamp_sane" Logs.UUIDBased.prop_TimeStamp_sane
+	, qctest "prop_addLog_sane" Logs.UUIDBased.prop_addLog_sane
 	]
 
 blackbox :: Test
@@ -341,22 +341,22 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 test_trust :: Test
 test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 	git_annex "trust" ["-q", repo] @? "trust failed"
-	trustcheck Trust.Trusted "trusted 1"
+	trustcheck Logs.Trust.Trusted "trusted 1"
 	git_annex "trust" ["-q", repo] @? "trust of trusted failed"
-	trustcheck Trust.Trusted "trusted 2"
+	trustcheck Logs.Trust.Trusted "trusted 2"
 	git_annex "untrust" ["-q", repo] @? "untrust failed"
-	trustcheck Trust.UnTrusted "untrusted 1"
+	trustcheck Logs.Trust.UnTrusted "untrusted 1"
 	git_annex "untrust" ["-q", repo] @? "untrust of untrusted failed"
-	trustcheck Trust.UnTrusted "untrusted 2"
+	trustcheck Logs.Trust.UnTrusted "untrusted 2"
 	git_annex "semitrust" ["-q", repo] @? "semitrust failed"
-	trustcheck Trust.SemiTrusted "semitrusted 1"
+	trustcheck Logs.Trust.SemiTrusted "semitrusted 1"
 	git_annex "semitrust" ["-q", repo] @? "semitrust of semitrusted failed"
-	trustcheck Trust.SemiTrusted "semitrusted 2"
+	trustcheck Logs.Trust.SemiTrusted "semitrusted 2"
 	where
 		repo = "origin"
 		trustcheck expected msg = do
 			present <- annexeval $ do
-				l <- Trust.trustGet expected
+				l <- Logs.Trust.trustGet expected
 				u <- Remote.nameToUUID repo
 				return $ u `elem` l
 			assertBool msg present
@@ -609,11 +609,11 @@ checkdangling f = do
 
 checklocationlog :: FilePath -> Bool -> Assertion
 checklocationlog f expected = do
-	thisuuid <- annexeval UUID.getUUID
+	thisuuid <- annexeval Logs.UUID.getUUID
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do
-			uuids <- annexeval $ LocationLog.keyLocations k
+			uuids <- annexeval $ Logs.Location.keyLocations k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
 				expected (thisuuid `elem` uuids)
 		_ -> assertFailure $ f ++ " failed to look up key"

From b4015064e1492c6591f69499d8d745989999ba53 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 16:25:51 -0400
Subject: [PATCH 2338/2835] break web log handling into a separate module

---
 Logs/Web.hs   | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 Remote/Web.hs | 34 +---------------------------------
 2 files changed, 50 insertions(+), 33 deletions(-)
 create mode 100644 Logs/Web.hs

diff --git a/Logs/Web.hs b/Logs/Web.hs
new file mode 100644
index 0000000000..ff8fbdb6ba
--- /dev/null
+++ b/Logs/Web.hs
@@ -0,0 +1,49 @@
+{- Web url logs.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Logs.Web (
+	URLString,
+	webUUID,
+	setUrl,
+	getUrls
+) where
+
+import Common.Annex
+import Logs.Presence
+import Logs.Location
+import Logs.UUID
+
+type URLString = String
+
+-- Dummy uuid for the whole web. Do not alter.
+webUUID :: UUID
+webUUID = "00000000-0000-0000-0000-000000000001"
+
+{- The urls for a key are stored in remote/web/hash/key.log 
+ - in the git-annex branch. -}
+urlLog :: Key -> FilePath
+urlLog key = "remote/web"  hashDirLower key  keyFile key ++ ".log"
+oldurlLog :: Key -> FilePath
+{- A bug used to store the urls elsewhere. -}
+oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
+
+getUrls :: Key -> Annex [URLString]
+getUrls key = do
+	us <- currentLog (urlLog key)
+	if null us
+		then currentLog (oldurlLog key)
+		else return us
+
+{- Records a change in an url for a key. -}
+setUrl :: Key -> URLString -> LogStatus -> Annex ()
+setUrl key url status = do
+	g <- gitRepo
+	addLog (urlLog key) =<< logNow status url
+
+	-- update location log to indicate that the web has the key, or not
+	us <- getUrls key
+	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 51373a49c3..e46937ba5f 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -14,13 +14,10 @@ import Common.Annex
 import Types.Remote
 import qualified Git
 import Config
-import Logs.Presence
-import Logs.Location
 import Logs.UUID
+import Logs.Web
 import qualified Utility.Url as Url
 
-type URLString = String
-
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "web",
@@ -35,10 +32,6 @@ remote = RemoteType {
 list :: Annex [Git.Repo]
 list = return [Git.repoRemoteNameSet Git.repoFromUnknown "web"]
 
--- Dummy uuid for the whole web. Do not alter.
-webUUID :: UUID
-webUUID = "00000000-0000-0000-0000-000000000001"
-
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
 	return Remote {
@@ -54,31 +47,6 @@ gen r _ _ =
 		repo = r
 	}
 
-{- The urls for a key are stored in remote/web/hash/key.log 
- - in the git-annex branch. -}
-urlLog :: Key -> FilePath
-urlLog key = "remote/web"  hashDirLower key  keyFile key ++ ".log"
-oldurlLog :: Key -> FilePath
-{- A bug used to store the urls elsewhere. -}
-oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
-
-getUrls :: Key -> Annex [URLString]
-getUrls key = do
-	us <- currentLog (urlLog key)
-	if null us
-		then currentLog (oldurlLog key)
-		else return us
-
-{- Records a change in an url for a key. -}
-setUrl :: Key -> URLString -> LogStatus -> Annex ()
-setUrl key url status = do
-	g <- gitRepo
-	addLog (urlLog key) =<< logNow status url
-
-	-- update location log to indicate that the web has the key, or not
-	us <- getUrls key
-	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
-
 downloadKey :: Key -> FilePath -> Annex Bool
 downloadKey key file = get =<< getUrls key
 	where

From ec169f84b1cc140b6d4c316fbd0e8407297d038a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 16:36:56 -0400
Subject: [PATCH 2339/2835] migrate: Copy url logs for keys when migrating.

---
 Command/AddUrl.hs  | 7 +++----
 Command/Migrate.hs | 9 +++++++++
 Logs/Web.hs        | 5 +++++
 Remote/Web.hs      | 5 +----
 debian/changelog   | 1 +
 5 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 4447dee812..2756af8807 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -13,12 +13,11 @@ import Common.Annex
 import Command
 import qualified Backend
 import qualified Utility.Url as Url
-import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
 import Annex.Content
-import Logs.Presence
+import Logs.Web
 
 command :: [Command]
 command = [repoCommand "addurl" (paramRepeating paramUrl) seek
@@ -58,14 +57,14 @@ download url file = do
 				Nothing -> stop
 				Just (key, _) -> do
 					moveAnnex key tmp
-					Remote.Web.setUrl key url InfoPresent
+					setUrlPresent key url
 					next $ Command.Add.cleanup file key True
 		else stop
 
 nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
 	let key = Backend.URL.fromUrl url
-	Remote.Web.setUrl key url InfoPresent
+	setUrlPresent key url
 	
 	next $ Command.Add.cleanup file key False
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 23ed6fd162..8167ac96eb 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -14,6 +14,7 @@ import qualified Types.Key
 import Annex.Content
 import qualified Command.Add
 import Backend
+import Logs.Web
 
 command :: [Command]
 command = [repoCommand "migrate" paramPaths seek
@@ -65,6 +66,14 @@ perform file oldkey newbackend = do
 				then do
 					-- Update symlink to use the new key.
 					liftIO $ removeFile file
+
+					-- If the old key had some
+					-- associated urls, record them for
+					-- the new key as well.
+					urls <- getUrls oldkey
+					when (not $ null urls) $
+						mapM_ (setUrlPresent newkey) urls
+
 					next $ Command.Add.cleanup file newkey True
 				else stop
 	where
diff --git a/Logs/Web.hs b/Logs/Web.hs
index ff8fbdb6ba..4c8ef7fc00 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -9,6 +9,7 @@ module Logs.Web (
 	URLString,
 	webUUID,
 	setUrl,
+	setUrlPresent,
 	getUrls
 ) where
 
@@ -31,6 +32,7 @@ oldurlLog :: Key -> FilePath
 {- A bug used to store the urls elsewhere. -}
 oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
 
+{- Gets all urls that a key might be available from. -}
 getUrls :: Key -> Annex [URLString]
 getUrls key = do
 	us <- currentLog (urlLog key)
@@ -47,3 +49,6 @@ setUrl key url status = do
 	-- update location log to indicate that the web has the key, or not
 	us <- getUrls key
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
+
+setUrlPresent :: Key -> URLString -> Annex ()
+setUrlPresent key url = setUrl key url InfoPresent
diff --git a/Remote/Web.hs b/Remote/Web.hs
index e46937ba5f..21b9818465 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -5,10 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Web (
-	remote,
-	setUrl
-) where
+module Remote.Web (remote) where
 
 import Common.Annex
 import Types.Remote
diff --git a/debian/changelog b/debian/changelog
index 6e3450a926..ce1489e9d0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20111012) UNRELEASED; urgency=low
 
   * A remote can have a annexUrl configured, that is used by git-annex
     instead of its usual url. (Similar to pushUrl.)
+  * migrate: Copy url logs for keys when migrating.
 
  -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
 

From ee9af605bc6c59365e6080e34d7c615978ba21d8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 17:47:03 -0400
Subject: [PATCH 2340/2835] break out non-log stuff to separate module

---
 Annex/Content.hs         |  2 +-
 Annex/UUID.hs            | 69 ++++++++++++++++++++++++++++++++++++++++
 Command/ConfigList.hs    |  2 +-
 Command/Fsck.hs          |  2 +-
 Command/Init.hs          |  1 +
 Command/InitRemote.hs    |  2 +-
 Command/Map.hs           |  1 +
 Command/Move.hs          |  2 +-
 Command/Semitrust.hs     |  1 -
 Command/Trust.hs         |  1 -
 Command/Untrust.hs       |  1 -
 Init.hs                  |  2 +-
 Logs/Location.hs         |  1 -
 Logs/Remote.hs           |  1 -
 Logs/Trust.hs            |  2 --
 Logs/UUID.hs             | 54 -------------------------------
 Logs/Web.hs              |  1 -
 Remote.hs                |  1 +
 Remote/Bup.hs            |  1 -
 Remote/Directory.hs      |  1 -
 Remote/Git.hs            |  2 +-
 Remote/Helper/Special.hs |  1 -
 Remote/Hook.hs           |  1 -
 Remote/Web.hs            |  1 -
 Types.hs                 |  4 ++-
 Utility/Ssh.hs           |  2 +-
 git-annex-shell.hs       |  2 +-
 test.hs                  |  3 +-
 28 files changed, 86 insertions(+), 78 deletions(-)
 create mode 100644 Annex/UUID.hs

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 9cf7ea8f21..aafdf6f2e6 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -23,7 +23,7 @@ module Annex.Content (
 
 import Common.Annex
 import Logs.Location
-import Logs.UUID
+import Annex.UUID
 import qualified Git
 import qualified Annex
 import qualified Annex.Queue
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
new file mode 100644
index 0000000000..39e296e5b7
--- /dev/null
+++ b/Annex/UUID.hs
@@ -0,0 +1,69 @@
+{- git-annex uuids
+ -
+ - Each git repository used by git-annex has an annex.uuid setting that
+ - uniquely identifies that repository.
+ -
+ - UUIDs of remotes are cached in git config, using keys named
+ - remote..annex-uuid
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.UUID (
+	getUUID,
+	getRepoUUID,
+	getUncachedUUID,
+	prepUUID,
+	genUUID
+) where
+
+import Common.Annex
+import qualified Git
+import qualified Build.SysConfig as SysConfig
+import Config
+
+configkey :: String
+configkey = "annex.uuid"
+
+{- Generates a UUID. There is a library for this, but it's not packaged,
+ - so use the command line tool. -}
+genUUID :: IO UUID
+genUUID = pOpen ReadFromPipe command params hGetLine
+	where
+		command = SysConfig.uuid
+		params = if command == "uuid"
+			-- request a random uuid be generated
+			then ["-m"]
+			-- uuidgen generates random uuid by default
+			else []
+
+getUUID :: Annex UUID
+getUUID = getRepoUUID =<< gitRepo
+
+{- Looks up a repo's UUID. May return "" if none is known. -}
+getRepoUUID :: Git.Repo -> Annex UUID
+getRepoUUID r = do
+	g <- gitRepo
+
+	let c = cached g
+	let u = getUncachedUUID r
+	
+	if c /= u && u /= ""
+		then do
+			updatecache g u
+			return u
+		else return c
+	where
+		cached g = Git.configGet g cachekey ""
+		updatecache g u = when (g /= r) $ setConfig cachekey u
+		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
+
+getUncachedUUID :: Git.Repo -> UUID
+getUncachedUUID r = Git.configGet r configkey ""
+
+{- Make sure that the repo has an annex.uuid setting. -}
+prepUUID :: Annex ()
+prepUUID = whenM (null <$> getUUID) $
+	setConfig configkey =<< liftIO genUUID
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index b50c759eee..43315f67ce 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -9,7 +9,7 @@ module Command.ConfigList where
 
 import Common.Annex
 import Command
-import Logs.UUID
+import Annex.UUID
 
 command :: [Command]
 command = [repoCommand "configlist" paramNothing seek
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 632570b110..1c1687a002 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -15,7 +15,7 @@ import qualified Types.Key
 import Annex.Content
 import Logs.Location
 import Logs.Trust
-import Logs.UUID
+import Annex.UUID
 import Utility.DataUnits
 import Utility.FileMode
 import Config
diff --git a/Command/Init.hs b/Command/Init.hs
index dcc6bfe6b2..3dd4493295 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -9,6 +9,7 @@ module Command.Init where
 
 import Common.Annex
 import Command
+import Annex.UUID
 import Logs.UUID
 import Init
 	
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 240528b879..073ba72f90 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -14,7 +14,7 @@ import Command
 import qualified Remote
 import qualified Logs.Remote
 import qualified Types.Remote as R
-import Logs.UUID
+import Annex.UUID
 
 command :: [Command]
 command = [repoCommand "initremote"
diff --git a/Command/Map.hs b/Command/Map.hs
index 5cbf51b27f..18cb915e3b 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -13,6 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Git
+import Annex.UUID
 import Logs.UUID
 import Logs.Trust
 import Utility.Ssh
diff --git a/Command/Move.hs b/Command/Move.hs
index 62f38224ca..a816aacde5 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -14,7 +14,7 @@ import qualified Annex
 import Logs.Location
 import Annex.Content
 import qualified Remote
-import Logs.UUID
+import Annex.UUID
 
 command :: [Command]
 command = [repoCommand "move" paramPaths seek
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index e13785a38f..5d60977eb4 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -10,7 +10,6 @@ module Command.Semitrust where
 import Common.Annex
 import Command
 import qualified Remote
-import Logs.UUID
 import Logs.Trust
 
 command :: [Command]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index fb7f47ec0e..eeeadc9afe 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Command
 import qualified Remote
 import Logs.Trust
-import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "trust" (paramRepeating paramRemote) seek
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 6f2b602038..f8bf498f24 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -10,7 +10,6 @@ module Command.Untrust where
 import Common.Annex
 import Command
 import qualified Remote
-import Logs.UUID
 import Logs.Trust
 
 command :: [Command]
diff --git a/Init.hs b/Init.hs
index 00a3d12fab..43840a1081 100644
--- a/Init.hs
+++ b/Init.hs
@@ -15,7 +15,7 @@ import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import Annex.Version
-import Logs.UUID
+import Annex.UUID
 
 initialize :: Annex ()
 initialize = do
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 4e8b2b5357..8868912dbb 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -24,7 +24,6 @@ module Logs.Location (
 import Common.Annex
 import qualified Git
 import qualified Annex.Branch
-import Logs.UUID
 import Logs.Presence
 
 {- Log a change in the presence of a key's value in a repository. -}
diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index 47c2d7472b..e2b04bf471 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -21,7 +21,6 @@ import Data.Char
 import Common.Annex
 import qualified Annex.Branch
 import Types.Remote
-import Logs.UUID
 import Logs.UUIDBased
 
 {- Filename of remote.log. -}
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 6966ffdd6e..372d8b3609 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -19,8 +19,6 @@ import Common.Annex
 import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
-
-import Logs.UUID
 import Logs.UUIDBased
 
 {- Filename of trust.log. -}
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index baf6650015..8a93b43fef 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -14,12 +14,6 @@
  -}
 
 module Logs.UUID (
-	UUID,
-	getUUID,
-	getRepoUUID,
-	getUncachedUUID,
-	prepUUID,
-	genUUID,
 	describeUUID,
 	uuidMap
 ) where
@@ -28,61 +22,13 @@ import qualified Data.Map as M
 import Data.Time.Clock.POSIX
 
 import Common.Annex
-import qualified Git
 import qualified Annex.Branch
-import Types.UUID
-import qualified Build.SysConfig as SysConfig
-import Config
 import Logs.UUIDBased
 
-configkey :: String
-configkey = "annex.uuid"
-
 {- Filename of uuid.log. -}
 logfile :: FilePath
 logfile = "uuid.log"
 
-{- Generates a UUID. There is a library for this, but it's not packaged,
- - so use the command line tool. -}
-genUUID :: IO UUID
-genUUID = pOpen ReadFromPipe command params hGetLine
-	where
-		command = SysConfig.uuid
-		params = if command == "uuid"
-			-- request a random uuid be generated
-			then ["-m"]
-			-- uuidgen generates random uuid by default
-			else []
-
-getUUID :: Annex UUID
-getUUID = getRepoUUID =<< gitRepo
-
-{- Looks up a repo's UUID. May return "" if none is known. -}
-getRepoUUID :: Git.Repo -> Annex UUID
-getRepoUUID r = do
-	g <- gitRepo
-
-	let c = cached g
-	let u = getUncachedUUID r
-	
-	if c /= u && u /= ""
-		then do
-			updatecache g u
-			return u
-		else return c
-	where
-		cached g = Git.configGet g cachekey ""
-		updatecache g u = when (g /= r) $ setConfig cachekey u
-		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
-
-getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = Git.configGet r configkey ""
-
-{- Make sure that the repo has an annex.uuid setting. -}
-prepUUID :: Annex ()
-prepUUID = whenM (null <$> getUUID) $
-	setConfig configkey =<< liftIO genUUID
-
 {- Records a description for a uuid in the log. -}
 describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
diff --git a/Logs/Web.hs b/Logs/Web.hs
index 4c8ef7fc00..605797079f 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -16,7 +16,6 @@ module Logs.Web (
 import Common.Annex
 import Logs.Presence
 import Logs.Location
-import Logs.UUID
 
 type URLString = String
 
diff --git a/Remote.hs b/Remote.hs
index 87c4e23b26..d1714f7751 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -36,6 +36,7 @@ import Common.Annex
 import Types.Remote
 import qualified Annex
 import Config
+import Annex.UUID
 import Logs.UUID
 import Logs.Trust
 import Logs.Location
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index dfc9116882..8d36245a94 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -15,7 +15,6 @@ import System.Process
 import Common.Annex
 import Types.Remote
 import qualified Git
-import Logs.UUID
 import Config
 import Utility.Ssh
 import Remote.Helper.Special
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 270c78f838..e8cf05a0e5 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -15,7 +15,6 @@ import Common.Annex
 import Utility.CopyFile
 import Types.Remote
 import qualified Git
-import Logs.UUID
 import Config
 import Utility.FileMode
 import Remote.Helper.Special
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 42d1b5858d..10183522fe 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -17,7 +17,7 @@ import Utility.Ssh
 import Types.Remote
 import qualified Git
 import qualified Annex
-import Logs.UUID
+import Annex.UUID
 import qualified Annex.Content
 import qualified Utility.Url as Url
 import Config
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 5603d13aa2..52f2dbf954 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -12,7 +12,6 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
-import Logs.UUID
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 2c6b50c7da..8b6a6cecfc 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -15,7 +15,6 @@ import System.Exit
 import Common.Annex
 import Types.Remote
 import qualified Git
-import Logs.UUID
 import Config
 import Annex.Content
 import Remote.Helper.Special
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 21b9818465..63963c5309 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Types.Remote
 import qualified Git
 import Config
-import Logs.UUID
 import Logs.Web
 import qualified Utility.Url as Url
 
diff --git a/Types.hs b/Types.hs
index 6353f6da68..703edb5c86 100644
--- a/Types.hs
+++ b/Types.hs
@@ -8,9 +8,11 @@
 module Types (
 	Annex,
 	Backend,
-	Key
+	Key,
+	UUID
 ) where
 
 import Annex
 import Types.Backend
 import Types.Key
+import Types.UUID
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 1847ff2440..34e4390f6b 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -13,7 +13,7 @@ import qualified Git
 import Utility.SafeCommand
 import Types
 import Config
-import Logs.UUID
+import Annex.UUID
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index f19abe6c3f..72e130ff08 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -13,7 +13,7 @@ import qualified Git
 import CmdLine
 import Command
 import Options
-import Logs.UUID
+import Annex.UUID
 
 import qualified Command.ConfigList
 import qualified Command.InAnnex
diff --git a/test.hs b/test.hs
index 6a6ad441e5..7d4c4293e1 100644
--- a/test.hs
+++ b/test.hs
@@ -23,6 +23,7 @@ import Common
 
 import qualified Utility.SafeCommand
 import qualified Annex
+import qualified Annex.UUID
 import qualified Backend
 import qualified Git
 import qualified Locations
@@ -609,7 +610,7 @@ checkdangling f = do
 
 checklocationlog :: FilePath -> Bool -> Assertion
 checklocationlog f expected = do
-	thisuuid <- annexeval Logs.UUID.getUUID
+	thisuuid <- annexeval Annex.UUID.getUUID
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do

From 1480d71adb1d6acf2cc8863064902244a31f099b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 18:45:32 -0400
Subject: [PATCH 2341/2835] fix

---
 Remote/S3real.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index a754731281..40d7d905d5 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -21,7 +21,7 @@ import Common.Annex
 import Types.Remote
 import Types.Key
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Remote.Helper.Special
 import Remote.Helper.Encryptable

From 52c8244219dd90102818282b8b09186f2ce93a0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 19:06:35 -0400
Subject: [PATCH 2342/2835] git-annex-shell: GIT_ANNEX_SHELL_READONLY and
 GIT_ANNEX_SHELL_LIMITED environment variables can be set to limit what
 commands can be run.

This could be used by eg, gitolite.
---
 debian/changelog         |  3 +++
 doc/git-annex-shell.mdwn | 24 +++++++++++++++++++----
 git-annex-shell.hs       | 42 ++++++++++++++++++++++++++++++++--------
 3 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ce1489e9d0..4e0a1e21e4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (3.20111012) UNRELEASED; urgency=low
   * A remote can have a annexUrl configured, that is used by git-annex
     instead of its usual url. (Similar to pushUrl.)
   * migrate: Copy url logs for keys when migrating.
+  * git-annex-shell: GIT_ANNEX_SHELL_READONLY and GIT_ANNEX_SHELL_LIMITED
+    environment variables can be set to limit what commands can be run.
+    This could be used by eg, gitolite.
 
  -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
 
diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 1fc9647c88..fc5bc6c2d6 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -19,6 +19,10 @@ user's restricted login shell.
 
 Any command not listed below is passed through to git-shell.
 
+Note that the directory parameter should be an absolute path, otherwise
+it is assumed to be relative to the user's home directory. Also the
+first "/~/" or "/~user/" is expanded to the specified home directory.
+
 * configlist directory
 
   This outputs a subset of the git configuration, in the same form as
@@ -44,11 +48,23 @@ Any command not listed below is passed through to git-shell.
 
 # OPTIONS
 
-Same as git-annex or git-shell, depending on the command being run.
+Most options are the same as in git-annex. The ones specific
+to git-annex-shell are:
 
-Note that the directory parameter should be an absolute path, otherwise
-it is assumed to be relative to the user's home directory. Also the
-first "/~/" or "/~user/" is expanded to the specified home directory.
+* --uuid=UUID
+
+  git-annex uses this to specify the UUID of the repository it was expecting
+  git-annex-shell to access, as a sanity check.
+
+# ENVIRONMENT
+
+* GIT_ANNEX_SHELL_READONLY
+
+  If set, disallows any command that could modify the repository.
+
+* GIT_ANNEX_SHELL_LIMITED
+
+  If set, disallows running git-shell to handle unknown commands.
 
 # SEE ALSO
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 72e130ff08..41cb72d7e2 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -21,21 +21,29 @@ import qualified Command.DropKey
 import qualified Command.RecvKey
 import qualified Command.SendKey
 
-cmds :: [Command]
-cmds = map adddirparam $ concat
+cmds_readonly :: [Command]
+cmds_readonly = concat
 	[ Command.ConfigList.command
 	, Command.InAnnex.command
-	, Command.DropKey.command
-	, Command.RecvKey.command
 	, Command.SendKey.command
 	]
+
+cmds_notreadonly :: [Command]
+cmds_notreadonly = concat
+	[ Command.RecvKey.command
+	, Command.DropKey.command
+	]
+
+cmds :: [Command]
+cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly
 	where
 		adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c }
 
 options :: [OptDescr (Annex ())]
-options = uuid : commonOptions
+options = commonOptions ++
+	[ Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
+	]
 	where
-		uuid = Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
 		check expected = do
 			u <- getUUID
 			when (u /= expected) $ error $
@@ -67,12 +75,14 @@ builtins :: [String]
 builtins = map cmdname cmds
 
 builtin :: String -> String -> [String] -> IO ()
-builtin cmd dir params =
+builtin cmd dir params = do
+	checkNotReadOnly cmd
 	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
 		dispatch (cmd : filterparams params) cmds options header
 
 external :: [String] -> IO ()
-external params =
+external params = do
+	checkNotLimited
 	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $
 		error "git-shell failed"
 
@@ -85,3 +95,19 @@ filterparams (a:as) = a:filterparams as
 
 failure :: IO ()
 failure = error $ "bad parameters\n\n" ++ usage header cmds options
+
+checkNotLimited :: IO ()
+checkNotLimited = checkEnv "GIT_ANNEX_SHELL_LIMITED"
+
+checkNotReadOnly :: String -> IO ()
+checkNotReadOnly cmd
+	| cmd `elem` map cmdname cmds_readonly = return ()
+	| otherwise = checkEnv "GIT_ANNEX_SHELL_READONLY"
+
+checkEnv :: String -> IO ()
+checkEnv var = catch check (const $ return ())
+	where
+		check = do
+			val <- getEnv var
+			when (not $ null val) $
+				error $ "Action blocked by " ++ var

From 91366c896d9c9cb4519b451a64ed4d1e0ff52cb3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 16 Oct 2011 00:04:26 -0400
Subject: [PATCH 2343/2835] clean Annex stuff out of Utility/

---
 {Utility => Annex}/Ssh.hs |  2 +-
 Command/AddUrl.hs         |  2 +-
 Command/Map.hs            |  2 +-
 Locations.hs              |  2 +-
 Remote/Bup.hs             |  2 +-
 Remote/Git.hs             |  4 ++--
 Remote/Web.hs             |  4 +++-
 Utility/Url.hs            | 12 ++++--------
 8 files changed, 14 insertions(+), 16 deletions(-)
 rename {Utility => Annex}/Ssh.hs (98%)

diff --git a/Utility/Ssh.hs b/Annex/Ssh.hs
similarity index 98%
rename from Utility/Ssh.hs
rename to Annex/Ssh.hs
index 34e4390f6b..851c7c06b6 100644
--- a/Utility/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.Ssh where
+module Annex.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 2756af8807..f32b5b86a9 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -48,7 +48,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Url.download url tmp
+	ok <- liftIO $ Url.download url tmp
 	if ok
 		then do
 			[(backend, _)] <- Backend.chooseBackends [file]
diff --git a/Command/Map.hs b/Command/Map.hs
index 18cb915e3b..48cba63f9e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -16,7 +16,7 @@ import qualified Git
 import Annex.UUID
 import Logs.UUID
 import Logs.Trust
-import Utility.Ssh
+import Annex.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Locations.hs b/Locations.hs
index 4579fe05b7..ceb6246b98 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -127,7 +127,7 @@ isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
  -     is one to one.
  - ":" is escaped to "&c", because despite it being 2011, people still care
  -     about FAT.
- - -}
+ -}
 keyFile :: Key -> FilePath
 keyFile key = replace "/" "%" $ replace ":" "&c" $
 	replace "%" "&s" $ replace "&" "&a"  $ show key
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 8d36245a94..48014f1dad 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -16,7 +16,7 @@ import Common.Annex
 import Types.Remote
 import qualified Git
 import Config
-import Utility.Ssh
+import Annex.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 10183522fe..8857d821d2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -13,7 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Utility.CopyFile
 import Utility.RsyncFile
-import Utility.Ssh
+import Annex.Ssh
 import Types.Remote
 import qualified Git
 import qualified Annex
@@ -164,7 +164,7 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| Git.repoIsHttp r = Url.download (keyUrl r key) file
+	| Git.repoIsHttp r = liftIO $ Url.download (keyUrl r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
 
 {- Tries to copy a key's content to a remote's annex. -}
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 63963c5309..3fea94531a 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -49,7 +49,9 @@ downloadKey key file = get =<< getUrls key
 		get [] = do
 			warning "no known url"
 			return False
-		get urls = anyM (`Url.download` file) urls
+		get urls = do
+			showOutput -- make way for download progress bar
+			liftIO $ anyM (`Url.download` file) urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 6ddeecc14f..b5f5b78c00 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -12,13 +12,10 @@ module Utility.Url (
 ) where
 
 import Control.Applicative
-import Control.Monad.State (liftIO)
 import qualified Network.Browser as Browser
 import Network.HTTP
 import Network.URI
 
-import Types
-import Messages
 import Utility.SafeCommand
 import Utility
 
@@ -38,13 +35,12 @@ exists url =
 {- Used to download large files, such as the contents of keys.
  - Uses wget or curl program for its progress bar. (Wget has a better one,
  - so is preferred.) -}
-download :: URLString -> FilePath -> Annex Bool
+download :: URLString -> FilePath -> IO Bool
 download url file = do
-	showOutput -- make way for program's progress bar
-	e <- liftIO $ inPath "wget"
+	e <- inPath "wget"
 	if e
 		then
-			liftIO $ boolSystem "wget"
+			boolSystem "wget"
 				[Params "-c -O", File file, File url]
 		else
 			-- Uses the -# progress display, because the normal
@@ -52,7 +48,7 @@ download url file = do
 			-- the remainder to download as the whole file,
 			-- and not indicating how much percent was
 			-- downloaded before the resume.
-			liftIO $ boolSystem "curl" 
+			boolSystem "curl" 
 				[Params "-L -C - -# -o", File file, File url]
 
 {- Downloads a small file. -}

From 23f2a12816e250f6780f80443ef6ec31c13fca9e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 16 Oct 2011 00:31:25 -0400
Subject: [PATCH 2344/2835] broke up Utility

---
 Command/Unused.hs    |   1 +
 Common.hs            |   4 +-
 Init.hs              |   1 +
 Remote/Git.hs        |   1 +
 Remote/Rsync.hs      |   1 -
 Remote/S3real.hs     |   1 -
 Remote/Web.hs        |   1 +
 Upgrade/V1.hs        |   1 +
 Upgrade/V2.hs        |   1 +
 Utility.hs           | 106 -------------------------------------------
 Utility/Misc.hs      |  29 ++++++++++++
 Utility/Monad.hs     |  26 +++++++++++
 Utility/Path.hs      |  22 +++++++++
 Utility/RsyncFile.hs |   2 +-
 Utility/TempFile.hs  |  39 ++++++++++++++++
 Utility/Url.hs       |   2 +-
 16 files changed, 126 insertions(+), 112 deletions(-)
 delete mode 100644 Utility.hs
 create mode 100644 Utility/Misc.hs
 create mode 100644 Utility/Monad.hs
 create mode 100644 Utility/TempFile.hs

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 874b0ca061..a901747521 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,6 +16,7 @@ import Common.Annex
 import Command
 import Annex.Content
 import Utility.FileMode
+import Utility.TempFile
 import Logs.Location
 import qualified Annex
 import qualified Git
diff --git a/Common.hs b/Common.hs
index e88342ae4a..2e1e4d996c 100644
--- a/Common.hs
+++ b/Common.hs
@@ -15,7 +15,7 @@ module Common (
 	module System.Posix.IO,
 	module System.Posix.Process,
 	module System.Exit,
-	module Utility,
+	module Utility.Misc,
 	module Utility.Conditional,
 	module Utility.SafeCommand,
 	module Utility.Path,
@@ -40,7 +40,7 @@ import System.Posix.IO
 import System.Posix.Process hiding (executeFile)
 import System.Exit
 
-import Utility
+import Utility.Misc
 import Utility.Conditional
 import Utility.SafeCommand
 import Utility.Path
diff --git a/Init.hs b/Init.hs
index 43840a1081..6e024e9fc5 100644
--- a/Init.hs
+++ b/Init.hs
@@ -12,6 +12,7 @@ module Init (
 ) where
 
 import Common.Annex
+import Utility.TempFile
 import qualified Git
 import qualified Annex.Branch
 import Annex.Version
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 8857d821d2..5d31770a22 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -20,6 +20,7 @@ import qualified Annex
 import Annex.UUID
 import qualified Annex.Content
 import qualified Utility.Url as Url
+import Utility.TempFile
 import Config
 import Init
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 3216567479..e79762a382 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -13,7 +13,6 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
-import Logs.UUID
 import Config
 import Annex.Content
 import Remote.Helper.Special
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 40d7d905d5..89b0326379 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -21,7 +21,6 @@ import Common.Annex
 import Types.Remote
 import Types.Key
 import qualified Git
-import Logs.UUID
 import Config
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 3fea94531a..393932d478 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -13,6 +13,7 @@ import qualified Git
 import Config
 import Logs.Web
 import qualified Utility.Url as Url
+import Utility.Monad
 
 remote :: RemoteType Annex
 remote = RemoteType {
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 6c6531acea..331328e816 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -21,6 +21,7 @@ import qualified Git.LsFiles as LsFiles
 import Backend
 import Annex.Version
 import Utility.FileMode
+import Utility.TempFile
 import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index d6334ed654..1ad41266a7 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -12,6 +12,7 @@ import qualified Git
 import qualified Annex.Branch
 import Logs.Location
 import Annex.Content
+import Utility.TempFile
 
 olddir :: Git.Repo -> FilePath
 olddir g
diff --git a/Utility.hs b/Utility.hs
deleted file mode 100644
index 8ef60a0814..0000000000
--- a/Utility.hs
+++ /dev/null
@@ -1,106 +0,0 @@
-{- general purpose utility functions
- -
- - Copyright 2010-2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Utility (
-	hGetContentsStrict,
-	readFileStrict,
-	readMaybe,
-	viaTmp,
-	withTempFile,
-	dirContents,
-	myHomeDir,
-	catchBool,
-	inPath,
-	firstM,
-	anyM
-) where
-
-import Control.Applicative
-import IO (bracket)
-import System.IO
-import System.Posix.Process hiding (executeFile)
-import System.Posix.User
-import System.FilePath
-import System.Directory
-import Utility.Path
-import Data.Maybe
-import Control.Monad (liftM)
-
-{- A version of hgetContents that is not lazy. Ensures file is 
- - all read before it gets closed. -}
-hGetContentsStrict :: Handle -> IO String
-hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
-
-{- A version of readFile that is not lazy. -}
-readFileStrict :: FilePath -> IO String
-readFileStrict f = readFile f >>= \s -> length s `seq` return s
-
-{- Attempts to read a value from a String. -}
-readMaybe :: (Read a) => String -> Maybe a
-readMaybe s = case reads s of
-	((x,_):_) -> Just x
-	_ -> Nothing
-
-{- Runs an action like writeFile, writing to a tmp file first and
- - then moving it into place. -}
-viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()
-viaTmp a file content = do
-	pid <- getProcessID
-        let tmpfile = file ++ ".tmp" ++ show pid
-	createDirectoryIfMissing True (parentDir file)
-	a tmpfile content
-	renameFile tmpfile file
-
-{- Runs an action with a temp file, then removes the file. -}
-withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
-withTempFile template a = bracket create remove use
-	where
-		create = do
-			tmpdir <- catch getTemporaryDirectory (const $ return ".")
-			openTempFile tmpdir template
-		remove (name, handle) = do
-			hClose handle
-			catchBool (removeFile name >> return True)
-		use (name, handle) = a name handle
-
-{- Lists the contents of a directory.
- - Unlike getDirectoryContents, paths are not relative to the directory. -}
-dirContents :: FilePath -> IO [FilePath]
-dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
-	where
-		notcruft "." = False
-		notcruft ".." = False
-		notcruft _ = True
-
-{- Current user's home directory. -}
-myHomeDir :: IO FilePath
-myHomeDir = homeDirectory <$> (getUserEntryForID =<< getEffectiveUserID)
-
-{- Catches IO errors and returns a Bool -}
-catchBool :: IO Bool -> IO Bool
-catchBool = flip catch (const $ return False)
-
-{- Return the first value from a list, if any, satisfying the given
- - predicate -}
-firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
-firstM _ [] = return Nothing
-firstM p (x:xs) = do
-	q <- p x
-	if q
-		then return (Just x)
-		else firstM p xs
-
-{- Returns true if any value in the list satisfies the preducate,
- - stopping once one is found. -}
-anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
-anyM p = liftM isJust . firstM p
-
-{- Checks if a command is available in PATH. -}
-inPath :: String -> IO Bool
-inPath command = getSearchPath >>= anyM indir
-	where
-		indir d = doesFileExist $ d  command
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
new file mode 100644
index 0000000000..bc18347746
--- /dev/null
+++ b/Utility/Misc.hs
@@ -0,0 +1,29 @@
+{- misc utility functions
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Misc where
+
+import System.IO
+
+{- A version of hgetContents that is not lazy. Ensures file is 
+ - all read before it gets closed. -}
+hGetContentsStrict :: Handle -> IO String
+hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
+
+{- A version of readFile that is not lazy. -}
+readFileStrict :: FilePath -> IO String
+readFileStrict f = readFile f >>= \s -> length s `seq` return s
+
+{- Attempts to read a value from a String. -}
+readMaybe :: (Read a) => String -> Maybe a
+readMaybe s = case reads s of
+	((x,_):_) -> Just x
+	_ -> Nothing
+
+{- Catches IO errors and returns a Bool -}
+catchBool :: IO Bool -> IO Bool
+catchBool = flip catch (const $ return False)
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
new file mode 100644
index 0000000000..9523e17165
--- /dev/null
+++ b/Utility/Monad.hs
@@ -0,0 +1,26 @@
+{- monadic stuff
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Monad where
+
+import Data.Maybe
+import Control.Monad (liftM)
+
+{- Return the first value from a list, if any, satisfying the given
+ - predicate -}
+firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
+firstM _ [] = return Nothing
+firstM p (x:xs) = do
+	q <- p x
+	if q
+		then return (Just x)
+		else firstM p xs
+
+{- Returns true if any value in the list satisfies the preducate,
+ - stopping once one is found. -}
+anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
+anyM p = liftM isJust . firstM p
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 1c68b87bbc..38e7bd05ca 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -14,6 +14,9 @@ import System.Directory
 import Data.List
 import Data.Maybe
 import Control.Applicative
+import System.Posix.User
+
+import Utility.Monad
 
 {- Returns the parent directory of a path. Parent of / is "" -}
 parentDir :: FilePath -> FilePath
@@ -112,3 +115,22 @@ preserveOrder (l:ls) new = found ++ preserveOrder ls rest
  -}
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
 runPreserveOrder a files = preserveOrder files <$> a files
+
+{- Lists the contents of a directory.
+ - Unlike getDirectoryContents, paths are not relative to the directory. -}
+dirContents :: FilePath -> IO [FilePath]
+dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
+	where
+		notcruft "." = False
+		notcruft ".." = False
+		notcruft _ = True
+
+{- Current user's home directory. -}
+myHomeDir :: IO FilePath
+myHomeDir = homeDirectory <$> (getUserEntryForID =<< getEffectiveUserID)
+
+{- Checks if a command is available in PATH. -}
+inPath :: String -> IO Bool
+inPath command = getSearchPath >>= anyM indir
+	where
+		indir d = doesFileExist $ d  command
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index b6c2267e87..056bd8d114 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -1,4 +1,4 @@
-{- git-annex file copying with rsync
+{- file copying with rsync
  -
  - Copyright 2010 Joey Hess 
  -
diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs
new file mode 100644
index 0000000000..1e823c10ef
--- /dev/null
+++ b/Utility/TempFile.hs
@@ -0,0 +1,39 @@
+{- temp file functions
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.TempFile where
+
+import IO (bracket)
+import System.IO
+import System.Posix.Process hiding (executeFile)
+import System.Directory
+
+import Utility.Misc
+import Utility.Path
+
+{- Runs an action like writeFile, writing to a temp file first and
+ - then moving it into place. The temp file is stored in the same
+ - directory as the final file to avoid cross-device renames. -}
+viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()
+viaTmp a file content = do
+	pid <- getProcessID
+        let tmpfile = file ++ ".tmp" ++ show pid
+	createDirectoryIfMissing True (parentDir file)
+	a tmpfile content
+	renameFile tmpfile file
+
+{- Runs an action with a temp file, then removes the file. -}
+withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
+withTempFile template a = bracket create remove use
+	where
+		create = do
+			tmpdir <- catch getTemporaryDirectory (const $ return ".")
+			openTempFile tmpdir template
+		remove (name, handle) = do
+			hClose handle
+			catchBool (removeFile name >> return True)
+		use (name, handle) = a name handle
diff --git a/Utility/Url.hs b/Utility/Url.hs
index b5f5b78c00..617fe3f4d0 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -17,7 +17,7 @@ import Network.HTTP
 import Network.URI
 
 import Utility.SafeCommand
-import Utility
+import Utility.Path
 
 type URLString = String
 

From 66fa4c947c30ca9848121912229f3e84a855a74f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 16 Oct 2011 01:03:38 -0400
Subject: [PATCH 2345/2835] correct spelling of "gibibyte"

---
 Utility/DataUnits.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index 0baa5dd896..e7552f52fb 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -84,7 +84,7 @@ memoryUnits =
 	, Unit (p 6) "EiB" "exbibyte"
 	, Unit (p 5) "PiB" "pebibyte"
 	, Unit (p 4) "TiB" "tebibyte"
-	, Unit (p 3) "GiB" "gigabyte"
+	, Unit (p 3) "GiB" "gibibyte"
 	, Unit (p 2) "MiB" "mebibyte"
 	, Unit (p 1) "KiB" "kibibyte"
 	, Unit (p 0) "B" "byte"

From 617bdc740f76e0b5cb8d73a8b122cd2b3e6fe961 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 13:56:36 -0400
Subject: [PATCH 2346/2835] reorg

---
 doc/cheatsheet.mdwn                              | 16 ----------------
 doc/tips.mdwn                                    |  4 ++++
 .../Internet_Archive_via_S3.mdwn                 |  0
 .../migrating_data_to_a_new_backend.mdwn         |  0
 .../powerful_file_matching.mdwn                  |  0
 .../recover_data_from_lost+found.mdwn            |  0
 .../untrusted_repositories.mdwn                  |  0
 doc/{walkthrough => tips}/using_Amazon_S3.mdwn   |  0
 .../using_the_SHA1_backend.mdwn                  |  0
 doc/{walkthrough => tips}/using_the_web.mdwn     |  0
 .../what_to_do_when_you_lose_a_repository.mdwn   |  0
 doc/walkthrough/more.mdwn                        |  3 +--
 12 files changed, 5 insertions(+), 18 deletions(-)
 delete mode 100644 doc/cheatsheet.mdwn
 create mode 100644 doc/tips.mdwn
 rename doc/{walkthrough => tips}/Internet_Archive_via_S3.mdwn (100%)
 rename doc/{walkthrough => tips}/migrating_data_to_a_new_backend.mdwn (100%)
 rename doc/{walkthrough => tips}/powerful_file_matching.mdwn (100%)
 rename doc/{walkthrough => tips}/recover_data_from_lost+found.mdwn (100%)
 rename doc/{walkthrough => tips}/untrusted_repositories.mdwn (100%)
 rename doc/{walkthrough => tips}/using_Amazon_S3.mdwn (100%)
 rename doc/{walkthrough => tips}/using_the_SHA1_backend.mdwn (100%)
 rename doc/{walkthrough => tips}/using_the_web.mdwn (100%)
 rename doc/{walkthrough => tips}/what_to_do_when_you_lose_a_repository.mdwn (100%)

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
deleted file mode 100644
index 65cbad187c..0000000000
--- a/doc/cheatsheet.mdwn
+++ /dev/null
@@ -1,16 +0,0 @@
-A supplement to the [[walkthrough]].
-
-[[!toc]]
-
-[[!inline feeds=no show=0 template=walkthrough pagenames="""
-	walkthrough/using_Amazon_S3
-	walkthrough/using_bup
-	walkthrough/using_the_web
-	walkthrough/using_the_SHA1_backend
-	walkthrough/migrating_data_to_a_new_backend
-	walkthrough/untrusted_repositories
-	walkthrough/what_to_do_when_you_lose_a_repository
-	walkthrough/recover_data_from_lost+found
-	walkthrough/Internet_Archive_via_S3
-	walkthrough/powerful_file_matching
-"""]]
diff --git a/doc/tips.mdwn b/doc/tips.mdwn
new file mode 100644
index 0000000000..eda84c8672
--- /dev/null
+++ b/doc/tips.mdwn
@@ -0,0 +1,4 @@
+This page is a place to document tips and techniques for using git-annex.
+
+[[!inline pages="tips/* and !tips/*/*" archive="yes"
+rootpage="tips" postformtext="Add a new tip about:" show=0]]
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/tips/Internet_Archive_via_S3.mdwn
similarity index 100%
rename from doc/walkthrough/Internet_Archive_via_S3.mdwn
rename to doc/tips/Internet_Archive_via_S3.mdwn
diff --git a/doc/walkthrough/migrating_data_to_a_new_backend.mdwn b/doc/tips/migrating_data_to_a_new_backend.mdwn
similarity index 100%
rename from doc/walkthrough/migrating_data_to_a_new_backend.mdwn
rename to doc/tips/migrating_data_to_a_new_backend.mdwn
diff --git a/doc/walkthrough/powerful_file_matching.mdwn b/doc/tips/powerful_file_matching.mdwn
similarity index 100%
rename from doc/walkthrough/powerful_file_matching.mdwn
rename to doc/tips/powerful_file_matching.mdwn
diff --git a/doc/walkthrough/recover_data_from_lost+found.mdwn b/doc/tips/recover_data_from_lost+found.mdwn
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found.mdwn
rename to doc/tips/recover_data_from_lost+found.mdwn
diff --git a/doc/walkthrough/untrusted_repositories.mdwn b/doc/tips/untrusted_repositories.mdwn
similarity index 100%
rename from doc/walkthrough/untrusted_repositories.mdwn
rename to doc/tips/untrusted_repositories.mdwn
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/tips/using_Amazon_S3.mdwn
similarity index 100%
rename from doc/walkthrough/using_Amazon_S3.mdwn
rename to doc/tips/using_Amazon_S3.mdwn
diff --git a/doc/walkthrough/using_the_SHA1_backend.mdwn b/doc/tips/using_the_SHA1_backend.mdwn
similarity index 100%
rename from doc/walkthrough/using_the_SHA1_backend.mdwn
rename to doc/tips/using_the_SHA1_backend.mdwn
diff --git a/doc/walkthrough/using_the_web.mdwn b/doc/tips/using_the_web.mdwn
similarity index 100%
rename from doc/walkthrough/using_the_web.mdwn
rename to doc/tips/using_the_web.mdwn
diff --git a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
similarity index 100%
rename from doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
rename to doc/tips/what_to_do_when_you_lose_a_repository.mdwn
diff --git a/doc/walkthrough/more.mdwn b/doc/walkthrough/more.mdwn
index 1eaf9009f6..0a4a5b94e8 100644
--- a/doc/walkthrough/more.mdwn
+++ b/doc/walkthrough/more.mdwn
@@ -1,4 +1,3 @@
 So ends the walkthrough. By now you should be able to use git-annex.
 
-Want more? See the [[cheatsheet]] for info about all of git-annex's hidden
-features.
+Want more? See [[tips]] for lots more features and advice.

From 9f30134300b2b119895389f73d413320dc897359 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:16:05 -0400
Subject: [PATCH 2347/2835] new tip for gitolite and git-annex

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 71 +++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex.mdwn

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
new file mode 100644
index 0000000000..c4df42cae6
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -0,0 +1,71 @@
+[Gitolite](https://github.com/sitaramc/gitolite) is a git repository
+manager. Here's how to add git-annex support to gitolite, so you can
+`git annex copy` files to a gitolite repository, and `git annex get`
+files from it.
+
+A nice feature of using gitolite with git-annex is that users can be given
+read-only access to a repository, and this allows them to `git annex get`
+file contents, but not change anything.
+
+First, you need new enough versions:
+
+* gitolite 2.2 is needed -- this version contains a git-annex-shell ADC
+  and supports "ua" ADCs.
+* git-annex 3.20111016 or newer needs to be installed on the gitolite
+  server. Don't install an older version, it wouldn't be secure!
+
+And here's how to set it up. The examples are for gitolite as installed
+on Debian with apt-get, but the changes described can be made to any
+gitolite installation, just with different paths.
+
+1. Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
+
+	echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+
+2. Make the ADC directory, and a "ua" subdirectory.
+
+	mkdir -p /usr/local/lib/gitolite/adc/ua
+
+3. Install the git-annex-shell ADC into the "ua" subdirectory and make it
+   executable.
+
+	cd /usr/local/lib/gitolite/adc/ua/
+	wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
+	chmod +x git-annex-shell
+
+4. Now all gitolite repositories can be used with git-annex just as any
+   ssh remote normally would be used. For example:
+
+
+# git clone gitolite@localhost:testing
+Cloning into testing...
+Receiving objects: 100% (18/18), done.
+# cd testing
+# >git annex init
+init  ok
+# cp /etc/passwd my-cool-big-file
+# git annex add my-cool-big-file
+add my-cool-big-file ok
+(Recording state in git...)
+# git commit -m added
+[master d36c8b4] added
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 120000 my-cool-big-file
+# git push --all
+Counting objects: 17, done.
+Delta compression using up to 2 threads.
+Compressing objects: 100% (12/12), done.
+Writing objects: 100% (14/14), 1.39 KiB, done.
+Total 14 (delta 0), reused 1 (delta 0)
+To gitolite@localhost:testing
+   c552a38..db4653e  git-annex -> git-annex
+   29cd204..d36c8b4  master -> master
+# git annex copy --to origin
+copy my-cool-big-file (checking origin...) (to origin...) 
+WORM-s2502-m1318875140--my-cool-big-file
+        2502 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)
+
+sent 2606 bytes  received 31 bytes  1758.00 bytes/sec
+total size is 2502  speedup is 0.95
+ok
+


From 81fe6f775c02630421aba6d62e84e9849a9869c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:17:18 -0400
Subject: [PATCH 2348/2835] close

---
 doc/todo/gitolite_and_gitosis_support.mdwn | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn
index 12e26243e4..2fca839863 100644
--- a/doc/todo/gitolite_and_gitosis_support.mdwn
+++ b/doc/todo/gitolite_and_gitosis_support.mdwn
@@ -33,18 +33,7 @@ I have posted an RFC for modifying gitolite to the
 As I don't write python, someone else is needed to work on gitosis.
 --[[Joey]] 
 
-## readonly commands
-
-* git-annex-shell configlist $directory
-* git-annex-shell inannex $directory [$key ...]
-* git-annex-shell sendkey $directory $key
-
-## read-write commands
-
-* git-annex-shell dropkey $directory [$key ...]
-* git-annex-shell recvkey $directory $key
-
-## other git-annex-shell parameters
-
-All parameters like --uuid=foo and --force are safe and need to be allowed
-through.
+> [[done]]; support for gitolite is in its `pu` branch, and some changes
+> made to git-annefor gitolite is in its `pu` branch, and some changes
+> made to git-annex. Word is gitosis is not being maintained so I won't
+> worry about try to support it. --[[Joey]] 

From a6633f857bdcbdd3924a5b75337c1af0cf68d6cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:19:58 -0400
Subject: [PATCH 2349/2835] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index c4df42cae6..4dc6d8ae50 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -19,23 +19,23 @@ on Debian with apt-get, but the changes described can be made to any
 gitolite installation, just with different paths.
 
 1. Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
-
+   
 	echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
 
 2. Make the ADC directory, and a "ua" subdirectory.
-
+   
 	mkdir -p /usr/local/lib/gitolite/adc/ua
 
 3. Install the git-annex-shell ADC into the "ua" subdirectory and make it
    executable.
-
+   
 	cd /usr/local/lib/gitolite/adc/ua/
 	wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
 	chmod +x git-annex-shell
 
 4. Now all gitolite repositories can be used with git-annex just as any
    ssh remote normally would be used. For example:
-
+   
 
 # git clone gitolite@localhost:testing
 Cloning into testing...

From 9e3783f8fedfc54ee4fbf96c1c5e267650993b4d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:20:36 -0400
Subject: [PATCH 2350/2835] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index 4dc6d8ae50..bf3d61434b 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -20,18 +20,24 @@ gitolite installation, just with different paths.
 
 1. Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
    
-	echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+
+echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+

 
 2. Make the ADC directory, and a "ua" subdirectory.
-   
-	mkdir -p /usr/local/lib/gitolite/adc/ua
+
+
   
+mkdir -p /usr/local/lib/gitolite/adc/ua
+

 
 3. Install the git-annex-shell ADC into the "ua" subdirectory and make it
    executable.
-   
-	cd /usr/local/lib/gitolite/adc/ua/
-	wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
-	chmod +x git-annex-shell
+
+
   
+cd /usr/local/lib/gitolite/adc/ua/
+wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
+chmod +x git-annex-shell
+

 
 4. Now all gitolite repositories can be used with git-annex just as any
    ssh remote normally would be used. For example:

From 1b76324034ce5770a6c5ccf2de5c08e81673032f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:21:44 -0400
Subject: [PATCH 2351/2835] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index bf3d61434b..745bdf450c 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -18,20 +18,20 @@ And here's how to set it up. The examples are for gitolite as installed
 on Debian with apt-get, but the changes described can be made to any
 gitolite installation, just with different paths.
 
-1. Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
+Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
    
 
 echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
 

 
-2. Make the ADC directory, and a "ua" subdirectory.
+Make the ADC directory, and a "ua" subdirectory.
 
 
   
 mkdir -p /usr/local/lib/gitolite/adc/ua
 

 
-3. Install the git-annex-shell ADC into the "ua" subdirectory and make it
-   executable.
+Install the git-annex-shell ADC into the "ua" subdirectory and make it
+executable.
 
 
   
 cd /usr/local/lib/gitolite/adc/ua/
@@ -39,8 +39,8 @@ wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
 chmod +x git-annex-shell
 

 
-4. Now all gitolite repositories can be used with git-annex just as any
-   ssh remote normally would be used. For example:
+Now all gitolite repositories can be used with git-annex just as any
+ssh remote normally would be used. For example:
    
 
 # git clone gitolite@localhost:testing

From 9c0930a2b2d7b7471a7b154ac4c892ef05687521 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:22:25 -0400
Subject: [PATCH 2352/2835] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index 745bdf450c..90bc0195f7 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -19,7 +19,7 @@ on Debian with apt-get, but the changes described can be made to any
 gitolite installation, just with different paths.
 
 Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
-   
+
 
 echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
 

@@ -41,7 +41,7 @@ chmod +x git-annex-shell
 
 Now all gitolite repositories can be used with git-annex just as any
 ssh remote normally would be used. For example:
-   
+
 
 # git clone gitolite@localhost:testing
 Cloning into testing...

From 5df717de437aa89da7be7a1b3030e1a39c69f87b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:23:27 -0400
Subject: [PATCH 2353/2835] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index 90bc0195f7..a51d6539b6 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -47,7 +47,7 @@ ssh remote normally would be used. For example:
 Cloning into testing...
 Receiving objects: 100% (18/18), done.
 # cd testing
-# >git annex init
+# git annex init
 init  ok
 # cp /etc/passwd my-cool-big-file
 # git annex add my-cool-big-file

From fa3f68ee63ec2700e76332d0439c3a5859791706 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:28:30 -0400
Subject: [PATCH 2354/2835] add link

---
 doc/index.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 4b7159cd53..5bd42074f5 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -7,6 +7,7 @@ To get a feel for it, see the [[walkthrough]].
 
 * **[[download]]**
 * [[install]]
+* [[tips]]
 * [[bugs]]
 * [[todo]]
 * [[forum]]

From 89e0e0e474fede529819dd614b5680918fe47333 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Mon, 17 Oct 2011 18:51:40 +0000
Subject: [PATCH 2355/2835]

---
 .../confusion_with_remotes__44___map.mdwn     | 113 ++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map.mdwn

diff --git a/doc/forum/confusion_with_remotes__44___map.mdwn b/doc/forum/confusion_with_remotes__44___map.mdwn
new file mode 100644
index 0000000000..0ae75d4e99
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map.mdwn
@@ -0,0 +1,113 @@
+I'm starting out with git-annex and running into some confusion with setting up the remotes.
+
+I have three systems I'm trying to set up (domains edited):
+
+* psychosis: ssh://psychosis.foo.com/vid
+* bacon: ssh://bucket.foo.com/vid
+* bucket: ssh://bucket.bar.org/vid
+
+And one bare repository so that I can have a single place to push/pull:
+
+* origin: https://git.foo.com/jim/vid.git
+
+On psychosis:
+
+    psychosis$ git config --list | grep ^remote | sort
+    remote.bacon.annex-uuid=8f1f0898-f8c1-11e0-9bf2-b387af26ee63
+    remote.bacon.fetch=+refs/heads/*:refs/remotes/bacon/*
+    remote.bacon.url=ssh://bucket.foo.com/vid
+    remote.bucket.annex-uuid=82814942-f8e0-11e0-b053-e70a61e98e19
+    remote.bucket.fetch=+refs/heads/*:refs/remotes/bucket/*
+    remote.bucket.url=ssh://bucket.bar.org/vid
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    
+    psychosis$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- here (psychosis)
+            82814942-f8e0-11e0-b053-e70a61e98e19 -- bucket
+    local annex keys: 2256
+    local annex size: 449 gigabytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+**First point of confusion**: Why doesn't "bacon" show up in "git annex status"?  I can "git annex copy --to bacon filename" and it will copy it there.  Is there some step of setting it up that I missed?  I basically just did "git remote add bacon ssh://bucket.foo.com/vid".
+
+Now I've started setting up the remotes on each host:
+
+On bacon:
+   
+    bacon$ git config --list | grep ^remote | sort
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    remote.psychosis.annex-uuid=09c0b436-f8de-11e0-842f-b7644539d57f
+    remote.psychosis.fetch=+refs/heads/*:refs/remotes/psychosis/*
+    remote.psychosis.url=ssh://psychosis.foo.com/vid
+    
+    bacon$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- psychosis
+            8f1f0898-f8c1-11e0-9bf2-b387af26ee63 -- here (bacon)
+    temporary directory size: 366 megabytes (clean up with git-annex unused)
+    local annex keys: 1
+    local annex size: 308 bytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+On bucket:
+
+    bucket$ git config --list | grep ^remote | sort
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    remote.psychosis.annex-uuid=09c0b436-f8de-11e0-842f-b7644539d57f
+    remote.psychosis.fetch=+refs/heads/*:refs/remotes/psychosis/*
+    remote.psychosis.url=ssh://psychosis.foo.com/vid
+
+    bucket$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- psychosis
+            82814942-f8e0-11e0-b053-e70a61e98e19 -- here (bucket)
+    temporary directory size: 183 megabytes (clean up with git-annex unused)
+    local annex keys: 3
+    local annex size: 550 megabytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+But I'm getting weird results if I try to show the map from psychosis:
+
+    psychosis$ git annex map
+    $ git annex map
+    map /vid/tv ok
+    map bacon (sshing...) 
+    ok
+    map bucket (sshing...) 
+    ok
+    map origin 
+    failed
+    map psychosis (sshing...) 
+    jim@psychosis.foo.com's password: 
+    ok
+    map psychosis (sshing...) 
+    jim@psychosis.foo.com's password: 
+    ok
+    
+      running: dot -Tx11 map.dot
+
+**Second confusion**: it's as if psychosis was considered a new remote each time?
+The generated map has psychosis listed with several redundant links:
+
+![Map](http://jim.sh/~jim/tmp/map.png)
+
+Is this some bug or do I just need to be hit with the clue bat?

From 4004b6860637d59dcab6786f20994269f192ba4a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 17 Oct 2011 19:01:21 +0000
Subject: [PATCH 2356/2835] Added a comment

---
 ...comment_1_a38ded23b7f288292a843abcb1a56f38._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment b/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment
new file mode 100644
index 0000000000..97de93d9ec
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-10-17T19:01:21Z"
+ content="""
+My guess is that psychosis has not pulled the git-annex branch since bacon was set up (or that bacon's git-annex branch has not been pushed to origin). git-annex status only shows remotes present in git-annex:uuid.log This may be a bug.
+
+The duplicate links in the map I don't quite understand. I only see duplicate links in my maps when I have the same repository configured as two different git remotes (for example, because the same repository can be accessed two different ways). You don't seem to have that in your config.
+"""]]

From 126bf2b5c10ab802df65c2f07d6a1edc8a9936e6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 17 Oct 2011 19:02:50 +0000
Subject: [PATCH 2357/2835] Added a comment

---
 .../comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment b/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment
new file mode 100644
index 0000000000..a61b126c0c
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-10-17T19:02:50Z"
+ content="""
+Actually, there is a hint that, while you ran the git annex map on psychosis, it decided to ssh to itself two times. That seems to be where the duplicate links came from, I guess you must have some git remotes you did not show.
+"""]]

From fcbc6f901d3005bf3c64fe539391ba1c2bd2d9fc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Mon, 17 Oct 2011 19:50:07 +0000
Subject: [PATCH 2358/2835] Added a comment

---
 ..._18531754089c991b6caefc57a5c17fe9._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment b/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment
new file mode 100644
index 0000000000..4c77222619
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 3"
+ date="2011-10-17T19:50:06Z"
+ content="""
+No extra remotes (that I'm aware of); that output was only edited to change hostnames.
+
+On all three hosts, \"git push origin\" and \"git pull origin\" say everything is up to date.
+
+I'm using git-annex 3.20111011 on all hosts (although some were running 3.20110928 when I created the repositories).
+
+Regarding the multiple links, I've put a copy of the dot file [here](http://jim.sh/~jim/tmp/map.dot).
+It shows psychosis in three separate subgraphs, that are just getting rendered together as one,
+if that helps clarify anything.
+
+Wait, I just realized you said \"the git-annex branch\".  My origin only has \"master\".
+Do you mean the one specifically named \"git-annex\"?  I thought that was something that
+gets managed automatically, or is it something I need to manually check out and deal with?
+
+Any other info I could provide?
+
+
+"""]]

From 89536e97fbe3f5024c3f194119be9ab7416a8c13 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Mon, 17 Oct 2011 20:36:57 +0000
Subject: [PATCH 2359/2835] Added a comment

---
 ...omment_4_3b89b6d1518267fcbc050c9de038b9ca._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment b/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment
new file mode 100644
index 0000000000..f6e5993c8e
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 4"
+ date="2011-10-17T20:36:51Z"
+ content="""
+Ok, after pushing the \"git-annex\" branch to origin, then \"git annex status\" knows all repositories on all hosts, so that part makes sense now.  Thanks for the tip.  But the \"git annex map\" output hasn't changed.
+
+
+
+"""]]

From bacb2e1881c9d4066ffe0be9a73c0add71f312d9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Tue, 18 Oct 2011 04:59:15 +0000
Subject: [PATCH 2360/2835] Added a comment

---
 ...t_5_27801584325d259fa490f67273f2ff71._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment b/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment
new file mode 100644
index 0000000000..77a2c4adbe
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 5"
+ date="2011-10-18T04:59:13Z"
+ content="""
+I think:
+
+* The first extra edge is because bucket had \"ssh://psychosis.foo.com/vid/\", while
+bacon had \"ssh://psychosis.foo.com/vid\" with no trailing slash.  That got lost in the hostname/path editing I did, sorry.
+Maybe those should be considered matching?
+* The second extra edge is because, when running \"git annex map\" from psychosis, it doesn't recognize the remote's
+remote URL as pointing back to itself.
+
+For the second case, after the \"spurious\" SSH, it could still recognize that the repositories are the same by the duplicated annex uuid, which currently shows up in `map.dot` twice.  I wonder what it would take to avoid the spurious SSH -- maybe some config that lists \"alternate\" URLs that should be considered the same as the current repository?  Or actually list URLs in uuid.log?  Fortunately, I think this only affects the map, so it's not a big problem.
+"""]]

From aa7da919a7588e733ee568b397f2b16b7e665a2f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 22 Oct 2011 01:18:27 +0000
Subject: [PATCH 2361/2835] Added a comment

---
 .../comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment b/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment
new file mode 100644
index 0000000000..412937f3fc
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-10-22T01:18:27Z"
+ content="""
+Hmm, I don't see the spurious ssh edge in the dot file -- that is, I don't see any ssh:// uris in it?
+"""]]

From 3c5a0e78c032f11097cebda5079e09666c54f2fc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Sat, 22 Oct 2011 05:25:48 +0000
Subject: [PATCH 2362/2835] Added a comment

---
 ..._9a456f61f956a3d5e81e723d5a90794c._comment | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment b/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment
new file mode 100644
index 0000000000..85ede3a89c
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 7"
+ date="2011-10-22T05:25:47Z"
+ content="""
+I think that's because the SSH was successful (I entered the password and let it connect), so it got the UUID and put that in the .dot instead.  The same UUID (for psychosis) then ended up in two different \"subgraph\" stanzas, and Graphviz just plotted them together as one node.
+
+
Maybe this will clarify:
+
+
On psychosis, run \"git annex map\" and press ^C at the ssh password prompt: [map-nossh.dot](http://jim.sh/~jim/tmp/map-nossh.dot)
+![Map](http://jim.sh/~jim/tmp/map-nossh.png)
+
+
On psychosis, run \"git annex map\" and type the correct password: [map-goodssh.dot](http://jim.sh/~jim/tmp/map-goodssh.dot)
+![Map](http://jim.sh/~jim/tmp/map-goodssh.png)
+
+As I see it:
+
+* psychosis (\"localhost\") connects to each of its remotes
+* some of them point back to ssh://psychosis
+* psychosis doesn't know that ssh://psychosis is itself, so it tries to connect
+* if successful:
+  * psychosis gets put twice in the .dot as if it was two different hosts, one \"local\" and one \"ssh://psychosis\"
+  * graphviz recognizes it as the same node because the UUID is the same, but graphviz still draws the extra connecting lines
+* if unsuccessful:
+  * ssh://psychosis is shown as an additional host that can't be reached
+"""]]

From 721f236e3a96a23d79dcf86ea7ef3d85d7270305 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 23 Oct 2011 14:37:33 +0000
Subject: [PATCH 2363/2835]

---
 ...ot_run_when_branch_git-annex_is_checked_out.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn

diff --git a/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn b/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
new file mode 100644
index 0000000000..d140219364
--- /dev/null
+++ b/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
@@ -0,0 +1,13 @@
+Running `git annex uninit` in a repo which has branch git-annex checked out will result in:
+
+    error: Cannot delete the branch 'git-annex' which you are currently on.
+    git-annex: git [Param "-D",Param "git-annex"] failed
+
+and trying to checkout branch master afterwards results in:
+
+    error: The following untracked working tree files would be overwritten by checkout:
+
+Both of which is logical. The best thing would be if git-annex refused to run uninit while in branch git-annex.
+
+
+Richard

From 962fed915ff2b4c652a879c4b7a01086c680c6e0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 23 Oct 2011 15:00:49 +0000
Subject: [PATCH 2364/2835] Added a comment

---
 .../comment_1_4c46a193915eab8f308a04175cb2e40a._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment

diff --git a/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment b/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment
new file mode 100644
index 0000000000..3e28a28cb0
--- /dev/null
+++ b/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-10-23T15:00:48Z"
+ content="""
+Having run into the same issue again, I still think git-annex should ensure no merges take place. The clutter introduced by a .gitnomerge is neglible, imo.
+"""]]

From e2853b3fec1c34bd420b9e1753be4aa1afeb675e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 25 Oct 2011 11:39:15 -0700
Subject: [PATCH 2365/2835] update

---
 debian/changelog     | 2 +-
 doc/todo/smudge.mdwn | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 4e0a1e21e4..0ebda22dc5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111012) UNRELEASED; urgency=low
   * migrate: Copy url logs for keys when migrating.
   * git-annex-shell: GIT_ANNEX_SHELL_READONLY and GIT_ANNEX_SHELL_LIMITED
     environment variables can be set to limit what commands can be run.
-    This could be used by eg, gitolite.
+    This is used by gitolite's new git-annex support!
 
  -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
 
diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 55de5f5789..6103ffa61e 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -11,6 +11,10 @@ version of the file during a merge.
 So every `git status` would need to read the entire content of all
 available files, and checksum them, which is too expensive.
 
+> Update from GitTogether: Peff thinks a new interface could be added to
+> git to handle this sort of case in an efficient way.. just needs someone
+> to do the work. --[[Joey]] 
+
 ----
 
 The clean filter is run when files are staged for commit. So a user could copy

From 270c1af087449573c17eb4911ca2dc3db3fb1fc1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 25 Oct 2011 13:46:01 -0700
Subject: [PATCH 2366/2835] releasing version 3.20111025

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 0ebda22dc5..81dec6ad06 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111012) UNRELEASED; urgency=low
+git-annex (3.20111025) unstable; urgency=low
 
   * A remote can have a annexUrl configured, that is used by git-annex
     instead of its usual url. (Similar to pushUrl.)
@@ -7,7 +7,7 @@ git-annex (3.20111012) UNRELEASED; urgency=low
     environment variables can be set to limit what commands can be run.
     This is used by gitolite's new git-annex support!
 
- -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
+ -- Joey Hess   Tue, 25 Oct 2011 13:03:08 -0700
 
 git-annex (3.20111011) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index e9ec089677..c3706808c0 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111012
+Version: 3.20111025
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 570aae9bbcdd6a7d5972288d322da3872d0a17be Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 25 Oct 2011 13:46:22 -0700
Subject: [PATCH 2367/2835] add news item for git-annex 3.20111025

---
 doc/news/version_3.20110902.mdwn | 9 ---------
 doc/news/version_3.20111025.mdwn | 8 ++++++++
 2 files changed, 8 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_3.20110902.mdwn
 create mode 100644 doc/news/version_3.20111025.mdwn

diff --git a/doc/news/version_3.20110902.mdwn b/doc/news/version_3.20110902.mdwn
deleted file mode 100644
index e354874ea3..0000000000
--- a/doc/news/version_3.20110902.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 3.20110902 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Set EMAIL when running test suite so that git does not need to be
-     configured first. Closes: #[638998](http://bugs.debian.org/638998)
-   * The wget command will now be used in preference to curl, if available.
-   * init: Make description an optional parameter.
-   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
-   * unused --remote: Reduced memory use to 1/4th what was used before.
-   * Add --json switch, to produce machine-consumable output."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111025.mdwn b/doc/news/version_3.20111025.mdwn
new file mode 100644
index 0000000000..90e4227bae
--- /dev/null
+++ b/doc/news/version_3.20111025.mdwn
@@ -0,0 +1,8 @@
+git-annex 3.20111025 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * A remote can have a annexUrl configured, that is used by git-annex
+     instead of its usual url. (Similar to pushUrl.)
+   * migrate: Copy url logs for keys when migrating.
+   * git-annex-shell: GIT\_ANNEX\_SHELL\_READONLY and GIT\_ANNEX\_SHELL\_LIMITED
+     environment variables can be set to limit what commands can be run.
+     This is used by gitolite's new git-annex support!"""]]
\ No newline at end of file

From 12a0a45fb1ddb464d7863cdd707dd4b328ae5df5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 25 Oct 2011 21:40:30 +0000
Subject: [PATCH 2368/2835]

---
 doc/todo/support_fsck_in_bare_repos.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/support_fsck_in_bare_repos.mdwn

diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
new file mode 100644
index 0000000000..b126a8170b
--- /dev/null
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -0,0 +1,10 @@
+What is says on the tin:
+
+
+    22:56:54 < RichiH> joeyh_: by the way, i have been thinking about fsck on bare repos
+    22:57:37 < RichiH> joeyh_: the best i could come with is to have a bare and a non-bare access the same repo store
+    22:58:00 < RichiH> joeyh_: alternatively, with the SHA* backend, you have all the information to verify that the local data is correct
+    22:58:41 < RichiH> and verifying that would already be a plus. if there  really _is_ a problem, having the SHA is enough to track issues down
+    23:09:50 < joeyh_> oh, I think I have code that fsck could use on bare repos already.. just a matter of wiring it up
+    23:10:42 < joeyh_> feel free to reopen a bug or whatever so I remember.. the unused command's branch content enumeration could be used in a bare repo
+    23:14:51 < joeyh_> unused/dropunused could work in bare repos too btw

From c8fdffa9f844f6884b064233d2cb7d4b44cc62e4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 26 Oct 2011 15:46:43 +0000
Subject: [PATCH 2369/2835]

---
 doc/bugs/fails_to_handle_lot_of_files.mdwn | 441 +++++++++++++++++++++
 1 file changed, 441 insertions(+)
 create mode 100644 doc/bugs/fails_to_handle_lot_of_files.mdwn

diff --git a/doc/bugs/fails_to_handle_lot_of_files.mdwn b/doc/bugs/fails_to_handle_lot_of_files.mdwn
new file mode 100644
index 0000000000..5c5a6c8fd1
--- /dev/null
+++ b/doc/bugs/fails_to_handle_lot_of_files.mdwn
@@ -0,0 +1,441 @@
+git-annex version: 3.20111011
+local repository version: 3
+default repository version: 3
+supported repository versions: 3
+upgrade supported from repository versions: 0 1 2
+
+I just created a new remote on a USB drive and wanted to copy my files over. git-annex wasn't too happy about that ;)
+I included a few OK transfers as there was an error before git-annex ran into a wall. As I could easily access that temp file after it aborted, I suspect something went wrong internally before git-annex started to throw those errors.
+
+Please note the "_n TIMES_" comments. It's how often I got the same error message...
+
+
+
+    git annex copy . --to USB --fast
+    
+    copy redacted.JPG (to USB...) 
+    redacted
+         4035668 100%   77.91MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4036374 bytes  received 31 bytes  8072810.00 bytes/sec
+    total size is 4035668  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18002094 100%   74.19MB/s    0:00:00 (xfer#1, to-check=0/1)
+    WARNING: redacted failed verification -- update retained (will try again).
+    redacted
+        18002094 100%   19.60MB/s    0:00:00 (xfer#2, to-check=0/1)
+    rsync: open "copy_target/.git/annex/tmp/redacted_E13" failed: Permission denied (13)
+    
+    sent 36008841 bytes  received 52 bytes  24005928.67 bytes/sec
+    total size is 18002094  speedup is 0.50
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1070) [sender=3.0.8]
+    
+      rsync failed -- run git annex again to resume file transfer
+    failed
+    copy redacted.JPG (to USB...) 
+    redacted
+         3687111 100%   39.16MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 3687773 bytes  received 31 bytes  2458536.00 bytes/sec
+    total size is 3687111  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17877177 100%   79.15MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17879573 bytes  received 31 bytes  11919736.00 bytes/sec
+    total size is 17877177  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         3694921 100%   40.14MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 3695583 bytes  received 31 bytes  2463742.67 bytes/sec
+    total size is 3694921  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17875448 100%   71.20MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17877844 bytes  received 31 bytes  11918583.33 bytes/sec
+    total size is 17875448  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         3833377 100%   62.49MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 3834055 bytes  received 31 bytes  2556057.33 bytes/sec
+    total size is 3833377  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17938200 100%   65.43MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17940604 bytes  received 31 bytes  11960423.33 bytes/sec
+    total size is 17938200  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4512557 100%   83.77MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4513319 bytes  received 31 bytes  3008900.00 bytes/sec
+    total size is 4512557  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18001641 100%   76.16MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18004053 bytes  received 31 bytes  12002722.67 bytes/sec
+    total size is 18001641  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4394272 100%   50.11MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4395022 bytes  received 31 bytes  8790106.00 bytes/sec
+    total size is 4394272  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18095781 100%   73.30MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18098205 bytes  received 31 bytes  12065490.67 bytes/sec
+    total size is 18095781  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4683795 100%   65.23MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4684577 bytes  received 31 bytes  9369216.00 bytes/sec
+    total size is 4683795  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18172801 100%   74.25MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18175233 bytes  received 31 bytes  36350528.00 bytes/sec
+    total size is 18172801  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4486231 100%   77.22MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4486989 bytes  received 31 bytes  8974040.00 bytes/sec
+    total size is 4486231  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17860427 100%   68.56MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17862823 bytes  received 31 bytes  35725708.00 bytes/sec
+    total size is 17860427  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4499768 100%   36.41MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4500530 bytes  received 31 bytes  9001122.00 bytes/sec
+    total size is 4499768  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17840132 100%   74.48MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17842524 bytes  received 31 bytes  11895036.67 bytes/sec
+    total size is 17840132  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4358032 100%   75.00MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4358774 bytes  received 31 bytes  8717610.00 bytes/sec
+    total size is 4358032  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18084753 100%   61.48MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18087173 bytes  received 31 bytes  12058136.00 bytes/sec
+    total size is 18084753  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4270213 100%   68.49MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4270947 bytes  received 31 bytes  2847318.67 bytes/sec
+    total size is 4270213  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17661246 100%   68.34MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17663614 bytes  received 31 bytes  11775763.33 bytes/sec
+    total size is 17661246  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4538305 100%   63.19MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4539071 bytes  received 31 bytes  9078204.00 bytes/sec
+    total size is 4538305  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18672466 100%   68.90MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18674958 bytes  received 31 bytes  12449992.67 bytes/sec
+    total size is 18672466  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4453445 100%   73.96MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4454199 bytes  received 31 bytes  8908460.00 bytes/sec
+    total size is 4453445  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18495494 100%   59.28MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18497966 bytes  received 31 bytes  12331998.00 bytes/sec
+    total size is 18495494  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4255858 100%   70.66MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4256588 bytes  received 31 bytes  1702647.60 bytes/sec
+    total size is 4255858  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18376531 100%   69.15MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18378987 bytes  received 31 bytes  36758036.00 bytes/sec
+    total size is 18376531  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4013365 100%   48.67MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4014067 bytes  received 31 bytes  8028196.00 bytes/sec
+    total size is 4013365  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17606341 100%   51.73MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17608705 bytes  received 31 bytes  11739157.33 bytes/sec
+    total size is 17606341  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4179869 100%   74.62MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4180591 bytes  received 31 bytes  8361244.00 bytes/sec
+    total size is 4179869  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18382569 100%   67.05MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18385025 bytes  received 31 bytes  12256704.00 bytes/sec
+    total size is 18382569  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4318363 100%   44.91MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4319101 bytes  received 31 bytes  8638264.00 bytes/sec
+    total size is 4318363  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17715958 100%   72.69MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17718334 bytes  received 31 bytes  11812243.33 bytes/sec
+    total size is 17715958  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4241893 100%   65.81MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4242623 bytes  received 31 bytes  8485308.00 bytes/sec
+    total size is 4241893  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17717287 100%   71.77MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17719663 bytes  received 31 bytes  11813129.33 bytes/sec
+    total size is 17717287  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4488380 100%   49.99MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4489138 bytes  received 31 bytes  2992779.33 bytes/sec
+    total size is 4488380  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17770208 100%   38.80MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17772592 bytes  received 31 bytes  11848415.33 bytes/sec
+    total size is 17770208  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4603958 100%   76.48MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4604732 bytes  received 31 bytes  9209526.00 bytes/sec
+    total size is 4603958  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18744380 100%   74.66MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18746884 bytes  received 31 bytes  12497943.33 bytes/sec
+    total size is 18744380  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4592098 100%   79.06MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4592872 bytes  received 31 bytes  3061935.33 bytes/sec
+    total size is 4592098  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18746205 100%   43.00MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18748709 bytes  received 31 bytes  12499160.00 bytes/sec
+    total size is 18746205  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         7493353 100%   80.85MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 7494479 bytes  received 31 bytes  14989020.00 bytes/sec
+    total size is 7493353  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        19496768 100%   81.77MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 19499360 bytes  received 31 bytes  12999594.00 bytes/sec
+    total size is 19496768  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         5462482 100%   82.19MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 5463360 bytes  received 31 bytes  10926782.00 bytes/sec
+    total size is 5462482  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        19669815 100%   80.37MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 19672431 bytes  received 31 bytes  13114974.67 bytes/sec
+    total size is 19669815  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         5449487 100%   57.40MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 5450365 bytes  received 31 bytes  3633597.33 bytes/sec
+    total size is 5449487  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        19633259 100%   74.18MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 19635871 bytes  received 31 bytes  13090601.33 bytes/sec
+    total size is 19633259  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         5392184 100%   62.33MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 5393054 bytes  received 31 bytes  3595390.00 bytes/sec
+    total size is 5392184  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18912104 100%   65.00MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18914628 bytes  received 31 bytes  12609772.67 bytes/sec
+    total size is 18912104  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4869300 100%   80.92MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4870106 bytes  received 31 bytes  9740274.00 bytes/sec
+    total size is 4869300  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        20178932 100%   68.13MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 20181608 bytes  received 31 bytes  13454426.00 bytes/sec
+    total size is 20178932  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4995425 100%   86.05MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4996247 bytes  received 31 bytes  9992556.00 bytes/sec
+    total size is 4995425  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        19970679 100%   76.36MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 19973331 bytes  received 31 bytes  13315574.67 bytes/sec
+    total size is 19970679  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         7905795 100%   66.45MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 7906973 bytes  received 31 bytes  15814008.00 bytes/sec
+    total size is 7905795  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        21234069 100%   78.07MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 21236877 bytes  received 31 bytes  8494763.20 bytes/sec
+    total size is 21234069  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         7963979 100%   62.51MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 7965165 bytes  received 31 bytes  5310130.67 bytes/sec
+    total size is 7963979  speedup is 1.00
+    git ["--git-dir=copy_target/.git","--work-tree=copy_target","update-index","-z","--index-info"]: Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable)
+    
+    git-annex: user error (git ["--git-dir=copy_target/.git","--work-tree=copy_target","update-index","-z","--index-info"]: Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))
+    failed
+    _506 TIMES_ (user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))) failed
+    _11 TIMES_ copy foo (createPipe: resource exhausted (Too many open files)) failed
+    _2 TIMES_ (user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))) failed
+    _8574 TIMES_: copy foo (createPipe: resource exhausted (Too many open files)) failed
+    git-annex: createPipe: resource exhausted (Too many open files)
+    failed
+    git-annex: 9101 failed
+    
+    % ls copy_target/.git/annex/tmp/redacted_E13 copy_target/.git/annex/tmp/SHA512E-redacted_E13 # works
+    % find source -type l | wc -l
+    13554
+    % find copy_target -type l | wc -l
+    13554
+    % find copy_target/.git/annex/objects -type f | wc -l
+    4455
+    % find source -type f | wc -l
+    13554

From c37a70fbbc114b4deb89998571e03009240b8931 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 26 Oct 2011 15:49:10 +0000
Subject: [PATCH 2370/2835]

---
 doc/bugs/fails_to_handle_lot_of_files.mdwn | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/bugs/fails_to_handle_lot_of_files.mdwn b/doc/bugs/fails_to_handle_lot_of_files.mdwn
index 5c5a6c8fd1..bc5d92aeca 100644
--- a/doc/bugs/fails_to_handle_lot_of_files.mdwn
+++ b/doc/bugs/fails_to_handle_lot_of_files.mdwn
@@ -1,8 +1,8 @@
-git-annex version: 3.20111011
-local repository version: 3
-default repository version: 3
-supported repository versions: 3
-upgrade supported from repository versions: 0 1 2
+    git-annex version: 3.20111011
+    local repository version: 3
+    default repository version: 3
+    supported repository versions: 3
+    upgrade supported from repository versions: 0 1 2
 
 I just created a new remote on a USB drive and wanted to copy my files over. git-annex wasn't too happy about that ;)
 I included a few OK transfers as there was an error before git-annex ran into a wall. As I could easily access that temp file after it aborted, I suspect something went wrong internally before git-annex started to throw those errors.

From c65293977edd5fdd8f25ae4cdafa3d21003f4b0f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 26 Oct 2011 17:16:53 +0000
Subject: [PATCH 2371/2835] Added a comment

---
 .../comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment

diff --git a/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment b/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment
new file mode 100644
index 0000000000..587b1fd97c
--- /dev/null
+++ b/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-10-26T17:16:52Z"
+ content="""
+After another run, i am at 8909 files in the remote, now.
+"""]]

From c5c682c131ece6e4c05e86206f01838c6abe2f0d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 26 Oct 2011 18:22:37 +0000
Subject: [PATCH 2372/2835] Added a comment

---
 .../comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment

diff --git a/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment b/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment
new file mode 100644
index 0000000000..8e83fc19f4
--- /dev/null
+++ b/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-10-26T18:22:34Z"
+ content="""
+In case this matters, I just realized that this disk has been formatted with NTFS instead of a sane FS.
+"""]]

From c3df1c82b9b5bbc67a845667895adf0ca94ac6dd Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 27 Oct 2011 16:31:15 +0000
Subject: [PATCH 2373/2835]

---
 ...ishlist:_support_drop__44___find_on_special_remotes.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn

diff --git a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
new file mode 100644
index 0000000000..2dc735e99f
--- /dev/null
+++ b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
@@ -0,0 +1,6 @@
+Currently there is no way to drop files, or list what files are available, on a special remote.
+It would be good if "git annex drop" and "git annex find" supported the --from argument.
+
+For commands that don't support the --from argument, it would also be nice to print an error.
+Currently running "git annex drop --from usbdrive" doesn't behave as hoped and instead drops
+all content from the local annex.

From 5bc132b95cebbc5133082e08ba1c45227c679b7b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 27 Oct 2011 16:33:47 +0000
Subject: [PATCH 2374/2835]

---
 ...ishlist:_query_things_like_description__44___trust_level.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn

diff --git a/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn b/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
new file mode 100644
index 0000000000..6404393a89
--- /dev/null
+++ b/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
@@ -0,0 +1 @@
+It would be helpful to have a way to query things like a repository's description and trust level, without having to poke in the git-annex branch.  For example, "git annex describe ." currently clears the description but could print the current one instead.

From 2aa1c779ef218b5f0b253976d93adb333dfc26a0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 27 Oct 2011 17:09:33 +0000
Subject: [PATCH 2375/2835] Added a comment

---
 ...comment_1_14311384788312b96e550749ab7de9ea._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_1_14311384788312b96e550749ab7de9ea._comment

diff --git a/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_1_14311384788312b96e550749ab7de9ea._comment b/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_1_14311384788312b96e550749ab7de9ea._comment
new file mode 100644
index 0000000000..3ac4ba2678
--- /dev/null
+++ b/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_1_14311384788312b96e550749ab7de9ea._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-10-27T17:09:33Z"
+ content="""
+`git annex describe` only sets the description to avoid complication. Imagine using it in a script for example.
+
+`git annex status` shows the description. It does not show the trust level because I have not thought of a visually pleasing and compact way to show it in the repository list there.. suggestions appreciated, since the same list is used by `whereis`, and showing trust levels there would be particularly useful.
+"""]]

From 48d39dd35492952ac7f20da08a4168f43f782c95 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 27 Oct 2011 17:13:44 +0000
Subject: [PATCH 2376/2835] Added a comment

---
 .../comment_1_f11ed642a83d965076778a162f701e84._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/wishlist:_support_drop__44___find_on_special_remotes/comment_1_f11ed642a83d965076778a162f701e84._comment

diff --git a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes/comment_1_f11ed642a83d965076778a162f701e84._comment b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes/comment_1_f11ed642a83d965076778a162f701e84._comment
new file mode 100644
index 0000000000..6028933b40
--- /dev/null
+++ b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes/comment_1_f11ed642a83d965076778a162f701e84._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-10-27T17:13:43Z"
+ content="""
+Well, I don't think you mean \"special remotes\", but just any old remote (special or not).
+"""]]

From b3e660504fce7ed0eab48b4e7deeeb9503318986 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 12:58:01 -0400
Subject: [PATCH 2377/2835] avoid showing parens for empty descriptions

---
 Remote.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Remote.hs b/Remote.hs
index d1714f7751..49fa63cf92 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -137,6 +137,7 @@ prettyPrintUUIDs desc uuids = do
 	where
 		addname d n
 			| d == n = d
+			| null d = n
 			| otherwise = n ++ " (" ++ d ++ ")"
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m

From 2eefc580703199d3ecca58d453f07747b565d476 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 12:59:55 -0400
Subject: [PATCH 2378/2835] status too

---
 doc/todo/support_fsck_in_bare_repos.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
index b126a8170b..e6980fa281 100644
--- a/doc/todo/support_fsck_in_bare_repos.mdwn
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -8,3 +8,5 @@ What is says on the tin:
     23:09:50 < joeyh_> oh, I think I have code that fsck could use on bare repos already.. just a matter of wiring it up
     23:10:42 < joeyh_> feel free to reopen a bug or whatever so I remember.. the unused command's branch content enumeration could be used in a bare repo
     23:14:51 < joeyh_> unused/dropunused could work in bare repos too btw
+
+> Also `status`'s total annex keys/size could be handled for bare repos. --[[Joey]] 

From 373cad993d9f12a9dfb7473fdba78e92af6eff99 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 14:04:18 -0400
Subject: [PATCH 2379/2835] Sped up some operations on remotes that are on the
 same host.

Specifically, disabled trying to update the git-annex branch on the remote,
since that data is never used by operations that act on such remotes.

Also, when copying content to such a remote, skip committing the presence
information changes to its git-annex branch. Leaving it in the journal there
is ok: Any command run on the remote that needs the info will flush the
journal.

This may partially solve this bug:
http://git-annex.branchable.com/bugs/fails_to_handle_lot_of_files/
Although I still see unreaped git processes piling up when doing a copy --to.
---
 Annex/Branch.hs  | 12 ++++++++++--
 Remote/Git.hs    | 16 +++++++++-------
 debian/changelog |  6 ++++++
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index aea0d2bffe..24da8120c4 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -8,6 +8,7 @@
 module Annex.Branch (
 	create,
 	update,
+	disableUpdate,
 	get,
 	change,
 	commit,
@@ -136,7 +137,7 @@ update = onceonly $ do
 	c <- filterM changedbranch =<< siblingBranches
 	let (refs, branches) = unzip c
 	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
-		when dirty $ stageJournalFiles
+		when dirty stageJournalFiles
 		g <- gitRepo
 		unless (null branches) $ do
 			showSideAction $ "merging " ++
@@ -164,8 +165,15 @@ update = onceonly $ do
 			return $ not $ L.null diffs
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
 			r <- a
-			Annex.changeState setupdated
+			disableUpdate
 			return r
+
+{- Avoids updating the branch. A useful optimisation when the branc
+ - is known to have not changed, or git-annex won't be relying on info
+ - from it. -}
+disableUpdate :: Annex ()
+disableUpdate = Annex.changeState setupdated
+	where
 		setupdated s = s { Annex.branchstate = new }
 			where
 				new = old { branchUpdated = True }
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 5d31770a22..e77b005725 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -19,6 +19,7 @@ import qualified Git
 import qualified Annex
 import Annex.UUID
 import qualified Annex.Content
+import qualified Annex.Branch
 import qualified Utility.Url as Url
 import Utility.TempFile
 import Config
@@ -133,7 +134,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = safely checklocal
 	where
-		checklocal = onLocal r (Annex.Content.inAnnex key)
+		checklocal = onLocal r $ Annex.Content.inAnnex key
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
@@ -147,7 +148,11 @@ inAnnex r key
 onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
 	annex <- Annex.new r
-	Annex.eval annex a
+	Annex.eval annex $ do
+		-- No need to update the branch; its data is not used
+		-- for anything onLocal is used to do.
+		Annex.Branch.disableUpdate
+		a
 
 keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
@@ -175,11 +180,8 @@ copyToRemote r key
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
-		liftIO $ onLocal r $ do
-			ok <- Annex.Content.getViaTmp key $
-				rsyncOrCopyFile r keysrc
-			Annex.Content.saveState
-			return ok
+		liftIO $ onLocal r $ Annex.Content.getViaTmp key $
+			rsyncOrCopyFile r keysrc
 	| Git.repoIsSsh r = do
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
diff --git a/debian/changelog b/debian/changelog
index 81dec6ad06..b17de57ff5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20111026) UNRELEASED; urgency=low
+
+  * Sped up some operations on remotes that are on the same host.
+
+ -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
+
 git-annex (3.20111025) unstable; urgency=low
 
   * A remote can have a annexUrl configured, that is used by git-annex

From c30366e95aa8967112cec0103393ea2b39ddf1c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 14:38:59 -0400
Subject: [PATCH 2380/2835] improve config reading when operating on remote on
 same host

Before the config was read each time onLocal was called, and entirely
redundantly since it's read for same-host remotes on startup.

Also a minor bug fix: When rsyncing to a same-host remote, use the
rsync-options from the repository that the user ran git-annex in, not those of
the receiving repository.
---
 Annex.hs      |  1 +
 Remote/Git.hs | 26 ++++++++++++++++----------
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 6877537740..b9e71c9314 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -12,6 +12,7 @@ module Annex (
 	AnnexState(..),
 	OutputType(..),
 	new,
+	newState,
 	run,
 	eval,
 	getState,
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e77b005725..4285be653d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -57,7 +57,8 @@ gen r u _ = do
 	 - cached UUID value. -}
 	let cheap = not $ Git.repoIsUrl r
 	r' <- case (cheap, u) of
-		(True, _) -> tryGitConfigRead r
+		(True, _) -> do
+			tryGitConfigRead r
 		(False, "") -> tryGitConfigRead r
 		_ -> return r
 
@@ -147,8 +148,12 @@ inAnnex r key
  - monad using that repository. -}
 onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
-	annex <- Annex.new r
-	Annex.eval annex $ do
+	-- Avoid re-reading the repository's configuration if it was
+	-- already read.
+	state <- if (M.null $ Git.configMap r)
+		then Annex.new r
+		else return $ Annex.newState r
+	Annex.eval state $ do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
 		Annex.Branch.disableUpdate
@@ -168,7 +173,9 @@ dropKey r key
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
-	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
+	| not $ Git.repoIsUrl r = do
+		params <- rsyncParams r
+		rsyncOrCopyFile params (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
 	| Git.repoIsHttp r = liftIO $ Url.download (keyUrl r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
@@ -179,9 +186,10 @@ copyToRemote r key
 	| not $ Git.repoIsUrl r = do
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
+		params <- rsyncParams r
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ Annex.Content.getViaTmp key $
-			rsyncOrCopyFile r keysrc
+			rsyncOrCopyFile params keysrc
 	| Git.repoIsSsh r = do
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
@@ -200,15 +208,13 @@ rsyncHelper p = do
 
 {- Copys a file with rsync unless both locations are on the same
  - filesystem. Then cp could be faster. -}
-rsyncOrCopyFile :: Git.Repo -> FilePath -> FilePath -> Annex Bool
-rsyncOrCopyFile r src dest = do
+rsyncOrCopyFile :: [CommandParam] -> FilePath -> FilePath -> Annex Bool
+rsyncOrCopyFile rsyncparams src dest = do
 	ss <- liftIO $ getFileStatus $ parentDir src
 	ds <- liftIO $ getFileStatus $ parentDir dest
 	if deviceID ss == deviceID ds
 		then liftIO $ copyFileExternal src dest
-		else do
-			params <- rsyncParams r
-			rsyncHelper $ params ++ [Param src, Param dest]
+		else rsyncHelper $ rsyncparams ++ [Param src, Param dest]
 
 {- Generates rsync parameters that ssh to the remote and asks it
  - to either receive or send the key's content. -}

From f84d66fa15bc746517ba61f2c05beade59c846e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 14:48:53 -0400
Subject: [PATCH 2381/2835] reap in onLocal

Each onLocal call involves a new Annex state, so needs to clean up after it.
---
 Remote/Git.hs                              | 4 +++-
 debian/changelog                           | 2 ++
 doc/bugs/fails_to_handle_lot_of_files.mdwn | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 4285be653d..fd3a612436 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -157,7 +157,9 @@ onLocal r a = do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
 		Annex.Branch.disableUpdate
-		a
+		ret <- a
+		liftIO $ Git.reap
+		return ret
 
 keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
diff --git a/debian/changelog b/debian/changelog
index b17de57ff5..c32ad39503 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (3.20111026) UNRELEASED; urgency=low
 
   * Sped up some operations on remotes that are on the same host.
+  * copy --to: Fixed leak when ran on many files and a remote on the same
+    host.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/fails_to_handle_lot_of_files.mdwn b/doc/bugs/fails_to_handle_lot_of_files.mdwn
index bc5d92aeca..470a5180f0 100644
--- a/doc/bugs/fails_to_handle_lot_of_files.mdwn
+++ b/doc/bugs/fails_to_handle_lot_of_files.mdwn
@@ -439,3 +439,7 @@ Please note the "_n TIMES_" comments. It's how often I got the same error messag
     4455
     % find source -type f | wc -l
     13554
+
+> Fixed unreaped process leak.
+> (This has nothing to do with NTFS). Ran test with 10k files
+> [[done]] --[[Joey]] 

From 83d11c03c44779c000759040eca8797e70e4765c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 15:24:58 -0400
Subject: [PATCH 2382/2835] wording

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index c32ad39503..e177298da8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,7 @@
 git-annex (3.20111026) UNRELEASED; urgency=low
 
   * Sped up some operations on remotes that are on the same host.
-  * copy --to: Fixed leak when ran on many files and a remote on the same
+  * copy --to: Fixed leak when copying many files to a remote on the same
     host.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400

From 66194684acaf8dc5c72e5a163465b42050cf9212 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 15:47:11 -0400
Subject: [PATCH 2383/2835] uninit: Add guard against being run with the
 git-annex branch checked out.

---
 Command/Uninit.hs                               | 17 ++++++++++++++++-
 debian/changelog                                |  1 +
 ...un_when_branch_git-annex_is_checked_out.mdwn |  2 ++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 7b3b4fe329..8214c4208e 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,6 +7,8 @@
 
 module Command.Uninit where
 
+import qualified Data.ByteString.Lazy.Char8 as B
+
 import Common.Annex
 import Command
 import qualified Git
@@ -21,7 +23,20 @@ command = [repoCommand "uninit" paramPaths seek
         "de-initialize git-annex and clean out repository"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit startUnannex, withNothing start]
+seek = [withNothing startCheck, withFilesInGit startUnannex, withNothing start]
+
+startCheck :: CommandStart
+startCheck = do
+	b <- current_branch	
+	when (b == Annex.Branch.name) $ error $
+		"cannot uninit when the " ++ b ++ " branch is checked out"
+	stop
+	where
+		current_branch = do
+			g <- gitRepo
+			b <- liftIO $
+				Git.pipeRead g [Params "rev-parse --abbrev-ref HEAD"]
+			return $ head $ lines $ B.unpack b
 
 startUnannex :: FilePath -> CommandStart
 startUnannex file = do
diff --git a/debian/changelog b/debian/changelog
index e177298da8..dd72fc296d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * Sped up some operations on remotes that are on the same host.
   * copy --to: Fixed leak when copying many files to a remote on the same
     host.
+  * uninit: Add guard against being run with the git-annex branch checked out.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn b/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
index d140219364..e4e407ec87 100644
--- a/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
+++ b/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
@@ -11,3 +11,5 @@ Both of which is logical. The best thing would be if git-annex refused to run un
 
 
 Richard
+
+> [[done]] --[[Joey]] 

From 5b74b130a39d8c45e7d24520d838d6c1635582c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 16:31:35 -0400
Subject: [PATCH 2384/2835] refactored and generalized pre-command sanity
 checking

---
 CmdLine.hs            |  7 +------
 Command.hs            | 36 ++++++++++++++++++++++--------------
 Command/Add.hs        |  2 +-
 Command/AddUrl.hs     |  2 +-
 Command/ConfigList.hs |  2 +-
 Command/Copy.hs       |  2 +-
 Command/Describe.hs   |  2 +-
 Command/Drop.hs       |  2 +-
 Command/DropKey.hs    |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  2 +-
 Command/FromKey.hs    |  2 +-
 Command/Fsck.hs       |  2 +-
 Command/Get.hs        |  2 +-
 Command/InAnnex.hs    |  2 +-
 Command/Init.hs       |  3 +--
 Command/InitRemote.hs |  7 +++----
 Command/Lock.hs       |  2 +-
 Command/Map.hs        |  3 ++-
 Command/Merge.hs      |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/Move.hs       |  2 +-
 Command/PreCommit.hs  |  3 ++-
 Command/RecvKey.hs    |  2 +-
 Command/Semitrust.hs  |  2 +-
 Command/SendKey.hs    |  2 +-
 Command/SetKey.hs     |  2 +-
 Command/Status.hs     |  2 +-
 Command/Trust.hs      |  2 +-
 Command/Unannex.hs    |  3 ++-
 Command/Uninit.hs     | 14 +++++++-------
 Command/Unlock.hs     |  6 ++++--
 Command/Untrust.hs    |  2 +-
 Command/Unused.hs     |  2 +-
 Command/Upgrade.hs    |  2 +-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  2 +-
 38 files changed, 73 insertions(+), 67 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index b1c9c17285..1037401e00 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -21,7 +21,6 @@ import qualified Git
 import Annex.Content
 import Command
 import Options
-import Init
 
 {- Runs the passed command line. -}
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
@@ -41,7 +40,7 @@ parseCmd argv header cmds options = do
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			checkCmdEnviron command
+			checkCommand command
 			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
@@ -53,10 +52,6 @@ parseCmd argv header cmds options = do
 		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds
 		usagemsg = "\n\n" ++ usage header cmds options
 
-{- Checks that the command can be run in the current environment. -}
-checkCmdEnviron :: Command -> Annex ()
-checkCmdEnviron command = when (cmdusesrepo command) ensureInitialized
-
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
 usage header cmds options =
diff --git a/Command.hs b/Command.hs
index f282791fb3..d19dad2601 100644
--- a/Command.hs
+++ b/Command.hs
@@ -18,42 +18,38 @@ import Logs.Location
 import Config
 import Backend
 import Limit
+import Init
 
-{- A command runs in four stages.
+{- A command runs in these stages.
  -
- - 0. The seek stage takes the parameters passed to the command,
+ - a. The check stage is run once and should error out if anything
+ -    prevents the command from running. -}
+type CommandCheck = Annex ()
+{- b. The seek stage takes the parameters passed to the command,
  -    looks through the repo to find the ones that are relevant
  -    to that command (ie, new files to add), and generates
  -    a list of start stage actions. -}
 type CommandSeek = [String] -> Annex [CommandStart]
-{- 1. The start stage is run before anything is printed about the
+{- c. The start stage is run before anything is printed about the
   -   command, is passed some input, and can early abort it
   -   if the input does not make sense. It should run quickly and
   -   should not modify Annex state. -}
 type CommandStart = Annex (Maybe CommandPerform)
-{- 2. The perform stage is run after a message is printed about the command
+{- d. The perform stage is run after a message is printed about the command
  -    being run, and it should be where the bulk of the work happens. -}
 type CommandPerform = Annex (Maybe CommandCleanup)
-{- 3. The cleanup stage is run only if the perform stage succeeds, and it
+{- e. The cleanup stage is run only if the perform stage succeeds, and it
  -    returns the overall success/fail of the command. -}
 type CommandCleanup = Annex Bool
 
 data Command = Command {
-	cmdusesrepo :: Bool,
 	cmdname :: String,
 	cmdparams :: String,
+	cmdcheck :: CommandCheck,
 	cmdseek :: [CommandSeek],
 	cmddesc :: String
 }
 
-{- Most commands operate on files in a git repo. -}
-repoCommand :: String -> String -> [CommandSeek] -> String -> Command
-repoCommand = Command True
-
-{- Others can run anywhere. -}
-standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command
-standaloneCommand = Command False
-
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
 next a = return $ Just a
@@ -62,6 +58,18 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
+needsNothing :: CommandCheck
+needsNothing = return ()
+
+{- Most commands will check this, as they need to be run in an initialized
+ - repo. -}
+needsRepo :: CommandCheck
+needsRepo = ensureInitialized
+
+{- Checks that the command can be run in the current environment. -}
+checkCommand :: Command -> Annex ()
+checkCommand Command { cmdcheck = check } = check
+
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
 prepCommand :: Command -> [String] -> Annex [Annex Bool]
diff --git a/Command/Add.hs b/Command/Add.hs
index bfddd72ee7..255e787b74 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -19,7 +19,7 @@ import Utility.Touch
 import Backend
 
 command :: [Command]
-command = [repoCommand "add" paramPaths seek "add files to annex"]
+command = [Command "add" paramPaths needsRepo seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index f32b5b86a9..8deb79541f 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -20,7 +20,7 @@ import Annex.Content
 import Logs.Web
 
 command :: [Command]
-command = [repoCommand "addurl" (paramRepeating paramUrl) seek
+command = [Command "addurl" (paramRepeating paramUrl) needsRepo seek
 	"add urls to annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 43315f67ce..35a939b38f 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.UUID
 
 command :: [Command]
-command = [repoCommand "configlist" paramNothing seek
+command = [Command "configlist" paramNothing needsRepo seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
diff --git a/Command/Copy.hs b/Command/Copy.hs
index d7625ccdb0..2f10d981c0 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -11,7 +11,7 @@ import Command
 import qualified Command.Move
 
 command :: [Command]
-command = [repoCommand "copy" paramPaths seek
+command = [Command "copy" paramPaths needsRepo seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 65cd8d0bf5..9184ede9cf 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.UUID
 
 command :: [Command]
-command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
+command = [Command "describe" (paramPair paramRemote paramDesc) needsRepo seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index dc858fb29b..7309c2acdb 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -17,7 +17,7 @@ import Annex.Content
 import Config
 
 command :: [Command]
-command = [repoCommand "drop" paramPaths seek
+command = [Command "drop" paramPaths needsRepo seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index fde6ce02ea..9e35548569 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -14,7 +14,7 @@ import Logs.Location
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "dropkey" (paramRepeating paramKey) seek
+command = [Command "dropkey" (paramRepeating paramKey) needsRepo seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 0050685565..019fab0769 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -21,7 +21,7 @@ import Types.Key
 type UnusedMap = M.Map String Key
 
 command :: [Command]
-command = [repoCommand "dropunused" (paramRepeating paramNumber) seek
+command = [Command "dropunused" (paramRepeating paramNumber) needsRepo seek
 	"drop unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Find.hs b/Command/Find.hs
index 98501078e8..5b13c742ad 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Limit
 
 command :: [Command]
-command = [repoCommand "find" paramPaths seek "lists available files"]
+command = [Command "find" paramPaths needsRepo seek "lists available files"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 5b6f1f7a47..5e58f07332 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -13,7 +13,7 @@ import qualified Annex.Queue
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "fix" paramPaths seek
+command = [Command "fix" paramPaths needsRepo seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 1b05d71fb8..30243964e5 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Types.Key
 
 command :: [Command]
-command = [repoCommand "fromkey" paramPath seek
+command = [Command "fromkey" paramPath needsRepo seek
 	"adds a file using a specific key"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 1c1687a002..0098a822db 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -21,7 +21,7 @@ import Utility.FileMode
 import Config
 
 command :: [Command]
-command = [repoCommand "fsck" paramPaths seek "check for problems"]
+command = [Command "fsck" paramPaths needsRepo seek "check for problems"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/Get.hs b/Command/Get.hs
index acf7e07228..d9596c3fe8 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -15,7 +15,7 @@ import Annex.Content
 import qualified Command.Move
 
 command :: [Command]
-command = [repoCommand "get" paramPaths seek
+command = [Command "get" paramPaths needsRepo seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 773693b65f..b4db849c9b 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "inannex" (paramRepeating paramKey) seek
+command = [Command "inannex" (paramRepeating paramKey) needsRepo seek
 		"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Init.hs b/Command/Init.hs
index 3dd4493295..06bdf4ad56 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -14,8 +14,7 @@ import Logs.UUID
 import Init
 	
 command :: [Command]
-command = [standaloneCommand "init" paramDesc seek
-		"initialize git-annex"]
+command = [Command "init" paramDesc needsNothing seek "initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 073ba72f90..8f97199b77 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -17,10 +17,9 @@ import qualified Types.Remote as R
 import Annex.UUID
 
 command :: [Command]
-command = [repoCommand "initremote"
-	(paramPair paramName $
-		paramOptional $ paramRepeating paramKeyValue) seek
-	"sets up a special (non-git) remote"]
+command = [Command "initremote"
+	(paramPair paramName $ 	paramOptional $ paramRepeating paramKeyValue)
+	needsRepo seek "sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index c6c66a1585..bf3b125592 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -13,7 +13,7 @@ import qualified Annex.Queue
 import Backend
 	
 command :: [Command]
-command = [repoCommand "lock" paramPaths seek "undo unlock command"]
+command = [Command "lock" paramPaths needsRepo seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
diff --git a/Command/Map.hs b/Command/Map.hs
index 48cba63f9e..6fbc6930ba 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -23,7 +23,8 @@ import qualified Utility.Dot as Dot
 data Link = Link Git.Repo Git.Repo
 
 command :: [Command]
-command = [repoCommand "map" paramNothing seek "generate map of repositories"]
+command = [Command "map" paramNothing needsNothing seek
+	"generate map of repositories"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Merge.hs b/Command/Merge.hs
index eef2f3857a..2b7162946a 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -12,7 +12,7 @@ import Command
 import qualified Annex.Branch
 
 command :: [Command]
-command = [repoCommand "merge" paramNothing seek
+command = [Command "merge" paramNothing needsRepo seek
 		"auto-merge remote changes into git-annex branch"]
 
 seek :: [CommandSeek]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 8167ac96eb..e3956c5aa7 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -17,7 +17,7 @@ import Backend
 import Logs.Web
 
 command :: [Command]
-command = [repoCommand "migrate" paramPaths seek
+command = [Command "migrate" paramPaths needsRepo seek
 	"switch data to different backend"]
 
 seek :: [CommandSeek]
diff --git a/Command/Move.hs b/Command/Move.hs
index a816aacde5..ae5e0e1d45 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -17,7 +17,7 @@ import qualified Remote
 import Annex.UUID
 
 command :: [Command]
-command = [repoCommand "move" paramPaths seek
+command = [Command "move" paramPaths needsRepo seek
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index b6323e2b79..50bc2662e1 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -13,7 +13,8 @@ import qualified Command.Fix
 import Backend
 
 command :: [Command]
-command = [repoCommand "pre-commit" paramPaths seek "run by git pre-commit hook"]
+command = [Command "pre-commit" paramPaths needsRepo seek
+	"run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index d3b77d8ac1..9dc436a681 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
-command = [repoCommand "recvkey" paramKey seek
+command = [Command "recvkey" paramKey needsRepo seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 5d60977eb4..f6a2f639c7 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
+command = [Command "semitrust" (paramRepeating paramRemote) needsRepo seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index ad47cd009f..e8ba3ae79c 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
-command = [repoCommand "sendkey" paramKey seek
+command = [Command "sendkey" paramKey needsRepo seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index b707e0b918..51f344f20f 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -13,7 +13,7 @@ import Logs.Location
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "setkey" paramPath seek
+command = [Command "setkey" paramPath needsRepo seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
diff --git a/Command/Status.hs b/Command/Status.hs
index 70282b79ee..155e53ee2c 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -39,7 +39,7 @@ data StatInfo = StatInfo
 type StatState = StateT StatInfo Annex
 
 command :: [Command]
-command = [repoCommand "status" paramNothing seek
+command = [Command "status" paramNothing needsRepo seek
 	"shows status information about the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index eeeadc9afe..1af458630f 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [repoCommand "trust" (paramRepeating paramRemote) seek
+command = [Command "trust" (paramRepeating paramRemote) needsRepo seek
 	"trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 083984d0c5..cdaa790c0f 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -19,7 +19,8 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 
 command :: [Command]
-command = [repoCommand "unannex" paramPaths seek "undo accidential add command"]
+command = [Command "unannex" paramPaths needsRepo seek
+	"undo accidential add command"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 8214c4208e..60e86cc039 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -19,18 +19,15 @@ import qualified Annex.Branch
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "uninit" paramPaths seek 
+command = [Command "uninit" paramPaths check seek 
         "de-initialize git-annex and clean out repository"]
 
-seek :: [CommandSeek]
-seek = [withNothing startCheck, withFilesInGit startUnannex, withNothing start]
-
-startCheck :: CommandStart
-startCheck = do
+check :: Annex ()
+check = do
+	needsRepo
 	b <- current_branch	
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ b ++ " branch is checked out"
-	stop
 	where
 		current_branch = do
 			g <- gitRepo
@@ -38,6 +35,9 @@ startCheck = do
 				Git.pipeRead g [Params "rev-parse --abbrev-ref HEAD"]
 			return $ head $ lines $ B.unpack b
 
+seek :: [CommandSeek]
+seek = [withFilesInGit startUnannex, withNothing start]
+
 startUnannex :: FilePath -> CommandStart
 startUnannex file = do
 	-- Force fast mode before running unannex. This way, if multiple
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 9b568b5a6b..c89b61de70 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -15,9 +15,11 @@ import Utility.FileMode
 
 command :: [Command]
 command =
-	[ repoCommand "unlock" paramPaths seek "unlock files for modification"
-	, repoCommand "edit" paramPaths seek "same as unlock"
+	[ c "unlock" "unlock files for modification"
+	, c "edit" "same as unlock"
 	]
+	where
+		c n = Command n paramPaths needsRepo seek
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index f8bf498f24..7d65c1af95 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [repoCommand "untrust" (paramRepeating paramRemote) seek
+command = [Command "untrust" (paramRepeating paramRemote) needsRepo seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a901747521..5cef829d6e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -28,7 +28,7 @@ import qualified Annex.Branch
 import Annex.CatFile
 
 command :: [Command]
-command = [repoCommand "unused" paramNothing seek
+command = [Command "unused" paramNothing needsRepo seek
 	"look for unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 90d3a4e95b..77d15c9306 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -13,7 +13,7 @@ import Upgrade
 import Annex.Version
 
 command :: [Command]
-command = [standaloneCommand "upgrade" paramNothing seek
+command = [Command "upgrade" paramNothing needsNothing seek
 	"upgrade repository layout"]
 
 seek :: [CommandSeek]
diff --git a/Command/Version.hs b/Command/Version.hs
index 5ac87099bb..dae9a31d3b 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -13,7 +13,7 @@ import qualified Build.SysConfig as SysConfig
 import Annex.Version
 
 command :: [Command]
-command = [standaloneCommand "version" paramNothing seek "show version info"]
+command = [Command "version" paramNothing needsNothing seek "show version info"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index b1646ae69a..71b3ad96b1 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -14,7 +14,7 @@ import Remote
 import Logs.Trust
 
 command :: [Command]
-command = [repoCommand "whereis" paramPaths seek
+command = [Command "whereis" paramPaths needsRepo seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]

From 1826b3bd67b8d3e146fd833d4cb500b03d1bf096 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 18:01:46 -0400
Subject: [PATCH 2385/2835] cleanup

---
 Annex/Branch.hs | 2 +-
 test.hs         | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 24da8120c4..645a2de762 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -168,7 +168,7 @@ update = onceonly $ do
 			disableUpdate
 			return r
 
-{- Avoids updating the branch. A useful optimisation when the branc
+{- Avoids updating the branch. A useful optimisation when the branch
  - is known to have not changed, or git-annex won't be relying on info
  - from it. -}
 disableUpdate :: Annex ()
diff --git a/test.hs b/test.hs
index 7d4c4293e1..0ef7449162 100644
--- a/test.hs
+++ b/test.hs
@@ -31,7 +31,6 @@ import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
 import qualified Logs.Location
-import qualified Logs.UUID
 import qualified Logs.UUIDBased
 import qualified Logs.Trust
 import qualified Logs.Remote

From c879eb873ec2fda1ec7d76c0de27d9ace57ba6e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 18:03:36 -0400
Subject: [PATCH 2386/2835] do commit location changes to remote in copy --to

test suite pointed out that if a file was copied from B to A, and then
A cloned, the clone ought to immediatly know it can get the file from A.
---
 Remote/Git.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index fd3a612436..b0138901d7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -190,8 +190,11 @@ copyToRemote r key
 		let keysrc = gitAnnexLocation g key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
-		liftIO $ onLocal r $ Annex.Content.getViaTmp key $
-			rsyncOrCopyFile params keysrc
+		liftIO $ onLocal r $ do
+			ok <- Annex.Content.getViaTmp key $
+				rsyncOrCopyFile params keysrc
+			Annex.Content.saveState
+			return ok
 	| Git.repoIsSsh r = do
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key

From b955238ec7464b12c793d543fc51308c8b213e08 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 18:56:54 -0400
Subject: [PATCH 2387/2835] Fail if --from or --to is passed to commands that
 do not support them.

---
 Command.hs            | 32 ++++++++++++++++++++++----------
 Command/Add.hs        |  2 +-
 Command/AddUrl.hs     |  2 +-
 Command/ConfigList.hs |  2 +-
 Command/Describe.hs   |  2 +-
 Command/Drop.hs       |  2 +-
 Command/DropKey.hs    |  2 +-
 Command/DropUnused.hs |  4 ++--
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  2 +-
 Command/FromKey.hs    |  2 +-
 Command/Fsck.hs       |  2 +-
 Command/Get.hs        |  2 +-
 Command/InAnnex.hs    |  2 +-
 Command/Init.hs       |  2 +-
 Command/InitRemote.hs |  2 +-
 Command/Lock.hs       |  2 +-
 Command/Map.hs        |  2 +-
 Command/Merge.hs      |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/PreCommit.hs  |  2 +-
 Command/RecvKey.hs    |  2 +-
 Command/Semitrust.hs  |  2 +-
 Command/SendKey.hs    |  2 +-
 Command/SetKey.hs     |  2 +-
 Command/Status.hs     |  2 +-
 Command/Trust.hs      |  2 +-
 Command/Unannex.hs    |  2 +-
 Command/Uninit.hs     |  3 +--
 Command/Unlock.hs     |  2 +-
 Command/Untrust.hs    |  2 +-
 Command/Unused.hs     |  2 +-
 Command/Upgrade.hs    |  2 +-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  2 +-
 debian/changelog      |  1 +
 36 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/Command.hs b/Command.hs
index d19dad2601..5690118c40 100644
--- a/Command.hs
+++ b/Command.hs
@@ -22,8 +22,8 @@ import Init
 
 {- A command runs in these stages.
  -
- - a. The check stage is run once and should error out if anything
- -    prevents the command from running. -}
+ - a. The check stage runs checks, that error out if
+ -    anything prevents the command from running. -}
 type CommandCheck = Annex ()
 {- b. The seek stage takes the parameters passed to the command,
  -    looks through the repo to find the ones that are relevant
@@ -58,14 +58,6 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
-needsNothing :: CommandCheck
-needsNothing = return ()
-
-{- Most commands will check this, as they need to be run in an initialized
- - repo. -}
-needsRepo :: CommandCheck
-needsRepo = ensureInitialized
-
 {- Checks that the command can be run in the current environment. -}
 checkCommand :: Command -> Annex ()
 checkCommand Command { cmdcheck = check } = check
@@ -239,3 +231,23 @@ autoCopies key vs numcopiesattr a = do
 			(_, have) <- trustPartition UnTrusted =<< keyLocations key
 			if length have `vs` needed then a else stop
 		else a
+
+{- Checks -}
+defaultChecks :: CommandCheck
+defaultChecks = noFrom >> noTo >> needsRepo
+
+noChecks :: CommandCheck
+noChecks = return ()
+
+needsRepo :: CommandCheck
+needsRepo = ensureInitialized
+
+noFrom :: CommandCheck
+noFrom = do
+	v <- Annex.getState Annex.fromremote
+	unless (v == Nothing) $ error "cannot use --from with this command"
+
+noTo :: CommandCheck
+noTo = do
+	v <- Annex.getState Annex.toremote
+	unless (v == Nothing) $ error "cannot use --to with this command"
diff --git a/Command/Add.hs b/Command/Add.hs
index 255e787b74..33b636bed7 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -19,7 +19,7 @@ import Utility.Touch
 import Backend
 
 command :: [Command]
-command = [Command "add" paramPaths needsRepo seek "add files to annex"]
+command = [Command "add" paramPaths defaultChecks seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 8deb79541f..72e29ff60e 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -20,7 +20,7 @@ import Annex.Content
 import Logs.Web
 
 command :: [Command]
-command = [Command "addurl" (paramRepeating paramUrl) needsRepo seek
+command = [Command "addurl" (paramRepeating paramUrl) defaultChecks seek
 	"add urls to annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 35a939b38f..645d1523ca 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.UUID
 
 command :: [Command]
-command = [Command "configlist" paramNothing needsRepo seek
+command = [Command "configlist" paramNothing defaultChecks seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 9184ede9cf..cd5da302e9 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.UUID
 
 command :: [Command]
-command = [Command "describe" (paramPair paramRemote paramDesc) needsRepo seek
+command = [Command "describe" (paramPair paramRemote paramDesc) defaultChecks seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 7309c2acdb..02f44915ce 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -17,7 +17,7 @@ import Annex.Content
 import Config
 
 command :: [Command]
-command = [Command "drop" paramPaths needsRepo seek
+command = [Command "drop" paramPaths defaultChecks seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 9e35548569..3e666b9aba 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -14,7 +14,7 @@ import Logs.Location
 import Annex.Content
 
 command :: [Command]
-command = [Command "dropkey" (paramRepeating paramKey) needsRepo seek
+command = [Command "dropkey" (paramRepeating paramKey) defaultChecks seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 019fab0769..1236fb8230 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -21,8 +21,8 @@ import Types.Key
 type UnusedMap = M.Map String Key
 
 command :: [Command]
-command = [Command "dropunused" (paramRepeating paramNumber) needsRepo seek
-	"drop unused file content"]
+command = [Command "dropunused" (paramRepeating paramNumber) (noTo >> needsRepo)
+	seek "drop unused file content"]
 
 seek :: [CommandSeek]
 seek = [withUnusedMaps]
diff --git a/Command/Find.hs b/Command/Find.hs
index 5b13c742ad..291904ec03 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Limit
 
 command :: [Command]
-command = [Command "find" paramPaths needsRepo seek "lists available files"]
+command = [Command "find" paramPaths defaultChecks seek "lists available files"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 5e58f07332..090558d521 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -13,7 +13,7 @@ import qualified Annex.Queue
 import Annex.Content
 
 command :: [Command]
-command = [Command "fix" paramPaths needsRepo seek
+command = [Command "fix" paramPaths defaultChecks seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 30243964e5..a3e96b20da 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Types.Key
 
 command :: [Command]
-command = [Command "fromkey" paramPath needsRepo seek
+command = [Command "fromkey" paramPath defaultChecks seek
 	"adds a file using a specific key"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0098a822db..d025095b1b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -21,7 +21,7 @@ import Utility.FileMode
 import Config
 
 command :: [Command]
-command = [Command "fsck" paramPaths needsRepo seek "check for problems"]
+command = [Command "fsck" paramPaths defaultChecks seek "check for problems"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/Get.hs b/Command/Get.hs
index d9596c3fe8..21ff73bc4e 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -15,7 +15,7 @@ import Annex.Content
 import qualified Command.Move
 
 command :: [Command]
-command = [Command "get" paramPaths needsRepo seek
+command = [Command "get" paramPaths (noTo >> needsRepo) seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index b4db849c9b..7a5735b742 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.Content
 
 command :: [Command]
-command = [Command "inannex" (paramRepeating paramKey) needsRepo seek
+command = [Command "inannex" (paramRepeating paramKey) defaultChecks seek
 		"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Init.hs b/Command/Init.hs
index 06bdf4ad56..6646432002 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -14,7 +14,7 @@ import Logs.UUID
 import Init
 	
 command :: [Command]
-command = [Command "init" paramDesc needsNothing seek "initialize git-annex"]
+command = [Command "init" paramDesc noChecks seek "initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 8f97199b77..cea1acc8d8 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -19,7 +19,7 @@ import Annex.UUID
 command :: [Command]
 command = [Command "initremote"
 	(paramPair paramName $ 	paramOptional $ paramRepeating paramKeyValue)
-	needsRepo seek "sets up a special (non-git) remote"]
+	defaultChecks seek "sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index bf3b125592..8f0bd78ebe 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -13,7 +13,7 @@ import qualified Annex.Queue
 import Backend
 	
 command :: [Command]
-command = [Command "lock" paramPaths needsRepo seek "undo unlock command"]
+command = [Command "lock" paramPaths defaultChecks seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
diff --git a/Command/Map.hs b/Command/Map.hs
index 6fbc6930ba..05cc9d794c 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -23,7 +23,7 @@ import qualified Utility.Dot as Dot
 data Link = Link Git.Repo Git.Repo
 
 command :: [Command]
-command = [Command "map" paramNothing needsNothing seek
+command = [Command "map" paramNothing noChecks seek
 	"generate map of repositories"]
 
 seek :: [CommandSeek]
diff --git a/Command/Merge.hs b/Command/Merge.hs
index 2b7162946a..33d4c8ffc5 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -12,7 +12,7 @@ import Command
 import qualified Annex.Branch
 
 command :: [Command]
-command = [Command "merge" paramNothing needsRepo seek
+command = [Command "merge" paramNothing defaultChecks seek
 		"auto-merge remote changes into git-annex branch"]
 
 seek :: [CommandSeek]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index e3956c5aa7..ac8f042ba2 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -17,7 +17,7 @@ import Backend
 import Logs.Web
 
 command :: [Command]
-command = [Command "migrate" paramPaths needsRepo seek
+command = [Command "migrate" paramPaths defaultChecks seek
 	"switch data to different backend"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 50bc2662e1..5dac4f5339 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -13,7 +13,7 @@ import qualified Command.Fix
 import Backend
 
 command :: [Command]
-command = [Command "pre-commit" paramPaths needsRepo seek
+command = [Command "pre-commit" paramPaths defaultChecks seek
 	"run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 9dc436a681..3415de526e 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
-command = [Command "recvkey" paramKey needsRepo seek
+command = [Command "recvkey" paramKey defaultChecks seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index f6a2f639c7..4f61531ff0 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [Command "semitrust" (paramRepeating paramRemote) needsRepo seek
+command = [Command "semitrust" (paramRepeating paramRemote) defaultChecks seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index e8ba3ae79c..5118a009b4 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
-command = [Command "sendkey" paramKey needsRepo seek
+command = [Command "sendkey" paramKey defaultChecks seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 51f344f20f..a60f539973 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -13,7 +13,7 @@ import Logs.Location
 import Annex.Content
 
 command :: [Command]
-command = [Command "setkey" paramPath needsRepo seek
+command = [Command "setkey" paramPath defaultChecks seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
diff --git a/Command/Status.hs b/Command/Status.hs
index 155e53ee2c..df79d4a7fe 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -39,7 +39,7 @@ data StatInfo = StatInfo
 type StatState = StateT StatInfo Annex
 
 command :: [Command]
-command = [Command "status" paramNothing needsRepo seek
+command = [Command "status" paramNothing defaultChecks seek
 	"shows status information about the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 1af458630f..17b689c345 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [Command "trust" (paramRepeating paramRemote) needsRepo seek
+command = [Command "trust" (paramRepeating paramRemote) defaultChecks seek
 	"trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index cdaa790c0f..76100c8c3c 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -19,7 +19,7 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 
 command :: [Command]
-command = [Command "unannex" paramPaths needsRepo seek
+command = [Command "unannex" paramPaths defaultChecks seek
 	"undo accidential add command"]
 
 seek :: [CommandSeek]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 60e86cc039..b2046ec414 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -19,12 +19,11 @@ import qualified Annex.Branch
 import Annex.Content
 
 command :: [Command]
-command = [Command "uninit" paramPaths check seek 
+command = [Command "uninit" paramPaths (check >> defaultChecks) seek 
         "de-initialize git-annex and clean out repository"]
 
 check :: Annex ()
 check = do
-	needsRepo
 	b <- current_branch	
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ b ++ " branch is checked out"
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index c89b61de70..be1d052984 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -19,7 +19,7 @@ command =
 	, c "edit" "same as unlock"
 	]
 	where
-		c n = Command n paramPaths needsRepo seek
+		c n = Command n paramPaths defaultChecks seek
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 7d65c1af95..5a2505a10f 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [Command "untrust" (paramRepeating paramRemote) needsRepo seek
+command = [Command "untrust" (paramRepeating paramRemote) defaultChecks seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5cef829d6e..d2b45ed6b7 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -28,7 +28,7 @@ import qualified Annex.Branch
 import Annex.CatFile
 
 command :: [Command]
-command = [Command "unused" paramNothing needsRepo seek
+command = [Command "unused" paramNothing (noTo >> needsRepo) seek
 	"look for unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 77d15c9306..9ca3c8d2be 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -13,7 +13,7 @@ import Upgrade
 import Annex.Version
 
 command :: [Command]
-command = [Command "upgrade" paramNothing needsNothing seek
+command = [Command "upgrade" paramNothing noChecks seek
 	"upgrade repository layout"]
 
 seek :: [CommandSeek]
diff --git a/Command/Version.hs b/Command/Version.hs
index dae9a31d3b..905a48a51e 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -13,7 +13,7 @@ import qualified Build.SysConfig as SysConfig
 import Annex.Version
 
 command :: [Command]
-command = [Command "version" paramNothing needsNothing seek "show version info"]
+command = [Command "version" paramNothing noChecks seek "show version info"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 71b3ad96b1..06a894fd3f 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -14,7 +14,7 @@ import Remote
 import Logs.Trust
 
 command :: [Command]
-command = [Command "whereis" paramPaths needsRepo seek
+command = [Command "whereis" paramPaths defaultChecks seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]
diff --git a/debian/changelog b/debian/changelog
index dd72fc296d..d100bd4e85 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * copy --to: Fixed leak when copying many files to a remote on the same
     host.
   * uninit: Add guard against being run with the git-annex branch checked out.
+  * Fail if --from or --to is passed to commands that do not support them.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From 2888562724acd06ed2acd5c5a318a88d9e33d484 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 18:59:25 -0400
Subject: [PATCH 2388/2835] update

---
 ...t:_support_drop__44___find_on_special_remotes.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
index 2dc735e99f..982f398ae9 100644
--- a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
+++ b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
@@ -1,6 +1,17 @@
 Currently there is no way to drop files, or list what files are available, on a special remote.
 It would be good if "git annex drop" and "git annex find" supported the --from argument.
 
+> I agree, drop should support --from.
+> 
+> To find files *believed* to be present in a given remote, use
+> `git annex find --in remote`
+> Note that it might show out of date info, since it does not actually go
+> check the current contents of the remote. The only reason to support
+> `find --from` would be to always check, but I don't think that's needed.
+> --[[Joey]] 
+
 For commands that don't support the --from argument, it would also be nice to print an error.
 Currently running "git annex drop --from usbdrive" doesn't behave as hoped and instead drops
 all content from the local annex.
+
+> This is done now. --[[Joey]] 

From f66f97c90e5692ab34cb95e6facdec194a72456b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 19:04:12 -0400
Subject: [PATCH 2389/2835] document the little-known get --from

---
 doc/git-annex.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 1b8464b314..3dcc117056 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -72,6 +72,9 @@ subdirectories).
   will involve copying them from another repository, or downloading them,
   or transferring them from some kind of key-value store.
 
+  Normally git-annex will choose which repository to copy the content from,
+  but you can override this using the --from option.
+
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 

From 33e18d3d02865ac0677fc1f22de2352b92f184a8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 19:10:10 -0400
Subject: [PATCH 2390/2835] cleanup

---
 Command/Drop.hs    | 11 ++++-------
 Command/Unannex.hs |  7 +------
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index 02f44915ce..a84d2efcc3 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -23,8 +23,6 @@ command = [Command "drop" paramPaths defaultChecks seek
 seek :: [CommandSeek]
 seek = [withNumCopies start]
 
-{- Indicates a file's content is not wanted anymore, and should be removed
- - if it's safe to do so. -}
 start :: FilePath -> Maybe Int -> CommandStart
 start file numcopies = isAnnexed file $ \(key, _) -> do
 	present <- inAnnex key
@@ -36,7 +34,7 @@ start file numcopies = isAnnexed file $ \(key, _) -> do
 
 perform :: Key -> Maybe Int -> CommandPerform
 perform key numcopies = do
-	success <- dropKey key numcopies
+	success <- canDropKey key numcopies
 	if success
 		then next $ cleanup key
 		else stop
@@ -48,10 +46,9 @@ cleanup key = do
 	return True
 
 {- Checks remotes to verify that enough copies of a key exist to allow
- - for a key to be safely removed (with no data loss), and fails with an
- - error if not. -}
-dropKey :: Key -> Maybe Int -> Annex Bool
-dropKey key numcopiesM = do
+ - for a key to be safely removed (with no data loss). -}
+canDropKey :: Key -> Maybe Int -> Annex Bool
+canDropKey key numcopiesM = do
 	force <- Annex.getState Annex.force
 	if force || numcopiesM == Just 0
 		then return True
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 76100c8c3c..b39dc0a5f0 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -9,7 +9,6 @@ module Command.Unannex where
 
 import Common.Annex
 import Command
-import qualified Command.Drop
 import qualified Annex
 import qualified Annex.Queue
 import Utility.FileMode
@@ -44,11 +43,7 @@ start file = isAnnexed file $ \(key, _) -> do
 		else stop
 
 perform :: FilePath -> Key -> CommandPerform
-perform file key = do
-	ok <- Command.Drop.dropKey key (Just 0) -- always remove
-	if ok
-		then next $ cleanup file key
-		else stop
+perform file key = next $ cleanup file key
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do

From 6c31e3a8c3c5ab92ca2e84b4c166f32d02a50f4f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Oct 2011 17:26:38 -0400
Subject: [PATCH 2391/2835] drop --from is now supported to remove file content
 from a remote.

---
 Command/Drop.hs                               | 125 +++++++++++++-----
 Command/DropUnused.hs                         |   5 +-
 Command/Move.hs                               |  25 +---
 Remote.hs                                     |  15 ++-
 debian/changelog                              |   1 +
 ...rt_drop__44___find_on_special_remotes.mdwn |   1 +
 doc/git-annex.mdwn                            |   5 +-
 7 files changed, 114 insertions(+), 63 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index a84d2efcc3..27049fc67d 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -11,76 +11,131 @@ import Common.Annex
 import Command
 import qualified Remote
 import qualified Annex
+import Annex.UUID
 import Logs.Location
 import Logs.Trust
 import Annex.Content
 import Config
 
 command :: [Command]
-command = [Command "drop" paramPaths defaultChecks seek
+command = [Command "drop" paramPaths (noTo >> needsRepo) seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
 
 start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, _) -> do
+start file numcopies = isAnnexed file $ \(key, _) ->
+	autoCopies key (>) numcopies $ do
+		from <- Annex.getState Annex.fromremote
+		case from of
+			Nothing -> startLocal file numcopies key
+			Just name -> do
+				remote <- Remote.byName name
+				u <- getUUID
+				if Remote.uuid remote == u
+					then startLocal file numcopies key
+					else startRemote file numcopies key remote
+
+startLocal :: FilePath -> Maybe Int -> Key -> CommandStart
+startLocal file numcopies key = do
 	present <- inAnnex key
 	if present
-		then autoCopies key (>) numcopies $ do
+		then do
 			showStart "drop" file
-			next $ perform key numcopies
+			next $ performLocal key numcopies
 		else stop
 
-perform :: Key -> Maybe Int -> CommandPerform
-perform key numcopies = do
-	success <- canDropKey key numcopies
+startRemote :: FilePath -> Maybe Int -> Key -> Remote.Remote Annex -> CommandStart
+startRemote file numcopies key remote = do
+	showStart "drop" file
+	next $ performRemote key numcopies remote
+
+performLocal :: Key -> Maybe Int -> CommandPerform
+performLocal key numcopies = do
+	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
+	untrusteduuids <- trustGet UnTrusted
+	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
+	success <- canDropKey key numcopies trusteduuids tocheck []
 	if success
-		then next $ cleanup key
+		then next $ cleanupLocal key
 		else stop
 
-cleanup :: Key -> CommandCleanup
-cleanup key = do
+performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
+performRemote key numcopies remote = do
+	-- Filter the remote it's being dropped from out of the lists of
+	-- places assumed to have the key, and places to check.
+	-- When the local repo has the key, that's one additional copy.
+	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
+	present <- inAnnex key
+	u <- getUUID
+	let have = filter (/= uuid) $
+		if present then u:trusteduuids else trusteduuids
+	untrusteduuids <- trustGet UnTrusted
+	let tocheck = filter (/= remote) $
+		Remote.remotesWithoutUUID remotes (have++untrusteduuids)
+	success <- canDropKey key numcopies have tocheck [uuid]
+	if success
+		then next $ cleanupRemote key remote
+		else stop
+	where
+		uuid = Remote.uuid remote
+
+cleanupLocal :: Key -> CommandCleanup
+cleanupLocal key = do
 	whenM (inAnnex key) $ removeAnnex key
 	logStatus key InfoMissing
 	return True
 
-{- Checks remotes to verify that enough copies of a key exist to allow
- - for a key to be safely removed (with no data loss). -}
-canDropKey :: Key -> Maybe Int -> Annex Bool
-canDropKey key numcopiesM = do
+cleanupRemote :: Key -> Remote.Remote Annex -> CommandCleanup
+cleanupRemote key remote = do
+	ok <- Remote.removeKey remote key
+	-- better safe than sorry: assume the remote dropped the key
+	-- even if it seemed to fail; the failure could have occurred
+	-- after it really dropped it
+	Remote.remoteHasKey remote key False
+	return ok
+
+{- Checks specified remotes to verify that enough copies of a key exist to
+ - allow it to be safely removed (with no data loss). Can be provided with
+ - some locations where the key is known/assumed to be present. -}
+canDropKey :: Key -> Maybe Int -> [UUID] -> [Remote.Remote Annex] -> [UUID] -> Annex Bool
+canDropKey key numcopiesM have check skip = do
 	force <- Annex.getState Annex.force
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
-			untrusteduuids <- trustGet UnTrusted
-			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
-			numcopies <- getNumCopies numcopiesM
-			findcopies numcopies trusteduuids tocheck []
+			need <- getNumCopies numcopiesM
+			findCopies key need skip have check
+
+findCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote.Remote Annex] -> Annex Bool
+findCopies key need skip = helper []
 	where
-		findcopies need have [] bad
+		helper bad have []
 			| length have >= need = return True
-			| otherwise = notEnoughCopies need have bad
-		findcopies need have (r:rs) bad
+			| otherwise = notEnoughCopies key need have skip bad
+		helper bad have (r:rs)
 			| length have >= need = return True
 			| otherwise = do
 				let u = Remote.uuid r
 				let duplicate = u `elem` have
 				haskey <- Remote.hasKey r key
 				case (duplicate, haskey) of
-					(False, Right True)	-> findcopies need (u:have) rs bad
-					(False, Left _)		-> findcopies need have rs (r:bad)
-					_			-> findcopies need have rs bad
-		notEnoughCopies need have bad = do
-			unsafe
-			showLongNote $
-				"Could only verify the existence of " ++
-				show (length have) ++ " out of " ++ show need ++ 
-				" necessary copies"
-			Remote.showTriedRemotes bad
-			Remote.showLocations key have
-			hint
-			return False
+					(False, Right True) -> helper bad (u:have) rs
+					(False, Left _)     -> helper (r:bad) have rs
+					_                   -> helper bad have rs
+
+notEnoughCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote.Remote Annex] -> Annex Bool
+notEnoughCopies key need have skip bad = do
+	unsafe
+	showLongNote $
+		"Could only verify the existence of " ++
+		show (length have) ++ " out of " ++ show need ++ 
+		" necessary copies"
+	Remote.showTriedRemotes bad
+	Remote.showLocations key (have++skip)
+	hint
+	return False
+	where
 		unsafe = showNote "unsafe"
 		hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)"
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 1236fb8230..46f2dc9f7f 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -13,7 +13,6 @@ import Common.Annex
 import Command
 import qualified Annex
 import qualified Command.Drop
-import qualified Command.Move
 import qualified Remote
 import qualified Git
 import Types.Key
@@ -56,8 +55,8 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 		dropremote name = do
 			r <- Remote.byName name
 			showAction $ "from " ++ Remote.name r
-			next $ Command.Move.fromCleanup r True key
-		droplocal = Command.Drop.perform key (Just 0) -- force drop
+			next $ Command.Drop.cleanupRemote key r
+		droplocal = Command.Drop.performLocal key (Just 0) -- force drop
 
 performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
diff --git a/Command/Move.hs b/Command/Move.hs
index ae5e0e1d45..2a7402a0d6 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Command
 import qualified Command.Drop
 import qualified Annex
-import Logs.Location
 import Annex.Content
 import qualified Remote
 import Annex.UUID
@@ -49,18 +48,6 @@ showMoveAction :: Bool -> FilePath -> Annex ()
 showMoveAction True file = showStart "move" file
 showMoveAction False file = showStart "copy" file
 
-{- Used to log a change in a remote's having a key. The change is logged
- - in the local repo, not on the remote. The process of transferring the
- - key to the remote, or removing the key from it *may* log the change
- - on the remote, but this cannot be relied on. -}
-remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()
-remoteHasKey remote key present	= do
-	let remoteuuid = Remote.uuid remote
-	g <- gitRepo
-	logChange g key remoteuuid status
-	where
-		status = if present then InfoPresent else InfoMissing
-
 {- Moves (or copies) the content of an annexed file to a remote.
  -
  - If the remote already has the content, it is still removed from
@@ -108,9 +95,9 @@ toPerform dest move key = do
 		Right True -> next $ toCleanup dest move key
 toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 toCleanup dest move key = do
-	remoteHasKey dest key True
+	Remote.remoteHasKey dest key True
 	if move
-		then Command.Drop.cleanup key
+		then Command.Drop.cleanupLocal key
 		else return True
 
 {- Moves (or copies) the content of an annexed file from a remote
@@ -140,11 +127,5 @@ fromPerform src move key = do
 				then next $ fromCleanup src move key
 				else stop -- fail
 fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
-fromCleanup src True key = do
-	ok <- Remote.removeKey src key
-	-- better safe than sorry: assume the src dropped the key
-	-- even if it seemed to fail; the failure could have occurred
-	-- after it really dropped it
-	remoteHasKey src key False
-	return ok
+fromCleanup src True key = Command.Drop.cleanupRemote key src
 fromCleanup _ False _ = return True
diff --git a/Remote.hs b/Remote.hs
index 49fa63cf92..6ce4fe0186 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -25,7 +25,8 @@ module Remote (
 	nameToUUID,
 	showTriedRemotes,
 	showLocations,
-	forceTrust
+	forceTrust,
+	remoteHasKey
 ) where
 
 import qualified Data.Map as M
@@ -225,3 +226,15 @@ forceTrust level remotename = do
 	r <- nameToUUID remotename
 	Annex.changeState $ \s ->
 		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
+
+{- Used to log a change in a remote's having a key. The change is logged
+ - in the local repo, not on the remote. The process of transferring the
+ - key to the remote, or removing the key from it *may* log the change
+ - on the remote, but this cannot always be relied on. -}
+remoteHasKey :: Remote Annex -> Key -> Bool -> Annex ()
+remoteHasKey remote key present	= do
+	let remoteuuid = uuid remote
+	g <- gitRepo
+	logChange g key remoteuuid status
+	where
+		status = if present then InfoPresent else InfoMissing
diff --git a/debian/changelog b/debian/changelog
index d100bd4e85..237abb83ff 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     host.
   * uninit: Add guard against being run with the git-annex branch checked out.
   * Fail if --from or --to is passed to commands that do not support them.
+  * drop --from is now supported to remove file content from a remote.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
index 982f398ae9..24cacbf71c 100644
--- a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
+++ b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
@@ -2,6 +2,7 @@ Currently there is no way to drop files, or list what files are available, on a
 It would be good if "git annex drop" and "git annex find" supported the --from argument.
 
 > I agree, drop should support --from.
+>> [[done]] --[[Joey]] 
 > 
 > To find files *believed* to be present in a given remote, use
 > `git annex find --in remote`
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3dcc117056..dc0b49ab2c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -80,8 +80,9 @@ subdirectories).
   Drops the content of annexed files from this repository. 
 
   git-annex will refuse to drop content if it cannot verify it is
-  safe to do so. At least one copy of content needs to exist in another
-  remote. This can be overridden with the --force switch.
+  safe to do so. This can be overridden with the --force switch.
+
+  To drop content from a remote, specify --from.
 
 * move [path ...]
   

From ab738a403a0b5f34b9c679c6c9a92b279babdf42 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Oct 2011 19:49:01 -0400
Subject: [PATCH 2392/2835] status: Now always shows the current repository,
 even when it does not appear in uuid.log.

---
 Logs/UUID.hs     | 13 +++++++++++--
 debian/changelog |  2 ++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 8a93b43fef..c05c4e3481 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -24,6 +24,7 @@ import Data.Time.Clock.POSIX
 import Common.Annex
 import qualified Annex.Branch
 import Logs.UUIDBased
+import qualified Annex.UUID
 
 {- Filename of uuid.log. -}
 logfile :: FilePath
@@ -36,6 +37,14 @@ describeUUID uuid desc = do
 	Annex.Branch.change logfile $
 		showLog id . changeLog ts uuid desc . parseLog Just
 
-{- Read the uuidLog into a simple Map -}
+{- Read the uuidLog into a simple Map.
+ -
+ - The UUID of the current repository is included explicitly, since
+ - it may not have been described and so otherwise would not appear. -}
 uuidMap :: Annex (M.Map UUID String)
-uuidMap = (simpleMap . parseLog Just) <$> Annex.Branch.get logfile
+uuidMap = do
+	m <- (simpleMap . parseLog Just) <$> Annex.Branch.get logfile
+	u <- Annex.UUID.getUUID
+	return $ M.insertWith' preferold u "" m
+	where
+		preferold = flip const
diff --git a/debian/changelog b/debian/changelog
index 237abb83ff..4a873af945 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * uninit: Add guard against being run with the git-annex branch checked out.
   * Fail if --from or --to is passed to commands that do not support them.
   * drop --from is now supported to remove file content from a remote.
+  * status: Now always shows the current repository, even when it does not
+    appear in uuid.log.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From aeb4e285eb2b3b880ab6b91c5b1f7fa29d7182cb Mon Sep 17 00:00:00 2001
From: "http://nicolas-schodet.myopenid.com/"
 
Date: Sat, 29 Oct 2011 01:01:38 +0000
Subject: [PATCH 2393/2835] as far as I know, --bwlimit does not accept unit,
 it always takes kilobytes

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index dc0b49ab2c..27b4cead8b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -542,7 +542,7 @@ Here are all the supported configuration settings.
 * `remote..annex-bup-split-options`
 
   Options to pass to bup split when storing content in this remote.
-  For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k"
+  For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100"
   (There is no corresponding option for bup join.)
 
 * `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`

From f65100b408c9b2efe35af340f9438b3e84d69028 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Oct 2011 21:24:24 -0400
Subject: [PATCH 2394/2835] Revert "as far as I know, --bwlimit does not accept
 unit, it always takes kilobytes"

This reverts commit aeb4e285eb2b3b880ab6b91c5b1f7fa29d7182cb.

bup's --bwlimit does take a unit, unlike rsync's.
---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 27b4cead8b..dc0b49ab2c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -542,7 +542,7 @@ Here are all the supported configuration settings.
 * `remote..annex-bup-split-options`
 
   Options to pass to bup split when storing content in this remote.
-  For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100"
+  For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k"
   (There is no corresponding option for bup join.)
 
 * `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`

From 7588b042daaecec5de167971b3123d08db849942 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 15:27:06 +0000
Subject: [PATCH 2395/2835]

---
 doc/bugs/uninit_does_not_work_in_old_repos.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/uninit_does_not_work_in_old_repos.mdwn

diff --git a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
new file mode 100644
index 0000000000..a6f7858639
--- /dev/null
+++ b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
@@ -0,0 +1,6 @@
+As uninit does not need to actually write out any data, just remove it, it should be possible to uninit in old stores.
+
+    % git annex uninit
+    git-annex: Repository version 2 is not supported. Upgrade this repository: git-annex upgrade
+
+If the repo happens to be broken, this essentially locks in data.

From ce4029c973ac6cab7f30f7452723a0750b0b7ea5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 15:30:10 +0000
Subject: [PATCH 2396/2835] Added a comment

---
 .../comment_1_bc0619c6e17139df74639448aa6a0f72._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/uninit_does_not_work_in_old_repos/comment_1_bc0619c6e17139df74639448aa6a0f72._comment

diff --git a/doc/bugs/uninit_does_not_work_in_old_repos/comment_1_bc0619c6e17139df74639448aa6a0f72._comment b/doc/bugs/uninit_does_not_work_in_old_repos/comment_1_bc0619c6e17139df74639448aa6a0f72._comment
new file mode 100644
index 0000000000..7a1ea582b0
--- /dev/null
+++ b/doc/bugs/uninit_does_not_work_in_old_repos/comment_1_bc0619c6e17139df74639448aa6a0f72._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-10-29T15:30:09Z"
+ content="""
+After upgrading the repo, I still have to commit the changes, else git-annex won't let me uninit. Arguably a Good Thing, but I wanted to document it here.
+"""]]

From 0dcbe51ed2e966376f506b1f780c37632bd81860 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 15:36:43 +0000
Subject: [PATCH 2397/2835]

---
 ...t_annex_upgrade_output_is_inconsistent_and_spammy.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn

diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
new file mode 100644
index 0000000000..89701144a4
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
@@ -0,0 +1,8 @@
+Upgrading from v1 to v3:
+
+    upgrade . (v1 to v2...) (moving content...) (updating symlinks...) (moving location logs...) (v2 to v3...) ..............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
+      git-annex branch created
+      Be sure to push this branch when pushing to remotes.
+    ok
+
+A whirly would be preferable, imo.

From 4d7802bff71d62ceb674eee6c957c74a1e85a2eb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 12:45:47 -0400
Subject: [PATCH 2398/2835] responsen

---
 ..._annex_upgrade_output_is_inconsistent_and_spammy.mdwn | 7 +++++++
 doc/bugs/uninit_does_not_work_in_old_repos.mdwn          | 9 +++++++++
 2 files changed, 16 insertions(+)

diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
index 89701144a4..3d75483dc3 100644
--- a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
+++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
@@ -6,3 +6,10 @@ Upgrading from v1 to v3:
     ok
 
 A whirly would be preferable, imo.
+
+> Erm, I'm pretty sure you were the one who asked for there to be some
+> progress dots, Richard.
+> 
+> I'm not particularly interested in implementing a whirley that would only
+> be used in this one place, in code that very few users are going to run
+> again. I could remove the dots.. --[[Joey]]
diff --git a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
index a6f7858639..1e1b1fec73 100644
--- a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
+++ b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
@@ -4,3 +4,12 @@ As uninit does not need to actually write out any data, just remove it, it shoul
     git-annex: Repository version 2 is not supported. Upgrade this repository: git-annex upgrade
 
 If the repo happens to be broken, this essentially locks in data.
+
+> No, because you can always check out the version of git-annex you need
+> for that repository.
+> 
+> uninit, as implemented, runs unannex on every file and then does some
+> cleanup. The cleanup does not need to write state, but the unannex does.
+> And it depends on the object directory layout, which has changed between
+> versions. So supporting old versions in this code would complicate it
+> quite a lot. I don't want to go there. --[[Joey]]

From 39c304be4376eab387851eb94414654ce61a14ee Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 16:57:10 +0000
Subject: [PATCH 2399/2835]

---
 ...led_checksum_when_less_copies_than_required_are_found.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
new file mode 100644
index 0000000000..860a61d2cb
--- /dev/null
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -0,0 +1,4 @@
+    (checksum...) failed
+    fsck foo (fixing location log) 
+      Only 1 of 2 trustworthy copies exist of foo
+      Back it up with git-annex copy.

From 36355e815e0a9c95a4aaea5b091e5c1f06ad6999 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 17:03:27 +0000
Subject: [PATCH 2400/2835] Added a comment

---
 ...comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment

diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment
new file mode 100644
index 0000000000..4f9565517d
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-10-29T17:03:26Z"
+ content="""
+I could dig it out, but I am sure I said dots are fine and a whirly better.
+
+Still, WONTFIX is fine.
+"""]]

From 3868a65663231b929a1c09b02185e85dc70e59e5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 17:07:44 +0000
Subject: [PATCH 2401/2835]

---
 doc/bugs/uninit_does_not_work_in_old_repos.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
index 1e1b1fec73..0e5175abe3 100644
--- a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
+++ b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
@@ -13,3 +13,5 @@ If the repo happens to be broken, this essentially locks in data.
 > And it depends on the object directory layout, which has changed between
 > versions. So supporting old versions in this code would complicate it
 > quite a lot. I don't want to go there. --[[Joey]]
+
+>>Requiring a version upgrade for unannex is fine. Yet, I see a problem when a git repo is broken; you are stuck without being able to uninit. In this case an uninit that does nothing but undo the symlinking would be useful. -- Richard

From 978ab987d50211ae43864409c4ad9fb77d7f6974 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 13:16:10 -0400
Subject: [PATCH 2402/2835] pebak

---
 ...led_checksum_when_less_copies_than_required_are_found.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 860a61d2cb..101fc6e36b 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -2,3 +2,7 @@
     fsck foo (fixing location log) 
       Only 1 of 2 trustworthy copies exist of foo
       Back it up with git-annex copy.
+
+> You've given me severely partial output, and no test case, but until
+> it says "fsck foo", the output is pertaining to some other file than foo.
+> As far as I can see, there is no bug here. [[done]] --[[Joey]]

From ad3b46221463b9cb5fa3b0abaef9cbfcf3facdae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 13:17:37 -0400
Subject: [PATCH 2403/2835] sheesh. seriously?

---
 Upgrade/V2.hs                                                   | 1 -
 .../git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn    | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 1ad41266a7..67f0205c14 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -78,7 +78,6 @@ inject source dest = do
 	new <- liftIO (readFile $ olddir g  source)
 	Annex.Branch.change dest $ \prev -> 
 		unlines $ nub $ lines prev ++ lines new
-	showProgress
 
 logFiles :: FilePath -> Annex [FilePath]
 logFiles dir = return . filter (".log" `isSuffixOf`)
diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
index 3d75483dc3..ec8a10915e 100644
--- a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
+++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
@@ -12,4 +12,4 @@ A whirly would be preferable, imo.
 > 
 > I'm not particularly interested in implementing a whirley that would only
 > be used in this one place, in code that very few users are going to run
-> again. I could remove the dots.. --[[Joey]]
+> again. I could remove the dots.. [[done]] --[[Joey]]

From 0dfe750c0b2f5b11fba82a077055580864cc1c27 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 13:21:28 -0400
Subject: [PATCH 2404/2835] close

---
 doc/bugs/uninit_does_not_work_in_old_repos.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
index 0e5175abe3..d3df061487 100644
--- a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
+++ b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
@@ -15,3 +15,6 @@ If the repo happens to be broken, this essentially locks in data.
 > quite a lot. I don't want to go there. --[[Joey]]
 
 >>Requiring a version upgrade for unannex is fine. Yet, I see a problem when a git repo is broken; you are stuck without being able to uninit. In this case an uninit that does nothing but undo the symlinking would be useful. -- Richard
+
+>>> As I said, version 2 of git-annex is still there for people who need
+>>> it for whatever reason. [[done]] --[[Joey]]

From e9d142e9e8ae7095b0ab19ef70d0b5edac33947e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 17:27:48 +0000
Subject: [PATCH 2405/2835]

---
 ..._to_provide_UUID_when_running___96__git_annex_init__96__.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn

diff --git a/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn b/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
new file mode 100644
index 0000000000..a1a8f2827b
--- /dev/null
+++ b/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
@@ -0,0 +1 @@
+As there's no way to permanently hide remotes and I have to recreate two repos now, I would love to be able to re-use the old UUIDs to remove clutter.

From 75e99b16f9ef4ee3bc35791f76e57525375e3364 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 17:45:07 +0000
Subject: [PATCH 2406/2835]

---
 ...ed_checksum_when_less_copies_than_required_are_found.mdwn | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 101fc6e36b..a5bde3c1ac 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -5,4 +5,7 @@
 
 > You've given me severely partial output, and no test case, but until
 > it says "fsck foo", the output is pertaining to some other file than foo.
-> As far as I can see, there is no bug here. [[done]] --[[Joey]]
+> As far as I can see, there is no bug here. --[[Joey]]
+
+>> Sorry, I thought it would be obvious, but that's no excuse for not providing additional explanation.
+>> The problem is that fsck tells me a file's fsck has failed without printing extra details. In this case, the checksum is OK while I don't have enough copies to satisfy the fsck. The fact that I don't have enough copies is obviously relevant, but I would still like to know if the checksums are OK. -- Richard

From 0d92aca1aabbbeb2d50d91807312ff6039971751 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 14:17:02 -0400
Subject: [PATCH 2407/2835] responsen

---
 ...ksum_when_less_copies_than_required_are_found.mdwn | 11 +++++++++--
 ..._UUID_when_running___96__git_annex_init__96__.mdwn |  4 ++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index a5bde3c1ac..3f1d28d3da 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -7,5 +7,12 @@
 > it says "fsck foo", the output is pertaining to some other file than foo.
 > As far as I can see, there is no bug here. --[[Joey]]
 
->> Sorry, I thought it would be obvious, but that's no excuse for not providing additional explanation.
->> The problem is that fsck tells me a file's fsck has failed without printing extra details. In this case, the checksum is OK while I don't have enough copies to satisfy the fsck. The fact that I don't have enough copies is obviously relevant, but I would still like to know if the checksums are OK. -- Richard
+>> Sorry, I thought it would be obvious, but that's no excuse for not
+>> providing additional explanation. The problem is that fsck tells me a
+>> file's fsck has failed without printing extra details. In this case, the
+>> checksum is OK while I don't have enough copies to satisfy the fsck. The
+>> fact that I don't have enough copies is obviously relevant, but I would
+>> still like to know if the checksums are OK. -- Richard
+
+>>> I think you're misreading the truncated output you posted. The actual,
+>>> full output would make much more sense. --[[Joey]]
diff --git a/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn b/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
index a1a8f2827b..0dc9ec08a2 100644
--- a/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
+++ b/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
@@ -1 +1,5 @@
 As there's no way to permanently hide remotes and I have to recreate two repos now, I would love to be able to re-use the old UUIDs to remove clutter.
+
+> git-annex already provides a way to do this: Copy `.git/config` from the
+> original repo (or use `git-config` to set `annex.uuid`) *before* running
+> `git annex init`. [[done]] --[[Joey]]

From d46b8c053e0fbea824a9f9faec4dc58d3f777a07 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 18:28:14 +0000
Subject: [PATCH 2408/2835] Added a comment

---
 ..._342d1ac07573c7ef4e27f003a692e261._comment | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_2_342d1ac07573c7ef4e27f003a692e261._comment

diff --git a/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_2_342d1ac07573c7ef4e27f003a692e261._comment b/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_2_342d1ac07573c7ef4e27f003a692e261._comment
new file mode 100644
index 0000000000..3bb92919f9
--- /dev/null
+++ b/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_2_342d1ac07573c7ef4e27f003a692e261._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-10-29T18:28:13Z"
+ content="""
+Possible solutions:
+
+This:
+
+    trusted repositories:
+            UUID -- foo
+    semi-trusted repositories:
+            UUID -- bar
+    untrusted repositories:
+            UUID -- baz
+
+or this:
+
+            UUID -- trusted      -- foo
+            UUID -- semi-trusted -- bar
+            UUID -- untrusted    -- baz
+
+or this:
+
+    known repositories (!/*/X):
+            UUID -- ! foo
+            UUID -- * bar
+            UUID -- X baz
+
+If you want to reformat this output, putting 'here', 'origin', etc into fixed formatting might make sense, as well. -- Richard
+"""]]

From b7d2fd8186a8bcd4615d17b9bf3a96b010b55f68 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 19:09:19 +0000
Subject: [PATCH 2409/2835]

---
 ...iled_checksum_when_less_copies_than_required_are_found.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 3f1d28d3da..57c80d7ca3 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -16,3 +16,6 @@
 
 >>> I think you're misreading the truncated output you posted. The actual,
 >>> full output would make much more sense. --[[Joey]]
+
+>>>> No. I have a total of 14908 annex keys, 3333 of which are on a remote. The only message other than 'checksum OK' and the above is 'git-annex: 11577 failed'.
+>>>> I checked several files manually, their checksums are OK so `git annex status` is reporting those files as completely failed when they "only" miss copies. -- Richard

From f97c783283847c6cc4516673fe638b4d551e671d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 15:19:05 -0400
Subject: [PATCH 2410/2835] clean up check selection code

This new approach allows filtering out checks from the default set that are
not appropriate for a command, rather than having to list every check
that is appropriate. It also reduces some boilerplate.

Haskell does not define Eq for functions, so I had to go a long way around
with each check having a unique id. Meh.
---
 CmdLine.hs            |  6 ++--
 Command.hs            | 47 +++++++++++++++++++------------
 Command/Add.hs        |  4 +--
 Command/AddUrl.hs     |  5 ++--
 Command/ConfigList.hs |  6 ++--
 Command/Copy.hs       |  5 ++--
 Command/Describe.hs   |  4 +--
 Command/Drop.hs       |  4 +--
 Command/DropKey.hs    |  4 +--
 Command/DropUnused.hs |  4 +--
 Command/Find.hs       |  4 +--
 Command/Fix.hs        |  4 +--
 Command/FromKey.hs    |  5 ++--
 Command/Fsck.hs       |  4 +--
 Command/Get.hs        |  6 ++--
 Command/InAnnex.hs    |  6 ++--
 Command/Init.hs       |  5 ++--
 Command/InitRemote.hs |  8 +++---
 Command/Lock.hs       |  4 +--
 Command/Map.hs        |  6 ++--
 Command/Merge.hs      |  4 +--
 Command/Migrate.hs    |  5 ++--
 Command/Move.hs       |  5 ++--
 Command/PreCommit.hs  |  5 ++--
 Command/RecvKey.hs    |  4 +--
 Command/Semitrust.hs  |  4 +--
 Command/SendKey.hs    |  4 +--
 Command/SetKey.hs     |  4 +--
 Command/Status.hs     |  4 +--
 Command/Trust.hs      |  5 ++--
 Command/Unannex.hs    |  5 ++--
 Command/Uninit.hs     |  4 +--
 Command/Unlock.hs     |  6 ++--
 Command/Untrust.hs    |  4 +--
 Command/Unused.hs     |  4 +--
 Command/Upgrade.hs    |  6 ++--
 Command/Version.hs    |  5 ++--
 Command/Whereis.hs    |  4 +--
 GitAnnex.hs           | 64 +++++++++++++++++++++----------------------
 git-annex-shell.hs    | 10 +++----
 40 files changed, 154 insertions(+), 143 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 1037401e00..9f1ded498c 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -38,10 +38,10 @@ parseCmd argv header cmds options = do
 	when (null params) $ error $ "missing command" ++ usagemsg
 	case lookupCmd (head params) of
 		[] -> error $ "unknown command" ++ usagemsg
-		[command] -> do
+		[cmd] -> do
 			_ <- sequence flags
-			checkCommand command
-			prepCommand command (drop 1 params)
+			checkCommand cmd
+			prepCommand cmd (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
 		getopt = case getOpt Permute options argv of
diff --git a/Command.hs b/Command.hs
index 5690118c40..b039403ca0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -24,7 +24,9 @@ import Init
  -
  - a. The check stage runs checks, that error out if
  -    anything prevents the command from running. -}
-type CommandCheck = Annex ()
+data CommandCheck = CommandCheck { idCheck :: Int, runCheck :: Annex () }
+instance Eq CommandCheck where
+	a == b = idCheck a == idCheck b
 {- b. The seek stage takes the parameters passed to the command,
  -    looks through the repo to find the ones that are relevant
  -    to that command (ie, new files to add), and generates
@@ -43,9 +45,9 @@ type CommandPerform = Annex (Maybe CommandCleanup)
 type CommandCleanup = Annex Bool
 
 data Command = Command {
+	cmdcheck :: [CommandCheck],
 	cmdname :: String,
 	cmdparams :: String,
-	cmdcheck :: CommandCheck,
 	cmdseek :: [CommandSeek],
 	cmddesc :: String
 }
@@ -58,9 +60,9 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
-{- Checks that the command can be run in the current environment. -}
-checkCommand :: Command -> Annex ()
-checkCommand Command { cmdcheck = check } = check
+{- Generates a command with the common checks. -}
+command :: String -> String -> [CommandSeek] -> String -> Command
+command = Command commonChecks
 
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
@@ -232,22 +234,33 @@ autoCopies key vs numcopiesattr a = do
 			if length have `vs` needed then a else stop
 		else a
 
-{- Checks -}
-defaultChecks :: CommandCheck
-defaultChecks = noFrom >> noTo >> needsRepo
+{- Common checks for commands, and an interface to selectively remove them,
+ - or add others. -}
+commonChecks :: [CommandCheck]
+commonChecks = [fromOpt, toOpt, repoExists]
 
-noChecks :: CommandCheck
-noChecks = return ()
+repoExists :: CommandCheck
+repoExists = CommandCheck 0 ensureInitialized
 
-needsRepo :: CommandCheck
-needsRepo = ensureInitialized
-
-noFrom :: CommandCheck
-noFrom = do
+fromOpt :: CommandCheck
+fromOpt = CommandCheck 1 $ do
 	v <- Annex.getState Annex.fromremote
 	unless (v == Nothing) $ error "cannot use --from with this command"
 
-noTo :: CommandCheck
-noTo = do
+toOpt :: CommandCheck
+toOpt = CommandCheck 2 $ do
 	v <- Annex.getState Annex.toremote
 	unless (v == Nothing) $ error "cannot use --to with this command"
+
+checkCommand :: Command -> Annex ()
+checkCommand Command { cmdcheck = c } = sequence_ $ map runCheck c
+
+dontCheck :: CommandCheck -> Command -> Command
+dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
+
+addCheck :: Annex () -> Command -> Command
+addCheck check cmd = mutateCheck cmd $
+	\c -> CommandCheck (length c + 100) check : c
+
+mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command
+mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c }
diff --git a/Command/Add.hs b/Command/Add.hs
index 33b636bed7..82287be0bf 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -18,8 +18,8 @@ import Annex.Content
 import Utility.Touch
 import Backend
 
-command :: [Command]
-command = [Command "add" paramPaths defaultChecks seek "add files to annex"]
+def :: [Command]
+def = [command "add" paramPaths seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 72e29ff60e..e974d06a14 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -19,9 +19,8 @@ import qualified Backend.URL
 import Annex.Content
 import Logs.Web
 
-command :: [Command]
-command = [Command "addurl" (paramRepeating paramUrl) defaultChecks seek
-	"add urls to annex"]
+def :: [Command]
+def = [command "addurl" (paramRepeating paramUrl) seek "add urls to annex"]
 
 seek :: [CommandSeek]
 seek = [withStrings start]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 645d1523ca..cbc6e801b0 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -11,9 +11,9 @@ import Common.Annex
 import Command
 import Annex.UUID
 
-command :: [Command]
-command = [Command "configlist" paramNothing defaultChecks seek
-		"outputs relevant git configuration"]
+def :: [Command]
+def = [command "configlist" paramNothing seek
+	"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 2f10d981c0..8316b7cabf 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -10,8 +10,9 @@ module Command.Copy where
 import Command
 import qualified Command.Move
 
-command :: [Command]
-command = [Command "copy" paramPaths needsRepo seek
+def :: [Command]
+def = [dontCheck toOpt $ dontCheck fromOpt $ 
+	command "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Describe.hs b/Command/Describe.hs
index cd5da302e9..882a0e1bb9 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Remote
 import Logs.UUID
 
-command :: [Command]
-command = [Command "describe" (paramPair paramRemote paramDesc) defaultChecks seek
+def :: [Command]
+def = [command "describe" (paramPair paramRemote paramDesc) seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 27049fc67d..2267bd9416 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -17,8 +17,8 @@ import Logs.Trust
 import Annex.Content
 import Config
 
-command :: [Command]
-command = [Command "drop" paramPaths (noTo >> needsRepo) seek
+def :: [Command]
+def = [dontCheck fromOpt $ command "drop" paramPaths seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 3e666b9aba..d00bb6c83e 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -13,8 +13,8 @@ import qualified Annex
 import Logs.Location
 import Annex.Content
 
-command :: [Command]
-command = [Command "dropkey" (paramRepeating paramKey) defaultChecks seek
+def :: [Command]
+def = [command "dropkey" (paramRepeating paramKey) seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 46f2dc9f7f..d2eb3df710 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -19,8 +19,8 @@ import Types.Key
 
 type UnusedMap = M.Map String Key
 
-command :: [Command]
-command = [Command "dropunused" (paramRepeating paramNumber) (noTo >> needsRepo)
+def :: [Command]
+def = [dontCheck fromOpt $ command "dropunused" (paramRepeating paramNumber)
 	seek "drop unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Find.hs b/Command/Find.hs
index 291904ec03..46364c9876 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -12,8 +12,8 @@ import Command
 import Annex.Content
 import Limit
 
-command :: [Command]
-command = [Command "find" paramPaths defaultChecks seek "lists available files"]
+def :: [Command]
+def = [command "find" paramPaths seek "lists available files"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 090558d521..c46ddc7ee0 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Annex.Queue
 import Annex.Content
 
-command :: [Command]
-command = [Command "fix" paramPaths defaultChecks seek
+def :: [Command]
+def = [command "fix" paramPaths seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index a3e96b20da..fe9b5c96a0 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -13,9 +13,8 @@ import qualified Annex.Queue
 import Annex.Content
 import Types.Key
 
-command :: [Command]
-command = [Command "fromkey" paramPath defaultChecks seek
-	"adds a file using a specific key"]
+def :: [Command]
+def = [command "fromkey" paramPath seek "adds a file using a specific key"]
 
 seek :: [CommandSeek]
 seek = [withFilesMissing start]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index d025095b1b..5d2e2ee50b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -20,8 +20,8 @@ import Utility.DataUnits
 import Utility.FileMode
 import Config
 
-command :: [Command]
-command = [Command "fsck" paramPaths defaultChecks seek "check for problems"]
+def :: [Command]
+def = [command "fsck" paramPaths seek "check for problems"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/Get.hs b/Command/Get.hs
index 21ff73bc4e..4a0908bdc8 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -14,9 +14,9 @@ import qualified Remote
 import Annex.Content
 import qualified Command.Move
 
-command :: [Command]
-command = [Command "get" paramPaths (noTo >> needsRepo) seek
-		"make content of annexed files available"]
+def :: [Command]
+def = [dontCheck fromOpt $ command "get" paramPaths seek
+	"make content of annexed files available"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 7a5735b742..9c169d0d78 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -11,9 +11,9 @@ import Common.Annex
 import Command
 import Annex.Content
 
-command :: [Command]
-command = [Command "inannex" (paramRepeating paramKey) defaultChecks seek
-		"checks if keys are present in the annex"]
+def :: [Command]
+def = [command "inannex" (paramRepeating paramKey) seek
+	"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
 seek = [withKeys start]
diff --git a/Command/Init.hs b/Command/Init.hs
index 6646432002..e2a6eb7b03 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -13,8 +13,9 @@ import Annex.UUID
 import Logs.UUID
 import Init
 	
-command :: [Command]
-command = [Command "init" paramDesc noChecks seek "initialize git-annex"]
+def :: [Command]
+def = [dontCheck repoExists $
+	command "init" paramDesc seek "initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index cea1acc8d8..4ba5b07873 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -16,10 +16,10 @@ import qualified Logs.Remote
 import qualified Types.Remote as R
 import Annex.UUID
 
-command :: [Command]
-command = [Command "initremote"
-	(paramPair paramName $ 	paramOptional $ paramRepeating paramKeyValue)
-	defaultChecks seek "sets up a special (non-git) remote"]
+def :: [Command]
+def = [command "initremote"
+	(paramPair paramName $ paramOptional $ paramRepeating paramKeyValue)
+	seek "sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 8f0bd78ebe..329fd3eff7 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Annex.Queue
 import Backend
 	
-command :: [Command]
-command = [Command "lock" paramPaths defaultChecks seek "undo unlock command"]
+def :: [Command]
+def = [command "lock" paramPaths seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
diff --git a/Command/Map.hs b/Command/Map.hs
index 05cc9d794c..7e61d2e9e8 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -22,9 +22,9 @@ import qualified Utility.Dot as Dot
 -- a link from the first repository to the second (its remote)
 data Link = Link Git.Repo Git.Repo
 
-command :: [Command]
-command = [Command "map" paramNothing noChecks seek
-	"generate map of repositories"]
+def :: [Command]
+def = [dontCheck repoExists $
+	command "map" paramNothing seek "generate map of repositories"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Merge.hs b/Command/Merge.hs
index 33d4c8ffc5..c1f7e899af 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -11,8 +11,8 @@ import Common.Annex
 import Command
 import qualified Annex.Branch
 
-command :: [Command]
-command = [Command "merge" paramNothing defaultChecks seek
+def :: [Command]
+def = [command "merge" paramNothing seek
 		"auto-merge remote changes into git-annex branch"]
 
 seek :: [CommandSeek]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index ac8f042ba2..a68582996b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -16,9 +16,8 @@ import qualified Command.Add
 import Backend
 import Logs.Web
 
-command :: [Command]
-command = [Command "migrate" paramPaths defaultChecks seek
-	"switch data to different backend"]
+def :: [Command]
+def = [command "migrate" paramPaths seek "switch data to different backend"]
 
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
diff --git a/Command/Move.hs b/Command/Move.hs
index 2a7402a0d6..5a3ea7172a 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -15,8 +15,9 @@ import Annex.Content
 import qualified Remote
 import Annex.UUID
 
-command :: [Command]
-command = [Command "move" paramPaths needsRepo seek
+def :: [Command]
+def = [dontCheck toOpt $ dontCheck fromOpt $
+	command "move" paramPaths seek
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 5dac4f5339..1949de113f 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -12,9 +12,8 @@ import qualified Command.Add
 import qualified Command.Fix
 import Backend
 
-command :: [Command]
-command = [Command "pre-commit" paramPaths defaultChecks seek
-	"run by git pre-commit hook"]
+def :: [Command]
+def = [command "pre-commit" paramPaths seek "run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 3415de526e..5243fa9d4b 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -13,8 +13,8 @@ import CmdLine
 import Annex.Content
 import Utility.RsyncFile
 
-command :: [Command]
-command = [Command "recvkey" paramKey defaultChecks seek
+def :: [Command]
+def = [command "recvkey" paramKey seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 4f61531ff0..f8c3062131 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Remote
 import Logs.Trust
 
-command :: [Command]
-command = [Command "semitrust" (paramRepeating paramRemote) defaultChecks seek
+def :: [Command]
+def = [command "semitrust" (paramRepeating paramRemote) seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 5118a009b4..318ea56d03 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -12,8 +12,8 @@ import Command
 import Annex.Content
 import Utility.RsyncFile
 
-command :: [Command]
-command = [Command "sendkey" paramKey defaultChecks seek
+def :: [Command]
+def = [command "sendkey" paramKey seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index a60f539973..9c31abb083 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -12,8 +12,8 @@ import Command
 import Logs.Location
 import Annex.Content
 
-command :: [Command]
-command = [Command "setkey" paramPath defaultChecks seek
+def :: [Command]
+def = [command "setkey" paramPath seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
diff --git a/Command/Status.hs b/Command/Status.hs
index df79d4a7fe..b5f4956dbf 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -38,8 +38,8 @@ data StatInfo = StatInfo
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
 
-command :: [Command]
-command = [Command "status" paramNothing defaultChecks seek
+def :: [Command]
+def = [command "status" paramNothing seek
 	"shows status information about the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 17b689c345..d976b86a8f 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -12,9 +12,8 @@ import Command
 import qualified Remote
 import Logs.Trust
 
-command :: [Command]
-command = [Command "trust" (paramRepeating paramRemote) defaultChecks seek
-	"trust a repository"]
+def :: [Command]
+def = [command "trust" (paramRepeating paramRemote) seek "trust a repository"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index b39dc0a5f0..825f819390 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -17,9 +17,8 @@ import Annex.Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 
-command :: [Command]
-command = [Command "unannex" paramPaths defaultChecks seek
-	"undo accidential add command"]
+def :: [Command]
+def = [command "unannex" paramPaths seek "undo accidential add command"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index b2046ec414..5a6ee0be25 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -18,8 +18,8 @@ import Init
 import qualified Annex.Branch
 import Annex.Content
 
-command :: [Command]
-command = [Command "uninit" paramPaths (check >> defaultChecks) seek 
+def :: [Command]
+def = [addCheck check $ command "uninit" paramPaths seek 
         "de-initialize git-annex and clean out repository"]
 
 check :: Annex ()
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index be1d052984..7ecaf0b7fa 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -13,13 +13,13 @@ import Annex.Content
 import Utility.CopyFile
 import Utility.FileMode
 
-command :: [Command]
-command =
+def :: [Command]
+def =
 	[ c "unlock" "unlock files for modification"
 	, c "edit" "same as unlock"
 	]
 	where
-		c n = Command n paramPaths defaultChecks seek
+		c n = command n paramPaths seek
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 5a2505a10f..e16040e6bb 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Remote
 import Logs.Trust
 
-command :: [Command]
-command = [Command "untrust" (paramRepeating paramRemote) defaultChecks seek
+def :: [Command]
+def = [command "untrust" (paramRepeating paramRemote) seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index d2b45ed6b7..11c3f429ec 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -27,8 +27,8 @@ import qualified Remote
 import qualified Annex.Branch
 import Annex.CatFile
 
-command :: [Command]
-command = [Command "unused" paramNothing (noTo >> needsRepo) seek
+def :: [Command]
+def = [dontCheck fromOpt $ command "unused" paramNothing seek
 	"look for unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 9ca3c8d2be..b39fcd99c2 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -12,9 +12,9 @@ import Command
 import Upgrade
 import Annex.Version
 
-command :: [Command]
-command = [Command "upgrade" paramNothing noChecks seek
-	"upgrade repository layout"]
+def :: [Command]
+def = [dontCheck repoExists $ -- because an old version may not seem to exist
+	command "upgrade" paramNothing seek "upgrade repository layout"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Version.hs b/Command/Version.hs
index 905a48a51e..5a45fd77f2 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -12,8 +12,9 @@ import Command
 import qualified Build.SysConfig as SysConfig
 import Annex.Version
 
-command :: [Command]
-command = [Command "version" paramNothing noChecks seek "show version info"]
+def :: [Command]
+def = [dontCheck repoExists $
+	command "version" paramNothing seek "show version info"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 06a894fd3f..7799af08c6 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -13,8 +13,8 @@ import Command
 import Remote
 import Logs.Trust
 
-command :: [Command]
-command = [Command "whereis" paramPaths defaultChecks seek
+def :: [Command]
+def = [command "whereis" paramPaths seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 3cd7207d3e..c07e727fa0 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -54,38 +54,38 @@ import qualified Command.Version
 
 cmds :: [Command]
 cmds = concat
-	[ Command.Add.command
-	, Command.Get.command
-	, Command.Drop.command
-	, Command.Move.command
-	, Command.Copy.command
-	, Command.Unlock.command
-	, Command.Lock.command
-	, Command.Init.command
-	, Command.Describe.command
-	, Command.InitRemote.command
-	, Command.Unannex.command
-	, Command.Uninit.command
-	, Command.PreCommit.command
-	, Command.Trust.command
-	, Command.Untrust.command
-	, Command.Semitrust.command
-	, Command.AddUrl.command
-	, Command.FromKey.command
-	, Command.DropKey.command
-	, Command.SetKey.command
-	, Command.Fix.command
-	, Command.Fsck.command
-	, Command.Unused.command
-	, Command.DropUnused.command
-	, Command.Find.command
-	, Command.Whereis.command
-	, Command.Merge.command
-	, Command.Status.command
-	, Command.Migrate.command
-	, Command.Map.command
-	, Command.Upgrade.command
-	, Command.Version.command
+	[ Command.Add.def
+	, Command.Get.def
+	, Command.Drop.def
+	, Command.Move.def
+	, Command.Copy.def
+	, Command.Unlock.def
+	, Command.Lock.def
+	, Command.Init.def
+	, Command.Describe.def
+	, Command.InitRemote.def
+	, Command.Unannex.def
+	, Command.Uninit.def
+	, Command.PreCommit.def
+	, Command.Trust.def
+	, Command.Untrust.def
+	, Command.Semitrust.def
+	, Command.AddUrl.def
+	, Command.FromKey.def
+	, Command.DropKey.def
+	, Command.SetKey.def
+	, Command.Fix.def
+	, Command.Fsck.def
+	, Command.Unused.def
+	, Command.DropUnused.def
+	, Command.Find.def
+	, Command.Whereis.def
+	, Command.Merge.def
+	, Command.Status.def
+	, Command.Migrate.def
+	, Command.Map.def
+	, Command.Upgrade.def
+	, Command.Version.def
 	]
 
 options :: [Option]
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 41cb72d7e2..328d7b1006 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -23,15 +23,15 @@ import qualified Command.SendKey
 
 cmds_readonly :: [Command]
 cmds_readonly = concat
-	[ Command.ConfigList.command
-	, Command.InAnnex.command
-	, Command.SendKey.command
+	[ Command.ConfigList.def
+	, Command.InAnnex.def
+	, Command.SendKey.def
 	]
 
 cmds_notreadonly :: [Command]
 cmds_notreadonly = concat
-	[ Command.RecvKey.command
-	, Command.DropKey.command
+	[ Command.RecvKey.def
+	, Command.DropKey.def
 	]
 
 cmds :: [Command]

From 6c3b87f0de14d4545dc02484e8258fd863dcaf53 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 15:40:32 -0400
Subject: [PATCH 2411/2835] add a tip

---
 ..._to_do_when_a_repository_is_corrupted.mdwn | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn

diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn b/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn
new file mode 100644
index 0000000000..80cb046d90
--- /dev/null
+++ b/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn
@@ -0,0 +1,22 @@
+A git-annex repository on a removable USB drive is great, until the cable
+falls out at the wrong time and git's repository gets trashed. The way
+git checksums everything and the poor quality of USB media makes this
+perhaps more likely than you would expect. If this happens to you,
+here's a way to recover that makes the most of whatever data is left
+on the drive.
+
+* First, run `git fsck`. If it does not report any problems, your data
+  is fine, and you don't need to proceed further.
+* So `git fsck` says the git repository is corrupted. But probably the data
+  git-annex stored is fine. Your first step is to clone another copy
+  of the git repository from somewhere else. Let's call this clone
+  "$good", and the corrupted repository "$bad".
+* Preserve your git configuration changes, and the `annex.uuid` setting:
+  `mv $bad/.git/config $good/.git/config`
+* Move annexed data into the new repository: `mkdir $good/.git/annex; mv
+  $bad/.git/annex/objects $good/.git/annex/objects`
+* Reinitalize git-annex: `cd $good; git annex init`
+* Check for any problems with the annexed data: `cd $good; git annex fsck`
+* Now you can remove the corrupted repository, the new one is ready to use.
+
+--[[Joey]]

From a93aa2e51ebd88db2e6a88e8dc76e9a34d01b362 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 15:46:16 -0400
Subject: [PATCH 2412/2835] responsen

---
 ..._checksum_when_less_copies_than_required_are_found.mdwn | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 57c80d7ca3..faf67c243e 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -18,4 +18,9 @@
 >>> full output would make much more sense. --[[Joey]]
 
 >>>> No. I have a total of 14908 annex keys, 3333 of which are on a remote. The only message other than 'checksum OK' and the above is 'git-annex: 11577 failed'.
->>>> I checked several files manually, their checksums are OK so `git annex status` is reporting those files as completely failed when they "only" miss copies. -- Richard
+>>>> I checked several files manually, their checksums are OK so `git annex
+>>>> fsck` is reporting those files as completely failed when they "only" miss copies. -- Richard
+
+>>>>> fsck considers not enough copies to be a failure condition; it prints
+>>>>> error messages about it etc. That has nothing to do with checksums.
+>>>>> --[[Joey]]

From 3f322161788b3fc648083f9970cb31cdff03eda7 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 20:02:21 +0000
Subject: [PATCH 2413/2835]

---
 ...sum_when_less_copies_than_required_are_found.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index faf67c243e..95848456d1 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -24,3 +24,15 @@
 >>>>> fsck considers not enough copies to be a failure condition; it prints
 >>>>> error messages about it etc. That has nothing to do with checksums.
 >>>>> --[[Joey]]
+
+>>>>>> I get that. Still, I think it would be _extremely_ useful to know what failures occurred, exactly. Not having enough copies is Not Good, yet not having enough copies and a locally correct file is _lot_ better than having not enough copies and a broken file. I.e. I would prefer:
+
+    (checksum...) OK
+    Not enough copies: Only 1 of 2 trustworthy copies exist of foo
+
+>>>>>> or similar and at the end
+
+    git-annex: 0 wrong checksums
+    git-annex: 11577 with too few copies
+
+>>>>>> In the end, it comes down to the distinction of different failure classes. -- Richard

From 36f63ab19e54f51561dd9f2946c68037a8e99791 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 16:21:34 -0400
Subject: [PATCH 2414/2835] getting tired of repeating myself

---
 ...d_checksum_when_less_copies_than_required_are_found.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 95848456d1..1f5667a11c 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -36,3 +36,9 @@
     git-annex: 11577 with too few copies
 
 >>>>>> In the end, it comes down to the distinction of different failure classes. -- Richard
+
+>>>>>>> For the third, and final time: 
+>>>>>>> # You are misreading the truncated output you posted
+>>>>>>> The "checksum" line is regarding **different** file than the 
+>>>>>>> not enough copies message. fsck does not attempt to checksum a file
+>>>>>>> that is not present. [[done]] --[[Joey]]

From fef2cf739872b905bbdf493f9f3ba7124400c633 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 16:45:06 -0400
Subject: [PATCH 2415/2835] refactor

---
 Command/Fsck.hs | 36 ++++++++++++++----------------------
 1 file changed, 14 insertions(+), 22 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 5d2e2ee50b..9f3ae02636 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -32,14 +32,17 @@ start file numcopies = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	next $ perform key file backend numcopies
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
-perform key file backend numcopies = do
-	-- the location log is checked first, so that if it has bad data
-	-- that gets corrected
-	locationlogok <- verifyLocationLog key file
-	backendok <- fsckKey backend key (Just file) numcopies
-	if locationlogok && backendok
-		then next $ return True
-		else stop
+perform key file backend numcopies = check =<< sequence
+	-- order matters
+	[ verifyLocationLog key file
+	, checkKeySize key
+	, checkKeyNumCopies key file numcopies
+	, (Types.Backend.fsckKey backend) key
+	]
+	where
+		check vs
+			| all (== True) vs = next $ return True
+			| otherwise = stop
 
 {- Checks that the location log reflects the current status of the key,
    in this repository only. -}
@@ -77,14 +80,6 @@ verifyLocationLog key file = do
 			showNote "fixing location log"
 			logChange g key u s
 
-{- Checks a key for problems. -}
-fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-fsckKey backend key file numcopies = do
-	size_ok <- checkKeySize key
-	copies_ok <- checkKeyNumCopies key file numcopies
-	backend_ok <- (Types.Backend.fsckKey backend) key
-	return $ size_ok && copies_ok && backend_ok
-
 {- The size of the data for a key is checked against the size encoded in
  - the key's metadata, if available. -}
 checkKeySize :: Key -> Annex Bool
@@ -108,7 +103,7 @@ checkKeySize key = do
 					return False
 
 
-checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
 	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< keyLocations key
@@ -116,12 +111,9 @@ checkKeyNumCopies key file numcopies = do
 	if present < needed
 		then do
 			ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations
-			warning $ missingNote (filename file key) present needed ppuuids
+			warning $ missingNote file present needed ppuuids
 			return False
 		else return True
-	where
-		filename Nothing k = show k
-		filename (Just f) _ = f
 
 missingNote :: String -> Int -> Int -> String -> String
 missingNote file 0 _ [] = 

From a183487cd5477add59f470a42b18ec0c233bfb06 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 21:36:06 +0000
Subject: [PATCH 2416/2835]

---
 ...um_when_less_copies_than_required_are_found.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 1f5667a11c..fe6536b6a7 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -42,3 +42,16 @@
 >>>>>>> The "checksum" line is regarding **different** file than the 
 >>>>>>> not enough copies message. fsck does not attempt to checksum a file
 >>>>>>> that is not present. [[done]] --[[Joey]]
+
+
+>>>>>>>> I realized early on that I pasted the wrong cross-passage, but as there is a ton of the same output, I didn't think it would matter. I wasn't aware that it does not try to checksum when there aren't enough copies. To be fair, you only just mentioned that.
+>>>>>>>> Personally, I think that's a bug as it makes ensuring local correctness before copying a file to remotes impossible.
+>>>>>>>> Either way, I really didn't know it actually _skipped_ checksumming; that part was missing.
+>>>>>>>> For the benefit of anyone else who might read this, this is the correct order:
+
+    fsck foo (fixing location log) 
+      Only 1 of 2 trustworthy copies exist of foo
+      Back it up with git-annex copy.
+    (checksum...) failed
+
+>>>>>>>> If you would like to keep things this way, fine. I think it's less than ideal, but I don't want to argue, either. -- Richard

From 2566eb85fe2bbd0f9d1798d50ca0d88970a4490c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 17:49:37 -0400
Subject: [PATCH 2417/2835] fsck: Now works in bare repositories.

Checks location log information, and file contents.

Does not check that numcopies is satisfied, as .gitattributes information
about numcopies is not available in a bare repository. In practice, that
should not be a problem, since fsck is also run in a checkout and will
check numcopies there.
---
 Command.hs                                    | 25 +++++-----
 Command/Fsck.hs                               | 49 ++++++++++++++++---
 Command/Unused.hs                             | 10 +---
 Logs/Location.hs                              | 13 +++++
 debian/changelog                              |  4 ++
 ...t_annex_fsck_is_a_no-op_in_bare_repos.mdwn |  2 +
 doc/todo/support_fsck_in_bare_repos.mdwn      |  5 +-
 7 files changed, 77 insertions(+), 31 deletions(-)

diff --git a/Command.hs b/Command.hs
index b039403ca0..4e312e66d4 100644
--- a/Command.hs
+++ b/Command.hs
@@ -81,18 +81,6 @@ doCommand = start
 		success = return True
 		failure = showEndFail >> return False
 
-notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
-notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
-
-isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
-isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
-
-notBareRepo :: Annex a -> Annex a
-notBareRepo a = do
-	whenM (Git.repoIsLocalBare <$> gitRepo) $
-		error "You cannot run this subcommand in a bare repository."
-	a
-
 {- These functions find appropriate files or other things based on a
    user's parameters, and prepare actions operating on them. -}
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
@@ -168,7 +156,18 @@ runFilteredGen a d fs = do
 			ok <- matcher f
 			if ok then a v else stop
 
-{- filter out symlinks -}	
+notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
+notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
+
+isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
+isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
+
+notBareRepo :: Annex a -> Annex a
+notBareRepo a = do
+	whenM (Git.repoIsLocalBare <$> gitRepo) $
+		error "You cannot run this subcommand in a bare repository."
+	a
+
 notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
 
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 9f3ae02636..6f184a760c 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010,2011 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -12,6 +12,8 @@ import Command
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
+import qualified Backend
+import qualified Git
 import Annex.Content
 import Logs.Location
 import Logs.Trust
@@ -24,30 +26,61 @@ def :: [Command]
 def = [command "fsck" paramPaths seek "check for problems"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start]
+seek = [withNumCopies start, withBarePresentKeys startBare]
 
 start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = notBareRepo $ isAnnexed file $ \(key, backend) -> do
+start file numcopies = isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
 	next $ perform key file backend numcopies
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
-perform key file backend numcopies = check =<< sequence
+perform key file backend numcopies = check
 	-- order matters
 	[ verifyLocationLog key file
 	, checkKeySize key
 	, checkKeyNumCopies key file numcopies
 	, (Types.Backend.fsckKey backend) key
 	]
+
+{- To fsck a bare repository, fsck each key in the location log. -}
+withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek
+withBarePresentKeys a params = do
+	bare <- Git.repoIsLocalBare <$> gitRepo
+	if bare
+		then do
+			unless (null params) $ do
+				error "fsck should be run without parameters in a bare repository"
+			liftM (map a) loggedKeys
+		else return []
+
+startBare :: Key -> CommandStart
+startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of
+	Nothing -> stop
+	Just backend -> do
+		showStart "fsck" (show key)
+		next $ performBare key backend
+
+{- Note that numcopies cannot be checked in a bare repository, because
+ - getting the numcopies value requires a working copy with .gitattributes
+ - files. -}
+performBare :: Key -> Backend Annex -> CommandPerform
+performBare key backend = check
+	[ verifyLocationLog key (show key)
+	, checkKeySize key
+	, (Types.Backend.fsckKey backend) key
+	]
+
+check :: [Annex Bool] -> CommandPerform	
+check s = sequence s >>= dispatch
 	where
-		check vs
+		dispatch vs
 			| all (== True) vs = next $ return True
 			| otherwise = stop
 
 {- Checks that the location log reflects the current status of the key,
    in this repository only. -}
-verifyLocationLog :: Key -> FilePath -> Annex Bool
-verifyLocationLog key file = do
+verifyLocationLog :: Key -> String -> Annex Bool
+verifyLocationLog key desc = do
 	g <- gitRepo
 	present <- inAnnex key
 	
@@ -69,7 +102,7 @@ verifyLocationLog key file = do
 		(False, True) -> do
 				fix g u InfoMissing
 				warning $
-					"** Based on the location log, " ++ file
+					"** Based on the location log, " ++ desc
 					++ "\n** was expected to be present, " ++
 					"but its content is missing."
 				return False
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 11c3f429ec..a6cced27ff 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -67,19 +67,11 @@ checkRemoteUnused name = do
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
 	showAction "checking for unused data"
-	remotehas <- filterM isthere =<< loggedKeys
+	remotehas <- loggedKeysFor (Remote.uuid r)
 	remoteunused <- excludeReferenced remotehas
 	let list = number 0 remoteunused
 	writeUnusedFile "" list
 	unless (null remoteunused) $ showLongNote $ remoteUnusedMsg r list
-	where
-		{- This should run strictly to avoid the filterM
-		 - building many thunks containing keyLocations data. -}
-		isthere k = do
-			us <- keyLocations k
-			let !there = uuid `elem` us
-			return there
-		uuid = Remote.uuid r
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 8868912dbb..8855cf63b5 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -17,6 +17,7 @@ module Logs.Location (
 	readLog,
 	keyLocations,
 	loggedKeys,
+	loggedKeysFor,
 	logFile,
 	logFileKey
 ) where
@@ -44,6 +45,18 @@ keyLocations = currentLog . logFile
 loggedKeys :: Annex [Key]
 loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Annex.Branch.files
 
+{- Finds all keys that have location log information indicating
+ - they are present for the specified repository. -}
+loggedKeysFor :: UUID -> Annex [Key]
+loggedKeysFor u = filterM isthere =<< loggedKeys
+	where
+		{- This should run strictly to avoid the filterM
+		 - building many thunks containing keyLocations data. -}
+		isthere k = do
+			us <- keyLocations k
+			let !there = u `elem` us
+			return there
+
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
 logFile key = hashDirLower key ++ keyFile key ++ ".log"
diff --git a/debian/changelog b/debian/changelog
index 4a873af945..42454fe8f7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,10 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * drop --from is now supported to remove file content from a remote.
   * status: Now always shows the current repository, even when it does not
     appear in uuid.log.
+  * fsck: Now works in bare repositories. Checks location log information,
+    and file contents. Does not check that numcopies is satisfied, as
+    .gitattributes information about numcopies is not available in a bare
+    repository.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
index 3fd1a8082c..9a044860ae 100644
--- a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -17,3 +17,5 @@ See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html
 >>> While storing the data is no longer an issue in bare repos, fsck would
 >>> need a special mode that examines all the location logs, since it
 >>> cannot run thru the checked out files. --[[Joey]] 
+
+>>>> [[done]]! --[[Joey]] 
diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
index e6980fa281..53331a4f51 100644
--- a/doc/todo/support_fsck_in_bare_repos.mdwn
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -1,6 +1,5 @@
 What is says on the tin:
 
-
     22:56:54 < RichiH> joeyh_: by the way, i have been thinking about fsck on bare repos
     22:57:37 < RichiH> joeyh_: the best i could come with is to have a bare and a non-bare access the same repo store
     22:58:00 < RichiH> joeyh_: alternatively, with the SHA* backend, you have all the information to verify that the local data is correct
@@ -10,3 +9,7 @@ What is says on the tin:
     23:14:51 < joeyh_> unused/dropunused could work in bare repos too btw
 
 > Also `status`'s total annex keys/size could be handled for bare repos. --[[Joey]] 
+
+>> Fsck is done. Rest not done yet. --[[Joey]]
+
+[[!meta title="support status, unused, dropunused in bare repos"]]

From 61000904d74ffd4745dd6808bcfa88289affc169 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 18:47:53 -0400
Subject: [PATCH 2418/2835] refactor

---
 Command.hs      | 10 ++++++++--
 Command/Fsck.hs | 12 +++++-------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/Command.hs b/Command.hs
index 4e312e66d4..32f6743f36 100644
--- a/Command.hs
+++ b/Command.hs
@@ -149,22 +149,28 @@ backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
 runFilteredGen :: (b -> Annex (Maybe a)) -> (b -> FilePath) -> Annex [b] -> Annex [Annex (Maybe a)]
 runFilteredGen a d fs = do
 	matcher <- Limit.getMatcher
-	liftM (map $ proc matcher) fs
+	runActions (proc matcher) fs
 	where
 		proc matcher v = do
 			let f = d v
 			ok <- matcher f
 			if ok then a v else stop
 
+runActions :: (b -> Annex (Maybe a)) -> Annex [b] -> Annex [Annex (Maybe a)]
+runActions a fs = liftM (map a) fs
+
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
 
 isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
 isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
+isBareRepo :: Annex Bool
+isBareRepo = Git.repoIsLocalBare <$> gitRepo
+
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
-	whenM (Git.repoIsLocalBare <$> gitRepo) $
+	whenM isBareRepo $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 6f184a760c..1f30d2eb63 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -13,7 +13,6 @@ import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
 import qualified Backend
-import qualified Git
 import Annex.Content
 import Logs.Location
 import Logs.Trust
@@ -44,14 +43,13 @@ perform key file backend numcopies = check
 
 {- To fsck a bare repository, fsck each key in the location log. -}
 withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek
-withBarePresentKeys a params = do
-	bare <- Git.repoIsLocalBare <$> gitRepo
-	if bare
-		then do
+withBarePresentKeys a params = isBareRepo >>= go
+	where
+		go False = return []
+		go True = do
 			unless (null params) $ do
 				error "fsck should be run without parameters in a bare repository"
-			liftM (map a) loggedKeys
-		else return []
+			runActions a loggedKeys
 
 startBare :: Key -> CommandStart
 startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of

From c102e63595d502c2424552b29e338ab71cb4a098 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 19:03:43 -0400
Subject: [PATCH 2419/2835] status: clean up for bare repositories

The backend usage graph shows present keys as well as keys found in the
repository tree, so it will also be populated for bare repositories.

Changed wording to "visible annex keys", which explains why it's 0 in
a bare repository (no keys visible as no tree), and also why it varies
depending on which branch is checked out. This seemed better than doing
something expensive to look up keys from the git-annex branch.
---
 Command/Status.hs                        | 20 ++++++++++----------
 doc/todo/support_fsck_in_bare_repos.mdwn |  2 +-
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index b5f4956dbf..53d64d0428 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -57,8 +57,8 @@ stats =
 	, bad_data_size
 	, local_annex_keys
 	, local_annex_size
-	, total_annex_keys
-	, total_annex_size
+	, visible_annex_keys
+	, visible_annex_size
 	, backend_usage
 	]
 
@@ -99,16 +99,16 @@ local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
 	keySizeSum <$> cachedKeysPresent
 
-total_annex_size :: Stat
-total_annex_size = stat "total annex size" $
-	keySizeSum <$> cachedKeysReferenced
-
 local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $
 	show . S.size <$> cachedKeysPresent
 
-total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" $
+visible_annex_size :: Stat
+visible_annex_size = stat "visible annex size" $
+	keySizeSum <$> cachedKeysReferenced
+
+visible_annex_keys :: Stat
+visible_annex_keys = stat "visible annex keys" $
 	show . S.size <$> cachedKeysReferenced
 
 tmp_size :: Stat
@@ -118,9 +118,9 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
+backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced <*> cachedKeysPresent
 	where
-		usage ks = pp "" $ reverse . sort $ map swap $ splits $ S.toList ks
+		usage a b = pp "" $ reverse . sort $ map swap $ splits $ S.toList $ S.union a b
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
index 53331a4f51..31481a4a70 100644
--- a/doc/todo/support_fsck_in_bare_repos.mdwn
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -12,4 +12,4 @@ What is says on the tin:
 
 >> Fsck is done. Rest not done yet. --[[Joey]]
 
-[[!meta title="support status, unused, dropunused in bare repos"]]
+[[!meta title="support unused, dropunused in bare repos"]]

From 22e9f445ab74041321d731fd6290f2922384753b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 19:16:45 -0400
Subject: [PATCH 2420/2835] unused, dropunused: Now work in bare repositories.

Turned out I had already done all the work needed to support this when
unused started checking all branches.
---
 Command/DropUnused.hs                    |  2 +-
 Command/Unused.hs                        |  2 +-
 debian/changelog                         |  1 +
 doc/bare_repositories.mdwn               | 20 ++++++++------------
 doc/todo/support_fsck_in_bare_repos.mdwn |  2 ++
 5 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index d2eb3df710..07ae1b48c3 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -35,7 +35,7 @@ withUnusedMaps params = do
 	return $ map (start (unused, unusedbad, unusedtmp)) params
 
 start :: (UnusedMap, UnusedMap, UnusedMap) -> FilePath -> CommandStart
-start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
+start (unused, unusedbad, unusedtmp) s = search
 	[ (unused, perform)
 	, (unusedbad, performOther gitAnnexBadLocation)
 	, (unusedtmp, performOther gitAnnexTmpLocation)
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a6cced27ff..7e9ffa01f5 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -36,7 +36,7 @@ seek = [withNothing start]
 
 {- Finds unused content in the annex. -} 
 start :: CommandStart
-start = notBareRepo $ do
+start = do
 	from <- Annex.getState Annex.fromremote
 	let (name, action) = case from of
 		Nothing -> (".", checkUnused)
diff --git a/debian/changelog b/debian/changelog
index 42454fe8f7..e553877def 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     and file contents. Does not check that numcopies is satisfied, as
     .gitattributes information about numcopies is not available in a bare
     repository.
+  * unused, dropunused: Now work in bare repositories.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index c5663d84cf..f2a3ea4e94 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -7,16 +7,12 @@ Of course, for that to work, the bare repository has to be on a system with
 [[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't
 use git-annex to store stuff there.
 
-Known to work ok:
+It took a while, but bare repositories are now supported exactly as well
+as non-bare repositories. Except for these caveats:
 
-* `git annex move --to` and `--from`, when pointed at a bare repository.
-* `git annex copy` ditto.
-* `git annex drop` can check that a bare repository has a copy of data
-  that is being dropped.
-* `git annex get` can transfer data from a bare repository.
-* Most other stuff (ie, init, describe, trust, etc.)
-
-There are a few caveats to keep in mind when using bare repositories:
-
-* A few subcommands, like `unused` cannot be run in a bare repository.
-  Those subcommands will refuse to do anything.
+* `git annex fsck` works in a bare repository, but does not display
+  warnings about insufficient
+  [[copies]]. To get those warnings, just run it in one of the non-bare
+  checkouts.
+* `git annex unused` in a bare repository only knows about keys used in
+  branches that have been pushed to the bare repository. So use it with care..
diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
index 31481a4a70..32ced467e0 100644
--- a/doc/todo/support_fsck_in_bare_repos.mdwn
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -12,4 +12,6 @@ What is says on the tin:
 
 >> Fsck is done. Rest not done yet. --[[Joey]]
 
+>>> all [[done]]! --[[Joey]] 
+
 [[!meta title="support unused, dropunused in bare repos"]]

From ef5330120c0b522ff159a5b3caba7a926236947b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 19:30:48 -0400
Subject: [PATCH 2421/2835] bare cleanup

---
 Command/Add.hs             | 2 +-
 Command/AddUrl.hs          | 2 +-
 doc/bare_repositories.mdwn | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 82287be0bf..cd18f6c725 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -29,7 +29,7 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
 start :: BackendFile -> CommandStart
-start p@(_, file) = notAnnexed file $ do
+start p@(_, file) = notBareRepo $ notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
 	if isSymbolicLink s || not (isRegularFile s)
 		then stop
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e974d06a14..4382a9c07a 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -26,7 +26,7 @@ seek :: [CommandSeek]
 seek = [withStrings start]
 
 start :: String -> CommandStart
-start s = do
+start s = notBareRepo $ do
 	let u = parseURI s
 	case u of
 		Nothing -> error $ "bad url " ++ s
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index f2a3ea4e94..3bc0a22cbd 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -16,3 +16,5 @@ as non-bare repositories. Except for these caveats:
   checkouts.
 * `git annex unused` in a bare repository only knows about keys used in
   branches that have been pushed to the bare repository. So use it with care..
+* Commands that need a work tree, like `git annex add` won't work in a bare
+  repository, of course.

From 4e9be0d1f86893a469b33b763b55edfe75bdb3aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 23:48:46 -0400
Subject: [PATCH 2422/2835] refactoring and cleanup

No code changes.
---
 Checks.hs          |  45 +++++++++
 CmdLine.hs         |   1 -
 Command.hs         | 233 ++++++---------------------------------------
 Command/FromKey.hs |   1 +
 Command/Fsck.hs    |   2 +-
 Command/SetKey.hs  |   1 +
 Config.hs          |  12 +++
 GitAnnex.hs        |   1 -
 Options.hs         |  39 +++++++-
 Seek.hs            | 117 +++++++++++++++++++++++
 Types/Command.hs   |  45 +++++++++
 git-annex-shell.hs |   1 -
 12 files changed, 288 insertions(+), 210 deletions(-)
 create mode 100644 Checks.hs
 create mode 100644 Seek.hs
 create mode 100644 Types/Command.hs

diff --git a/Checks.hs b/Checks.hs
new file mode 100644
index 0000000000..cd172c6091
--- /dev/null
+++ b/Checks.hs
@@ -0,0 +1,45 @@
+{- git-annex command checks
+ -
+ - Common sanity checks for commands, and an interface to selectively
+ - remove them, or add others.
+ - 
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Checks where
+
+import Common.Annex
+import Types.Command
+import Init
+import qualified Annex
+
+commonChecks :: [CommandCheck]
+commonChecks = [fromOpt, toOpt, repoExists]
+
+repoExists :: CommandCheck
+repoExists = CommandCheck 0 ensureInitialized
+
+fromOpt :: CommandCheck
+fromOpt = CommandCheck 1 $ do
+	v <- Annex.getState Annex.fromremote
+	unless (v == Nothing) $ error "cannot use --from with this command"
+
+toOpt :: CommandCheck
+toOpt = CommandCheck 2 $ do
+	v <- Annex.getState Annex.toremote
+	unless (v == Nothing) $ error "cannot use --to with this command"
+
+checkCommand :: Command -> Annex ()
+checkCommand Command { cmdcheck = c } = sequence_ $ map runCheck c
+
+dontCheck :: CommandCheck -> Command -> Command
+dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
+
+addCheck :: Annex () -> Command -> Command
+addCheck check cmd = mutateCheck cmd $
+	\c -> CommandCheck (length c + 100) check : c
+
+mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command
+mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c }
diff --git a/CmdLine.hs b/CmdLine.hs
index 9f1ded498c..fffd343f0d 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -20,7 +20,6 @@ import qualified Annex.Queue
 import qualified Git
 import Annex.Content
 import Command
-import Options
 
 {- Runs the passed command line. -}
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
diff --git a/Command.hs b/Command.hs
index 32f6743f36..74b1ff21c6 100644
--- a/Command.hs
+++ b/Command.hs
@@ -5,52 +5,38 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Command where
+module Command (
+	module Types.Command,
+	module Seek,
+	module Checks,
+	module Options,
+	command,
+	next,
+	stop,
+	prepCommand,
+	doCommand,
+	notAnnexed,
+	isAnnexed,
+	notBareRepo,
+	isBareRepo,
+	autoCopies
+) where
 
 import Common.Annex
 import qualified Backend
 import qualified Annex
 import qualified Git
-import qualified Git.LsFiles as LsFiles
-import Types.Key
+import Types.Command
 import Logs.Trust
 import Logs.Location
 import Config
-import Backend
-import Limit
-import Init
+import Seek
+import Checks
+import Options
 
-{- A command runs in these stages.
- -
- - a. The check stage runs checks, that error out if
- -    anything prevents the command from running. -}
-data CommandCheck = CommandCheck { idCheck :: Int, runCheck :: Annex () }
-instance Eq CommandCheck where
-	a == b = idCheck a == idCheck b
-{- b. The seek stage takes the parameters passed to the command,
- -    looks through the repo to find the ones that are relevant
- -    to that command (ie, new files to add), and generates
- -    a list of start stage actions. -}
-type CommandSeek = [String] -> Annex [CommandStart]
-{- c. The start stage is run before anything is printed about the
-  -   command, is passed some input, and can early abort it
-  -   if the input does not make sense. It should run quickly and
-  -   should not modify Annex state. -}
-type CommandStart = Annex (Maybe CommandPerform)
-{- d. The perform stage is run after a message is printed about the command
- -    being run, and it should be where the bulk of the work happens. -}
-type CommandPerform = Annex (Maybe CommandCleanup)
-{- e. The cleanup stage is run only if the perform stage succeeds, and it
- -    returns the overall success/fail of the command. -}
-type CommandCleanup = Annex Bool
-
-data Command = Command {
-	cmdcheck :: [CommandCheck],
-	cmdname :: String,
-	cmdparams :: String,
-	cmdseek :: [CommandSeek],
-	cmddesc :: String
-}
+{- Generates a command with the common checks. -}
+command :: String -> String -> [CommandSeek] -> String -> Command
+command = Command commonChecks
 
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
@@ -60,15 +46,14 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
-{- Generates a command with the common checks. -}
-command :: String -> String -> [CommandSeek] -> String -> Command
-command = Command commonChecks
-
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
 prepCommand :: Command -> [String] -> Annex [Annex Bool]
-prepCommand Command { cmdseek = seek } params =
-	return . map doCommand . concat =<< mapM (\s -> s params) seek
+prepCommand cmd ps = return . map doCommand =<< seekCommand cmd ps
+
+{- Runs a command through the seek stage. -}
+seekCommand :: Command -> [String] -> Annex [CommandStart]
+seekCommand Command { cmdseek = seek } ps = concat <$> mapM (\s -> s ps) seek
 
 {- Runs a command through the start, perform and cleanup stages -}
 doCommand :: CommandStart -> CommandCleanup
@@ -81,147 +66,20 @@ doCommand = start
 		success = return True
 		failure = showEndFail >> return False
 
-{- These functions find appropriate files or other things based on a
-   user's parameters, and prepare actions operating on them. -}
-withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
-withFilesInGit a params = do
-	repo <- gitRepo
-	runFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
-withAttrFilesInGit attr a params = do
-	repo <- gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	runFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
-withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
-withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
-	where
-		go (file, v) = a file (readMaybe v)
-withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
-withBackendFilesInGit a params = do
-	repo <- gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	backendPairs a files
-withFilesMissing :: (String -> CommandStart) -> CommandSeek
-withFilesMissing a params = runFiltered a $ liftIO $ filterM missing params
-	where
-		missing = liftM not . doesFileExist
-withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
-withFilesNotInGit a params = do
-	repo <- gitRepo
-	force <- Annex.getState Annex.force
-	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
-	backendPairs a newfiles
-withWords :: ([String] -> CommandStart) -> CommandSeek
-withWords a params = return [a params]
-withStrings :: (String -> CommandStart) -> CommandSeek
-withStrings a params = return $ map a params
-withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
-withFilesToBeCommitted a params = do
-	repo <- gitRepo
-	runFiltered a $
-		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
-withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
-withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
-withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
-withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
-withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
-withFilesUnlocked' typechanged a params = do
-	-- unlocked files have changed type from a symlink to a regular file
-	repo <- gitRepo
-	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
-	unlockedfiles <- liftIO $ filterM notSymlink $
-		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
-	backendPairs a unlockedfiles
-withKeys :: (Key -> CommandStart) -> CommandSeek
-withKeys a params = return $ map (a . parse) params
-	where
-		parse p = fromMaybe (error "bad key") $ readKey p
-withNothing :: CommandStart -> CommandSeek
-withNothing a [] = return [a]
-withNothing _ _ = error "This command takes no parameters."
-
-runFiltered :: (FilePath -> Annex (Maybe a)) -> Annex [FilePath] -> Annex [Annex (Maybe a)]
-runFiltered a = runFilteredGen a id
-
-backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
-backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
-
-runFilteredGen :: (b -> Annex (Maybe a)) -> (b -> FilePath) -> Annex [b] -> Annex [Annex (Maybe a)]
-runFilteredGen a d fs = do
-	matcher <- Limit.getMatcher
-	runActions (proc matcher) fs
-	where
-		proc matcher v = do
-			let f = d v
-			ok <- matcher f
-			if ok then a v else stop
-
-runActions :: (b -> Annex (Maybe a)) -> Annex [b] -> Annex [Annex (Maybe a)]
-runActions a fs = liftM (map a) fs
-
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
 
 isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
 isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
-isBareRepo :: Annex Bool
-isBareRepo = Git.repoIsLocalBare <$> gitRepo
-
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
 	whenM isBareRepo $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
-notSymlink :: FilePath -> IO Bool
-notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
-
-{- Descriptions of params used in usage messages. -}
-paramPaths :: String
-paramPaths = paramOptional $ paramRepeating paramPath -- most often used
-paramPath :: String
-paramPath = "PATH"
-paramKey :: String
-paramKey = "KEY"
-paramDesc :: String
-paramDesc = "DESC"
-paramUrl :: String
-paramUrl = "URL"
-paramNumber :: String
-paramNumber = "NUMBER"
-paramRemote :: String
-paramRemote = "REMOTE"
-paramGlob :: String
-paramGlob = "GLOB"
-paramName :: String
-paramName = "NAME"
-paramUUID :: String
-paramUUID = "UUID"
-paramType :: String
-paramType = "TYPE"
-paramKeyValue :: String
-paramKeyValue = "K=V"
-paramNothing :: String
-paramNothing = ""
-paramRepeating :: String -> String
-paramRepeating s = s ++ " ..."
-paramOptional :: String -> String
-paramOptional s = "[" ++ s ++ "]"
-paramPair :: String -> String -> String
-paramPair a b = a ++ " " ++ b
-
-{- The Key specified by the --key parameter. -}
-cmdlineKey :: Annex Key
-cmdlineKey  = do
-	k <- Annex.getState Annex.defaultkey
-	case k of
-		Nothing -> nokey
-		Just "" -> nokey
-		Just kstring -> maybe badkey return $ readKey kstring
-	where
-		nokey = error "please specify the key with --key"
-		badkey = error "bad key"
+isBareRepo :: Annex Bool
+isBareRepo = Git.repoIsLocalBare <$> gitRepo
 
 {- Used for commands that have an auto mode that checks the number of known
  - copies of a key.
@@ -238,34 +96,3 @@ autoCopies key vs numcopiesattr a = do
 			(_, have) <- trustPartition UnTrusted =<< keyLocations key
 			if length have `vs` needed then a else stop
 		else a
-
-{- Common checks for commands, and an interface to selectively remove them,
- - or add others. -}
-commonChecks :: [CommandCheck]
-commonChecks = [fromOpt, toOpt, repoExists]
-
-repoExists :: CommandCheck
-repoExists = CommandCheck 0 ensureInitialized
-
-fromOpt :: CommandCheck
-fromOpt = CommandCheck 1 $ do
-	v <- Annex.getState Annex.fromremote
-	unless (v == Nothing) $ error "cannot use --from with this command"
-
-toOpt :: CommandCheck
-toOpt = CommandCheck 2 $ do
-	v <- Annex.getState Annex.toremote
-	unless (v == Nothing) $ error "cannot use --to with this command"
-
-checkCommand :: Command -> Annex ()
-checkCommand Command { cmdcheck = c } = sequence_ $ map runCheck c
-
-dontCheck :: CommandCheck -> Command -> Command
-dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
-
-addCheck :: Annex () -> Command -> Command
-addCheck check cmd = mutateCheck cmd $
-	\c -> CommandCheck (length c + 100) check : c
-
-mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command
-mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c }
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index fe9b5c96a0..4e4644708f 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -12,6 +12,7 @@ import Command
 import qualified Annex.Queue
 import Annex.Content
 import Types.Key
+import Config
 
 def :: [Command]
 def = [command "fromkey" paramPath seek "adds a file using a specific key"]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 1f30d2eb63..073652d2ce 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -49,7 +49,7 @@ withBarePresentKeys a params = isBareRepo >>= go
 		go True = do
 			unless (null params) $ do
 				error "fsck should be run without parameters in a bare repository"
-			runActions a loggedKeys
+			prepStart a loggedKeys
 
 startBare :: Key -> CommandStart
 startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 9c31abb083..0c70d12b09 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Command
 import Logs.Location
 import Annex.Content
+import Config
 
 def :: [Command]
 def = [command "setkey" paramPath seek
diff --git a/Config.hs b/Config.hs
index f4c3843af8..f994002b93 100644
--- a/Config.hs
+++ b/Config.hs
@@ -10,6 +10,7 @@ module Config where
 import Common.Annex
 import qualified Git
 import qualified Annex
+import Types.Key (readKey)
 
 type ConfigKey = String
 
@@ -92,3 +93,14 @@ getNumCopies v =
 			return $ read $ Git.configGet g config "1"
 		config = "annex.numcopies"
 
+{- The Key specified by the --key parameter. -}
+cmdlineKey :: Annex Key
+cmdlineKey  = do
+	k <- Annex.getState Annex.defaultkey
+	case k of
+		Nothing -> nokey
+		Just "" -> nokey
+		Just kstring -> maybe badkey return $ readKey kstring
+	where
+		nokey = error "please specify the key with --key"
+		badkey = error "bad key"
diff --git a/GitAnnex.hs b/GitAnnex.hs
index c07e727fa0..89fb4e5919 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -13,7 +13,6 @@ import Common.Annex
 import qualified Git
 import CmdLine
 import Command
-import Options
 import Types.TrustLevel
 import qualified Annex
 import qualified Remote
diff --git a/Options.hs b/Options.hs
index 0c7b4d5f41..a8c165a816 100644
--- a/Options.hs
+++ b/Options.hs
@@ -1,6 +1,6 @@
-{- git-annex dashed options
+{- git-annex command-line options
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -12,7 +12,6 @@ import System.Log.Logger
 
 import Common.Annex
 import qualified Annex
-import Command
 import Limit
 
 {- Each dashed command-line option results in generation of an action
@@ -59,3 +58,37 @@ matcherOptions =
 	where
 		longopt o = Option [] [o] $ NoArg $ addToken o
 		shortopt o = Option o [] $ NoArg $ addToken o
+
+{- Descriptions of params used in usage messages. -}
+paramPaths :: String
+paramPaths = paramOptional $ paramRepeating paramPath -- most often used
+paramPath :: String
+paramPath = "PATH"
+paramKey :: String
+paramKey = "KEY"
+paramDesc :: String
+paramDesc = "DESC"
+paramUrl :: String
+paramUrl = "URL"
+paramNumber :: String
+paramNumber = "NUMBER"
+paramRemote :: String
+paramRemote = "REMOTE"
+paramGlob :: String
+paramGlob = "GLOB"
+paramName :: String
+paramName = "NAME"
+paramUUID :: String
+paramUUID = "UUID"
+paramType :: String
+paramType = "TYPE"
+paramKeyValue :: String
+paramKeyValue = "K=V"
+paramNothing :: String
+paramNothing = ""
+paramRepeating :: String -> String
+paramRepeating s = s ++ " ..."
+paramOptional :: String -> String
+paramOptional s = "[" ++ s ++ "]"
+paramPair :: String -> String -> String
+paramPair a b = a ++ " " ++ b
diff --git a/Seek.hs b/Seek.hs
new file mode 100644
index 0000000000..4ae943157b
--- /dev/null
+++ b/Seek.hs
@@ -0,0 +1,117 @@
+{- git-annex command seeking
+ - 
+ - These functions find appropriate files or other things based on
+ - the values a user passes to a command, and prepare actions operating
+ - on them.
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Seek where
+
+import Common.Annex
+import Types.Command
+import Types.Key
+import Backend
+import qualified Annex
+import qualified Git
+import qualified Git.LsFiles as LsFiles
+import qualified Limit
+
+withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
+withFilesInGit a params = do
+	repo <- gitRepo
+	prepFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+
+withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
+withAttrFilesInGit attr a params = do
+	repo <- gitRepo
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+	prepFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
+
+withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
+withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
+	where
+		go (file, v) = a file (readMaybe v)
+
+withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
+withBackendFilesInGit a params = do
+	repo <- gitRepo
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+	prepBackendPairs a files
+
+withFilesMissing :: (String -> CommandStart) -> CommandSeek
+withFilesMissing a params = prepFiltered a $ liftIO $ filterM missing params
+	where
+		missing = liftM not . doesFileExist
+
+withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesNotInGit a params = do
+	repo <- gitRepo
+	force <- Annex.getState Annex.force
+	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
+	prepBackendPairs a newfiles
+
+withWords :: ([String] -> CommandStart) -> CommandSeek
+withWords a params = return [a params]
+
+withStrings :: (String -> CommandStart) -> CommandSeek
+withStrings a params = return $ map a params
+
+withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
+withFilesToBeCommitted a params = do
+	repo <- gitRepo
+	prepFiltered a $
+		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
+
+withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
+
+withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
+
+withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlocked' typechanged a params = do
+	-- unlocked files have changed type from a symlink to a regular file
+	repo <- gitRepo
+	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
+	unlockedfiles <- liftIO $ filterM notSymlink $
+		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
+	prepBackendPairs a unlockedfiles
+
+withKeys :: (Key -> CommandStart) -> CommandSeek
+withKeys a params = return $ map (a . parse) params
+	where
+		parse p = fromMaybe (error "bad key") $ readKey p
+
+withNothing :: CommandStart -> CommandSeek
+withNothing a [] = return [a]
+withNothing _ _ = error "This command takes no parameters."
+
+
+prepFiltered :: (FilePath -> CommandStart) -> Annex [FilePath] -> Annex [CommandStart]
+prepFiltered a = prepFilteredGen a id
+
+prepBackendPairs :: (BackendFile -> CommandStart) -> CommandSeek
+prepBackendPairs a fs = prepFilteredGen a snd (chooseBackends fs)
+
+prepFilteredGen :: (b -> CommandStart) -> (b -> FilePath) -> Annex [b] -> Annex [CommandStart]
+prepFilteredGen a d fs = do
+	matcher <- Limit.getMatcher
+	prepStart (proc matcher) fs
+	where
+		proc matcher v = do
+			let f = d v
+			ok <- matcher f
+			if ok then a v else return Nothing
+
+{- Generates a list of CommandStart actions that will be run to perform a
+ - command, using a list (ie of files) coming from an action. The list
+ - will be produced and consumed lazily. -}
+prepStart :: (b -> CommandStart) -> Annex [b] -> Annex [CommandStart]
+prepStart a fs = liftM (map a) fs
+
+notSymlink :: FilePath -> IO Bool
+notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
diff --git a/Types/Command.hs b/Types/Command.hs
new file mode 100644
index 0000000000..d39876a7ae
--- /dev/null
+++ b/Types/Command.hs
@@ -0,0 +1,45 @@
+{- git-annex command data types
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.Command where
+
+import Types
+
+{- A command runs in these stages.
+ -
+ - a. The check stage runs checks, that error out if
+ -    anything prevents the command from running. -}
+data CommandCheck = CommandCheck { idCheck :: Int, runCheck :: Annex () }
+{- b. The seek stage takes the parameters passed to the command,
+ -    looks through the repo to find the ones that are relevant
+ -    to that command (ie, new files to add), and generates
+ -    a list of start stage actions. -}
+type CommandSeek = [String] -> Annex [CommandStart]
+{- c. The start stage is run before anything is printed about the
+  -   command, is passed some input, and can early abort it
+  -   if the input does not make sense. It should run quickly and
+  -   should not modify Annex state. -}
+type CommandStart = Annex (Maybe CommandPerform)
+{- d. The perform stage is run after a message is printed about the command
+ -    being run, and it should be where the bulk of the work happens. -}
+type CommandPerform = Annex (Maybe CommandCleanup)
+{- e. The cleanup stage is run only if the perform stage succeeds, and it
+ -    returns the overall success/fail of the command. -}
+type CommandCleanup = Annex Bool
+
+{- A command is defined by specifying these things. -}
+data Command = Command {
+	cmdcheck :: [CommandCheck],
+	cmdname :: String,
+	cmdparams :: String,
+	cmdseek :: [CommandSeek],
+	cmddesc :: String
+}
+
+{- CommandCheck functions can be compared using their unique id. -}
+instance Eq CommandCheck where
+	a == b = idCheck a == idCheck b
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 328d7b1006..10eeb454af 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -12,7 +12,6 @@ import Common.Annex
 import qualified Git
 import CmdLine
 import Command
-import Options
 import Annex.UUID
 
 import qualified Command.ConfigList

From ee715647540a8b5d81254ed60cdf7709f63f42af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 16:38:48 -0400
Subject: [PATCH 2423/2835] add command name to some output

---
 CmdLine.hs                                    | 34 +++++++++++--------
 ...or_bug:_errors_are_not_verbose_enough.mdwn |  2 ++
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index fffd343f0d..658b38ab1c 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -26,22 +26,25 @@ dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
 dispatch args cmds options header gitrepo = do
 	setupConsole
 	state <- Annex.new gitrepo
-	(actions, state') <- Annex.run state $ parseCmd args header cmds options
-	tryRun state' $ [startup] ++ actions ++ [shutdown]
+	((cmd, actions), state') <- Annex.run state $ parseCmd args header cmds options
+	tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
 
 {- Parses command line, stores configure flags, and returns a 
- - list of actions to be run in the Annex monad. -}
-parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool]
+ - list of actions to be run in the Annex monad and the Command
+ - being run. -}
+parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex (Command, [Annex Bool])
 parseCmd argv header cmds options = do
 	(flags, params) <- liftIO getopt
 	when (null params) $ error $ "missing command" ++ usagemsg
-	case lookupCmd (head params) of
-		[] -> error $ "unknown command" ++ usagemsg
+	let (c:rest) = params
+	case lookupCmd c of
+		[] -> error $ "unknown command " ++ c ++ " " ++ usagemsg
 		[cmd] -> do
 			_ <- sequence flags
 			checkCommand cmd
-			prepCommand cmd (drop 1 params)
-		_ -> error "internal error: multiple matching commands"
+			as <- prepCommand cmd rest
+			return (cmd, as)
+		_ -> error $ "internal error: multiple matching commands for " ++ c
 	where
 		getopt = case getOpt Permute options argv of
 			(flags, params, []) ->
@@ -70,10 +73,10 @@ usage header cmds options =
 {- Runs a list of Annex actions. Catches IO errors and continues
  - (but explicitly thrown errors terminate the whole command).
  -}
-tryRun :: Annex.AnnexState -> [Annex Bool] -> IO ()
+tryRun :: Annex.AnnexState -> Command -> [Annex Bool] -> IO ()
 tryRun = tryRun' 0
-tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
-tryRun' errnum state (a:as) = do
+tryRun' :: Integer -> Annex.AnnexState -> Command -> [Annex Bool] -> IO ()
+tryRun' errnum state cmd (a:as) = do
 	result <- try $ Annex.run state $ do
 		Annex.Queue.flushWhenFull
 		a
@@ -82,10 +85,11 @@ tryRun' errnum state (a:as) = do
 			Annex.eval state $ do
 				showErr err
 				showEndFail
-			tryRun' (errnum + 1) state as
-		Right (True,state') -> tryRun' errnum state' as
-		Right (False,state') -> tryRun' (errnum + 1) state' as
-tryRun' errnum _ [] = when (errnum > 0) $ error $ show errnum ++ " failed"
+			tryRun' (errnum + 1) state cmd as
+		Right (True,state') -> tryRun' errnum state' cmd as
+		Right (False,state') -> tryRun' (errnum + 1) state' cmd as
+tryRun' errnum _ cmd [] = when (errnum > 0) $
+		error $ cmdname cmd ++ ": " ++ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
diff --git a/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn b/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn
index 8def2e8c3f..a6620f4255 100644
--- a/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn
+++ b/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn
@@ -22,3 +22,5 @@ Better:
 
 
 etc pp.
+
+> [[done]] --[[Joey]] 

From 56080a0febac6e75e61a6e358f09e0b1e77b77bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 16:44:09 -0400
Subject: [PATCH 2424/2835] closures

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn                 | 3 +++
 ...:_more_descriptive_commit_messages_in_git-annex_branch.mdwn | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
index b5213b8aa5..13980dd292 100644
--- a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -12,3 +12,6 @@ Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?
 > dependency in the cabal file. Your system might be old/new/or broken,
 > perhaps it's time to provide some details about the version of haskell
 > and of `monad-control` you have installed? --[[Joey]] 
+
+>> Closing as apparently user error or a broken system.
+>> If you see this problem please do say. [[done]] --[[Joey]] 
diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
index aad119ffd3..3a891fc9b2 100644
--- a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -34,3 +34,6 @@ in my opinion, the messages should at least contain
 > 
 > It would be possible to make it *less* verbose, with an empty commit
 > message. :) --[[Joey]] 
+
+>> Closing as this is literally impossible to do without making
+>> git-annex worse. [[done]] --[[Joey]] 

From 3cf811ead0f1dcdcb4c8c6e7ea0c97f99220a38f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 16:49:49 -0400
Subject: [PATCH 2425/2835] update; status is no longer slow

---
 doc/todo/cache_key_info.mdwn | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/todo/cache_key_info.mdwn b/doc/todo/cache_key_info.mdwn
index 791584836d..d4352ccf7f 100644
--- a/doc/todo/cache_key_info.mdwn
+++ b/doc/todo/cache_key_info.mdwn
@@ -7,13 +7,14 @@ but that's out of scope -- and recent git-annex versions use queuing
 to save git add from piling up too much in the index.)
 
 But currently two git-annex commands are quite slow when annexes become large
-in quantity of files. These are unused and stats
+in quantity of files. These are unused and status.
 (Both have --fast versions that don't do as much).
+> (Update: status has become acceptably fast; most of its slowdown was due to using a bad data structure; scanning the tree is not particularly slow and it no longer looks at the git-annex branch.)
 
-Both are slow because both need two pieces of information that are not
+unused is slow because it needs two pieces of information that are not
 quick to look up, and require examining the whole repo, very seekily:
 
-1. The keys present in the annex. Found by looking thru .git/annex/objects.
+1. The keys present in the annex. Found by looking thru .git/annex/objects
 2. The keys referenced by files in git. Found by finding every file
    in git, and looking at its symlink.
 

From 1530eac31294347a83c2a7973aa2c27ede9184f3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 16:57:20 -0400
Subject: [PATCH 2426/2835] closures

---
 doc/todo/auto_remotes.mdwn                                  | 6 ++++++
 ...hlist:___34__git_annex_add__34___multiple_processes.mdwn | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn
index c12878950a..715dea7207 100644
--- a/doc/todo/auto_remotes.mdwn
+++ b/doc/todo/auto_remotes.mdwn
@@ -21,3 +21,9 @@ Question: When should git-annex update the remote.log?
 (If not just on init.) Whenever it reads in a repo's remotes?
 
 > This sounds useful and the log should be updated every time any remote is being accessed. A counter or timestamp (yes, distributed times may be wrong/different) could be used to auto-prune old entries via a global and per-remote config setting. -- RichiH
+
+---
+
+I no longer think I'd use this myself, I find that my repositories quickly
+grow the paths I actually use, somewhat organically. Unofficial paths
+across university quads come to mind. [[done]] --[[Joey]] 
diff --git a/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn
index f3d3837015..a04af05b42 100644
--- a/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn
+++ b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn
@@ -5,3 +5,6 @@ i'm in the process of managing my music collection with git-annex. The initial "
 Anyway, thanks for this wonderful piece of software.
 
 Jean-Baptiste
+
+> closing as dup of [[parallel possibilities]] (also see comments below)
+> [[done]] --[[Joey]] 

From e09dd6f306b3f69718c77a03364ee9e51a51bb3b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 20:04:15 -0400
Subject: [PATCH 2427/2835] cleanup

---
 Checks.hs  |  3 --
 CmdLine.hs | 86 ++++++++++++++++++++++++++----------------------------
 Command.hs | 21 +++++++------
 3 files changed, 51 insertions(+), 59 deletions(-)

diff --git a/Checks.hs b/Checks.hs
index cd172c6091..6a70fc52db 100644
--- a/Checks.hs
+++ b/Checks.hs
@@ -31,9 +31,6 @@ toOpt = CommandCheck 2 $ do
 	v <- Annex.getState Annex.toremote
 	unless (v == Nothing) $ error "cannot use --to with this command"
 
-checkCommand :: Command -> Annex ()
-checkCommand Command { cmdcheck = c } = sequence_ $ map runCheck c
-
 dontCheck :: CommandCheck -> Command -> Command
 dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
 
diff --git a/CmdLine.hs b/CmdLine.hs
index 658b38ab1c..af53abc628 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -21,45 +21,41 @@ import qualified Git
 import Annex.Content
 import Command
 
+type Params = [String]
+type Flags = [Annex ()]
+
 {- Runs the passed command line. -}
-dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
+dispatch :: Params -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
 dispatch args cmds options header gitrepo = do
 	setupConsole
 	state <- Annex.new gitrepo
-	((cmd, actions), state') <- Annex.run state $ parseCmd args header cmds options
+	(actions, state') <- Annex.run state $ do
+		sequence_ flags
+		prepCommand cmd params
 	tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
-
-{- Parses command line, stores configure flags, and returns a 
- - list of actions to be run in the Annex monad and the Command
- - being run. -}
-parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex (Command, [Annex Bool])
-parseCmd argv header cmds options = do
-	(flags, params) <- liftIO getopt
-	when (null params) $ error $ "missing command" ++ usagemsg
-	let (c:rest) = params
-	case lookupCmd c of
-		[] -> error $ "unknown command " ++ c ++ " " ++ usagemsg
-		[cmd] -> do
-			_ <- sequence flags
-			checkCommand cmd
-			as <- prepCommand cmd rest
-			return (cmd, as)
-		_ -> error $ "internal error: multiple matching commands for " ++ c
 	where
-		getopt = case getOpt Permute options argv of
-			(flags, params, []) ->
-				return (flags, params)
-			(_, _, errs) ->
-				ioError (userError (concat errs ++ usagemsg))
-		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds
-		usagemsg = "\n\n" ++ usage header cmds options
+		(flags, cmd, params) = parseCmd args cmds options header
+
+{- Parses command line, and returns actions to run to configure flags,
+ - the Command being run, and the remaining parameters for the command. -} 
+parseCmd :: Params -> [Command] -> [Option] -> String -> (Flags, Command, Params)
+parseCmd argv cmds options header = check $ getOpt Permute options argv
+	where
+		check (_, [], []) = err "missing command"
+		check (flags, name:rest, [])
+			| null matches = err $ "unknown command " ++ name
+			| otherwise = (flags, head matches, rest)
+			where
+				matches = filter (\c -> name == cmdname c) cmds
+		check (_, _, errs) = err $ concat errs
+		err msg = error $ msg ++ "\n\n" ++ usage header cmds options
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
-usage header cmds options =
-	usageInfo (header ++ "\n\nOptions:") options ++
-		"\nCommands:\n" ++ cmddescs
+usage header cmds options = usageInfo top options ++ commands
 	where
+		top = header ++ "\n\nOptions:"
+		commands = "\nCommands:\n" ++ cmddescs
 		cmddescs = unlines $ map (indent . showcmd) cmds
 		showcmd c =
 			cmdname c ++
@@ -73,23 +69,23 @@ usage header cmds options =
 {- Runs a list of Annex actions. Catches IO errors and continues
  - (but explicitly thrown errors terminate the whole command).
  -}
-tryRun :: Annex.AnnexState -> Command -> [Annex Bool] -> IO ()
+tryRun :: Annex.AnnexState -> Command -> [CommandCleanup] -> IO ()
 tryRun = tryRun' 0
-tryRun' :: Integer -> Annex.AnnexState -> Command -> [Annex Bool] -> IO ()
-tryRun' errnum state cmd (a:as) = do
-	result <- try $ Annex.run state $ do
-		Annex.Queue.flushWhenFull
-		a
-	case result of
-		Left err -> do
-			Annex.eval state $ do
-				showErr err
-				showEndFail
-			tryRun' (errnum + 1) state cmd as
-		Right (True,state') -> tryRun' errnum state' cmd as
-		Right (False,state') -> tryRun' (errnum + 1) state' cmd as
-tryRun' errnum _ cmd [] = when (errnum > 0) $
-		error $ cmdname cmd ++ ": " ++ show errnum ++ " failed"
+tryRun' :: Integer -> Annex.AnnexState -> Command -> [CommandCleanup] -> IO ()
+tryRun' errnum _ cmd []
+	| errnum > 0 = error $ cmdname cmd ++ ": " ++ show errnum ++ " failed"
+	| otherwise = return ()
+tryRun' errnum state cmd (a:as) = run >>= handle
+	where
+		run = try $ Annex.run state $ do
+			Annex.Queue.flushWhenFull
+			a
+		handle (Left err) = showerr err >> cont False state
+		handle (Right (success, state')) = cont success state'
+		cont success s = tryRun' (if success then errnum else errnum + 1) s cmd as
+		showerr err = Annex.eval state $ do
+			showErr err
+			showEndFail
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
diff --git a/Command.hs b/Command.hs
index 74b1ff21c6..c11b906103 100644
--- a/Command.hs
+++ b/Command.hs
@@ -46,25 +46,24 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
-{- Prepares a list of actions to run to perform a command, based on
- - the parameters passed to it. -}
-prepCommand :: Command -> [String] -> Annex [Annex Bool]
-prepCommand cmd ps = return . map doCommand =<< seekCommand cmd ps
-
-{- Runs a command through the seek stage. -}
-seekCommand :: Command -> [String] -> Annex [CommandStart]
-seekCommand Command { cmdseek = seek } ps = concat <$> mapM (\s -> s ps) seek
+{- Prepares to run a command via the check and seek stages, returning a
+ - list of actions to perform to run the command. -}
+prepCommand :: Command -> [String] -> Annex [CommandCleanup]
+prepCommand Command { cmdseek = seek, cmdcheck = c } params = do
+	sequence_ $ map runCheck c
+	map doCommand . concat <$> mapM (\s -> s params) seek
 
 {- Runs a command through the start, perform and cleanup stages -}
 doCommand :: CommandStart -> CommandCleanup
 doCommand = start
 	where
-		start   = stage $ maybe success perform
+		start   = stage $ maybe skip perform
 		perform = stage $ maybe failure cleanup
-		cleanup = stage $ \r -> showEndResult r >> return r
+		cleanup = stage $ status
 		stage = (=<<)
-		success = return True
+		skip = return True
 		failure = showEndFail >> return False
+		status r = showEndResult r >> return r
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file

From cc1ea8f84463c7e333bfa17a815f250d8d088841 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 12:33:41 -0400
Subject: [PATCH 2428/2835] Removed the setkey command, and added a setcontent
 command with a more useful interface.

---
 Command/Fsck.hs       |  7 ++++--
 Command/SetContent.hs | 56 +++++++++++++++++++++++++++++++++++++++++++
 Command/SetKey.hs     | 48 -------------------------------------
 GitAnnex.hs           |  9 +++----
 debian/changelog      |  2 ++
 doc/git-annex.mdwn    | 23 +++++++++++-------
 6 files changed, 80 insertions(+), 65 deletions(-)
 create mode 100644 Command/SetContent.hs
 delete mode 100644 Command/SetKey.hs

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 073652d2ce..8d6f03d76b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -38,7 +38,7 @@ perform key file backend numcopies = check
 	[ verifyLocationLog key file
 	, checkKeySize key
 	, checkKeyNumCopies key file numcopies
-	, (Types.Backend.fsckKey backend) key
+	, checkBackend backend key
 	]
 
 {- To fsck a bare repository, fsck each key in the location log. -}
@@ -65,7 +65,7 @@ performBare :: Key -> Backend Annex -> CommandPerform
 performBare key backend = check
 	[ verifyLocationLog key (show key)
 	, checkKeySize key
-	, (Types.Backend.fsckKey backend) key
+	, checkBackend backend key
 	]
 
 check :: [Annex Bool] -> CommandPerform	
@@ -134,6 +134,9 @@ checkKeySize key = do
 					return False
 
 
+checkBackend :: Backend Annex -> Key -> Annex Bool
+checkBackend backend key =  (Types.Backend.fsckKey backend) key
+
 checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
diff --git a/Command/SetContent.hs b/Command/SetContent.hs
new file mode 100644
index 0000000000..d62faa445f
--- /dev/null
+++ b/Command/SetContent.hs
@@ -0,0 +1,56 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.SetContent where
+
+import Common.Annex
+import Command
+import Logs.Location
+import Annex.Content
+import qualified Backend
+import qualified Command.Fsck
+
+def :: [Command]
+def = [command "setcontent" (paramPair paramPath paramPath) seek
+	"sets content of annexed file"]
+
+seek :: [CommandSeek]
+seek = [withWords start]
+
+start :: [FilePath] -> CommandStart
+start (src:dest:[]) = do
+	showStart "setkey" dest
+	next $ perform src dest
+start _ = error "specify a src file and a dest file"
+
+perform :: FilePath -> FilePath -> CommandPerform
+perform src dest = go =<< Backend.lookupFile dest
+	where
+		go Nothing = error "dest file is not in annex"
+		go (Just (key, backend)) = do
+			-- the file might be on a different filesystem,
+			-- so mv is used rather than simply calling
+			-- moveToObjectDir; disk space is also
+			-- checked this way.
+			ok <- getViaTmp key $ \tmp ->
+				if dest /= src
+					then liftIO $
+						boolSystem "mv" [File src, File tmp]
+					else return True
+			if ok
+				then next $ cleanup key backend
+				else error "mv failed!"
+
+cleanup :: Key -> Backend Annex -> CommandCleanup
+cleanup key backend = do
+	logStatus key InfoPresent
+
+	-- fsck the new content
+	size_ok <- Command.Fsck.checkKeySize key
+	backend_ok <- Command.Fsck.checkBackend backend key
+
+	return $ size_ok && backend_ok
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
deleted file mode 100644
index 0c70d12b09..0000000000
--- a/Command/SetKey.hs
+++ /dev/null
@@ -1,48 +0,0 @@
-{- git-annex command
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Command.SetKey where
-
-import Common.Annex
-import Command
-import Logs.Location
-import Annex.Content
-import Config
-
-def :: [Command]
-def = [command "setkey" paramPath seek
-	"sets annexed content for a key using a temp file"]
-
-seek :: [CommandSeek]
-seek = [withStrings start]
-
-{- Sets cached content for a key. -}
-start :: FilePath -> CommandStart
-start file = do
-	showStart "setkey" file
-	next $ perform file
-
-perform :: FilePath -> CommandPerform
-perform file = do
-	key <- cmdlineKey
-	-- the file might be on a different filesystem, so mv is used
-	-- rather than simply calling moveToObjectDir; disk space is also
-	-- checked this way.
-	ok <- getViaTmp key $ \dest ->
-		if dest /= file
-			then liftIO $
-				boolSystem "mv" [File file, File dest]
-			else return True
-	if ok
-		then next cleanup
-		else error "mv failed!"
-
-cleanup :: CommandCleanup
-cleanup = do
-	key <- cmdlineKey
-	logStatus key InfoPresent
-	return True
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 89fb4e5919..09f0a118c0 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -26,7 +26,7 @@ import qualified Command.Copy
 import qualified Command.Get
 import qualified Command.FromKey
 import qualified Command.DropKey
-import qualified Command.SetKey
+import qualified Command.SetContent
 import qualified Command.Fix
 import qualified Command.Init
 import qualified Command.Describe
@@ -63,6 +63,7 @@ cmds = concat
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def
+	, Command.SetContent.def
 	, Command.Unannex.def
 	, Command.Uninit.def
 	, Command.PreCommit.def
@@ -72,7 +73,6 @@ cmds = concat
 	, Command.AddUrl.def
 	, Command.FromKey.def
 	, Command.DropKey.def
-	, Command.SetKey.def
 	, Command.Fix.def
 	, Command.Fsck.def
 	, Command.Unused.def
@@ -89,9 +89,7 @@ cmds = concat
 
 options :: [Option]
 options = commonOptions ++
-	[ Option ['k'] ["key"] (ReqArg setkey paramKey)
-		"specify a key to use"
-	, Option ['t'] ["to"] (ReqArg setto paramRemote)
+	[ Option ['t'] ["to"] (ReqArg setto paramRemote)
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
@@ -116,7 +114,6 @@ options = commonOptions ++
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
-		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
 			g <- gitRepo
diff --git a/debian/changelog b/debian/changelog
index e553877def..91e4a18af1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,8 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     .gitattributes information about numcopies is not available in a bare
     repository.
   * unused, dropunused: Now work in bare repositories.
+  * Removed the setkey command, and added a setcontent command with a more
+    useful interface.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index dc0b49ab2c..abddaa0dea 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -274,6 +274,20 @@ subdirectories).
   However, if a backend changes the information it uses to construct a key,
   this can also be used to migrate files to use the new key format.
 
+* setcontent src dest
+
+  Makes the dest file, which must already be tracked by git-annex have the
+  content of the src file. The src file is removed. This can be useful if you
+  have obtained the content of a file from elsewhere and want to put it in
+  the local annex.
+
+  Automatically runs fsck on dest to check that the expected content was
+  provided.
+
+  Example:
+
+	git annex setcontent /tmp/foo.iso foo.iso
+
 * unannex [path ...]
 
   Use this to undo an accidental `git annex add` command. You can use
@@ -321,15 +335,6 @@ subdirectories).
 
 	git annex dropkey SHA1-s10-7da006579dd64330eb2456001fd01948430572f2
 
-* setkey file
-
-  This plumbing-level command sets the annexed data for a key to the
-  content of the specified file, and then removes the file.
-
-  Example:
-
-	git annex setkey --key=WORM-s3-m1287765018--file /tmp/file
-
 # OPTIONS
 
 * --force

From 380839299ea8ffa2c98ce6219e62b979ede0c93b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 12:47:13 -0400
Subject: [PATCH 2429/2835] The fromkey command now takes the key as its first
 parameter. The --key option is no longer used.

---
 Annex.hs                                      |  2 --
 Command/FromKey.hs                            | 22 +++++++++----------
 Command/SetContent.hs                         |  2 +-
 Config.hs                                     | 13 -----------
 debian/changelog                              |  2 ++
 ..._480a4f72445a636eab1b1c0f816d365c._comment |  3 ++-
 doc/git-annex.mdwn                            |  6 +----
 test.hs                                       | 20 ++++++++---------
 8 files changed, 27 insertions(+), 43 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index b9e71c9314..2c6402ac35 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -60,7 +60,6 @@ data AnnexState = AnnexState
 	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
-	, defaultkey :: Maybe String
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
 	, limit :: Either [Utility.Matcher.Token (FilePath -> Annex Bool)] (Utility.Matcher.Matcher (FilePath -> Annex Bool))
@@ -85,7 +84,6 @@ newState gitrepo = AnnexState
 	, catfilehandle = Nothing
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
-	, defaultkey = Nothing
 	, toremote = Nothing
 	, fromremote = Nothing
 	, limit = Left []
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 4e4644708f..b910dd1f09 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -12,26 +12,26 @@ import Command
 import qualified Annex.Queue
 import Annex.Content
 import Types.Key
-import Config
 
 def :: [Command]
-def = [command "fromkey" paramPath seek "adds a file using a specific key"]
+def = [command "fromkey" (paramPair paramKey paramPath) seek
+	"adds a file using a specific key"]
 
 seek :: [CommandSeek]
-seek = [withFilesMissing start]
+seek = [withWords start]
 
-start :: FilePath -> CommandStart
-start file = notBareRepo $ do
-	key <- cmdlineKey
+start :: [String] -> CommandStart
+start (keyname:file:[]) = notBareRepo $ do
+	let key = maybe (error "bad key") id $ readKey keyname
 	inbackend <- inAnnex key
 	unless inbackend $ error $
-		"key ("++keyName key++") is not present in backend"
+		"key ("++ keyname ++") is not present in backend"
 	showStart "fromkey" file
-	next $ perform file
+	next $ perform key file
+start _ = error "specify a key and a dest file"
 
-perform :: FilePath -> CommandPerform
-perform file = do
-	key <- cmdlineKey
+perform :: Key -> FilePath -> CommandPerform
+perform key file = do
 	link <- calcGitLink file key
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 	liftIO $ createSymbolicLink link file
diff --git a/Command/SetContent.hs b/Command/SetContent.hs
index d62faa445f..0ecfa34bd4 100644
--- a/Command/SetContent.hs
+++ b/Command/SetContent.hs
@@ -23,7 +23,7 @@ seek = [withWords start]
 
 start :: [FilePath] -> CommandStart
 start (src:dest:[]) = do
-	showStart "setkey" dest
+	showStart "setcontent" dest
 	next $ perform src dest
 start _ = error "specify a src file and a dest file"
 
diff --git a/Config.hs b/Config.hs
index f994002b93..bf929219df 100644
--- a/Config.hs
+++ b/Config.hs
@@ -10,7 +10,6 @@ module Config where
 import Common.Annex
 import qualified Git
 import qualified Annex
-import Types.Key (readKey)
 
 type ConfigKey = String
 
@@ -92,15 +91,3 @@ getNumCopies v =
 			g <- gitRepo
 			return $ read $ Git.configGet g config "1"
 		config = "annex.numcopies"
-
-{- The Key specified by the --key parameter. -}
-cmdlineKey :: Annex Key
-cmdlineKey  = do
-	k <- Annex.getState Annex.defaultkey
-	case k of
-		Nothing -> nokey
-		Just "" -> nokey
-		Just kstring -> maybe badkey return $ readKey kstring
-	where
-		nokey = error "please specify the key with --key"
-		badkey = error "bad key"
diff --git a/debian/changelog b/debian/changelog
index 91e4a18af1..25f18f7950 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -15,6 +15,8 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * unused, dropunused: Now work in bare repositories.
   * Removed the setkey command, and added a setcontent command with a more
     useful interface.
+  * The fromkey command now takes the key as its first parameter. The --key
+    option is no longer used.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
index 4a22d414b2..787173b0a6 100644
--- a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
@@ -4,5 +4,6 @@
  subject="comment 4"
  date="2011-05-14T16:29:35Z"
  content="""
-Although, if you really do want to shoot yourself in the foot, or know you have the old content, you can use `git-annex setkey`.
+Although, if you really do want to shoot yourself in the foot, or know you
+have the old content, you can use `git-annex setcontent`.
 """]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index abddaa0dea..42997e5342 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -318,7 +318,7 @@ subdirectories).
   This is meant to be called from git's pre-commit hook. `git annex init`
   automatically creates a pre-commit hook using this.
 
-* fromkey file
+* fromkey key file
 
   This plumbing-level command can be used to manually set up a file
   in the git repository to link to a specified key.
@@ -406,10 +406,6 @@ subdirectories).
   are in the annex, their backend is known and this option is not
   necessary.
 
-* --key=name
-
-  Specifies a key to operate on.
-
 * -c name=value
 
   Used to override git configuration settings. May be specified multiple times.
diff --git a/test.hs b/test.hs
index 0ef7449162..fa7657b581 100644
--- a/test.hs
+++ b/test.hs
@@ -88,7 +88,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	-- test order matters, later tests may rely on state from earlier
 	[ test_init
 	, test_add
-	, test_setkey
+	, test_setcontent
 	, test_unannex
 	, test_drop
 	, test_get
@@ -118,15 +118,15 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			writeFile annexedfile $ content annexedfile
 			git_annex "add" ["-q", annexedfile] @? "add failed"
 			annexed_present annexedfile
+			writeFile sha1annexedfile $ content sha1annexedfile
+			git_annex "add" ["-q", sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
+			annexed_present sha1annexedfile
 			writeFile ingitfile $ content ingitfile
 			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
 			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
 			unannexed ingitfile
 		sha1dup = TestCase $ intmpclonerepo $ do
-			writeFile sha1annexedfile $ content sha1annexedfile
-			git_annex "add" ["-q", sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
-			annexed_present sha1annexedfile
 			writeFile sha1annexedfiledup $ content sha1annexedfiledup
 			git_annex "add" ["-q", sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
 			annexed_present sha1annexedfiledup
@@ -140,15 +140,15 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			changeWorkingDirectory "dir"
 			git_annex "add" ["-q", "../dir2"] @? "add of ../subdir failed"
 
-test_setkey :: Test
-test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
+test_setcontent :: Test
+test_setcontent = "git-annex setcontent/fromkey" ~: TestCase $ intmpclonerepo $ do
+	git_annex "drop" ["-q", "--force", sha1annexedfile] @? "drop failed"
 	writeFile tmp $ content sha1annexedfile
 	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
 	let key = show $ fromJust r
-	git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed"
-	git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed"
-	boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
-	annexed_present sha1annexedfile
+	git_annex "setcontent" ["-q", tmp, sha1annexedfile] @? "setcontent failed"
+	git_annex "fromkey" ["-q", key, sha1annexedfiledup] @? "fromkey failed"
+	annexed_present sha1annexedfiledup
 	where
 		tmp = "tmpfile"
 

From 09861cf4f75e2bb9e5597f8b9ea39ab4a33bf4d8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 15:12:02 -0400
Subject: [PATCH 2430/2835] cleanup

---
 Command/SetContent.hs | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/Command/SetContent.hs b/Command/SetContent.hs
index 0ecfa34bd4..b63cc9119a 100644
--- a/Command/SetContent.hs
+++ b/Command/SetContent.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Command
 import Logs.Location
 import Annex.Content
-import qualified Backend
 import qualified Command.Fsck
 
 def :: [Command]
@@ -28,22 +27,18 @@ start (src:dest:[]) = do
 start _ = error "specify a src file and a dest file"
 
 perform :: FilePath -> FilePath -> CommandPerform
-perform src dest = go =<< Backend.lookupFile dest
+perform src dest = isAnnexed dest $ \(key, backend) -> do
+	unlessM (move key) $ error "mv failed!"
+	next $ cleanup key backend
 	where
-		go Nothing = error "dest file is not in annex"
-		go (Just (key, backend)) = do
-			-- the file might be on a different filesystem,
-			-- so mv is used rather than simply calling
-			-- moveToObjectDir; disk space is also
-			-- checked this way.
-			ok <- getViaTmp key $ \tmp ->
-				if dest /= src
-					then liftIO $
-						boolSystem "mv" [File src, File tmp]
-					else return True
-			if ok
-				then next $ cleanup key backend
-				else error "mv failed!"
+		-- the file might be on a different filesystem,
+		-- so mv is used rather than simply calling
+		-- moveToObjectDir; disk space is also
+		-- checked this way.
+		move key = getViaTmp key $ \tmp ->
+			if dest /= src
+			then liftIO $ boolSystem "mv" [File src, File tmp]
+			else return True
 
 cleanup :: Key -> Backend Annex -> CommandCleanup
 cleanup key backend = do

From 3d3e1c4c25f4bbefd0f5e3f445444f3293875a93 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 15:18:41 -0400
Subject: [PATCH 2431/2835] better command name

---
 Command/{SetContent.hs => Reinject.hs}                |  6 +++---
 GitAnnex.hs                                           |  4 ++--
 debian/changelog                                      |  2 +-
 ...omment_4_480a4f72445a636eab1b1c0f816d365c._comment |  3 +--
 doc/git-annex.mdwn                                    | 11 +++++------
 test.hs                                               |  8 ++++----
 6 files changed, 16 insertions(+), 18 deletions(-)
 rename Command/{SetContent.hs => Reinject.hs} (90%)

diff --git a/Command/SetContent.hs b/Command/Reinject.hs
similarity index 90%
rename from Command/SetContent.hs
rename to Command/Reinject.hs
index b63cc9119a..63309aa522 100644
--- a/Command/SetContent.hs
+++ b/Command/Reinject.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Command.SetContent where
+module Command.Reinject where
 
 import Common.Annex
 import Command
@@ -14,7 +14,7 @@ import Annex.Content
 import qualified Command.Fsck
 
 def :: [Command]
-def = [command "setcontent" (paramPair paramPath paramPath) seek
+def = [command "reinject" (paramPair paramPath paramPath) seek
 	"sets content of annexed file"]
 
 seek :: [CommandSeek]
@@ -22,7 +22,7 @@ seek = [withWords start]
 
 start :: [FilePath] -> CommandStart
 start (src:dest:[]) = do
-	showStart "setcontent" dest
+	showStart "reinject" dest
 	next $ perform src dest
 start _ = error "specify a src file and a dest file"
 
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 09f0a118c0..399b26ef7f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -26,7 +26,7 @@ import qualified Command.Copy
 import qualified Command.Get
 import qualified Command.FromKey
 import qualified Command.DropKey
-import qualified Command.SetContent
+import qualified Command.Reinject
 import qualified Command.Fix
 import qualified Command.Init
 import qualified Command.Describe
@@ -63,7 +63,7 @@ cmds = concat
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def
-	, Command.SetContent.def
+	, Command.Reinject.def
 	, Command.Unannex.def
 	, Command.Uninit.def
 	, Command.PreCommit.def
diff --git a/debian/changelog b/debian/changelog
index 25f18f7950..dc6e3c38aa 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,7 +13,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     .gitattributes information about numcopies is not available in a bare
     repository.
   * unused, dropunused: Now work in bare repositories.
-  * Removed the setkey command, and added a setcontent command with a more
+  * Removed the setkey command, and added a reinject command with a more
     useful interface.
   * The fromkey command now takes the key as its first parameter. The --key
     option is no longer used.
diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
index 787173b0a6..fcca0561d4 100644
--- a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
@@ -4,6 +4,5 @@
  subject="comment 4"
  date="2011-05-14T16:29:35Z"
  content="""
-Although, if you really do want to shoot yourself in the foot, or know you
-have the old content, you can use `git-annex setcontent`.
+Now available as `git-annex reinject`.
 """]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 42997e5342..fdd8dd1c19 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -274,19 +274,18 @@ subdirectories).
   However, if a backend changes the information it uses to construct a key,
   this can also be used to migrate files to use the new key format.
 
-* setcontent src dest
+* reinject src dest
 
-  Makes the dest file, which must already be tracked by git-annex have the
-  content of the src file. The src file is removed. This can be useful if you
-  have obtained the content of a file from elsewhere and want to put it in
-  the local annex.
+  Moves the src file into the annex as the content of the dest file.
+  This can be useful if you have obtained the content of a file from
+  elsewhere and want to put it in the local annex.
 
   Automatically runs fsck on dest to check that the expected content was
   provided.
 
   Example:
 
-	git annex setcontent /tmp/foo.iso foo.iso
+	git annex reinject /tmp/foo.iso foo.iso
 
 * unannex [path ...]
 
diff --git a/test.hs b/test.hs
index fa7657b581..d466d3ad32 100644
--- a/test.hs
+++ b/test.hs
@@ -88,7 +88,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	-- test order matters, later tests may rely on state from earlier
 	[ test_init
 	, test_add
-	, test_setcontent
+	, test_reinject
 	, test_unannex
 	, test_drop
 	, test_get
@@ -140,13 +140,13 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			changeWorkingDirectory "dir"
 			git_annex "add" ["-q", "../dir2"] @? "add of ../subdir failed"
 
-test_setcontent :: Test
-test_setcontent = "git-annex setcontent/fromkey" ~: TestCase $ intmpclonerepo $ do
+test_reinject :: Test
+test_reinject = "git-annex reinject/fromkey" ~: TestCase $ intmpclonerepo $ do
 	git_annex "drop" ["-q", "--force", sha1annexedfile] @? "drop failed"
 	writeFile tmp $ content sha1annexedfile
 	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
 	let key = show $ fromJust r
-	git_annex "setcontent" ["-q", tmp, sha1annexedfile] @? "setcontent failed"
+	git_annex "reinject" ["-q", tmp, sha1annexedfile] @? "reinject failed"
 	git_annex "fromkey" ["-q", key, sha1annexedfiledup] @? "fromkey failed"
 	annexed_present sha1annexedfiledup
 	where

From 00988bcf369671bdc3b78e95e3c2ae43f4835b1c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 15:40:57 -0400
Subject: [PATCH 2432/2835] fixed my build environment

---
 debian/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/changelog b/debian/changelog
index dc6e3c38aa..b17586bc68 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     useful interface.
   * The fromkey command now takes the key as its first parameter. The --key
     option is no longer used.
+  * Built without any filename containing .git being excluded. Closes: #647215
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From 3d2a9f84051e9dc705ba4bb4828af691e479ae0e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 16:46:51 -0400
Subject: [PATCH 2433/2835] cleanup

---
 Command/Add.hs        | 15 +++++++--------
 Command/AddUrl.hs     | 12 +++++-------
 Command/Describe.hs   | 10 +++-------
 Command/DropKey.hs    | 14 ++++++++------
 Command/DropUnused.hs |  3 +--
 Command/InitRemote.hs |  4 +++-
 Command/Migrate.hs    | 11 ++++++-----
 Command/Reinject.hs   | 12 ++++++------
 Command/Version.hs    | 11 ++++++-----
 Command/Whereis.hs    |  6 ++----
 Init.hs               |  4 +---
 11 files changed, 48 insertions(+), 54 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index cd18f6c725..a633db7b36 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -38,11 +38,10 @@ start p@(_, file) = notBareRepo $ notAnnexed file $ do
 			next $ perform p
 
 perform :: BackendFile -> CommandPerform
-perform (backend, file) = do
-	k <- Backend.genKey file backend
-	case k of
-		Nothing -> stop
-		Just (key, _) -> do
+perform (backend, file) = Backend.genKey file backend >>= go
+	where
+		go Nothing = stop
+		go (Just (key, _)) = do
 			handle (undo file key) $ moveAnnex key file
 			next $ cleanup file key True
 
@@ -75,9 +74,9 @@ cleanup file key hascontent = do
 	
 			-- touch the symlink to have the same mtime as the
 			-- file it points to
-			s <- liftIO $ getFileStatus file
-			let mtime = modificationTime s
-			liftIO $ touch file (TimeSpec mtime) False
+			liftIO $ do
+				mtime <- modificationTime <$> getFileStatus file
+				touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 4382a9c07a..b717e271d1 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -26,15 +26,14 @@ seek :: [CommandSeek]
 seek = [withStrings start]
 
 start :: String -> CommandStart
-start s = notBareRepo $ do
-	let u = parseURI s
-	case u of
-		Nothing -> error $ "bad url " ++ s
-		Just url -> do
+start s = notBareRepo $ go $ parseURI s
+	where
+		go Nothing = error $ "bad url " ++ s
+		go (Just url) = do
 			file <- liftIO $ url2file url
 			showStart "addurl" file
 			next $ perform s file
-			
+
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
 	fast <- Annex.getState Annex.fast
@@ -64,7 +63,6 @@ nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
 	let key = Backend.URL.fromUrl url
 	setUrlPresent key url
-	
 	next $ Command.Add.cleanup file key False
 
 url2file :: URI -> IO FilePath
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 882a0e1bb9..1cc81bcbd5 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -20,15 +20,11 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: [String] -> CommandStart
-start ws = do
-	let (name, description) =
-		case ws of
-			(n:d) -> (n,unwords d)
-			_ -> error "Specify a repository and a description."
-	
+start (name:description) = do
 	showStart "describe" name
 	u <- Remote.nameToUUID name
-	next $ perform u description
+	next $ perform u $ unwords description
+start _ = do error "Specify a repository and a description."	
 
 perform :: UUID -> String -> CommandPerform
 perform u description = do
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index d00bb6c83e..cac955b4ac 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -23,14 +23,16 @@ seek = [withKeys start]
 start :: Key -> CommandStart
 start key = do
 	present <- inAnnex key
-	force <- Annex.getState Annex.force
 	if not present
 		then stop
-		else if not force
-			then error "dropkey is can cause data loss; use --force if you're sure you want to do this"
-			else do
-				showStart "dropkey" (show key)
-				next $ perform key
+		else do
+			checkforced
+			showStart "dropkey" (show key)
+			next $ perform key
+	where
+		checkforced = 
+			unlessM (Annex.getState Annex.force) $
+				error "dropkey can cause data loss; use --force if you're sure you want to do this"
 
 perform :: Key -> CommandPerform
 perform key = do
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 07ae1b48c3..70dcc4cc72 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -71,8 +71,7 @@ readUnusedLog prefix = do
 	let f = gitAnnexUnusedLog prefix g
 	e <- liftIO $ doesFileExist f
 	if e
-		then return . M.fromList . map parse . lines
-			=<< liftIO (readFile f)
+		then M.fromList . map parse . lines <$> liftIO (readFile f)
 		else return M.empty
 	where
 		parse line = (head ws, fromJust $ readKey $ unwords $ tail ws)
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 4ba5b07873..600e17eb84 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -67,7 +67,9 @@ findByName name = do
 			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
-findByName' n m = if null matches then Nothing else Just $ head matches
+findByName' n m
+	| null matches = Nothing
+	| otherwise = Just $ head matches
 	where
 		matches = filter (matching . snd) $ M.toList m
 		matching c = case M.lookup nameKey c of
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index a68582996b..2d4d24a224 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -56,11 +56,7 @@ perform file oldkey newbackend = do
 	case k of
 		Nothing -> stop
 		Just (newkey, _) -> do
-			ok <- getViaTmpUnchecked newkey $ \t -> do
-				-- Make a hard link to the old backend's
-				-- cached key, to avoid wasting disk space.
-				liftIO $ unlessM (doesFileExist t) $ createLink src t
-				return True
+			ok <- link src newkey
 			if ok
 				then do
 					-- Update symlink to use the new key.
@@ -77,3 +73,8 @@ perform file oldkey newbackend = do
 				else stop
 	where
 		cleantmp t = whenM (doesFileExist t) $ removeFile t
+		link src newkey = getViaTmpUnchecked newkey $ \t -> do
+			-- Make a hard link to the old backend's
+			-- cached key, to avoid wasting disk space.
+			liftIO $ unlessM (doesFileExist t) $ createLink src t
+			return True	
diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 63309aa522..3de24f3fc3 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -21,9 +21,11 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: [FilePath] -> CommandStart
-start (src:dest:[]) = do
-	showStart "reinject" dest
-	next $ perform src dest
+start (src:dest:[])
+	| src == dest = stop
+	| otherwise = do
+		showStart "reinject" dest
+		next $ perform src dest
 start _ = error "specify a src file and a dest file"
 
 perform :: FilePath -> FilePath -> CommandPerform
@@ -36,9 +38,7 @@ perform src dest = isAnnexed dest $ \(key, backend) -> do
 		-- moveToObjectDir; disk space is also
 		-- checked this way.
 		move key = getViaTmp key $ \tmp ->
-			if dest /= src
-			then liftIO $ boolSystem "mv" [File src, File tmp]
-			else return True
+			liftIO $ boolSystem "mv" [File src, File tmp]
 
 cleanup :: Key -> Backend Annex -> CommandCleanup
 cleanup key backend = do
diff --git a/Command/Version.hs b/Command/Version.hs
index 5a45fd77f2..a584264828 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -21,12 +21,13 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion
 	v <- getVersion
-	liftIO $ putStrLn $ "local repository version: " ++ fromMaybe "unknown" v
-	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion
-	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions
-	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
+	liftIO $ do
+		putStrLn $ "git-annex version: " ++ SysConfig.packageversion
+		putStrLn $ "local repository version: " ++ fromMaybe "unknown" v
+		putStrLn $ "default repository version: " ++ defaultVersion
+		putStrLn $ "supported repository versions: " ++ vs supportedVersions
+		putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
 	stop
 	where
 		vs = join " "
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 7799af08c6..0681bfba1e 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -31,11 +31,9 @@ perform key = do
 	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
 	pp <- prettyPrintUUIDs "whereis" safelocations
-	unless (null safelocations) $
-		showLongNote pp
+	unless (null safelocations) $ showLongNote pp
 	pp' <- prettyPrintUUIDs "untrusted" untrustedlocations
-	unless (null untrustedlocations) $
-		showLongNote $ untrustedheader ++ pp'
+	unless (null untrustedlocations) $ showLongNote $ untrustedheader ++ pp'
 	if null safelocations then stop else next $ return True
 	where
 		copiesplural 1 = "copy"
diff --git a/Init.hs b/Init.hs
index 6e024e9fc5..1fac80820a 100644
--- a/Init.hs
+++ b/Init.hs
@@ -65,9 +65,7 @@ gitPreCommitHookUnWrite = unlessBare $ do
 				" Edit it to remove call to git annex."
 
 unlessBare :: Annex () -> Annex ()
-unlessBare a = do
-	g <- gitRepo
-	unless (Git.repoIsLocalBare g) a
+unlessBare = unlessM $ Git.repoIsLocalBare <$> gitRepo
 
 preCommitHook :: Annex FilePath
 preCommitHook = do

From c643136e32b99a650a6fdea0731ea5af275f6866 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 23:39:55 -0400
Subject: [PATCH 2434/2835] playing with >=>

Apparently in haskell if you teach a man to fish, he'll write
more pointfree code.
---
 Backend/SHA.hs  | 2 +-
 Command/Fsck.hs | 2 +-
 Limit.hs        | 4 ++--
 Utility/Misc.hs | 5 +++--
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 3a54a8871b..3f210df1f1 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -83,7 +83,7 @@ keyValue size file = do
 
 {- Extension preserving keys. -}
 keyValueE :: SHASize -> FilePath -> Annex (Maybe Key)
-keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
+keyValueE size file = keyValue >>= maybe (return Nothing) addE
 	where
 		addE k = return $ Just $ k
 			{ keyName = keyName k ++ extension
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 8d6f03d76b..d1abb29e37 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -69,7 +69,7 @@ performBare key backend = check
 	]
 
 check :: [Annex Bool] -> CommandPerform	
-check s = sequence s >>= dispatch
+check = sequence >=> dispatch
 	where
 		dispatch vs
 			| all (== True) vs = next $ return True
diff --git a/Limit.hs b/Limit.hs
index 490577e80f..3ae949bfb9 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -67,7 +67,7 @@ addExclude glob = addLimit $ return . notExcluded
 addIn :: String -> Annex ()
 addIn name = addLimit $ check $ if name == "." then inAnnex else inremote
 	where
-		check a f = Backend.lookupFile f >>= handle a
+		check a = Backend.lookupFile >=> handle a
 		handle _ Nothing = return False
 		handle a (Just (key, _)) = a key
 		inremote key = do
@@ -83,7 +83,7 @@ addCopies num =
 		Nothing -> error "bad number for --copies"
 		Just n -> addLimit $ check n
 	where
-		check n f = Backend.lookupFile f >>= handle n
+		check n = Backend.lookupFile >=> handle n
 		handle _ Nothing = return False
 		handle n (Just (key, _)) = do
 			us <- keyLocations key
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index bc18347746..b3bf226120 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -8,15 +8,16 @@
 module Utility.Misc where
 
 import System.IO
+import Control.Monad
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
 hGetContentsStrict :: Handle -> IO String
-hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
+hGetContentsStrict  = hGetContents >=> \s -> length s `seq` return s
 
 {- A version of readFile that is not lazy. -}
 readFileStrict :: FilePath -> IO String
-readFileStrict f = readFile f >>= \s -> length s `seq` return s
+readFileStrict = readFile >=> \s -> length s `seq` return s
 
 {- Attempts to read a value from a String. -}
 readMaybe :: (Read a) => String -> Maybe a

From e9286e7be7a8b68d9a0e02e909bf58c7936b12af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 12:02:04 -0400
Subject: [PATCH 2435/2835] point to new extension now in mercurial

---
 doc/not.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/not.mdwn b/doc/not.mdwn
index 2827dd12d0..ad278da0dd 100644
--- a/doc/not.mdwn
+++ b/doc/not.mdwn
@@ -43,7 +43,7 @@
   same repository, which git-annex does, and is an important feature for
   large-scale archiving.
 
-* git-annex is not the [Mercurial bfiles extension](http://mercurial.selenic.com/wiki/BfilesExtension).
+* git-annex is not the [Mercurial largefiles extension](http://mercurial.selenic.com/wiki/LargefilesExtension).
   Although mercurial and git have some of the same problems around large
   files, and both try to solve them in similar ways (standin files using
   mostly hashes of the real content).

From 8e249ea0bdbf0b0890d6e3d3ed1e30c1df36e136 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 13:32:19 -0400
Subject: [PATCH 2436/2835] add tip for mode of operation somewhat like
 mercurial bigfiles extension

---
 .../automatically_getting_files_on_checkout.mdwn  | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/tips/automatically_getting_files_on_checkout.mdwn

diff --git a/doc/tips/automatically_getting_files_on_checkout.mdwn b/doc/tips/automatically_getting_files_on_checkout.mdwn
new file mode 100644
index 0000000000..bbb3b302eb
--- /dev/null
+++ b/doc/tips/automatically_getting_files_on_checkout.mdwn
@@ -0,0 +1,15 @@
+Normally git-annex does not retrieve file contents when checking out a
+tree. In some use cases, it makes sense to always have the contents of
+files available after a `git checkout` or `git update`. This can be
+accomplished by installing the following as `.git/hooks/post-checkout`
+
+	#!/bin/sh
+	# Uses git-annex to get all files in the specified directories
+	# (relative to the top of the repository) on checkout.
+	dirs=.
+	top="$(git rev-parse --show-toplevel)"
+	for dir in "$dirs"; do git annex get $top/$dir"; done
+
+By default, all files in the whole repository will be made available. The
+`dirs` setting can be configured if you only want to get files in certian
+directories. 

From eec137f33a2b6982403d7dd9187500ad5a14b868 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 14:18:21 -0400
Subject: [PATCH 2437/2835] Record uuid when auto-initializing a remote so it
 shows in status.

---
 Backend/SHA.hs   |  2 +-
 Init.hs          |  2 ++
 Logs/UUID.hs     | 10 ++++++++++
 debian/changelog |  1 +
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 3f210df1f1..3a54a8871b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -83,7 +83,7 @@ keyValue size file = do
 
 {- Extension preserving keys. -}
 keyValueE :: SHASize -> FilePath -> Annex (Maybe Key)
-keyValueE size file = keyValue >>= maybe (return Nothing) addE
+keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 	where
 		addE k = return $ Just $ k
 			{ keyName = keyName k ++ extension
diff --git a/Init.hs b/Init.hs
index 1fac80820a..8c79002bcb 100644
--- a/Init.hs
+++ b/Init.hs
@@ -15,6 +15,7 @@ import Common.Annex
 import Utility.TempFile
 import qualified Git
 import qualified Annex.Branch
+import Logs.UUID
 import Annex.Version
 import Annex.UUID
 
@@ -24,6 +25,7 @@ initialize = do
 	Annex.Branch.create
 	setVersion
 	gitPreCommitHookWrite
+	getUUID >>= recordUUID
 
 uninitialize :: Annex ()
 uninitialize = gitPreCommitHookUnWrite
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index c05c4e3481..77cfb5ce0f 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -15,6 +15,7 @@
 
 module Logs.UUID (
 	describeUUID,
+	recordUUID,
 	uuidMap
 ) where
 
@@ -37,6 +38,15 @@ describeUUID uuid desc = do
 	Annex.Branch.change logfile $
 		showLog id . changeLog ts uuid desc . parseLog Just
 
+{- Records the uuid in the log, if it's not already there. -}
+recordUUID :: UUID -> Annex ()
+recordUUID u = go . M.lookup u =<< uuidMap 
+	where
+		go (Just "") = set
+		go Nothing = set
+		go _ = return ()
+		set = describeUUID u ""
+
 {- Read the uuidLog into a simple Map.
  -
  - The UUID of the current repository is included explicitly, since
diff --git a/debian/changelog b/debian/changelog
index b17586bc68..5beb32ee1f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -18,6 +18,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * The fromkey command now takes the key as its first parameter. The --key
     option is no longer used.
   * Built without any filename containing .git being excluded. Closes: #647215
+  * Record uuid when auto-initializing a remote so it shows in status.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From c33313c50b1a9424dd20e6e9995c46d60a48e540 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 14:24:44 -0400
Subject: [PATCH 2438/2835] tweak

---
 Command/Init.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index e2a6eb7b03..69cdc8e8a4 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -30,6 +30,5 @@ start ws = do
 perform :: String -> CommandPerform
 perform description = do
 	initialize
-	u <- getUUID
-	describeUUID u description
+	getUUID >>= describeUUID description
 	next $ return True

From 5f3dd3d246c757ee92aa8e68654071519364c0e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 15:09:19 -0400
Subject: [PATCH 2439/2835] ensure directory exists when locking journal

Fixes git annex init in a bare repository that already has a git-annex
branch.
---
 Annex/Branch.hs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 645a2de762..4c3192f531 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -250,10 +250,9 @@ files = withIndexUpdate $ do
 setJournalFile :: FilePath -> String -> Annex ()
 setJournalFile file content = do
 	g <- gitRepo
-	liftIO $ catch (write g) $ const $ do
+	liftIO $ doRedo (write g) $ do
 		createDirectoryIfMissing True $ gitAnnexJournalDir g
 		createDirectoryIfMissing True $ gitAnnexTmpDir g
-		write g
 	where
 		-- journal file is written atomically
 		write g = do
@@ -342,7 +341,13 @@ lockJournal a = do
 	bracketIO (lock file) unlock a
 	where
 		lock file = do
-			l <- createFile file stdFileMode
+			l <- doRedo (createFile file stdFileMode) $
+				createDirectoryIfMissing True $ takeDirectory file
 			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
 			return l
 		unlock = closeFd
+
+{- Runs an action, catching failure and running something to fix it up, and
+ - retrying if necessary. -}
+doRedo :: IO a -> IO b -> IO a
+doRedo a b = catch a $ const $ b >> a

From 7df8052a4b5289ed48f0bcb9aa95b319caed76d5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Thu, 3 Nov 2011 23:53:51 +0000
Subject: [PATCH 2440/2835]

---
 doc/forum/Recommended_number_of_repositories.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/forum/Recommended_number_of_repositories.mdwn

diff --git a/doc/forum/Recommended_number_of_repositories.mdwn b/doc/forum/Recommended_number_of_repositories.mdwn
new file mode 100644
index 0000000000..9e9f2838d6
--- /dev/null
+++ b/doc/forum/Recommended_number_of_repositories.mdwn
@@ -0,0 +1,4 @@
+With git it is easy to create one repository per project, and it almost always makes sense to do so.  When using git-annex, what is the recommended setup?
+
+Should one have a single annex containing all files, or is it recommended to create different repositories for things like 'photos', 'music', 'isos' ?
+

From 532ff19aa5a18821fdb36df60837ba9690d0c963 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Nov 2011 13:14:38 -0400
Subject: [PATCH 2441/2835] fix link

---
 doc/design/encryption.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn
index e5053134ee..647683bd9f 100644
--- a/doc/design/encryption.mdwn
+++ b/doc/design/encryption.mdwn
@@ -1,6 +1,6 @@
 This was the design doc for [[/encryption]] and is preserved for
 the curious. For an example of using git-annex with an encrypted S3 remote,
-see [[walkthrough/using_Amazon_S3]].
+see [[tips/using_Amazon_S3]].
 
 [[!toc]]
 

From 1089e85d48a0d3c455fc2f4139b82484b94b5bbe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Nov 2011 15:01:43 -0400
Subject: [PATCH 2442/2835] add changelog for bugfix

---
 debian/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 5beb32ee1f..e59b4f4048 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,6 +19,8 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     option is no longer used.
   * Built without any filename containing .git being excluded. Closes: #647215
   * Record uuid when auto-initializing a remote so it shows in status.
+  * Bugfix: Fixed git-annex init crash in a bare repository when there was
+    already an existing git-annex branch.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From ef3457196ace3669ddfa93039f2d3c15baf54713 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Nov 2011 15:21:45 -0400
Subject: [PATCH 2443/2835] use SHA256 by default

To get old behavior, add a .gitattributes containing: * annex.backend=WORM

I feel that SHA256 is a better default for most people, as long as their
systems are fast enough that checksumming their files isn't a problem.
git-annex should default to preserving the integrity of data as well as git
does. Checksum backends also work better with editing files via
unlock/lock.

I considered just using SHA1, but since that hash is believed to be somewhat
near to being broken, and git-annex deals with large files which would be a
perfect exploit medium, I decided to go to a SHA-2 hash.

SHA512 is annoyingly long when displayed, and git-annex displays it in a
few places (and notably it is shown in ls -l), so I picked the shorter
hash. Considered SHA224 as it's even shorter, but feel it's a bit weird.

I expect git-annex will use SHA-3 at some point in the future, but
probably not soon!

Note that systems without a sha256sum (or sha256) program will fall back to
defaulting to SHA1.
---
 Backend.hs                                    |  4 +--
 Backend/SHA.hs                                |  6 ++--
 debian/changelog                              |  3 ++
 doc/backends.mdwn                             | 32 +++++++++++--------
 doc/walkthrough/adding_files.mdwn             |  4 +--
 ...ing_file_content_between_repositories.mdwn |  2 +-
 doc/walkthrough/unused_data.mdwn              | 14 ++++----
 doc/walkthrough/using_ssh_remotes.mdwn        |  2 +-
 8 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index a09fc0e990..9a40e54598 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -26,12 +26,12 @@ import Types.Key
 import qualified Types.Backend as B
 
 -- When adding a new backend, import it here and add it to the list.
-import qualified Backend.WORM
 import qualified Backend.SHA
+import qualified Backend.WORM
 import qualified Backend.URL
 
 list :: [Backend Annex]
-list = Backend.WORM.backends ++ Backend.SHA.backends ++ Backend.URL.backends
+list = Backend.SHA.backends ++ Backend.WORM.backends ++ Backend.URL.backends
 
 {- List of backends in the order to try them when storing a new key. -}
 orderedList :: Annex [Backend Annex]
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 3a54a8871b..d449821172 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -16,12 +16,12 @@ import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
 
+-- order is slightly significant; want SHA256 first, and more general
+-- sizes earlier
 sizes :: [Int]
-sizes = [1, 256, 512, 224, 384]
+sizes = [256, 1, 512, 224, 384]
 
 backends :: [Backend Annex]
--- order is slightly significant; want sha1 first, and more general
--- sizes earlier
 backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
 genBackend :: SHASize -> Maybe (Backend Annex)
diff --git a/debian/changelog b/debian/changelog
index e59b4f4048..e74a190ba5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,8 @@
 git-annex (3.20111026) UNRELEASED; urgency=low
 
+  * The default backend used when adding files to the annex is changed
+    from WORM to SHA256.
+    To get old behavior, add a .gitattributes containing: * annex.backend=WORM
   * Sped up some operations on remotes that are on the same host.
   * copy --to: Fixed leak when copying many files to a remote on the same
     host.
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index ebcdedc2a7..2030d107a3 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -5,17 +5,19 @@ to retrieve the file's content (its value).
 Multiple pluggable key-value backends are supported, and a single repository
 can use different ones for different files.
 
-* `WORM` ("Write Once, Read Many") This assumes that any file with
-  the same basename, size, and modification time has the same content.
-  This is the default, and the least expensive backend.
-* `SHA1` -- This uses a key based on a sha1 checksum. This allows
+* `SHA256` -- The default backend for new files. This allows
   verifying that the file content is right, and can avoid duplicates of
   files with the same content. Its need to generate checksums
-  can make it slower for large files.
-* `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
-  checksums. Mostly useful for the very paranoid, or anyone who is
-  researching checksum collisions and wants to annex their colliding data. ;)
-* `SHA1E`, `SHA512E`, etc -- Variants that preserve filename extension as
+  can make it slower for large files. 
+* `WORM` ("Write Once, Read Many") This assumes that any file with
+  the same basename, size, and modification time has the same content.
+  This is the the least expensive backend, recommended for really large
+  files or slow systems.
+* `SHA512` -- Best currently available hash, for the very paranoid.
+* `SHA1` -- Smaller hash than `SHA256` for those who want a checksum
+   but are not concerned about security.
+* `SHA384`, `SHA224` -- Hashes for people who like unusual sizes.
+* `SHA256E`, `SHA1E`, etc -- Variants that preserve filename extension as
   part of the key. Useful for archival tasks where the filename extension
   contains metadata that should be preserved.
 
@@ -27,9 +29,11 @@ For finer control of what backend is used when adding different types of
 files, the `.gitattributes` file can be used. The `annex.backend`
 attribute can be set to the name of the backend to use for matching files.
 
-For example, to use the SHA1 backend for sound files, which tend to be
-smallish and might be modified or copied over time, you could set in
-`.gitattributes`:
+For example, to use the SHA256 backend for sound files, which tend to be
+smallish and might be modified or copied over time,
+while using the WORM backend for everything else, you could set
+in `.gitattributes`:
 
-	*.mp3 annex.backend=SHA1
-	*.ogg annex.backend=SHA1
+	* annex.backend=WORM
+	*.mp3 annex.backend=SHA256
+	*.ogg annex.backend=SHA256
diff --git a/doc/walkthrough/adding_files.mdwn b/doc/walkthrough/adding_files.mdwn
index 77a7fbc154..d1b5a04f77 100644
--- a/doc/walkthrough/adding_files.mdwn
+++ b/doc/walkthrough/adding_files.mdwn
@@ -2,8 +2,8 @@
 	# cp /tmp/big_file .
 	# cp /tmp/debian.iso .
 	# git annex add .
-	add big_file ok
-	add debian.iso ok
+	add big_file (checksum...) ok
+	add debian.iso (checksum...) ok
 	# git commit -a -m added
 
 When you add a file to the annex and commit it, only a symlink to
diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn
index 27dffe9138..3ffcc11750 100644
--- a/doc/walkthrough/moving_file_content_between_repositories.mdwn
+++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn
@@ -9,5 +9,5 @@ makes it very easy.
 	move my_cool_big_file (to usbdrive...) ok
 	# git annex move video/hackity_hack_and_kaxxt.mov --from fileserver
 	move video/hackity_hack_and_kaxxt.mov (from fileserver...)
-	WORM-s86050597-m1274316523--hackity_hack_and_kax 100%   82MB 199.1KB/s   07:02
+	SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e   100%   82MB 199.1KB/s   07:02
 	ok
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index e142b576c0..bd6c398710 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -1,8 +1,8 @@
-It's possible for data to accumulate in the annex that no files point to
-anymore. One way it can happen is if you `git rm` a file without 
-first calling `git annex drop`. And, when you modify an annexed file, the old
-content of the file remains in the annex. Another way is when migrating
-between key-value [[backends|backend]].
+It's possible for data to accumulate in the annex that no files in any
+branch point to anymore. One way it can happen is if you `git rm` a file
+without first calling `git annex drop`. And, when you modify an annexed
+file, the old content of the file remains in the annex. Another way is when
+migrating between key-value [[backends|backend]].
 
 This might be historical data you want to preserve, so git-annex defaults to
 preserving it. So from time to time, you may want to check for such data and
@@ -12,8 +12,8 @@ eliminate it to save space.
 	unused . (checking for unused data...) 
 	  Some annexed data is no longer used by any files in the repository.
 	    NUMBER  KEY
-	    1       WORM-s3-m1289672605--file
-	    2       WORM-s14-m1289672605--file
+	    1       SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e
+	    2       SHA1-s14--f1358ec1873d57350e3dc62054dc232bc93c2bd1
 	  (To see where data was previously used, try: git log --stat -S'KEY')
 	  (To remove unwanted data: git-annex dropunused NUMBER)
 	ok
diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn
index fbbbbe0701..60011a200b 100644
--- a/doc/walkthrough/using_ssh_remotes.mdwn
+++ b/doc/walkthrough/using_ssh_remotes.mdwn
@@ -13,7 +13,7 @@ Now you can get files and they will be transferred (using `rsync` via `ssh`):
 
 	# git annex get my_cool_big_file
 	get my_cool_big_file (getting UUID for origin...) (from origin...)
-	WORM-s2159-m1285650548--my_cool_big_file       100% 2159     2.1KB/s   00:00
+	SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e  100% 2159     2.1KB/s   00:00
 	ok
 
 When you drop files, git-annex will ssh over to the remote and make

From dabb6a9f265d8babd8885806bf5475a901bdbd90 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 4 Nov 2011 19:59:24 +0000
Subject: [PATCH 2444/2835] Added a comment: depends ...

---
 ...nt_1_3ef256230756be8a9679b107cdbfd018._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment

diff --git a/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment b/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment
new file mode 100644
index 0000000000..46ce0e8d53
--- /dev/null
+++ b/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="depends ..."
+ date="2011-11-04T19:59:24Z"
+ content="""
+It makes sense to have separate repositories when you have well-defined uses for them. 
+
+I have a separate repository just for music and podcasts, which I can put various places where I have no need of the overhead of a tree of other files.
+
+If you're using it for whatever arbitrary large files you accumulate, I find it's useful to have them in one repository. This way I can rearrange things as makes sense. It might make sense to have \"photos\" and \"isos\" as categories today, but next year you might prefer to move those under 2011/{photos,isos}. It would certainly make sense to have different repositories for home, work, etc.
+
+How to split repositories up for a home directory is a general problem that the [vcs-home](http://vcs-home.branchable.com)
+project has surely considered at one time or another.
+"""]]

From 502f86604fd90d6f52236b38612a39ec6e713be7 Mon Sep 17 00:00:00 2001
From: Valentin_Haenel 
Date: Fri, 4 Nov 2011 23:19:13 +0000
Subject: [PATCH 2445/2835] a recipe for setting up a bare remote

---
 doc/bare_repositories.mdwn | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index 3bc0a22cbd..bf56d81446 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -18,3 +18,27 @@ as non-bare repositories. Except for these caveats:
   branches that have been pushed to the bare repository. So use it with care..
 * Commands that need a work tree, like `git annex add` won't work in a bare
   repository, of course.
+
+***
+
+Here is a quick example of how to set this up, using `origin` as the remote name, and assuming `~/annex` contains an annex:
+
+On the server:
+
+    mkdir bare-annex
+    git init --bare
+    git annex init origin
+
+Now configure the remote and do the initial push:
+
+    cd ~/annex
+    git remote add origin example.com:bare-annex
+    git push origin master git-annex
+
+Now `git annex status` should show the configured bare remote. If it does not, you may have to pull from the remote first (older versions of `git-annex`)
+
+If you wish to configure git such that you can push/pull without arguments, set the upstream branch:
+
+    git branch master --set-upstream origin/master
+
+   

From 0bb798e3516d5b4e8cdcf059572c54641f7a643a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Nov 2011 19:38:17 -0400
Subject: [PATCH 2446/2835] Pass -t to rsync to preserve timestamps.

---
 Utility/RsyncFile.hs | 2 ++
 debian/changelog     | 1 +
 2 files changed, 3 insertions(+)

diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index 056bd8d114..c5006a30fa 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -35,6 +35,8 @@ rsyncServerParams =
 	[ Param "--server"
 	-- preserve permissions
 	, Param "-p"
+	-- preserve timestamps
+	, Param "-t"
 	-- allow resuming of transfers of big files
 	, Param "--inplace"
 	-- other options rsync normally uses in server mode
diff --git a/debian/changelog b/debian/changelog
index e74a190ba5..07bfca7a78 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -24,6 +24,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * Record uuid when auto-initializing a remote so it shows in status.
   * Bugfix: Fixed git-annex init crash in a bare repository when there was
     already an existing git-annex branch.
+  * Pass -t to rsync to preserve timestamps.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From 526c20d06831706ced573ee0ea5f066976d8ed03 Mon Sep 17 00:00:00 2001
From: Valentin_Haenel 
Date: Sat, 5 Nov 2011 16:31:18 +0000
Subject: [PATCH 2447/2835] add bug report

---
 ...problems_with_urls_containing___126__.mdwn | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn

diff --git a/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
new file mode 100644
index 0000000000..c6eb7bc73a
--- /dev/null
+++ b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
@@ -0,0 +1,44 @@
+I discovered a problem with `git annex map` and relative urls containing `~`.
+In this case i have a remote `noam` configured with the the following urls:
+
+    zsh» git remote show noam | head -3
+    * remote noam
+      Fetch URL: noam:bare-annex
+      Push  URL: noam:bare-annex
+
+If i try to run `git annex map` i get the following error:
+
+    zsh» git annex map
+    map /home/esc/annex ok
+    map noam (sshing...)
+    bash: line 0: cd: /~/bare-annex/: No such file or directory
+    Command ssh ["noam","cd '/~/bare-annex/' && git config --list"] failed; exit code 1
+    (sshing...)
+    ok
+
+      running: dot -Tx11 map.dot
+
+    ok
+
+If i run the failing command manually, i get:
+
+    zsh» ssh noam "cd ~/bare-annex && git config --list"
+    core.repositoryformatversion=0
+    core.filemode=true
+    core.bare=true
+    annex.uuid=f267f55c-0732-11e1-a93b-93119f9aaf54
+    annex.version=3
+
+Also i can change the remote url to an absolute one, in which case `git annex
+map` works too:
+
+    zsh» git remote set-url noam noam:/home/esc/bare-annex
+    zsh» git annex map
+    map /home/esc/annex ok
+    map noam (sshing...)
+    ok
+
+      running: dot -Tx11 map.dot
+
+    ok
+

From 0556dc812edf5b809da9836a4f31397af26e138a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 5 Nov 2011 15:55:19 -0400
Subject: [PATCH 2448/2835] releasing version 3.20111105

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 07bfca7a78..fdd3cb7f62 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111026) UNRELEASED; urgency=low
+git-annex (3.20111105) unstable; urgency=low
 
   * The default backend used when adding files to the annex is changed
     from WORM to SHA256.
@@ -26,7 +26,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     already an existing git-annex branch.
   * Pass -t to rsync to preserve timestamps.
 
- -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
+ -- Joey Hess   Sat, 05 Nov 2011 15:47:52 -0400
 
 git-annex (3.20111025) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index c3706808c0..f464c46d20 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111025
+Version: 3.20111105
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From cd267dea159677c554aa65e04bd20bc14f5b12eb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 5 Nov 2011 15:55:30 -0400
Subject: [PATCH 2449/2835] add news item for git-annex 3.20111105

---
 doc/news/version_3.20110906.mdwn |  4 ----
 doc/news/version_3.20111105.mdwn | 27 +++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 4 deletions(-)
 delete mode 100644 doc/news/version_3.20110906.mdwn
 create mode 100644 doc/news/version_3.20111105.mdwn

diff --git a/doc/news/version_3.20110906.mdwn b/doc/news/version_3.20110906.mdwn
deleted file mode 100644
index d91776ccaf..0000000000
--- a/doc/news/version_3.20110906.mdwn
+++ /dev/null
@@ -1,4 +0,0 @@
-git-annex 3.20110906 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Improve display of newlines around error and warning messages.
-   * Fix Makefile to work with cabal again."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111105.mdwn b/doc/news/version_3.20111105.mdwn
new file mode 100644
index 0000000000..663c2ef1fc
--- /dev/null
+++ b/doc/news/version_3.20111105.mdwn
@@ -0,0 +1,27 @@
+git-annex 3.20111105 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * The default backend used when adding files to the annex is changed
+     from WORM to SHA256.
+     To get old behavior, add a .gitattributes containing: * annex.backend=WORM
+   * Sped up some operations on remotes that are on the same host.
+   * copy --to: Fixed leak when copying many files to a remote on the same
+     host.
+   * uninit: Add guard against being run with the git-annex branch checked out.
+   * Fail if --from or --to is passed to commands that do not support them.
+   * drop --from is now supported to remove file content from a remote.
+   * status: Now always shows the current repository, even when it does not
+     appear in uuid.log.
+   * fsck: Now works in bare repositories. Checks location log information,
+     and file contents. Does not check that numcopies is satisfied, as
+     .gitattributes information about numcopies is not available in a bare
+     repository.
+   * unused, dropunused: Now work in bare repositories.
+   * Removed the setkey command, and added a reinject command with a more
+     useful interface.
+   * The fromkey command now takes the key as its first parameter. The --key
+     option is no longer used.
+   * Built without any filename containing .git being excluded. Closes: #[647215](http://bugs.debian.org/647215)
+   * Record uuid when auto-initializing a remote so it shows in status.
+   * Bugfix: Fixed git-annex init crash in a bare repository when there was
+     already an existing git-annex branch.
+   * Pass -t to rsync to preserve timestamps."""]]
\ No newline at end of file

From bf07e2c921f66965ca0be816975fc608a81c4276 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 6 Nov 2011 13:53:11 -0400
Subject: [PATCH 2450/2835] typo

---
 doc/git-union-merge.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
index 495612f36d..ed1778910b 100644
--- a/doc/git-union-merge.mdwn
+++ b/doc/git-union-merge.mdwn
@@ -15,7 +15,7 @@ The union merge will always succeed, but assumes that files can be merged
 simply by concacenating together lines from all the oldrefs, in any order.
 So, this is useful only for branches containing log-type data.
 
-That this does not touch the checked out working copy. It operates
+Note that this does not touch the checked out working copy. It operates
 entirely on git refs and branches.
 
 # EXAMPLE

From c99fb589097d96b4e10cd8b137f72881cdb93118 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 6 Nov 2011 15:18:45 -0400
Subject: [PATCH 2451/2835] merge: Use fast-forward merges when possible.

Thanks Valentin Haenel for a test case showing how non-fast-forward merges
could result in an ongoing pull/merge/push cycle.

While the git-annex branch is fast-forwarded, git-annex's index file is still
updated using the union merge strategy as before. There's no other way to
update the index that would be any faster.

It is possible that a union merge and a fast-forward result in different file
contents: Files should have the same lines, but a union merge may change
their order. If this happens, the next commit made to the git-annex branch
will have some unnecessary changes to line orders, but the consistency
of data should be preserved.

Note that when the journal contains changes, a fast-forward is never attempted,
which is fine, because committing those changes would be vanishingly unlikely
to leave the git-annex branch at a commit that already exists in one of
the remotes.

The real difficulty is handling the case where multiple remotes have all
changed. git-annex does find the best (ie, newest) one and fast forwards
to it. If the remotes are diverged, no fast-forward is done at all. It would
be possible to pick one, fast forward to it, and make a merge commit to
the rest, I see no benefit to adding that complexity.

Determining the best of N changed remotes requires N*2+1 calls to git-log, but
these are fast git-log calls, and N is typically small. Also, typically
some or all of the remote refs will be the same, and git-log is not called to
compare those. In the real world I expect this will almost always add only
1 git-log call to the merge process. (Which already makes N anyway.)
---
 Annex/Branch.hs                               | 88 ++++++++++++++-----
 debian/changelog                              |  8 ++
 ...making_annex-merge_try_a_fast-forward.mdwn |  6 ++
 3 files changed, 79 insertions(+), 23 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 4c3192f531..0095b586b4 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -117,28 +117,30 @@ commit message = whenM journalDirty $ lockJournal $ do
 	g <- gitRepo
 	withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
-{- Ensures that the branch is up-to-date; should be called before
- - data is read from it. Runs only once per git-annex run.
+{- Ensures that the branch is up-to-date; should be called before data is
+ - read from it. Runs only once per git-annex run.
  -
- - Before refs are merged into the index, it's
- - important to first stage the journal into the
- - index. Otherwise, any changes in the journal
- - would later get staged, and might overwrite
- - changes made during the merge.
+ - Before refs are merged into the index, it's important to first stage the
+ - journal into the index. Otherwise, any changes in the journal would
+ - later get staged, and might overwrite changes made during the merge.
  -
- - It would be cleaner to handle the merge by
- - updating the journal, not the index, with changes
- - from the branches.
+ - It would be cleaner to handle the merge by updating the journal, not the
+ - index, with changes from the branches.
+ -
+ - The index is always updated using a union merge, as that's the most
+ - efficient way to update it. However, if the branch can be
+ - fast-forwarded, that is then done, rather than adding an unnecessary
+ - commit to it.
  -}
 update :: Annex ()
 update = onceonly $ do
+	g <- gitRepo
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
-	c <- filterM changedbranch =<< siblingBranches
+	c <- filterM (changedBranch name . snd) =<< siblingBranches
 	let (refs, branches) = unzip c
 	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
 		when dirty stageJournalFiles
-		g <- gitRepo
 		unless (null branches) $ do
 			showSideAction $ "merging " ++
 				(unwords $ map Git.refDescribe branches) ++
@@ -150,24 +152,64 @@ update = onceonly $ do
 			 - modify the branch.
 			 -}
 			liftIO $ Git.UnionMerge.merge_index g branches
-		liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
+		ff <- if dirty then return False else tryFastForwardTo refs
+		unless ff $
+			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
 		invalidateCache
 	where
-		changedbranch (_, branch) = do
-			g <- gitRepo
-			-- checking with log to see if there have been changes
-			-- is less expensive than always merging
-			diffs <- liftIO $ Git.pipeRead g [
-				Param "log",
-				Param (name ++ ".." ++ branch),
-				Params "--oneline -n1"
-				]
-			return $ not $ L.null diffs
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
 			r <- a
 			disableUpdate
 			return r
 
+{- Checks if the second branch has any commits not present on the first
+ - branch. -}
+changedBranch :: String -> String -> Annex Bool
+changedBranch origbranch newbranch = do
+	g <- gitRepo
+	diffs <- liftIO $ Git.pipeRead g [
+		Param "log",
+		Param (origbranch ++ ".." ++ newbranch),
+		Params "--oneline -n1"
+		]
+	return $ not $ L.null diffs
+
+{- Given a set of refs that are all known to have commits not
+ - on the git-annex branch, tries to update the branch by a
+ - fast-forward.
+ -
+ - In order for that to be possible, one of the refs must contain
+ - every commit present in all the other refs, as well as in the
+ - git-annex branch.
+ -}
+tryFastForwardTo :: [String] -> Annex Bool
+tryFastForwardTo [] = return True
+tryFastForwardTo (first:rest) = do
+	-- First, check that the git-annex branch does not contain any
+	-- new commits that are in the first other branch. If it does,
+	-- cannot fast-forward.
+	diverged <- changedBranch first fullname
+	if diverged
+		then no_ff
+		else maybe no_ff do_ff =<< findbest first rest
+	where
+		no_ff = return False
+		do_ff branch = do
+			g <- gitRepo
+			liftIO $ Git.run g "update-ref" [Param fullname, Param branch]
+			return True
+		findbest c [] = return $ Just c
+		findbest c (r:rs)
+			| c == r = findbest c rs
+			| otherwise = do
+			better <- changedBranch c r
+			worse <- changedBranch r c
+			case (better, worse) of
+				(True, True) -> return Nothing -- divergent fail
+				(True, False) -> findbest r rs -- better
+				(False, True) -> findbest c rs -- worse
+				(False, False) -> findbest c rs -- same
+			
 {- Avoids updating the branch. A useful optimisation when the branch
  - is known to have not changed, or git-annex won't be relying on info
  - from it. -}
diff --git a/debian/changelog b/debian/changelog
index fdd3cb7f62..57be53c859 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,11 @@
+git-annex (3.20111106) UNRELEASED; urgency=low
+
+  * merge: Use fast-forward merges when possible.
+    Thanks Valentin Haenel for a test case showing how non-fast-forward
+    merges could result in an ongoing pull/merge/push cycle.
+
+ -- Joey Hess   Sun, 06 Nov 2011 14:57:57 -0400
+
 git-annex (3.20111105) unstable; urgency=low
 
   * The default backend used when adding files to the annex is changed
diff --git a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
index a2bd8c7479..41a5a2a581 100644
--- a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
+++ b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
@@ -27,3 +27,9 @@ But as sometimes annex-merge takes time, it would probably be worth it
 > 
 > Although, perhaps fast-forward merge would use slightly
 > less space. --[[Joey]]
+
+>> To avoid the ladder-merge between two repositories described at
+>> , seems a fast-forward should be detected and
+>> written to git, even if the index is still updated the current way.
+>> [[done]]
+>> --[[Joey]]

From f2299117152955d622d0ae0fcb396e56b1b7866d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 6 Nov 2011 15:33:15 -0400
Subject: [PATCH 2452/2835] optimization

The last commit added some git-log calls to a merge. This removes some,
by only merging branches that have unique refs.
---
 Annex/Branch.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 0095b586b4..163c9ec60f 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -240,15 +240,16 @@ hasOrigin = refExists originname
 hasSomeBranch :: Annex Bool
 hasSomeBranch = not . null <$> siblingBranches
 
-{- List of all git-annex (refs, branches), including the main one and any
- - from remotes. -}
+{- List of git-annex (refs, branches), including the main one and any
+ - from remotes. Duplicate refs are filtered out. -}
 siblingBranches :: Annex [(String, String)]
 siblingBranches = do
 	g <- gitRepo
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-	return $ map (pair . words . L.unpack) (L.lines r)
+	return $ nubBy uref $ map (pair . words . L.unpack) (L.lines r)
 	where
 		pair l = (head l, last l)
+		uref (a, _) (b, _) = a == b
 
 {- Applies a function to modifiy the content of a file. -}
 change :: FilePath -> (String -> String) -> Annex ()

From aae0417d94c4ae81d28f2301bc1fac61d4f499a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 11:50:30 -0400
Subject: [PATCH 2453/2835] Don't try to read config from repos with
 annex-ignore set.

---
 Remote/Git.hs    | 9 +++++----
 debian/changelog | 1 +
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index b0138901d7..0cd64c9215 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -56,10 +56,11 @@ gen r u _ = do
 	 - the config of an URL remote is only read when there is no
 	 - cached UUID value. -}
 	let cheap = not $ Git.repoIsUrl r
-	r' <- case (cheap, u) of
-		(True, _) -> do
-			tryGitConfigRead r
-		(False, "") -> tryGitConfigRead r
+	notignored <- repoNotIgnored r
+	r' <- case (cheap, notignored, u) of
+		(_, False, _) -> return r
+		(True, _, _) -> tryGitConfigRead r
+		(False, _, "") -> tryGitConfigRead r
 		_ -> return r
 
 	u' <- getRepoUUID r'
diff --git a/debian/changelog b/debian/changelog
index 57be53c859..c09ca6578a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20111106) UNRELEASED; urgency=low
   * merge: Use fast-forward merges when possible.
     Thanks Valentin Haenel for a test case showing how non-fast-forward
     merges could result in an ongoing pull/merge/push cycle.
+  * Don't try to read config from repos with annex-ignore set.
 
  -- Joey Hess   Sun, 06 Nov 2011 14:57:57 -0400
 

From 41eecb4601896f7ece2151ff79775dd591e91b37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 12:47:41 -0400
Subject: [PATCH 2454/2835] Bugfix: In the past two releases, git-annex init
 has written the uuid.log in the wrong format, with the UUID and description
 flipped.

This is my own damn fault for not making UUID a real type, and then relying
on the type checker to ensure my refactoring was correct -- which it wasn't!

I should probably add code to clean up bogus entries in the uuid.log, but
right now I want to get the fix out there to prevent people experiencing
this bug.

I should also make UUID a real data type.
---
 Command/Init.hs  | 3 ++-
 debian/changelog | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 69cdc8e8a4..e2a6eb7b03 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -30,5 +30,6 @@ start ws = do
 perform :: String -> CommandPerform
 perform description = do
 	initialize
-	getUUID >>= describeUUID description
+	u <- getUUID
+	describeUUID u description
 	next $ return True
diff --git a/debian/changelog b/debian/changelog
index c09ca6578a..57be7dd67b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20111106) UNRELEASED; urgency=low
     Thanks Valentin Haenel for a test case showing how non-fast-forward
     merges could result in an ongoing pull/merge/push cycle.
   * Don't try to read config from repos with annex-ignore set.
+  * Bugfix: In the past two releases, git-annex init has written the uuid.log
+    in the wrong format, with the UUID and description flipped.
 
  -- Joey Hess   Sun, 06 Nov 2011 14:57:57 -0400
 

From f8911cc69dc069b47da7c156270620ef6aa758b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:06:58 -0400
Subject: [PATCH 2455/2835] releasing version 3.20111107

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 57be7dd67b..0b4fe3486e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111106) UNRELEASED; urgency=low
+git-annex (3.20111107) unstable; urgency=low
 
   * merge: Use fast-forward merges when possible.
     Thanks Valentin Haenel for a test case showing how non-fast-forward
@@ -7,7 +7,7 @@ git-annex (3.20111106) UNRELEASED; urgency=low
   * Bugfix: In the past two releases, git-annex init has written the uuid.log
     in the wrong format, with the UUID and description flipped.
 
- -- Joey Hess   Sun, 06 Nov 2011 14:57:57 -0400
+ -- Joey Hess   Mon, 07 Nov 2011 12:47:44 -0400
 
 git-annex (3.20111105) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index f464c46d20..4b3a708a4e 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111105
+Version: 3.20111107
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 80787703c34512dec98a794424c63e6977737767 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:07:43 -0400
Subject: [PATCH 2456/2835] add news item for git-annex 3.20111107

---
 doc/news/version_3.20110915.mdwn | 10 ----------
 doc/news/version_3.20111107.mdwn |  8 ++++++++
 2 files changed, 8 insertions(+), 10 deletions(-)
 delete mode 100644 doc/news/version_3.20110915.mdwn
 create mode 100644 doc/news/version_3.20111107.mdwn

diff --git a/doc/news/version_3.20110915.mdwn b/doc/news/version_3.20110915.mdwn
deleted file mode 100644
index 77a97242d9..0000000000
--- a/doc/news/version_3.20110915.mdwn
+++ /dev/null
@@ -1,10 +0,0 @@
-git-annex 3.20110915 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * whereis: Show untrusted locations separately and do not include in
-     location count.
-   * Fix build without S3.
-   * addurl: Always use whole url as destination filename, rather than
-     only its file component.
-   * get, drop, copy: Added --auto option, which decides whether
-     to get/drop content as needed to work toward the configured numcopies.
-   * bugfix: drop and fsck did not honor --exclude"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111107.mdwn b/doc/news/version_3.20111107.mdwn
new file mode 100644
index 0000000000..17431bf219
--- /dev/null
+++ b/doc/news/version_3.20111107.mdwn
@@ -0,0 +1,8 @@
+git-annex 3.20111107 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * merge: Use fast-forward merges when possible.
+     Thanks Valentin Haenel for a test case showing how non-fast-forward
+     merges could result in an ongoing pull/merge/push cycle.
+   * Don't try to read config from repos with annex-ignore set.
+   * Bugfix: In the past two releases, git-annex init has written the uuid.log
+     in the wrong format, with the UUID and description flipped."""]]
\ No newline at end of file

From b7cd088433c171d878a69ef116b7a2efe1248b32 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:08:47 -0400
Subject: [PATCH 2457/2835] add

---
 .../centralized_git_repository_tutorial.mdwn  | 129 ++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 doc/tips/centralized_git_repository_tutorial.mdwn

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
new file mode 100644
index 0000000000..920ae5d8f7
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -0,0 +1,129 @@
+The [[walkthrough]] builds up a decentralized git repository setup, but
+git-annex can also be used with a centralized bare repository, just like
+git can.
+
+## set up the repository, and make a checkout
+
+In this tutorial, I'll set up a centralized repository hosted on
+GitHub. I've created a repository for technical talk videos, which you can
+fork, or make your own repository on GitHub (or elsewhere) now.
+
+On your laptop, [[install]] git-annex, and clone the repository:
+
+	# git clone git@github.com:joeyh/techtalks.git
+	# cd techtalks
+
+Let's tell git-annex that GitHub doesn't support running git-annex-shell there.
+This means you can't store annexed file *contents* on GitHub; it would
+really be better to host the bare repository on your own server, which
+would not have this limitation. (If you want to do that, check out
+[[using_gitolite_with_git-annex]].)
+
+	# git config remote.origin.annex-ignore true
+
+Tell git-annex to use the repository, and describe where this clone is
+located:
+
+	# git annex init 'my laptop'
+	init my laptop ok
+
+## add files to the repository
+
+Add some files, obtained however.
+
+	# youtube-dl -t 'http://www.youtube.com/watch?v=b9FagOVqxmI'
+	# git annex add *.mp4
+	add Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 (checksum) ok
+	(Recording state in git...)
+	# git commit -m "added a video. I have not watched it yet but it sounds interesting"
+
+This file is available directly from the web; so git-annex can download it:
+
+	# git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg
+	(downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...)
+	(checksum...) ok
+	(Recording state in git...)
+	# git commit -a -m 'added a screencast I made'
+
+Now push your changes back to the central repository. This first time,
+remember to push the git-annex branch, which is used to track the file
+contents.
+
+	# git push origin master git-annex
+	To git@github.com:joeyh/techtalks.git
+	 * [new branch]      master -> master
+	 * [new branch]      git-annex -> git-annex
+
+That push went fast, because it didn't upload large videos to GitHub.
+To check this, you can ask git-annex where the contents of the videos are:
+
+	# git annex whereis
+	whereis Haskell_Amuse_Bouche.mp4 (1 copy) 
+	  	767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
+	ok
+	whereis git-annex_coding_in_haskell.ogg (2 copies) 
+	  	00000000-0000-0000-0000-000000000001 -- web
+	   	767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
+	ok
+
+## make more checkouts
+
+So far you have a central repository, and a checkout on a laptop.
+Let's make another checkout that's used as a backup. You can put it anywhere
+you like, just make it be somewhere your laptop can access. A few options:
+
+* Put it on a USB drive that you can plug into the laptop.
+* Put it on a desktop.
+* Put it on some server in the local network.
+* Put it on a remote VPS.
+* All of the above!
+
+I'll use the VPS option, but these instructions should work for
+any of the above.
+
+	# ssh server
+	server# sudo apt-get install git-annex
+
+Clone the central repository as before. (If the clone fails, you need
+to add your server's ssh public key to github -- see
+[this page](http://help.github.com/ssh-issues/).)
+
+	server# git clone git@github.com:joeyh/techtalks.git
+	server# cd techtalks
+	server# git config remote.origin.annex-ignore true
+	server# git annex init 'backup'
+	init backup (merging origin/git-annex into git-annex...) ok
+
+Notice that the server does not have the contents of any of the files yet.
+If you run `ls`, you'll see broken symlinks. We want to populate this
+backup with the file contents, by copying them from your laptop.
+
+Back on your laptop, you need to configure a git remote for the backup.
+Adjust the url as needed to point to wherever the backup is. (If it
+was on a local USB drive, you'd use the path to the repository.)
+
+	# git remote add backup ssh://server/~/techtalks
+
+Now git-annex on your laptop knows how to reach the backup repository,
+and can do things like copy files to it:
+
+	# git annex copy --to backup git-annex_coding_in_haskell.ogg
+	copy git-annex_coding_in_haskell.ogg (checking backup...)
+	12877824   2%  255.11kB/s    00:00
+	ok
+
+You can also `git annex move` files to it, to free up space on your laptop.
+And then you can `git annex get` files back to your laptop later on, as
+desired.
+
+## take it farther
+
+Of course you can create as many checkouts as you desire. If you have a
+desktop matchine too, you can make a checkout there, and use `git remote
+add` to also let your desktop access the backup repository. 
+
+You can add remotes for each direct connection between machines you find you
+need -- so make the laptop have the desktop as a remote, and the desktop
+have the laptop as a remote, and then on either machine git-annex can
+access files stored on the other.

From 146995c4e18223542992e7c575a670b59b9e2efd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:18:16 -0400
Subject: [PATCH 2458/2835] update

---
 doc/tips/centralized_git_repository_tutorial.mdwn | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
index 920ae5d8f7..4cc7519d8b 100644
--- a/doc/tips/centralized_git_repository_tutorial.mdwn
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -1,12 +1,13 @@
 The [[walkthrough]] builds up a decentralized git repository setup, but
 git-annex can also be used with a centralized bare repository, just like
-git can.
+git can. This tutorial shows how to set up a centralized repository hosted on
+GitHub.
 
 ## set up the repository, and make a checkout
 
-In this tutorial, I'll set up a centralized repository hosted on
-GitHub. I've created a repository for technical talk videos, which you can
-fork, or make your own repository on GitHub (or elsewhere) now.
+I've created a repository for technical talk videos, which you can
+[fork on Github](https://github.com/joeyh/techtalks).
+Or make your own repository on GitHub (or elsewhere) now.
 
 On your laptop, [[install]] git-annex, and clone the repository:
 
@@ -115,7 +116,10 @@ and can do things like copy files to it:
 
 You can also `git annex move` files to it, to free up space on your laptop.
 And then you can `git annex get` files back to your laptop later on, as
-desired.
+desired. After you using git-annex to move files around, remember to push,
+which will broadcast its updated location information.
+
+	# git push
 
 ## take it farther
 

From 95b9d726f87dee07c9eab4042e6d7caba612c765 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:26:37 -0400
Subject: [PATCH 2459/2835] update

---
 .../centralized_git_repository_tutorial.mdwn  | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
index 4cc7519d8b..03c4c9afd5 100644
--- a/doc/tips/centralized_git_repository_tutorial.mdwn
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -14,6 +14,12 @@ On your laptop, [[install]] git-annex, and clone the repository:
 	# git clone git@github.com:joeyh/techtalks.git
 	# cd techtalks
 
+Tell git-annex to use the repository, and describe where this clone is
+located:
+
+	# git annex init 'my laptop'
+	init my laptop ok
+
 Let's tell git-annex that GitHub doesn't support running git-annex-shell there.
 This means you can't store annexed file *contents* on GitHub; it would
 really be better to host the bare repository on your own server, which
@@ -22,12 +28,6 @@ would not have this limitation. (If you want to do that, check out
 
 	# git config remote.origin.annex-ignore true
 
-Tell git-annex to use the repository, and describe where this clone is
-located:
-
-	# git annex init 'my laptop'
-	init my laptop ok
-
 ## add files to the repository
 
 Add some files, obtained however.
@@ -78,7 +78,6 @@ you like, just make it be somewhere your laptop can access. A few options:
 * Put it on a desktop.
 * Put it on some server in the local network.
 * Put it on a remote VPS.
-* All of the above!
 
 I'll use the VPS option, but these instructions should work for
 any of the above.
@@ -101,8 +100,8 @@ If you run `ls`, you'll see broken symlinks. We want to populate this
 backup with the file contents, by copying them from your laptop.
 
 Back on your laptop, you need to configure a git remote for the backup.
-Adjust the url as needed to point to wherever the backup is. (If it
-was on a local USB drive, you'd use the path to the repository.)
+Adjust the ssh url as needed to point to wherever the backup is. (If it
+was on a local USB drive, you'd use the path to the repository instead.)
 
 	# git remote add backup ssh://server/~/techtalks
 
@@ -116,7 +115,9 @@ and can do things like copy files to it:
 
 You can also `git annex move` files to it, to free up space on your laptop.
 And then you can `git annex get` files back to your laptop later on, as
-desired. After you using git-annex to move files around, remember to push,
+desired.
+
+After you use git-annex to move files around, remember to push, 
 which will broadcast its updated location information.
 
 	# git push

From 97ba3a118de0e37b3f7dac8fd299a727e6859319 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:29:00 -0400
Subject: [PATCH 2460/2835] update

---
 doc/tips/centralized_git_repository_tutorial.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
index 03c4c9afd5..977a583cca 100644
--- a/doc/tips/centralized_git_repository_tutorial.mdwn
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -47,6 +47,12 @@ This file is available directly from the web; so git-annex can download it:
 	(Recording state in git...)
 	# git commit -a -m 'added a screencast I made'
 
+Feel free the rename the files, etc, using normal git commands:
+
+	# git mv Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 Haskell_Amuse_Bouche.mp4
+	# git mv kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg git-annex_coding_in_haskell.ogg
+	# git commit -m 'better filenames'
+
 Now push your changes back to the central repository. This first time,
 remember to push the git-annex branch, which is used to track the file
 contents.

From b08f7c428b4bc9eabd95596d08594ddd1057a0bf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 14:00:23 -0400
Subject: [PATCH 2461/2835] better usage

---
 Command/Reinject.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 3de24f3fc3..1277edf906 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import qualified Command.Fsck
 
 def :: [Command]
-def = [command "reinject" (paramPair paramPath paramPath) seek
+def = [command "reinject" (paramPair "SRC" "DEST") seek
 	"sets content of annexed file"]
 
 seek :: [CommandSeek]

From 63a292324d20832b68c92f784828e55e644481cc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 14:46:01 -0400
Subject: [PATCH 2462/2835] add a UUID type

Should have done this a long time ago.
---
 Annex/Ssh.hs             |  5 ++---
 Annex/UUID.hs            | 17 ++++++++++-------
 Command/ConfigList.hs    |  2 +-
 Command/Map.hs           |  8 ++++----
 Logs/Location.hs         | 11 +++++------
 Logs/Trust.hs            |  5 ++---
 Logs/UUIDBased.hs        | 14 +++++++-------
 Logs/Web.hs              |  2 +-
 Remote.hs                | 15 ++++++++-------
 Remote/Bup.hs            |  8 ++++----
 Remote/Git.hs            |  2 +-
 Remote/Helper/Special.hs |  2 +-
 Remote/S3real.hs         |  2 +-
 Types.hs                 |  2 +-
 Types/Remote.hs          |  7 ++++---
 Types/UUID.hs            | 14 ++++++++++++--
 git-annex-shell.hs       |  4 ++--
 test.hs                  |  2 +-
 18 files changed, 67 insertions(+), 55 deletions(-)

diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 851c7c06b6..f8cd5d9bc8 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -45,9 +45,8 @@ git_annex_shell r command params
 		sshcmd uuid = unwords $
 			shellcmd : (map shellEscape $ toCommand shellopts) ++
 			uuidcheck uuid
-		uuidcheck uuid
-			| null uuid = []
-			| otherwise = ["--uuid", uuid]
+		uuidcheck NoUUID = []
+		uuidcheck (UUID u) = ["--uuid", u]
 
 {- Uses a supplied function (such as boolSystem) to run a git-annex-shell
  - command on a remote.
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 39e296e5b7..90189bc475 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -30,7 +30,7 @@ configkey = "annex.uuid"
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
 genUUID :: IO UUID
-genUUID = pOpen ReadFromPipe command params hGetLine
+genUUID = pOpen ReadFromPipe command params $ liftM read . hGetLine
 	where
 		command = SysConfig.uuid
 		params = if command == "uuid"
@@ -50,20 +50,23 @@ getRepoUUID r = do
 	let c = cached g
 	let u = getUncachedUUID r
 	
-	if c /= u && u /= ""
+	if c /= u && u /= NoUUID
 		then do
 			updatecache g u
 			return u
 		else return c
 	where
-		cached g = Git.configGet g cachekey ""
-		updatecache g u = when (g /= r) $ setConfig cachekey u
+		cached g = read $ Git.configGet g cachekey ""
+		updatecache g u = when (g /= r) $ storeUUID cachekey u
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = Git.configGet r configkey ""
+getUncachedUUID r = read $ Git.configGet r configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
-prepUUID = whenM (null <$> getUUID) $
-	setConfig configkey =<< liftIO genUUID
+prepUUID = whenM ((==) NoUUID <$> getUUID) $
+	storeUUID configkey =<< liftIO genUUID
+
+storeUUID :: String -> UUID -> Annex ()
+storeUUID configfield uuid = setConfig configfield (show uuid)
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index cbc6e801b0..fadcbb8435 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -21,5 +21,5 @@ seek = [withNothing start]
 start :: CommandStart
 start = do
 	u <- getUUID
-	liftIO $ putStrLn $ "annex.uuid=" ++ u
+	liftIO $ putStrLn $ "annex.uuid=" ++ show u
 	stop
diff --git a/Command/Map.hs b/Command/Map.hs
index 7e61d2e9e8..803324e999 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -62,7 +62,7 @@ drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 		others = map (unreachable . uuidnode) $
 			filter (`notElem` ruuids) (M.keys umap)
 		trusted = map (trustworthy . uuidnode) ts
-		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
+		uuidnode u = Dot.graphNode (show u) $ M.findWithDefault "" u umap
 
 hostname :: Git.Repo -> String
 hostname r
@@ -76,7 +76,7 @@ basehostname r = head $ split "." $ hostname r
  - or the remote name if not. -}
 repoName :: M.Map UUID String -> Git.Repo -> String
 repoName umap r
-	| null repouuid = fallback
+	| repouuid == NoUUID = fallback
 	| otherwise = M.findWithDefault fallback repouuid umap
 	where
 		repouuid = getUncachedUUID r
@@ -86,8 +86,8 @@ repoName umap r
 nodeId :: Git.Repo -> String
 nodeId r =
 	case getUncachedUUID r of
-		"" -> Git.repoLocation r
-		u -> u
+		NoUUID -> Git.repoLocation r
+		UUID u -> u
 
 {- A node representing a repo. -}
 node :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> String
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 8855cf63b5..602c46f310 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -29,16 +29,15 @@ import Logs.Presence
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
-logChange repo key u s
-	| null u = error $
-		"unknown UUID for " ++ Git.repoDescribe repo ++ 
-		" (have you run git annex init there?)"
-	| otherwise = addLog (logFile key) =<< logNow s u
+logChange _ key (UUID u) s = addLog (logFile key) =<< logNow s u
+logChange repo _ NoUUID _ = error $
+	"unknown UUID for " ++ Git.repoDescribe repo ++ 
+	" (have you run git annex init there?)"
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations = currentLog . logFile
+keyLocations key = map read <$> (currentLog . logFile) key
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 372d8b3609..53a1bca2c5 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -53,13 +53,12 @@ parseTrust s
 
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
-trustSet uuid level = do
-	when (null uuid) $
-		error "unknown UUID; cannot modify trust level"
+trustSet uuid@(UUID _) level = do
 	ts <- liftIO $ getPOSIXTime
 	Annex.Branch.change trustLog $
 		showLog show . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
+trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
 
 {- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
 trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 46fa80be0d..7184709fe2 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -50,9 +50,9 @@ showLog :: (a -> String) -> Log a -> String
 showLog shower = unlines . map showpair . M.toList
 	where
 		showpair (k, LogEntry (Date p) v) =
-			unwords [k, shower v, tskey ++ show p]
+			unwords [show k, shower v, tskey ++ show p]
 		showpair (k, LogEntry Unknown v) =
-			unwords [k, shower v]
+			unwords [show k, shower v]
 
 parseLog :: (String -> Maybe a) -> String -> Log a
 parseLog parser = M.fromListWith best . catMaybes . map pair . lines
@@ -61,7 +61,7 @@ parseLog parser = M.fromListWith best . catMaybes . map pair . lines
 			| null ws = Nothing
 			| otherwise = case parser $ unwords info of
 				Nothing -> Nothing
-				Just v -> Just (u, LogEntry c v)
+				Just v -> Just (read u, LogEntry c v)
 			where
 				ws = words line
 				u = head ws
@@ -103,8 +103,8 @@ prop_TimeStamp_sane = Unknown < Date 1
 prop_addLog_sane :: Bool
 prop_addLog_sane = newWins && newestWins
 	where
-		newWins = addLog "foo" (LogEntry (Date 1) "new") l == l2
-		newestWins = addLog "foo" (LogEntry (Date 1) "newest") l2 /= l2
+		newWins = addLog (UUID "foo") (LogEntry (Date 1) "new") l == l2
+		newestWins = addLog (UUID "foo") (LogEntry (Date 1) "newest") l2 /= l2
 
-		l = M.fromList [("foo", LogEntry (Date 0) "old")]
-		l2 = M.fromList [("foo", LogEntry (Date 1) "new")]
+		l = M.fromList [(UUID "foo", LogEntry (Date 0) "old")]
+		l2 = M.fromList [(UUID "foo", LogEntry (Date 1) "new")]
diff --git a/Logs/Web.hs b/Logs/Web.hs
index 605797079f..b52e347e5f 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -21,7 +21,7 @@ type URLString = String
 
 -- Dummy uuid for the whole web. Do not alter.
 webUUID :: UUID
-webUUID = "00000000-0000-0000-0000-000000000001"
+webUUID = UUID "00000000-0000-0000-0000-000000000001"
 
 {- The urls for a key are stored in remote/web/hash/key.log 
  - in the git-annex branch. -}
diff --git a/Remote.hs b/Remote.hs
index 6ce4fe0186..d4fbf36cf0 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -100,7 +100,7 @@ byName' n = do
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
 		else return $ Right $ head match
 	where
-		matching r = n == name r || n == uuid r
+		matching r = n == name r || read n == uuid r
 
 {- Looks up a remote by name (or by UUID, or even by description),
  - and returns its UUID. Finds even remotes that are not configured in
@@ -115,12 +115,13 @@ nameToUUID n = do
 	where
 		byDescription = do
 			m <- uuidMap
-			case M.lookup n $ transform swap m of
+			case M.lookup wantuuid $ transform swap m of
 				Just u -> return $ Just u
-				Nothing -> return $ M.lookup n $ transform double m
+				Nothing -> return $ M.lookup wantuuid $ transform double m
 		transform a = M.fromList . map a . M.toList
 		swap (a, b) = (b, a)
-		double (a, _) = (a, a)
+		double (a, _) = (show a, a)
+		wantuuid = read n
 
 {- Pretty-prints a list of UUIDs of remotes, for human display.
  -
@@ -143,8 +144,8 @@ prettyPrintUUIDs desc uuids = do
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
 		prettify m here u
-			| not (null d) = u ++ " -- " ++ d
-			| otherwise = u
+			| not (null d) = show u ++ " -- " ++ d
+			| otherwise = show u
 			where
 				ishere = here == u
 				n = findlog m u
@@ -153,7 +154,7 @@ prettyPrintUUIDs desc uuids = do
 					| ishere = addname n "here"
 					| otherwise = n
 		jsonify m here u = toJSObject
-			[ ("uuid", toJSON u)
+			[ ("uuid", toJSON $ show u)
 			, ("description", toJSON $ findlog m u)
 			, ("here", toJSON $ here == u)
 			]
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 48014f1dad..b613225b8f 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -161,13 +161,13 @@ storeBupUUID u buprepo = do
 		then do
 			showAction "storing uuid"
 			onBupRemote r boolSystem "git"
-				[Params $ "config annex.uuid " ++ u]
+				[Params $ "config annex.uuid " ++ show u]
 					>>! error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
 			let olduuid = Git.configGet r' "annex.uuid" ""
 			when (olduuid == "") $
-				Git.run r' "config" [Param "annex.uuid", Param u]
+				Git.run r' "config" [Param "annex.uuid", Param $ show u]
 
 onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a
 onBupRemote r a command params = do
@@ -192,8 +192,8 @@ getBupUUID r u
 	| otherwise = liftIO $ do
 		ret <- try $ Git.configRead r
 		case ret of
-			Right r' -> return (Git.configGet r' "annex.uuid" "", r')
-			Left _ -> return ("", r)
+			Right r' -> return (read $ Git.configGet r' "annex.uuid" "", r')
+			Left _ -> return (NoUUID, r)
 
 {- Converts a bup remote path spec into a Git.Repo. There are some
  - differences in path representation between git and bup. -}
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 0cd64c9215..4c76e8ce6d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -60,7 +60,7 @@ gen r u _ = do
 	r' <- case (cheap, notignored, u) of
 		(_, False, _) -> return r
 		(True, _, _) -> tryGitConfigRead r
-		(False, _, "") -> tryGitConfigRead r
+		(False, _, NoUUID) -> tryGitConfigRead r
 		_ -> return r
 
 	u' <- getRepoUUID r'
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 52f2dbf954..8a9a01a228 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -32,7 +32,7 @@ gitConfigSpecialRemote u c k v = do
 	g <- gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param $ show u]
 	where
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 89b0326379..1f5b2bd594 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -64,7 +64,7 @@ s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
 s3Setup u c = handlehost $ M.lookup "host" c
 	where
 		remotename = fromJust (M.lookup "name" c)
-		defbucket = remotename ++ "-" ++ u
+		defbucket = remotename ++ "-" ++ show u
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
diff --git a/Types.hs b/Types.hs
index 703edb5c86..fd77bfe575 100644
--- a/Types.hs
+++ b/Types.hs
@@ -9,7 +9,7 @@ module Types (
 	Annex,
 	Backend,
 	Key,
-	UUID
+	UUID(..)
 ) where
 
 import Annex
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 49f16bfdd7..0a4a0fa883 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -15,6 +15,7 @@ import Data.Ord
 
 import qualified Git
 import Types.Key
+import Types.UUID
 
 type RemoteConfig = M.Map String String
 
@@ -25,15 +26,15 @@ data RemoteType a = RemoteType {
 	-- enumerates remotes of this type
 	enumerate :: a [Git.Repo],
 	-- generates a remote of this type
-	generate :: Git.Repo -> String -> Maybe RemoteConfig -> a (Remote a),
+	generate :: Git.Repo -> UUID -> Maybe RemoteConfig -> a (Remote a),
 	-- initializes or changes a remote
-	setup :: String -> RemoteConfig -> a RemoteConfig
+	setup :: UUID -> RemoteConfig -> a RemoteConfig
 }
 
 {- An individual remote. -}
 data Remote a = Remote {
 	-- each Remote has a unique uuid
-	uuid :: String,
+	uuid :: UUID,
 	-- each Remote has a human visible name
 	name :: String,
 	-- Remotes have a use cost; higher is more expensive
diff --git a/Types/UUID.hs b/Types/UUID.hs
index eb3497fa94..f7232d0b97 100644
--- a/Types/UUID.hs
+++ b/Types/UUID.hs
@@ -7,5 +7,15 @@
 
 module Types.UUID where
 
--- might be nice to have a newtype, but lots of stuff treats uuids as strings
-type UUID = String
+-- A UUID is either an arbitrary opaque string, or UUID info may be missing.
+data UUID = NoUUID | UUID String
+	deriving (Eq, Ord)
+
+instance Show UUID where
+	show (UUID u) = u
+	show NoUUID = ""
+
+instance Read UUID where
+	readsPrec _ s
+		| null s = [(NoUUID, "")]
+		| otherwise = [(UUID s, "")]
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 10eeb454af..de3160953c 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -45,9 +45,9 @@ options = commonOptions ++
 	where
 		check expected = do
 			u <- getUUID
-			when (u /= expected) $ error $
+			when (u /= read expected) $ error $
 				"expected repository UUID " ++ expected
-					++ " but found UUID " ++ u
+					++ " but found UUID " ++ show u
 
 header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
diff --git a/test.hs b/test.hs
index d466d3ad32..d4c1366d09 100644
--- a/test.hs
+++ b/test.hs
@@ -614,7 +614,7 @@ checklocationlog f expected = do
 	case r of
 		Just (k, _) -> do
 			uuids <- annexeval $ Logs.Location.keyLocations k
-			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
+			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ show thisuuid)
 				expected (thisuuid `elem` uuids)
 		_ -> assertFailure $ f ++ " failed to look up key"
 

From 64bc4e4751c5738d3e6c44db9452c46b26245910 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 16:13:06 -0400
Subject: [PATCH 2463/2835] refactor

---
 Command/Init.hs | 6 +-----
 Init.hs         | 9 +++++----
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index e2a6eb7b03..a6d72e4226 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -9,8 +9,6 @@ module Command.Init where
 
 import Common.Annex
 import Command
-import Annex.UUID
-import Logs.UUID
 import Init
 	
 def :: [Command]
@@ -29,7 +27,5 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
-	initialize
-	u <- getUUID
-	describeUUID u description
+	initialize (Just description)
 	next $ return True
diff --git a/Init.hs b/Init.hs
index 8c79002bcb..8cec98c5fc 100644
--- a/Init.hs
+++ b/Init.hs
@@ -19,13 +19,14 @@ import Logs.UUID
 import Annex.Version
 import Annex.UUID
 
-initialize :: Annex ()
-initialize = do
+initialize :: Maybe String -> Annex ()
+initialize mdescription = do
 	prepUUID
 	Annex.Branch.create
 	setVersion
 	gitPreCommitHookWrite
-	getUUID >>= recordUUID
+	u <- getUUID
+	maybe (recordUUID u) (describeUUID u) mdescription
 
 uninitialize :: Annex ()
 uninitialize = gitPreCommitHookUnWrite
@@ -40,7 +41,7 @@ ensureInitialized = getVersion >>= maybe needsinit checkVersion
 		needsinit = do
 			annexed <- Annex.Branch.hasSomeBranch
 			if annexed
-				then initialize
+				then initialize Nothing
 				else error "First run: git-annex init"
 
 {- set up a git pre-commit hook, if one is not already present -}

From 3c263cc9ea7fe5dc5b07db4a66281cda752fc6b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 16:34:12 -0400
Subject: [PATCH 2464/2835] fix

---
 Remote.hs | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index d4fbf36cf0..1591512ef1 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -107,21 +107,19 @@ byName' n = do
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID -- special case for current repo
-nameToUUID n = do
-	res <- byName' n
-	case res of
-		Left e -> fromMaybe (error e) <$> byDescription
-		Right r -> return $ uuid r
+nameToUUID n = byName' n >>= go
 	where
-		byDescription = do
+		go (Right r) = return $ uuid r
+		go (Left e) = fromMaybe (error e) <$> bydescription
+		bydescription = do
 			m <- uuidMap
-			case M.lookup wantuuid $ transform swap m of
+			case M.lookup n $ transform swap m of
 				Just u -> return $ Just u
-				Nothing -> return $ M.lookup wantuuid $ transform double m
+				Nothing -> return $ byuuid m
+		byuuid m = M.lookup (read n) $ transform double m
 		transform a = M.fromList . map a . M.toList
 		swap (a, b) = (b, a)
-		double (a, _) = (show a, a)
-		wantuuid = read n
+		double (a, _) = (a, a)
 
 {- Pretty-prints a list of UUIDs of remotes, for human display.
  -

From 26d3c3b4977405127b52da0cddefe845b855f08f Mon Sep 17 00:00:00 2001
From: gernot 
Date: Mon, 7 Nov 2011 20:55:54 +0000
Subject: [PATCH 2465/2835]

---
 ...nnexed_file_to_a_.gitignored_location.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn

diff --git a/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
new file mode 100644
index 0000000000..ca2cf75854
--- /dev/null
+++ b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
@@ -0,0 +1,19 @@
+I just noticed that if you move a git-annex symlink to a location ignored by git, it simply works.  Upon committing that change, however, part of git-annex's `fix` function apparently tries to `git-add` the symlink.  This fails because the new, ignored location requires a `git-add --force`.
+
+Considering that git proper doesn't fail or warn, I think git-annex shouldn't either.
+
+This is the error message:
+
+	$ git mv annexed-file ignored-dir/
+	$ git commit
+	fix ignored-dir/annexed-file ok
+	(Recording state in git...)
+	The following paths are ignored by one of your .gitignore files:
+	ignored-dir
+	Use -f if you really want to add them.
+	fatal: no files added
+	Command xargs ["-0","git","--git-dir=/home/[...]/repo/.git","--work-tree=/home/[...]/repo","add","--"] failed; exit code 123
+
+	git-annex: user error (Command xargs ["-0","git","--git-dir=/home/[...]/repo/.git","--work-tree=/home/[...]/repo","add","--"] failed; exit code 123)
+	failed
+	git-annex: 1 failed

From faa4935047b7083e1970d13f51fbaa6fe7d0fe3d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 18:10:31 -0400
Subject: [PATCH 2466/2835] Handle a case where an annexed file is moved into a
 gitignored directory, by having fix --force add its change.

---
 Command/Fix.hs                                             | 2 +-
 debian/changelog                                           | 7 +++++++
 ...when_moving_annexed_file_to_a_.gitignored_location.mdwn | 2 ++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/Command/Fix.hs b/Command/Fix.hs
index c46ddc7ee0..b46d6e8ecd 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -39,5 +39,5 @@ perform file link = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	Annex.Queue.add "add" [Param "--"] [file]
+	Annex.Queue.add "add" [Param "--force", Param "--"] [file]
 	return True
diff --git a/debian/changelog b/debian/changelog
index 0b4fe3486e..265a010c45 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20111108) UNRELEASED; urgency=low
+
+  * Handle a case where an annexed file is moved into a gitignored directory,
+    by having fix --force add its change.
+
+ -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
+
 git-annex (3.20111107) unstable; urgency=low
 
   * merge: Use fast-forward merges when possible.
diff --git a/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
index ca2cf75854..34d05c0b19 100644
--- a/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
+++ b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
@@ -17,3 +17,5 @@ This is the error message:
 	git-annex: user error (Command xargs ["-0","git","--git-dir=/home/[...]/repo/.git","--work-tree=/home/[...]/repo","add","--"] failed; exit code 123)
 	failed
 	git-annex: 1 failed
+
+> Weird edge case.. ok, fixed. [[done]] --[[Joey]] 

From fdf988be6d2b3bb931a9eb3dcf3fbb83b1fb8c17 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 21:27:43 -0400
Subject: [PATCH 2467/2835] indent

---
 Utility/Misc.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index b3bf226120..4c4aa4c935 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -13,7 +13,7 @@ import Control.Monad
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
 hGetContentsStrict :: Handle -> IO String
-hGetContentsStrict  = hGetContents >=> \s -> length s `seq` return s
+hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s
 
 {- A version of readFile that is not lazy. -}
 readFileStrict :: FilePath -> IO String

From b11a63a860e8446cf3a4b35a5d8ef76329d5135c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 23:21:22 -0400
Subject: [PATCH 2468/2835] clean up read/show abuse

Avoid ever using read to parse a non-haskell formatted input string.

show :: Key is arguably still show abuse, but displaying Keys as filenames
is just too useful to give up.
---
 Annex/UUID.hs            |  8 +++---
 Command/ConfigList.hs    |  2 +-
 Command/Map.hs           |  3 ++-
 Common/Annex.hs          |  2 ++
 Crypto.hs                |  8 ++++--
 Logs/Location.hs         |  2 +-
 Logs/Presence.hs         | 54 +++++++++++++---------------------------
 Logs/Trust.hs            | 14 ++++++++---
 Logs/UUIDBased.hs        | 17 ++++++-------
 Remote.hs                | 10 ++++----
 Remote/Bup.hs            | 10 +++++---
 Remote/Helper/Special.hs |  2 +-
 Remote/S3real.hs         |  2 +-
 Types/Crypto.hs          |  8 ------
 Types/TrustLevel.hs      | 10 --------
 Types/UUID.hs            | 15 ++++++-----
 Upgrade/V1.hs            |  2 +-
 git-annex-shell.hs       |  4 +--
 18 files changed, 75 insertions(+), 98 deletions(-)

diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 90189bc475..d3d674dcc6 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -30,7 +30,7 @@ configkey = "annex.uuid"
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
 genUUID :: IO UUID
-genUUID = pOpen ReadFromPipe command params $ liftM read . hGetLine
+genUUID = pOpen ReadFromPipe command params $ liftM toUUID . hGetLine
 	where
 		command = SysConfig.uuid
 		params = if command == "uuid"
@@ -56,12 +56,12 @@ getRepoUUID r = do
 			return u
 		else return c
 	where
-		cached g = read $ Git.configGet g cachekey ""
+		cached g = toUUID $ Git.configGet g cachekey ""
 		updatecache g u = when (g /= r) $ storeUUID cachekey u
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = read $ Git.configGet r configkey ""
+getUncachedUUID r = toUUID $ Git.configGet r configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
@@ -69,4 +69,4 @@ prepUUID = whenM ((==) NoUUID <$> getUUID) $
 	storeUUID configkey =<< liftIO genUUID
 
 storeUUID :: String -> UUID -> Annex ()
-storeUUID configfield uuid = setConfig configfield (show uuid)
+storeUUID configfield = setConfig configfield . fromUUID
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index fadcbb8435..dcf4d15093 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -21,5 +21,5 @@ seek = [withNothing start]
 start :: CommandStart
 start = do
 	u <- getUUID
-	liftIO $ putStrLn $ "annex.uuid=" ++ show u
+	liftIO $ putStrLn $ "annex.uuid=" ++ fromUUID u
 	stop
diff --git a/Command/Map.hs b/Command/Map.hs
index 803324e999..11808ed636 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -62,7 +62,8 @@ drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 		others = map (unreachable . uuidnode) $
 			filter (`notElem` ruuids) (M.keys umap)
 		trusted = map (trustworthy . uuidnode) ts
-		uuidnode u = Dot.graphNode (show u) $ M.findWithDefault "" u umap
+		uuidnode u = Dot.graphNode (fromUUID u) $
+			M.findWithDefault "" u umap
 
 hostname :: Git.Repo -> String
 hostname r
diff --git a/Common/Annex.hs b/Common/Annex.hs
index 43f1ea0af3..f802ec2533 100644
--- a/Common/Annex.hs
+++ b/Common/Annex.hs
@@ -1,6 +1,7 @@
 module Common.Annex (
 	module Common,
 	module Types,
+	module Types.UUID,
 	module Annex,
 	module Locations,
 	module Messages,
@@ -8,6 +9,7 @@ module Common.Annex (
 
 import Common
 import Types
+import Types.UUID (toUUID, fromUUID)
 import Annex (gitRepo)
 import Locations
 import Messages
diff --git a/Crypto.hs b/Crypto.hs
index ced7c144c4..b3acb30a6e 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -102,14 +102,18 @@ describeCipher (EncryptedCipher _ (KeyIds ks)) =
 {- Stores an EncryptedCipher in a remote's configuration. -}
 storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig
 storeCipher c (EncryptedCipher t ks) = 
-	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (show ks) c
+	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (showkeys ks) c
+	where
+		showkeys (KeyIds l) = join "," l
 
 {- Extracts an EncryptedCipher from a remote's configuration. -}
 extractCipher :: RemoteConfig -> Maybe EncryptedCipher
 extractCipher c = 
 	case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of
-		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (read ks)
+		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (readkeys ks)
 		_ -> Nothing
+	where
+		readkeys = KeyIds . split ","
 
 {- Encrypts a Cipher to the specified KeyIds. -}
 encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 602c46f310..ff874a5964 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -37,7 +37,7 @@ logChange repo _ NoUUID _ = error $
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations key = map read <$> (currentLog . logFile) key
+keyLocations key = map toUUID <$> (currentLog . logFile) key
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/Logs/Presence.hs b/Logs/Presence.hs
index 7211eba039..f5e4f1ea94 100644
--- a/Logs/Presence.hs
+++ b/Logs/Presence.hs
@@ -16,6 +16,7 @@ module Logs.Presence (
 	addLog,
 	readLog,
 	parseLog,
+	showLog,
 	logNow,
 	compactLog,
 	currentLog,
@@ -36,41 +37,9 @@ data LogLine = LogLine {
 	info :: String
 } deriving (Eq)
 
-data LogStatus = InfoPresent | InfoMissing | Undefined
+data LogStatus = InfoPresent | InfoMissing
 	deriving (Eq)
 
-instance Show LogStatus where
-	show InfoPresent = "1"
-	show InfoMissing = "0"
-	show Undefined = "undefined"
-
-instance Read LogStatus where
-	readsPrec _ "1" = [(InfoPresent, "")]
-	readsPrec _ "0" = [(InfoMissing, "")]
-	readsPrec _ _   = [(Undefined, "")]
-
-instance Show LogLine where
-	show (LogLine d s i) = unwords [show d, show s, i]
-
-instance Read LogLine where
-	-- This parser is robust in that even unparsable log lines are
-	-- read without an exception being thrown.
-	-- Such lines have a status of Undefined.
-	readsPrec _ string = 
-		if length w >= 3
-			then maybe bad good pdate
-			else bad
-		where
-			w = words string
-			s = read $ w !! 1
-			i = w !! 2
-			pdate :: Maybe UTCTime
-			pdate = parseTime defaultTimeLocale "%s%Qs" $ head w
-
-			good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s i
-			bad = ret $ LogLine 0 Undefined ""
-			ret v = [(v, "")]
-
 addLog :: FilePath -> LogLine -> Annex ()
 addLog file line = Annex.Branch.change file $ \s -> 
 	showLog $ compactLog (line : parseLog s)
@@ -80,15 +49,26 @@ addLog file line = Annex.Branch.change file $ \s ->
 readLog :: FilePath -> Annex [LogLine]
 readLog file = parseLog <$> Annex.Branch.get file
 
+{- Parses a log file. Unparseable lines are ignored. -}
 parseLog :: String -> [LogLine]
-parseLog = filter parsable . map read . lines
+parseLog = mapMaybe (parseline . words) . lines
 	where
-		-- some lines may be unparseable, avoid them
-		parsable l = status l /= Undefined
+		parseline (a:b:c:_) = do
+			d <- parseTime defaultTimeLocale "%s%Qs" a
+			s <- parsestatus b
+			Just $ LogLine (utcTimeToPOSIXSeconds d) s c
+		parseline _ = Nothing
+		parsestatus "1" = Just InfoPresent
+		parsestatus "0" = Just InfoMissing
+		parsestatus _ = Nothing
 
 {- Generates a log file. -}
 showLog :: [LogLine] -> String
-showLog = unlines . map show
+showLog = unlines . map genline
+	where
+		genline (LogLine d s i) = unwords [show d, genstatus s, i]
+		genstatus InfoPresent = "1"
+		genstatus InfoMissing = "0"
 
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> String -> Annex LogLine
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 53a1bca2c5..8c4507dcb8 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -45,18 +45,26 @@ trustMap = do
 
 parseTrust :: String -> Maybe TrustLevel
 parseTrust s
-	| length w > 0 = readMaybe $ head w
+	| length w > 0 = Just $ parse $ head w
 	-- back-compat; the trust.log used to only list trusted repos
-	| otherwise = Just Trusted
+	| otherwise = Just $ Trusted
 	where
 		w = words s
+		parse "1" = Trusted
+		parse "0" = UnTrusted
+		parse _ = SemiTrusted
+
+showTrust :: TrustLevel -> String
+showTrust SemiTrusted = "?"
+showTrust UnTrusted = "0"
+showTrust Trusted = "1"
 
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid@(UUID _) level = do
 	ts <- liftIO $ getPOSIXTime
 	Annex.Branch.change trustLog $
-		showLog show . changeLog ts uuid level . parseLog parseTrust
+		showLog showTrust . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
 trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
 
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 7184709fe2..9609d73213 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -50,28 +50,27 @@ showLog :: (a -> String) -> Log a -> String
 showLog shower = unlines . map showpair . M.toList
 	where
 		showpair (k, LogEntry (Date p) v) =
-			unwords [show k, shower v, tskey ++ show p]
+			unwords [fromUUID k, shower v, tskey ++ show p]
 		showpair (k, LogEntry Unknown v) =
-			unwords [show k, shower v]
+			unwords [fromUUID k, shower v]
 
 parseLog :: (String -> Maybe a) -> String -> Log a
-parseLog parser = M.fromListWith best . catMaybes . map pair . lines
+parseLog parser = M.fromListWith best . catMaybes . map parse . lines
 	where
-		pair line
+		parse line
 			| null ws = Nothing
-			| otherwise = case parser $ unwords info of
-				Nothing -> Nothing
-				Just v -> Just (read u, LogEntry c v)
+			| otherwise = parser (unwords info) >>= makepair
 			where
+				makepair v = Just (toUUID u, LogEntry ts v)
 				ws = words line
 				u = head ws
 				end = last ws
-				c
+				ts
 					| tskey `isPrefixOf` end =
 						pdate $ tail $ dropWhile (/= '=') end
 					| otherwise = Unknown
 				info
-					| c == Unknown = drop 1 ws
+					| ts == Unknown = drop 1 ws
 					| otherwise = drop 1 $ init ws
 				pdate s = case parseTime defaultTimeLocale "%s%Qs" s of
 					Nothing -> Unknown
diff --git a/Remote.hs b/Remote.hs
index 1591512ef1..6d55cee243 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -100,7 +100,7 @@ byName' n = do
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
 		else return $ Right $ head match
 	where
-		matching r = n == name r || read n == uuid r
+		matching r = n == name r || toUUID n == uuid r
 
 {- Looks up a remote by name (or by UUID, or even by description),
  - and returns its UUID. Finds even remotes that are not configured in
@@ -116,7 +116,7 @@ nameToUUID n = byName' n >>= go
 			case M.lookup n $ transform swap m of
 				Just u -> return $ Just u
 				Nothing -> return $ byuuid m
-		byuuid m = M.lookup (read n) $ transform double m
+		byuuid m = M.lookup (toUUID n) $ transform double m
 		transform a = M.fromList . map a . M.toList
 		swap (a, b) = (b, a)
 		double (a, _) = (a, a)
@@ -142,8 +142,8 @@ prettyPrintUUIDs desc uuids = do
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
 		prettify m here u
-			| not (null d) = show u ++ " -- " ++ d
-			| otherwise = show u
+			| not (null d) = fromUUID u ++ " -- " ++ d
+			| otherwise = fromUUID u
 			where
 				ishere = here == u
 				n = findlog m u
@@ -152,7 +152,7 @@ prettyPrintUUIDs desc uuids = do
 					| ishere = addname n "here"
 					| otherwise = n
 		jsonify m here u = toJSObject
-			[ ("uuid", toJSON $ show u)
+			[ ("uuid", toJSON $ fromUUID u)
 			, ("description", toJSON $ findlog m u)
 			, ("here", toJSON $ here == u)
 			]
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index b613225b8f..3e621ce569 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -161,13 +161,15 @@ storeBupUUID u buprepo = do
 		then do
 			showAction "storing uuid"
 			onBupRemote r boolSystem "git"
-				[Params $ "config annex.uuid " ++ show u]
+				[Params $ "config annex.uuid " ++ v]
 					>>! error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
 			let olduuid = Git.configGet r' "annex.uuid" ""
-			when (olduuid == "") $
-				Git.run r' "config" [Param "annex.uuid", Param $ show u]
+			when (olduuid == "") $ Git.run r' "config"
+				[Param "annex.uuid", Param v]
+	where
+		v = fromUUID u
 
 onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a
 onBupRemote r a command params = do
@@ -192,7 +194,7 @@ getBupUUID r u
 	| otherwise = liftIO $ do
 		ret <- try $ Git.configRead r
 		case ret of
-			Right r' -> return (read $ Git.configGet r' "annex.uuid" "", r')
+			Right r' -> return (toUUID $ Git.configGet r' "annex.uuid" "", r')
 			Left _ -> return (NoUUID, r)
 
 {- Converts a bup remote path spec into a Git.Repo. There are some
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 8a9a01a228..38f24eb373 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -32,7 +32,7 @@ gitConfigSpecialRemote u c k v = do
 	g <- gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param $ show u]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param $ fromUUID u]
 	where
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 1f5b2bd594..1281c27861 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -64,7 +64,7 @@ s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
 s3Setup u c = handlehost $ M.lookup "host" c
 	where
 		remotename = fromJust (M.lookup "name" c)
-		defbucket = remotename ++ "-" ++ show u
+		defbucket = remotename ++ "-" ++ fromUUID u
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
diff --git a/Types/Crypto.hs b/Types/Crypto.hs
index a39a016b8b..a9d3dddc59 100644
--- a/Types/Crypto.hs
+++ b/Types/Crypto.hs
@@ -7,17 +7,9 @@
 
 module Types.Crypto where
 
-import Data.String.Utils
-
 -- XXX ideally, this would be a locked memory region
 newtype Cipher = Cipher String
 
 data EncryptedCipher = EncryptedCipher String KeyIds
 
 newtype KeyIds = KeyIds [String]
-
-instance Show KeyIds where
-	show (KeyIds ks) = join "," ks
-
-instance Read KeyIds where
-	readsPrec _ s = [(KeyIds (split "," s), "")]
diff --git a/Types/TrustLevel.hs b/Types/TrustLevel.hs
index 058ce4595c..ddb8e45e49 100644
--- a/Types/TrustLevel.hs
+++ b/Types/TrustLevel.hs
@@ -17,14 +17,4 @@ import Types.UUID
 data TrustLevel = SemiTrusted | UnTrusted | Trusted
 	deriving Eq
 
-instance Show TrustLevel where
-        show SemiTrusted = "?"
-        show UnTrusted = "0"
-        show Trusted = "1"
-
-instance Read TrustLevel where
-        readsPrec _ "1" = [(Trusted, "")]
-        readsPrec _ "0" = [(UnTrusted, "")]
-	readsPrec _ _ = [(SemiTrusted, "")]
-
 type TrustMap = M.Map UUID TrustLevel
diff --git a/Types/UUID.hs b/Types/UUID.hs
index f7232d0b97..767cd0dfe8 100644
--- a/Types/UUID.hs
+++ b/Types/UUID.hs
@@ -9,13 +9,12 @@ module Types.UUID where
 
 -- A UUID is either an arbitrary opaque string, or UUID info may be missing.
 data UUID = NoUUID | UUID String
-	deriving (Eq, Ord)
+	deriving (Eq, Ord, Show)
 
-instance Show UUID where
-	show (UUID u) = u
-	show NoUUID = ""
+fromUUID :: UUID -> String
+fromUUID (UUID u) = u
+fromUUID NoUUID = ""
 
-instance Read UUID where
-	readsPrec _ s
-		| null s = [(NoUUID, "")]
-		| otherwise = [(UUID s, "")]
+toUUID :: String -> UUID
+toUUID [] = NoUUID
+toUUID s = UUID s
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 331328e816..be9a977ad7 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -178,7 +178,7 @@ fileKey1 file = readKey1 $
 	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file
 
 writeLog1 :: FilePath -> [LogLine] -> IO ()
-writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
+writeLog1 file ls = viaTmp writeFile file (showLog ls)
 
 readLog1 :: FilePath -> IO [LogLine]
 readLog1 file = catch (parseLog <$> readFileStrict file) (const $ return [])
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index de3160953c..12cc65e4dd 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -45,9 +45,9 @@ options = commonOptions ++
 	where
 		check expected = do
 			u <- getUUID
-			when (u /= read expected) $ error $
+			when (u /= toUUID expected) $ error $
 				"expected repository UUID " ++ expected
-					++ " but found UUID " ++ show u
+					++ " but found UUID " ++ fromUUID u
 
 header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"

From 05b7608113a6b9abf92064884361f3e035ef3255 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 01:27:06 -0400
Subject: [PATCH 2469/2835] update

---
 doc/todo/git-annex_unused_eats_memory.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index fcb09a1af7..3e9942e98e 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -2,12 +2,14 @@
 (all keys with content present in the repository,
 with all keys used by files in the repository), and so
 uses more memory than git-annex typically needs; around
-60-80 mb when run in a repository with 80 thousand files.
+50 mb when run in a repository with 80 thousand files.
+
+(Used to be 80 mb, but implementation improved.)
 
 I would like to reduce this. One idea is to use a bloom filter. 
 For example, construct a bloom filter of all keys used by files in
 the repository. Then for each key with content present, check if it's
-in the bloom filter. Since there can be false negatives, this might
+in the bloom filter. Since there can be false positives, this might
 miss finding some unused keys. The probability/size of filter
 could be tunable.
 

From d35cd6ff262bc7013e5bb8fe63e8251c0e4ba8b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 12:16:02 -0400
Subject: [PATCH 2470/2835] wiki updates

---
 doc/distributed_version_control.mdwn                      | 8 ++++++++
 doc/meta.mdwn                                             | 5 +++++
 doc/special_remotes/web.mdwn                              | 2 +-
 ...he_web.mdwn => using_the_web_as_a_special_remote.mdwn} | 0
 4 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 doc/meta.mdwn
 rename doc/tips/{using_the_web.mdwn => using_the_web_as_a_special_remote.mdwn} (100%)

diff --git a/doc/distributed_version_control.mdwn b/doc/distributed_version_control.mdwn
index f9cdb7e99e..738b5adc2e 100644
--- a/doc/distributed_version_control.mdwn
+++ b/doc/distributed_version_control.mdwn
@@ -11,3 +11,11 @@ Each git-annex repository is responsible for storing some of the content,
 and can copy it to or from other repositories. [[Location_tracking]]
 information is committed to git, to let repositories inform other
 repositories what file contents they have available.
+
+--
+
+The [[tutorial]] walks you through creating a distributed set of git-annex
+repositories with no central repository.
+
+Prefer a central repository like GitHub? See the
+[[tips/centralized_git_repository_tutorial]].
diff --git a/doc/meta.mdwn b/doc/meta.mdwn
new file mode 100644
index 0000000000..f78eaf9813
--- /dev/null
+++ b/doc/meta.mdwn
@@ -0,0 +1,5 @@
+This wiki contains [[!pagecount pages="pages(*)"]]
+
+Broken links:
+
+[[!brokenlinks ]]
diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index a969fb071d..cd20a93bb1 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -1,5 +1,5 @@
 git-annex can use the WWW as a special remote, downloading urls to files.
-See [[walkthrough/using_the_web]] for usage examples.
+See [[tips/using_the_web_as_a_special_remote]] for usage examples.
 
 ## notes
 
diff --git a/doc/tips/using_the_web.mdwn b/doc/tips/using_the_web_as_a_special_remote.mdwn
similarity index 100%
rename from doc/tips/using_the_web.mdwn
rename to doc/tips/using_the_web_as_a_special_remote.mdwn

From 67c9f84a1f80a0b8335a1e6edce2cd143d71b4c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 12:23:03 -0400
Subject: [PATCH 2471/2835] fix broken links

---
 doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn   | 2 +-
 doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn       | 2 +-
 doc/bugs/fat_support.mdwn                                   | 2 --
 doc/bugs/git_annex_initremote_walks_.git-annex.mdwn         | 2 +-
 doc/distributed_version_control.mdwn                        | 2 +-
 .../getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn   | 4 +++-
 doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn   | 6 +++---
 doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn         | 6 ------
 doc/future_proofing.mdwn                                    | 2 +-
 doc/meta.mdwn                                               | 2 +-
 doc/special_remotes/S3.mdwn                                 | 4 ++--
 doc/upgrades/SHA_size.mdwn                                  | 2 +-
 doc/walkthrough/unused_data.mdwn                            | 2 +-
 13 files changed, 16 insertions(+), 22 deletions(-)
 delete mode 100644 doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn

diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
index 47fc77fa4c..69fbc816f0 100644
--- a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
@@ -1,4 +1,4 @@
-Conversation moved from [[walkthrough/recover_data_from_lost+found]]
+Conversation moved from [[tips/recover_data_from_lost+found]]
 to a proper bug. --[[Joey]]
 
 (Unfortunatly that scrambled the comment creation times and thus order.)
diff --git a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn
index e88bf07f4d..c6ef13f844 100644
--- a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn
+++ b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn
@@ -1,4 +1,4 @@
-I was trying out the example with the walkthrough [[walkthrough/using_the_URL_backend]]. I tried dropping files that I had after doing an "git annex get ." which have the URL backend associated with the files it fails with
+I was trying out the example with the walkthrough using_the_URL_backend. I tried dropping files that I had after doing an "git annex get ." which have the URL backend associated with the files it fails with
 
 
 
diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn
index 60633c29bf..70ee3b369c 100644
--- a/doc/bugs/fat_support.mdwn
+++ b/doc/bugs/fat_support.mdwn
@@ -8,8 +8,6 @@ be VFAT formatted:
 - Use of ":" in filenames of object files, also not supported.
   Could easily be fixed by reorganizing the object directory.
 
-[[!tag wishlist]]
-
 [[Done]]; in annex.version 2 repos, colons are entirely avoided in
 filenames. So a bare git clone can be put on VFAT, and git-annex
 used to move stuff --to and --from it, for sneakernet.
diff --git a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
index 2457057c81..acd369bded 100644
--- a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
+++ b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
@@ -1,4 +1,4 @@
-a [[!taglink minor]]  issue: `git annex initremote` (in particular, adding
 a key as described in [[encryption]] -- `git annex initremote my_remote
 encryption=my_key`) seems to iterate over the `.git-annex/???/???/*.log` files
diff --git a/doc/distributed_version_control.mdwn b/doc/distributed_version_control.mdwn
index 738b5adc2e..e7c858c696 100644
--- a/doc/distributed_version_control.mdwn
+++ b/doc/distributed_version_control.mdwn
@@ -14,7 +14,7 @@ repositories what file contents they have available.
 
 --
 
-The [[tutorial]] walks you through creating a distributed set of git-annex
+The [[walkthrough]] shows how to create a distributed set of git-annex
 repositories with no central repository.
 
 Prefer a central repository like GitHub? See the
diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn
index f5d5a66bfc..cc9091ae5b 100644
--- a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn
+++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn
@@ -2,7 +2,9 @@ I'm not sure if this is my stupidity or if it's a bug, but
 
     git annex copy --force --to REMOTE . 
 
-just zip's through really quickly and doesn't actually force a copy to a remote location. This is just following up on the [[git-annex directory hashing problems on osx]]. I want to just do a force copy of all my data to my portable disk to really make sure that the data is really there. I would similarly would want to make sure I can force a 
+just zip's through really quickly and doesn't actually force a copy to a
+remote location. This is just following up on the
+[[bugs/git-annex_directory_hashing_problems_on_osx]]. I want to just do a force copy of all my data to my portable disk to really make sure that the data is really there. I would similarly would want to make sure I can force a 
 
     git annex copy --force --from REMOTE .
 
diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn
index 47e8b85622..7fd31efbcf 100644
--- a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn
+++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn
@@ -1,11 +1,11 @@
-i think it would be useful to have a fourth kind of [[special remote]]s
+i think it would be useful to have a fourth kind of [[special_remotes]]
 that connects to a dumb storage using sftp or rsync. this can be emulated
 by using sshfs, but that means lots of round-trips through the system and
 is limited to platforms where sshfs is available.
 
 typical use cases are backups to storate shared between a group of people
-where each user only has limited access (sftp or rsync), when using [[bup]]
-is not an option.
+where each user only has limited access (sftp or rsync), when using
+[[special_remotes/bup]] is not an option.
 
 an alternative to implementing yet another special remote would be to have
 some kind of plugin system by which external programs can provide an
diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn
deleted file mode 100644
index 6616758730..0000000000
--- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex does not seem to support all kinds of urls that git does.
-
-> Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls_]].
-> --[[Joey]] 
-
->> And now all fixed! --[[Joey]] 
diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn
index 3937e92656..a7bcce37c9 100644
--- a/doc/future_proofing.mdwn
+++ b/doc/future_proofing.mdwn
@@ -34,4 +34,4 @@ problem:
   metadata. So if a filesystem is badly corrupted and all your annexed
   files end up in `lost+found`, they can easily be lifted back out into
   another clone of the repository. Even if the filenames are lost,
-  it's possible to [[walkthrough/recover_data_from_lost+found]].
+  it's possible to [[tips/recover_data_from_lost+found]].
diff --git a/doc/meta.mdwn b/doc/meta.mdwn
index f78eaf9813..30de003d20 100644
--- a/doc/meta.mdwn
+++ b/doc/meta.mdwn
@@ -1,4 +1,4 @@
-This wiki contains [[!pagecount pages="pages(*)"]]
+This wiki contains [[!pagecount pages="*"]]
 
 Broken links:
 
diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index 047798e238..d4d3d02388 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -1,8 +1,8 @@
 This special remote type stores file contents in a bucket in Amazon S3
 or a similar service.
 
-See [[walkthrough/using_Amazon_S3]] and
-[[walkthrough/Internet_Archive_via_S3]] for usage examples.
+See [[tips/using_Amazon_S3]] and
+[[tips/Internet_Archive_via_S3]] for usage examples.
 
 ## configuration
 
diff --git a/doc/upgrades/SHA_size.mdwn b/doc/upgrades/SHA_size.mdwn
index 319b91108a..97603ba913 100644
--- a/doc/upgrades/SHA_size.mdwn
+++ b/doc/upgrades/SHA_size.mdwn
@@ -15,6 +15,6 @@ to repository version 2, in each repository run:
 	git annex migrate
 	git commit -m 'migrated keys for v2'
 
-The usual caveats about [[walkthrough/migrating_data_to_a_new_backend]]
+The usual caveats about [[tips/migrating_data_to_a_new_backend]]
 apply; you will end up with unused keys that you can later clean up with
 `git annex unused`.
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index bd6c398710..518550ac02 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -2,7 +2,7 @@ It's possible for data to accumulate in the annex that no files in any
 branch point to anymore. One way it can happen is if you `git rm` a file
 without first calling `git annex drop`. And, when you modify an annexed
 file, the old content of the file remains in the annex. Another way is when
-migrating between key-value [[backends|backend]].
+migrating between key-value [[backends]].
 
 This might be historical data you want to preserve, so git-annex defaults to
 preserving it. So from time to time, you may want to check for such data and

From 2ff8915365099501382183af9855e739fc234861 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 12:24:56 -0400
Subject: [PATCH 2472/2835] fix

---
 .../git_annex_migrate_leaves_old_backend_versions_around.mdwn   | 2 +-
 doc/meta.mdwn                                                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn
index 7f586b5ff0..263338d64f 100644
--- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn
+++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn
@@ -2,7 +2,7 @@
 would be great if these were purged automatically somehow. 
 
 > Yes, this is an issue mentioned in the
-> [[walkthrough|walkthrough/migrating_data_to_a_new_backend]].
+> [[tips/migrating_data_to_a_new_backend]].
 > 
 > Since multiple files can point to the same content, it could be that
 > only one file has been migrated, and the content is still used. So
diff --git a/doc/meta.mdwn b/doc/meta.mdwn
index 30de003d20..5ee36f8c0b 100644
--- a/doc/meta.mdwn
+++ b/doc/meta.mdwn
@@ -1,4 +1,4 @@
-This wiki contains [[!pagecount pages="*"]]
+This wiki contains [[!pagecount pages="*"]] pages.
 
 Broken links:
 

From bf460a0a98d7e4c7f4eac525fcf300629db582b6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 15:34:10 -0400
Subject: [PATCH 2473/2835] reorder repo parameters last

Many functions took the repo as their first parameter. Changing it
consistently to be the last parameter allows doing some useful things with
currying, that reduce boilerplate.

In particular, g <- gitRepo is almost never needed now, instead
use inRepo to run an IO action in the repo, and fromRepo to get
a value from the repo.

This also provides more opportunities to use monadic and applicative
combinators.
---
 Annex.hs                 |  16 ++++++-
 Annex/Branch.hs          |  60 ++++++++++-------------
 Annex/CatFile.hs         |   3 +-
 Annex/Content.hs         |  32 +++++--------
 Annex/Queue.hs           |   3 +-
 Annex/UUID.hs            |  14 +++---
 Annex/Version.hs         |  10 ++--
 Backend.hs               |  19 +++-----
 Backend/SHA.hs           |   3 +-
 Command.hs               |   2 +-
 Command/Add.hs           |   4 +-
 Command/AddUrl.hs        |   3 +-
 Command/DropUnused.hs    |   8 ++--
 Command/Fsck.hs          |  20 ++++----
 Command/Map.hs           |   3 +-
 Command/Migrate.hs       |   7 ++-
 Command/SendKey.hs       |   3 +-
 Command/Unannex.hs       |  16 +++----
 Command/Uninit.hs        |  17 +++----
 Command/Unlock.hs        |   5 +-
 Command/Unused.hs        |  20 ++++----
 Common/Annex.hs          |   2 +-
 Config.hs                |  16 +++----
 Git.hs                   | 100 ++++++++++++++++++++-------------------
 Git/CatFile.hs           |   2 +-
 Git/LsFiles.hs           |  43 +++++++++--------
 Git/LsTree.hs            |   6 +--
 Git/Queue.hs             |   8 ++--
 Git/UnionMerge.hs        |  54 +++++++++++----------
 GitAnnex.hs              |   5 +-
 Init.hs                  |   6 +--
 Locations.hs             |  12 ++---
 Remote.hs                |  14 +++---
 Remote/Bup.hs            |  17 ++++---
 Remote/Directory.hs      |   6 +--
 Remote/Git.hs            |  19 +++-----
 Remote/Helper/Special.hs |  10 ++--
 Remote/Hook.hs           |   9 ++--
 Remote/Rsync.hs          |  13 ++---
 Remote/S3real.hs         |   7 ++-
 Remote/Web.hs            |   2 +-
 Seek.hs                  |  34 +++++++------
 Upgrade/V0.hs            |   3 +-
 Upgrade/V1.hs            |  28 +++++------
 Upgrade/V2.hs            |  40 ++++++++--------
 git-union-merge.hs       |   4 +-
 46 files changed, 338 insertions(+), 390 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 2c6402ac35..501e54c662 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -17,7 +17,9 @@ module Annex (
 	eval,
 	getState,
 	changeState,
-	gitRepo
+	gitRepo,
+	inRepo,
+	fromRepo,
 ) where
 
 import Control.Monad.IO.Control
@@ -114,6 +116,16 @@ getState = gets
 changeState :: (AnnexState -> AnnexState) -> Annex ()
 changeState = modify
 
-{- Returns the git repository being acted on -}
+{- Returns the annex's git repository. -}
 gitRepo :: Annex Git.Repo
 gitRepo = getState repo
+
+{- Runs an IO action in the annex's git repository. -}
+inRepo :: (Git.Repo -> IO a) -> Annex a
+inRepo a = do
+	g <- gitRepo
+	liftIO $ a g
+
+{- Extracts a value from the annex's git repisitory. -}
+fromRepo :: (Git.Repo -> a) -> Annex a
+fromRepo a = a <$> gitRepo
diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 163c9ec60f..189289ad39 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -56,21 +56,19 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.ls_tree g fullname >>= Git.UnionMerge.update_index g
+genIndex g = Git.UnionMerge.ls_tree fullname g >>= Git.UnionMerge.update_index g
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
-	g <- gitRepo
-	let f = index g
-
+	f <- fromRepo $ index
 	bracketIO (Git.useIndex f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
 			liftIO $ createDirectoryIfMissing True $ takeDirectory f
-			unless bootstrapping $ liftIO $ genIndex g
+			unless bootstrapping $ inRepo genIndex
 		a
 
 withIndexUpdate :: Annex a -> Annex a
@@ -103,19 +101,17 @@ getCache file = getState >>= go
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
 create = unlessM hasBranch $ do
-	g <- gitRepo
 	e <- hasOrigin
 	if e
-		then liftIO $ Git.run g "branch" [Param name, Param originname]
+		then inRepo $ Git.run "branch" [Param name, Param originname]
 		else withIndex' True $
-			liftIO $ Git.commit g "branch created" fullname []
+			inRepo $ Git.commit "branch created" fullname []
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
 	stageJournalFiles
-	g <- gitRepo
-	withIndex $ liftIO $ Git.commit g message fullname [fullname]
+	withIndex $ inRepo $ Git.commit message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before data is
  - read from it. Runs only once per git-annex run.
@@ -134,7 +130,6 @@ commit message = whenM journalDirty $ lockJournal $ do
  -}
 update :: Annex ()
 update = onceonly $ do
-	g <- gitRepo
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches
@@ -151,10 +146,10 @@ update = onceonly $ do
 			 - documentation advises users not to directly
 			 - modify the branch.
 			 -}
-			liftIO $ Git.UnionMerge.merge_index g branches
+			inRepo $ \g -> Git.UnionMerge.merge_index g branches
 		ff <- if dirty then return False else tryFastForwardTo refs
-		unless ff $
-			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
+		unless ff $ inRepo $
+			Git.commit "update" fullname (nub $ fullname:refs)
 		invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
@@ -165,14 +160,13 @@ update = onceonly $ do
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
 changedBranch :: String -> String -> Annex Bool
-changedBranch origbranch newbranch = do
-	g <- gitRepo
-	diffs <- liftIO $ Git.pipeRead g [
-		Param "log",
-		Param (origbranch ++ ".." ++ newbranch),
-		Params "--oneline -n1"
-		]
-	return $ not $ L.null diffs
+changedBranch origbranch newbranch = not . L.null <$> diffs
+	where
+		diffs = inRepo $ Git.pipeRead
+			[ Param "log"
+			, Param (origbranch ++ ".." ++ newbranch)
+			, Params "--oneline -n1"
+			]
 
 {- Given a set of refs that are all known to have commits not
  - on the git-annex branch, tries to update the branch by a
@@ -195,8 +189,7 @@ tryFastForwardTo (first:rest) = do
 	where
 		no_ff = return False
 		do_ff branch = do
-			g <- gitRepo
-			liftIO $ Git.run g "update-ref" [Param fullname, Param branch]
+			inRepo $ Git.run "update-ref" [Param fullname, Param branch]
 			return True
 		findbest c [] = return $ Just c
 		findbest c (r:rs)
@@ -223,10 +216,8 @@ disableUpdate = Annex.changeState setupdated
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
-refExists ref = do
-	g <- gitRepo
-	liftIO $ Git.runBool g "show-ref"
-		[Param "--verify", Param "-q", Param ref]
+refExists ref = inRepo $ Git.runBool "show-ref"
+	[Param "--verify", Param "-q", Param ref]
 
 {- Does the main git-annex branch exist? -}
 hasBranch :: Annex Bool
@@ -244,8 +235,7 @@ hasSomeBranch = not . null <$> siblingBranches
  - from remotes. Duplicate refs are filtered out. -}
 siblingBranches :: Annex [(String, String)]
 siblingBranches = do
-	g <- gitRepo
-	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+	r <- inRepo $ Git.pipeRead [Param "show-ref", Param name]
 	return $ nubBy uref $ map (pair . words . L.unpack) (L.lines r)
 	where
 		pair l = (head l, last l)
@@ -280,8 +270,7 @@ get file = fromcache =<< getCache file
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
-	g <- gitRepo
-	bfiles <- liftIO $ Git.pipeNullSplit g
+	bfiles <- inRepo $ Git.pipeNullSplit
 		[Params "ls-tree --name-only -r -z", Param fullname]
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
@@ -349,8 +338,8 @@ stageJournalFiles = do
 	where
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
-		git_hash_object g = Git.gitCommandLine g
-			[Param "hash-object", Param "-w", Param "--stdin-paths"]
+		git_hash_object g = Git.gitCommandLine
+			[Param "hash-object", Param "-w", Param "--stdin-paths"] g
 
 
 {- Checks if there are changes in the journal. -}
@@ -379,8 +368,7 @@ fileJournal = replace "//" "_" . replace "_" "/"
  - contention with other git-annex processes. -}
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
-	g <- gitRepo
-	let file = gitAnnexJournalLock g
+	file <- fromRepo $ gitAnnexJournalLock
 	bracketIO (lock file) unlock a
 	where
 		lock file = do
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 2707ed3ea3..a043e1ae3d 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -17,8 +17,7 @@ catFile :: String -> FilePath -> Annex String
 catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
 	where
 		startup = do
-			g <- gitRepo
-			h <- liftIO $ Git.CatFile.catFileStart g
+			h <- inRepo $ Git.CatFile.catFileStart
 			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
 			go h
 		go h = liftIO $ Git.CatFile.catFile h branch file
diff --git a/Annex/Content.hs b/Annex/Content.hs
index aafdf6f2e6..fc2c2d0921 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -37,18 +37,18 @@ import Config
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
 inAnnex key = do
-	g <- gitRepo
-	when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo"
-	liftIO $ doesFileExist $ gitAnnexLocation g key
+	whenM (fromRepo Git.repoIsUrl) $
+		error "inAnnex cannot check remote repo"
+	inRepo $ doesFileExist . gitAnnexLocation key
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
-	g <- gitRepo
 	cwd <- liftIO getCurrentDirectory
 	let absfile = fromMaybe whoops $ absNormPath cwd file
+	top <- fromRepo Git.workTree
 	return $ relPathDirToFile (parentDir absfile) 
-			(Git.workTree g)  ".git"  annexLocation key
+			top  ".git"  annexLocation key
 	where
 		whoops = error $ "unable to normalize " ++ file
 
@@ -65,8 +65,7 @@ logStatus key status = do
  - the annex as a key's content. -}
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
-	g <- gitRepo
-	let tmp = gitAnnexTmpLocation g key
+	tmp <- fromRepo $ gitAnnexTmpLocation key
 
 	-- Check that there is enough free disk space.
 	-- When the temp file already exists, count the space
@@ -84,8 +83,7 @@ getViaTmp key action = do
 
 prepTmp :: Key -> Annex FilePath
 prepTmp key = do
-	g <- gitRepo
-	let tmp = gitAnnexTmpLocation g key
+	tmp <- fromRepo $ gitAnnexTmpLocation key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	return tmp
 
@@ -162,8 +160,7 @@ checkDiskSpace' adjustment key = do
  -}
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
-	g <- gitRepo
-	let dest = gitAnnexLocation g key
+	dest <- fromRepo $ gitAnnexLocation key
 	let dir = parentDir dest
 	e <- liftIO $ doesFileExist dest
 	if e
@@ -177,8 +174,7 @@ moveAnnex key src = do
 
 withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
 withObjectLoc key a = do
-	g <- gitRepo
-	let file = gitAnnexLocation g key
+	file <- fromRepo $gitAnnexLocation key
 	let dir = parentDir file
 	a (dir, file)
 
@@ -201,9 +197,9 @@ fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
  - returns the file it was moved to. -}
 moveBad :: Key -> Annex FilePath
 moveBad key = do
-	g <- gitRepo
-	let src = gitAnnexLocation g key
-	let dest = gitAnnexBadDir g  takeFileName src
+	src <- fromRepo $ gitAnnexLocation key
+	bad <- fromRepo $ gitAnnexBadDir
+	let dest = bad  takeFileName src
 	liftIO $ do
 		createDirectoryIfMissing True (parentDir dest)
 		allowWrite (parentDir src)
@@ -214,9 +210,7 @@ moveBad key = do
 
 {- List of keys whose content exists in .git/annex/objects/ -}
 getKeysPresent :: Annex [Key]
-getKeysPresent = do
-	g <- gitRepo
-	getKeysPresent' $ gitAnnexObjectDir g
+getKeysPresent = getKeysPresent' =<< fromRepo gitAnnexObjectDir
 getKeysPresent' :: FilePath -> Annex [Key]
 getKeysPresent' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index 4c1182750a..f611cf02eb 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -34,8 +34,7 @@ flush silent = do
 	unless (0 == Git.Queue.size q) $ do
 		unless silent $
 			showSideAction "Recording state in git"
-		g <- gitRepo
-		q' <- liftIO $ Git.Queue.flush g q
+		q' <- inRepo $ Git.Queue.flush q
 		store q'
 
 store :: Git.Queue.Queue -> Annex ()
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index d3d674dcc6..6fc04c0f09 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -45,23 +45,23 @@ getUUID = getRepoUUID =<< gitRepo
 {- Looks up a repo's UUID. May return "" if none is known. -}
 getRepoUUID :: Git.Repo -> Annex UUID
 getRepoUUID r = do
-	g <- gitRepo
-
-	let c = cached g
+	c <- fromRepo cached
 	let u = getUncachedUUID r
 	
 	if c /= u && u /= NoUUID
 		then do
-			updatecache g u
+			updatecache u
 			return u
 		else return c
 	where
-		cached g = toUUID $ Git.configGet g cachekey ""
-		updatecache g u = when (g /= r) $ storeUUID cachekey u
+		cached g = toUUID $ Git.configGet cachekey "" g
+		updatecache u = do
+			g <- gitRepo
+			when (g /= r) $ storeUUID cachekey u
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = toUUID $ Git.configGet r configkey ""
+getUncachedUUID = toUUID . Git.configGet configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
diff --git a/Annex/Version.hs b/Annex/Version.hs
index 935f777abf..9e694faf14 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -26,12 +26,10 @@ versionField :: String
 versionField = "annex.version"
 
 getVersion :: Annex (Maybe Version)
-getVersion = do
-	g <- gitRepo
-	let v = Git.configGet g versionField ""
-	if not $ null v
-		then return $ Just v
-		else return Nothing
+getVersion = handle <$> fromRepo (Git.configGet versionField "")
+	where
+		handle [] = Nothing
+		handle v = Just v
 
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
diff --git a/Backend.hs b/Backend.hs
index 9a40e54598..f7990c22c1 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -47,10 +47,7 @@ orderedList = do
 			l' <- (lookupBackendName name :) <$> standard
 			Annex.changeState $ \s -> s { Annex.backends = l' }
 			return l'
-		standard = do
-			g <- gitRepo
-			return $ parseBackendList $
-				Git.configGet g "annex.backends" ""
+		standard = fromRepo $ parseBackendList . Git.configGet "annex.backends" ""
 		parseBackendList [] = list
 		parseBackendList s = map lookupBackendName $ words s
 
@@ -96,16 +93,14 @@ type BackendFile = (Maybe (Backend Annex), FilePath)
  - That can be configured on a per-file basis in the gitattributes file.
  -}
 chooseBackends :: [FilePath] -> Annex [BackendFile]
-chooseBackends fs = do
-	g <- gitRepo
-	forced <- Annex.getState Annex.forcebackend
-	if isJust forced
-		then do
+chooseBackends fs = Annex.getState Annex.forcebackend >>= go
+	where
+		go Nothing = do
+			pairs <- inRepo $ Git.checkAttr "annex.backend" fs
+			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
+		go (Just _) = do
 			l <- orderedList
 			return $ map (\f -> (Just $ head l, f)) fs
-		else do
-			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
-			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend Annex
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index d449821172..a3846a410b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -98,9 +98,8 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 {- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
-	g <- gitRepo
 	fast <- Annex.getState Annex.fast
-	let file = gitAnnexLocation g key
+	file <- fromRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	if not present || fast
 		then return True
diff --git a/Command.hs b/Command.hs
index c11b906103..c436c5b628 100644
--- a/Command.hs
+++ b/Command.hs
@@ -78,7 +78,7 @@ notBareRepo a = do
 	a
 
 isBareRepo :: Annex Bool
-isBareRepo = Git.repoIsLocalBare <$> gitRepo
+isBareRepo = fromRepo Git.repoIsLocalBare
 
 {- Used for commands that have an auto mode that checks the number of known
  - copies of a key.
diff --git a/Command/Add.hs b/Command/Add.hs
index a633db7b36..ab104b53cc 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -60,8 +60,8 @@ undo file key e = do
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do
-			g <- gitRepo
-			liftIO $ renameFile (gitAnnexLocation g key) file
+			src <- fromRepo $ gitAnnexLocation key
+			liftIO $ renameFile src file
 
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
 cleanup file key hascontent = do
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index b717e271d1..945848e9f7 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -41,10 +41,9 @@ perform url file = do
 
 download :: String -> FilePath -> CommandPerform
 download url file = do
-	g <- gitRepo
 	showAction $ "downloading " ++ url ++ " "
 	let dummykey = Backend.URL.fromUrl url
-	let tmp = gitAnnexTmpLocation g dummykey
+	tmp <- fromRepo $ gitAnnexTmpLocation dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	ok <- liftIO $ Url.download url tmp
 	if ok
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 70dcc4cc72..55c21f83bf 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -58,17 +58,15 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 			next $ Command.Drop.cleanupRemote key r
 		droplocal = Command.Drop.performLocal key (Just 0) -- force drop
 
-performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
+performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
-	g <- gitRepo
-	let f = filespec g key
+	f <- fromRepo $ filespec key
 	liftIO $ whenM (doesFileExist f) $ removeFile f
 	next $ return True
 
 readUnusedLog :: FilePath -> Annex UnusedMap
 readUnusedLog prefix = do
-	g <- gitRepo
-	let f = gitAnnexUnusedLog prefix g
+	f <- fromRepo $ gitAnnexUnusedLog prefix
 	e <- liftIO $ doesFileExist f
 	if e
 		then M.fromList . map parse . lines <$> liftIO (readFile f)
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index d1abb29e37..3feabeb9ee 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -79,26 +79,26 @@ check = sequence >=> dispatch
    in this repository only. -}
 verifyLocationLog :: Key -> String -> Annex Bool
 verifyLocationLog key desc = do
-	g <- gitRepo
 	present <- inAnnex key
 	
 	-- Since we're checking that a key's file is present, throw
 	-- in a permission fixup here too.
-	when present $ liftIO $ do
-		let f = gitAnnexLocation g key
-		preventWrite f
-		preventWrite (parentDir f)
+	when present $ do
+		f <- fromRepo $ gitAnnexLocation key
+		liftIO $ do
+			preventWrite f
+			preventWrite (parentDir f)
 
 	u <- getUUID
         uuids <- keyLocations key
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
-				fix g u InfoPresent
+				fix u InfoPresent
 				-- There is no data loss, so do not fail.
 				return True
 		(False, True) -> do
-				fix g u InfoMissing
+				fix u InfoMissing
 				warning $
 					"** Based on the location log, " ++ desc
 					++ "\n** was expected to be present, " ++
@@ -107,16 +107,16 @@ verifyLocationLog key desc = do
 		_ -> return True
 	
 	where
-		fix g u s = do
+		fix u s = do
 			showNote "fixing location log"
+			g <- gitRepo
 			logChange g key u s
 
 {- The size of the data for a key is checked against the size encoded in
  - the key's metadata, if available. -}
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
-	g <- gitRepo
-	let file = gitAnnexLocation g key
+	file <- fromRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	case (present, Types.Key.keySize key) of
 		(_, Nothing) -> return True
diff --git a/Command/Map.hs b/Command/Map.hs
index 11808ed636..f72cb107ad 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -31,8 +31,7 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	g <- gitRepo
-	rs <- spider g
+	rs <- spider =<< gitRepo
 
 	umap <- uuidMap
 	trusted <- trustGet Trusted
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 2d4d24a224..a823466dc7 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -42,14 +42,13 @@ upgradableKey key = isNothing $ Types.Key.keySize key
 
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
-	g <- gitRepo
-
 	-- Store the old backend's cached key in the new backend
 	-- (the file can't be stored as usual, because it's already a symlink).
 	-- The old backend's key is not dropped from it, because there may
 	-- be other files still pointing at that key.
-	let src = gitAnnexLocation g oldkey
-	let tmpfile = gitAnnexTmpDir g  takeFileName file
+	src <- fromRepo $ gitAnnexLocation oldkey
+	tmp <- fromRepo $ gitAnnexTmpDir
+	let tmpfile = tmp  takeFileName file
 	liftIO $ createLink src tmpfile
 	k <- Backend.genKey tmpfile $ Just newbackend
 	liftIO $ cleantmp tmpfile
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 318ea56d03..5737478671 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -21,8 +21,7 @@ seek = [withKeys start]
 
 start :: Key -> CommandStart
 start key = do
-	g <- gitRepo
-	let file = gitAnnexLocation g key
+	file <- fromRepo $ gitAnnexLocation key
 	whenM (inAnnex key) $
 		liftIO $ rsyncServerSend file -- does not return
 	warning "requested key is not present"
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 825f819390..d24f921a93 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -31,8 +31,8 @@ start file = isAnnexed file $ \(key, _) -> do
 		then do
 			force <- Annex.getState Annex.force
 			unless force $ do
-				g <- gitRepo
-				staged <- liftIO $ LsFiles.staged g [Git.workTree g]
+				top <- fromRepo Git.workTree
+				staged <- inRepo $ LsFiles.staged [top]
 				unless (null staged) $
 					error "This command cannot be run when there are already files staged for commit."
 				Annex.changeState $ \s -> s { Annex.force = True }
@@ -46,19 +46,19 @@ perform file key = next $ cleanup file key
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	g <- gitRepo
-
 	liftIO $ removeFile file
-	liftIO $ Git.run g "rm" [Params "--quiet --", File file]
+	inRepo $ Git.run "rm" [Params "--quiet --", File file]
 	-- git rm deletes empty directories; put them back
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 
 	fast <- Annex.getState Annex.fast
 	if fast
-		then liftIO $ do
+		then do
 			-- fast mode: hard link to content in annex
-			createLink (gitAnnexLocation g key) file
-			allowWrite file
+			src <- fromRepo $ gitAnnexLocation key
+			liftIO $ do
+				createLink src file
+				allowWrite file
 		else do
 			fromAnnex key file
 			logStatus key InfoMissing
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 5a6ee0be25..f317b7620d 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -28,11 +28,9 @@ check = do
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ b ++ " branch is checked out"
 	where
-		current_branch = do
-			g <- gitRepo
-			b <- liftIO $
-				Git.pipeRead g [Params "rev-parse --abbrev-ref HEAD"]
-			return $ head $ lines $ B.unpack b
+		current_branch = head . lines . B.unpack <$> revhead
+		revhead = inRepo $ Git.pipeRead 
+			[Params "rev-parse --abbrev-ref HEAD"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit startUnannex, withNothing start]
@@ -53,12 +51,11 @@ perform = next cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
-	g <- gitRepo
+	annexdir <- fromRepo $ gitAnnexDir
 	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
-	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
+	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
 	saveState
-	liftIO $ do
-		Git.run g "branch" [Param "-D", Param Annex.Branch.name]
-		exitSuccess
+	inRepo $ Git.run "branch" [Param "-D", Param Annex.Branch.name]
+	liftIO $ exitSuccess
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 7ecaf0b7fa..590b753111 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -37,9 +37,8 @@ perform dest key = do
 	
 	checkDiskSpace key
 
-	g <- gitRepo
-	let src = gitAnnexLocation g key
-	let tmpdest = gitAnnexTmpLocation g key
+	src <- fromRepo $ gitAnnexLocation key
+	tmpdest <- fromRepo $ gitAnnexTmpLocation key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showAction "copying"
 	ok <- liftIO $ copyFileExternal src tmpdest
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 7e9ffa01f5..9d56d1ff11 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -75,8 +75,8 @@ checkRemoteUnused' r = do
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
-	g <- gitRepo
-	liftIO $ viaTmp writeFile (gitAnnexUnusedLog prefix g) $
+	logfile <- fromRepo $ gitAnnexUnusedLog prefix
+	liftIO $ viaTmp writeFile logfile $
 		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
 
 table :: [(Int, Key)] -> [String]
@@ -147,8 +147,7 @@ unusedKeys = do
 excludeReferenced :: [Key] -> Annex [Key]
 excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
-	g <- gitRepo
-	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
+	c <- inRepo $ Git.pipeRead [Param "show-ref"]
 	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
 		(S.fromList l)
 	where
@@ -183,8 +182,8 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 {- List of keys referenced by symlinks in the git repo. -}
 getKeysReferenced :: Annex [Key]
 getKeysReferenced = do
-	g <- gitRepo
-	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
+	top <- fromRepo Git.workTree
+	files <- inRepo $ LsFiles.inRepo [top]
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
 
@@ -192,8 +191,7 @@ getKeysReferenced = do
 getKeysReferencedInGit :: String -> Annex [Key]
 getKeysReferencedInGit ref = do
 	showAction $ "checking " ++ Git.refDescribe ref
-	g <- gitRepo
-	findkeys [] =<< liftIO (LsTree.lsTree g ref)
+	findkeys [] =<< inRepo (LsTree.lsTree ref)
 	where
 		findkeys c [] = return c
 		findkeys c (l:ls)
@@ -217,16 +215,14 @@ staleKeysPrune dirspec present = do
 	let stale = contents `exclude` present
 	let dups = contents `exclude` stale
 
-	g <- gitRepo
-	let dir = dirspec g
+	dir <- fromRepo dirspec
 	liftIO $ forM_ dups $ \t -> removeFile $ dir  keyFile t
 
 	return stale
 
 staleKeys :: (Git.Repo -> FilePath) -> Annex [Key]
 staleKeys dirspec = do
-	g <- gitRepo
-	let dir = dirspec g
+	dir <- fromRepo dirspec
 	exists <- liftIO $ doesDirectoryExist dir
 	if not exists
 		then return []
diff --git a/Common/Annex.hs b/Common/Annex.hs
index f802ec2533..6b5bc31de2 100644
--- a/Common/Annex.hs
+++ b/Common/Annex.hs
@@ -10,6 +10,6 @@ module Common.Annex (
 import Common
 import Types
 import Types.UUID (toUUID, fromUUID)
-import Annex (gitRepo)
+import Annex (gitRepo, inRepo, fromRepo)
 import Locations
 import Messages
diff --git a/Config.hs b/Config.hs
index bf929219df..c24ab9d04d 100644
--- a/Config.hs
+++ b/Config.hs
@@ -16,19 +16,17 @@ type ConfigKey = String
 {- Changes a git config setting in both internal state and .git/config -}
 setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
-	g <- gitRepo
-	liftIO $ Git.run g "config" [Param k, Param value]
+	inRepo $ Git.run "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
-	g' <- liftIO $ Git.configRead g
-	Annex.changeState $ \s -> s { Annex.repo = g' }
+	newg <- inRepo $ Git.configRead
+	Annex.changeState $ \s -> s { Annex.repo = newg }
 
 {- Looks up a per-remote config setting in git config.
  - Failing that, tries looking for a global config option. -}
 getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
 getConfig r key def = do
-	g <- gitRepo
-	let def' = Git.configGet g ("annex." ++ key) def
-	return $ Git.configGet g (remoteConfig r key) def'
+	def' <- fromRepo $ Git.configGet ("annex." ++ key) def
+	fromRepo $ Git.configGet (remoteConfig r key) def'
 
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
@@ -87,7 +85,5 @@ getNumCopies v =
 	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
 	where
 		use (Just n) = return n
-		use Nothing = do
-			g <- gitRepo
-			return $ read $ Git.configGet g config "1"
+		use Nothing = read <$> fromRepo (Git.configGet config "1")
 		config = "annex.numcopies"
diff --git a/Git.hs b/Git.hs
index b05c3b2d5e..6fb6e8361c 100644
--- a/Git.hs
+++ b/Git.hs
@@ -188,13 +188,13 @@ repoRemoteName Repo { remoteName = Just name } = Just name
 repoRemoteName _ = Nothing
 
 {- Sets the name of a remote. -}
-repoRemoteNameSet :: Repo -> String -> Repo
-repoRemoteNameSet r n = r { remoteName = Just n }
+repoRemoteNameSet :: String -> Repo -> Repo
+repoRemoteNameSet n r = r { remoteName = Just n }
 
 {- Sets the name of a remote based on the git config key, such as
    "remote.foo.url". -}
-repoRemoteNameFromKey :: Repo -> String -> Repo
-repoRemoteNameFromKey r k = repoRemoteNameSet r basename
+repoRemoteNameFromKey :: String -> Repo -> Repo
+repoRemoteNameFromKey k = repoRemoteNameSet basename
 	where
 		basename = join "." $ reverse $ drop 1 $
 				reverse $ drop 1 $ split "." k
@@ -280,8 +280,8 @@ workTree Repo { location = Unknown } = undefined
  - is itself a symlink). But, if the cwd is "/tmp/repo/subdir",
  - it's best to refer to "../foo".
  -}
-workTreeFile :: Repo -> FilePath -> IO FilePath
-workTreeFile repo@(Repo { location = Dir d }) file = do
+workTreeFile :: FilePath -> Repo -> IO FilePath
+workTreeFile file repo@(Repo { location = Dir d }) = do
 	cwd <- getCurrentDirectory
 	let file' = absfile cwd
 	unless (inrepo file') $
@@ -296,7 +296,7 @@ workTreeFile repo@(Repo { location = Dir d }) file = do
 		absfile c = fromMaybe file $ secureAbsNormPath c file
 		inrepo f = absrepo `isPrefixOf` f
 		bad = error $ "bad repo" ++ repoDescribe repo
-workTreeFile repo _ = assertLocal repo $ error "internal"
+workTreeFile _ repo = assertLocal repo $ error "internal"
 
 {- Path of an URL repo. -}
 urlPath :: Repo -> String
@@ -350,23 +350,23 @@ urlAuthPart a Repo { location = Url u } = a auth
 urlAuthPart _ repo = assertUrl repo $ error "internal"
 
 {- Constructs a git command line operating on the specified repo. -}
-gitCommandLine :: Repo -> [CommandParam] -> [CommandParam]
-gitCommandLine repo@(Repo { location = Dir _ } ) params =
+gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
+gitCommandLine params repo@(Repo { location = Dir _ } ) =
 	-- force use of specified repo via --git-dir and --work-tree
 	[ Param ("--git-dir=" ++ gitDir repo)
 	, Param ("--work-tree=" ++ workTree repo)
 	] ++ params
-gitCommandLine repo _ = assertLocal repo $ error "internal"
+gitCommandLine _ repo = assertLocal repo $ error "internal"
 
 {- Runs git in the specified repo. -}
-runBool :: Repo -> String -> [CommandParam] -> IO Bool
-runBool repo subcommand params = assertLocal repo $
-	boolSystem "git" $ gitCommandLine repo $ Param subcommand : params
+runBool :: String -> [CommandParam] -> Repo -> IO Bool
+runBool subcommand params repo = assertLocal repo $
+	boolSystem "git" $ gitCommandLine (Param subcommand : params) repo
 
 {- Runs git in the specified repo, throwing an error if it fails. -}
-run :: Repo -> String -> [CommandParam] -> IO ()
-run repo subcommand params = assertLocal repo $
-	runBool repo subcommand params
+run :: String -> [CommandParam] -> Repo -> IO ()
+run subcommand params repo = assertLocal repo $
+	runBool subcommand params repo
 		>>! error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns its output, lazily. 
@@ -374,26 +374,26 @@ run repo subcommand params = assertLocal repo $
  - Note that this leaves the git process running, and so zombies will
  - result unless reap is called.
  -}
-pipeRead :: Repo -> [CommandParam] -> IO L.ByteString
-pipeRead repo params = assertLocal repo $ do
-	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine repo params
+pipeRead :: [CommandParam] -> Repo -> IO L.ByteString
+pipeRead params repo = assertLocal repo $ do
+	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo
 	hSetBinaryMode h True
 	L.hGetContents h
 
 {- Runs a git subcommand, feeding it input.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: Repo -> [CommandParam] -> L.ByteString -> IO PipeHandle
-pipeWrite repo params s = assertLocal repo $ do
-	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine repo params)
+pipeWrite :: [CommandParam] -> L.ByteString -> Repo -> IO PipeHandle
+pipeWrite params s repo = assertLocal repo $ do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
 	L.hPut h s
 	hClose h
 	return p
 
 {- Runs a git subcommand, feeding it input, and returning its output.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: Repo -> [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString)
-pipeWriteRead repo params s = assertLocal repo $ do
-	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine repo params)
+pipeWriteRead :: [CommandParam] -> L.ByteString -> Repo -> IO (PipeHandle, L.ByteString)
+pipeWriteRead params s repo = assertLocal repo $ do
+	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
 	hSetBinaryMode from True
 	L.hPut to s
 	hClose to
@@ -402,13 +402,13 @@ pipeWriteRead repo params s = assertLocal repo $ do
 
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it. -}
-pipeNullSplit :: Repo -> [CommandParam] -> IO [String]
-pipeNullSplit repo params = map L.unpack <$> pipeNullSplitB repo params
+pipeNullSplit :: [CommandParam] -> Repo -> IO [String]
+pipeNullSplit params repo = map L.unpack <$> pipeNullSplitB params repo
 
 {- For when Strings are not needed. -}
-pipeNullSplitB :: Repo -> [CommandParam] -> IO [L.ByteString]
-pipeNullSplitB repo params = filter (not . L.null) . L.split '\0' <$>
-	pipeRead repo params
+pipeNullSplitB ::[CommandParam] -> Repo -> IO [L.ByteString]
+pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
+	pipeRead params repo
 
 {- Reaps any zombie git processes. -}
 reap :: IO ()
@@ -448,15 +448,15 @@ shaSize = 40
 
 {- Commits the index into the specified branch, 
  - with the specified parent refs. -}
-commit :: Repo -> String -> String -> [String] -> IO ()
-commit g message newref parentrefs = do
+commit :: String -> String -> [String] -> Repo -> IO ()
+commit message newref parentrefs repo = do
 	tree <- getSha "write-tree" $ asString $
-		pipeRead g [Param "write-tree"]
+		pipeRead [Param "write-tree"] repo
 	sha <- getSha "commit-tree" $ asString $
-		ignorehandle $ pipeWriteRead g
+		ignorehandle $ pipeWriteRead
 			(map Param $ ["commit-tree", tree] ++ ps)
-			(L.pack message)
-	run g "update-ref" [Param newref, Param sha]
+			(L.pack message) repo
+	run "update-ref" [Param newref, Param sha] repo
 	where
 		ignorehandle a = snd <$> a
 		asString a = L.unpack <$> a
@@ -478,13 +478,13 @@ configRead r = assertLocal r $ error "internal"
 hConfigRead :: Repo -> Handle -> IO Repo
 hConfigRead repo h = do
 	val <- hGetContentsStrict h
-	configStore repo val
+	configStore val repo
 
 {- Stores a git config into a repo, returning the new version of the repo.
  - The git config may be multiple lines, or a single line. Config settings
  - can be updated inrementally. -}
-configStore :: Repo -> String -> IO Repo
-configStore repo s = do
+configStore :: String -> Repo -> IO Repo
+configStore s repo = do
 	let repo' = repo { config = configParse s `M.union` config repo }
 	rs <- configRemotes repo'
 	return $ repo' { remotes = rs }
@@ -507,13 +507,11 @@ configRemotes repo = mapM construct remotepairs
 		filterkeys f = filterconfig (\(k,_) -> f k)
 		remotepairs = filterkeys isremote
 		isremote k = startswith "remote." k && endswith ".url" k
-		construct (k,v) = do
-			r <- genRemote repo v
-			return $ repoRemoteNameFromKey r k
+		construct (k,v) = repoRemoteNameFromKey k <$> genRemote v repo
 
 {- Generates one of a repo's remotes using a given location (ie, an url). -}
-genRemote :: Repo -> String -> IO Repo
-genRemote repo = gen . calcloc
+genRemote :: String -> Repo -> IO Repo
+genRemote s repo = gen $ calcloc s
 	where
 		filterconfig f = filter f $ M.toList $ config repo
 		gen v	
@@ -549,8 +547,8 @@ configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 
 {- Returns a single git config setting, or a default value if not set. -}
-configGet :: Repo -> String -> String -> String
-configGet repo key defaultValue = 
+configGet :: String -> String -> Repo -> String
+configGet key defaultValue repo = 
 	M.findWithDefault defaultValue key (config repo)
 
 {- Access to raw config Map -}
@@ -558,8 +556,8 @@ configMap :: Repo -> M.Map String String
 configMap = config
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
-checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
-checkAttr repo attr files = do
+checkAttr :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
+checkAttr attr files repo = do
 	-- git check-attr needs relative filenames input; it will choke
 	-- on some absolute filenames. This also means it will output
 	-- all relative filenames.
@@ -574,7 +572,11 @@ checkAttr repo attr files = do
         hClose toh
 	(map topair . lines) <$> hGetContents fromh
 	where
-		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]
+		params = gitCommandLine 
+				[ Param "check-attr"
+				, Param attr
+				, Params "-z --stdin"
+				] repo
 		topair l = (file, value)
 			where 
 				file = decodeGitFile $ join sep $ take end bits
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 64857c66a9..51fa585a82 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -25,7 +25,7 @@ type CatFileHandle = (PipeHandle, Handle, Handle)
 {- Starts git cat-file running in batch mode in a repo and returns a handle. -}
 catFileStart :: Repo -> IO CatFileHandle
 catFileStart repo = hPipeBoth "git" $ toCommand $
-	Git.gitCommandLine repo [Param "cat-file", Param "--batch"]
+	Git.gitCommandLine [Param "cat-file", Param "--batch"] repo
 
 {- Stops git cat-file. -}
 catFileStop :: CatFileHandle -> IO ()
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 28e007a4dc..bceee26fcd 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -19,51 +19,52 @@ import Git
 import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
-inRepo :: Repo -> [FilePath] -> IO [FilePath]
-inRepo repo l = pipeNullSplit repo $ Params "ls-files --cached -z --" : map File l
+inRepo :: [FilePath] -> Repo -> IO [FilePath]
+inRepo l repo = pipeNullSplit (Params "ls-files --cached -z --" : map File l) repo
 
 {- Scans for files at the specified locations that are not checked into git. -}
-notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
-notInRepo repo include_ignored l = pipeNullSplit repo $
-		[Params "ls-files --others"] ++ exclude ++
-		[Params "-z --"] ++ map File l
+notInRepo :: Bool -> [FilePath] -> Repo -> IO [FilePath]
+notInRepo include_ignored l repo = pipeNullSplit params repo
 	where
+		params = [Params "ls-files --others"] ++ exclude ++
+			[Params "-z --"] ++ map File l
 		exclude
 			| include_ignored = []
 			| otherwise = [Param "--exclude-standard"]
 
 {- Returns a list of all files that are staged for commit. -}
-staged :: Repo -> [FilePath] -> IO [FilePath]
-staged repo l = staged' repo l []
+staged :: [FilePath] -> Repo -> IO [FilePath]
+staged = staged' []
 
 {- Returns a list of the files, staged for commit, that are being added,
  - moved, or changed (but not deleted), from the specified locations. -}
-stagedNotDeleted :: Repo -> [FilePath] -> IO [FilePath]
-stagedNotDeleted repo l = staged' repo l [Param "--diff-filter=ACMRT"]
+stagedNotDeleted :: [FilePath] -> Repo -> IO [FilePath]
+stagedNotDeleted = staged' [Param "--diff-filter=ACMRT"]
 
-staged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
-staged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+staged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
+staged' middle l = pipeNullSplit $ start ++ middle ++ end
 	where
 		start = [Params "diff --cached --name-only -z"]
 		end = Param "--" : map File l
 
 {- Returns a list of files that have unstaged changes. -}
-changedUnstaged :: Repo -> [FilePath] -> IO [FilePath]
-changedUnstaged repo l = pipeNullSplit repo $
-	Params "diff --name-only -z --" : map File l
+changedUnstaged :: [FilePath] -> Repo -> IO [FilePath]
+changedUnstaged l = pipeNullSplit params
+	where
+		params = Params "diff --name-only -z --" : map File l
 
 {- Returns a list of the files in the specified locations that are staged
  - for commit, and whose type has changed. -}
-typeChangedStaged :: Repo -> [FilePath] -> IO [FilePath]
-typeChangedStaged repo l = typeChanged' repo l [Param "--cached"]
+typeChangedStaged :: [FilePath] -> Repo -> IO [FilePath]
+typeChangedStaged = typeChanged' [Param "--cached"]
 
 {- Returns a list of the files in the specified locations whose type has
  - changed.  Files only staged for commit will not be included. -}
-typeChanged :: Repo -> [FilePath] -> IO [FilePath]
-typeChanged repo l = typeChanged' repo l []
+typeChanged :: [FilePath] -> Repo -> IO [FilePath]
+typeChanged = typeChanged' []
 
-typeChanged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
-typeChanged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+typeChanged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
+typeChanged' middle l = pipeNullSplit $ start ++ middle ++ end
 	where
 		start = [Params "diff --name-only --diff-filter=T -z"]
 		end = Param "--" : map File l
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index c072ef5be9..1fcdf13ed5 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -29,9 +29,9 @@ data TreeItem = TreeItem
 	} deriving Show
 
 {- Lists the contents of a Treeish -}
-lsTree :: Repo -> Treeish -> IO [TreeItem]
-lsTree repo t = map parseLsTree <$>
-	pipeNullSplitB repo [Params "ls-tree --full-tree -z -r --", File t]
+lsTree :: Treeish -> Repo -> IO [TreeItem]
+lsTree t repo = map parseLsTree <$>
+	pipeNullSplitB [Params "ls-tree --full-tree -z -r --", File t] repo
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 25b9ffad08..70c766d044 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -72,8 +72,8 @@ full :: Queue -> Bool
 full (Queue n _) = n > maxSize
 
 {- Runs a queue on a git repository. -}
-flush :: Repo -> Queue -> IO Queue
-flush repo (Queue _ m) = do
+flush :: Queue -> Repo -> IO Queue
+flush (Queue _ m) repo = do
 	forM_ (M.toList m) $ uncurry $ runAction repo
 	return empty
 
@@ -87,6 +87,6 @@ runAction :: Repo -> Action -> [FilePath] -> IO ()
 runAction repo action files =
 	pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 	where
-		params = toCommand $ gitCommandLine repo
-			(Param (getSubcommand action):getParams action)
+		params = toCommand $ gitCommandLine
+			(Param (getSubcommand action):getParams action) repo
 		feedxargs h = hPutStr h $ join "\0" files
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 859a66ca07..32966c846d 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -27,24 +27,25 @@ import Git
  -
  - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: Repo -> String -> String -> IO ()
-merge g x y = do
-	a <- ls_tree g x
-	b <- merge_trees g x y
-	update_index g (a++b)
+merge :: String -> String -> Repo -> IO ()
+merge x y repo = do
+	a <- ls_tree x repo
+	b <- merge_trees x y repo
+	update_index repo (a++b)
 
 {- Merges a list of branches into the index. Previously staged changed in
  - the index are preserved (and participate in the merge). -}
 merge_index :: Repo -> [String] -> IO ()
-merge_index g bs = update_index g =<< concat <$> mapM (merge_tree_index g) bs
+merge_index repo bs =
+	update_index repo =<< concat <$> mapM (\b -> merge_tree_index b repo) bs
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Repo -> [String] -> IO ()
-update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
+update_index repo l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
-		togit ps content = pipeWrite g (map Param ps) (L.pack content)
+		togit ps content = pipeWrite (map Param ps) (L.pack content) repo
 			>>= forceSuccess
 
 {- Generates a line suitable to be fed into update-index, to add
@@ -53,27 +54,28 @@ update_index_line :: String -> FilePath -> String
 update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 
 {- Gets the contents of a tree in a format suitable for update_index. -}
-ls_tree :: Repo -> String -> IO [String]
-ls_tree g x = pipeNullSplit g $ 
-	map Param ["ls-tree", "-z", "-r", "--full-tree", x]
+ls_tree :: String -> Repo -> IO [String]
+ls_tree x = pipeNullSplit params
+	where
+		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: Repo -> String -> String -> IO [String]
-merge_trees g x y = calc_merge g $ "diff-tree":diff_opts ++ [x, y]
+merge_trees :: String -> String -> Repo -> IO [String]
+merge_trees x y = calc_merge $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: Repo -> String -> IO [String]
-merge_tree_index g x = calc_merge g $ "diff-index":diff_opts ++ ["--cached", x]
+merge_tree_index :: String -> Repo -> IO [String]
+merge_tree_index x = calc_merge $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
 diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: Repo -> [String] -> IO [String]
-calc_merge g differ = do
-	diff <- pipeNullSplit g $ map Param differ
-	l <- mapM (mergeFile g) (pairs diff)
+calc_merge :: [String] -> Repo -> IO [String]
+calc_merge differ repo = do
+	diff <- pipeNullSplit (map Param differ) repo
+	l <- mapM (\p -> mergeFile p repo) (pairs diff)
 	return $ catMaybes l
 	where
 		pairs [] = []
@@ -81,9 +83,9 @@ calc_merge g differ = do
 		pairs (a:b:rest) = (a,b):pairs rest
 
 {- Injects some content into git, returning its hash. -}
-hashObject :: Repo -> L.ByteString -> IO String
-hashObject repo content = getSha subcmd $ do
-	(h, s) <- pipeWriteRead repo (map Param params) content
+hashObject :: L.ByteString -> Repo -> IO String
+hashObject content repo = getSha subcmd $ do
+	(h, s) <- pipeWriteRead (map Param params) content repo
 	L.length s `seq` do
 		forceSuccess h
 		reap -- XXX unsure why this is needed
@@ -95,13 +97,13 @@ hashObject repo content = getSha subcmd $ do
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
-mergeFile :: Repo -> (String, FilePath) -> IO (Maybe String)
-mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
+mergeFile :: (String, FilePath) -> Repo -> IO (Maybe String)
+mergeFile (info, file) repo = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- pipeRead g $ map Param ("show":shas)
-		sha <- hashObject g $ unionmerge content
+		content <- pipeRead (map Param ("show":shas)) repo
+		sha <- hashObject (unionmerge content) repo
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 399b26ef7f..f416b7bead 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -116,9 +116,8 @@ options = commonOptions ++
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
-			g <- gitRepo
-			g' <- liftIO $ Git.configStore g v
-			Annex.changeState $ \s -> s { Annex.repo = g' }
+			newg <- inRepo $ Git.configStore v
+			Annex.changeState $ \s -> s { Annex.repo = newg }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/Init.hs b/Init.hs
index 8cec98c5fc..d03e10031f 100644
--- a/Init.hs
+++ b/Init.hs
@@ -68,12 +68,10 @@ gitPreCommitHookUnWrite = unlessBare $ do
 				" Edit it to remove call to git annex."
 
 unlessBare :: Annex () -> Annex ()
-unlessBare = unlessM $ Git.repoIsLocalBare <$> gitRepo
+unlessBare = unlessM $ fromRepo $ Git.repoIsLocalBare
 
 preCommitHook :: Annex FilePath
-preCommitHook = do
-	g <- gitRepo
-	return $ Git.gitDir g ++ "/hooks/pre-commit"
+preCommitHook = () <$> fromRepo Git.gitDir <*> pure "hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 
diff --git a/Locations.hs b/Locations.hs
index ceb6246b98..83897f488f 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -65,8 +65,8 @@ annexLocation key = objectDir  hashDirMixed key  f  f
 		f = keyFile key
 
 {- Annexed file's absolute location in a repository. -}
-gitAnnexLocation :: Git.Repo -> Key -> FilePath
-gitAnnexLocation r key
+gitAnnexLocation :: Key -> Git.Repo -> FilePath
+gitAnnexLocation key r
 	| Git.repoIsLocalBare r = Git.workTree r  annexLocation key
 	| otherwise = Git.workTree r  ".git"  annexLocation key
 
@@ -88,16 +88,16 @@ gitAnnexTmpDir :: Git.Repo -> FilePath
 gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 
 {- The temp file to use for a given key. -}
-gitAnnexTmpLocation :: Git.Repo -> Key -> FilePath
-gitAnnexTmpLocation r key = gitAnnexTmpDir r  keyFile key
+gitAnnexTmpLocation :: Key -> Git.Repo -> FilePath
+gitAnnexTmpLocation key r = gitAnnexTmpDir r  keyFile key
 
 {- .git/annex/bad/ is used for bad files found during fsck -}
 gitAnnexBadDir :: Git.Repo -> FilePath
 gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
 
 {- The bad file to use for a given key. -}
-gitAnnexBadLocation :: Git.Repo -> Key -> FilePath
-gitAnnexBadLocation r key = gitAnnexBadDir r  keyFile key
+gitAnnexBadLocation :: Key -> Git.Repo -> FilePath
+gitAnnexBadLocation key r = gitAnnexBadDir r  keyFile key
 
 {- .git/annex/*unused is used to number possibly unused keys -}
 gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath
diff --git a/Remote.hs b/Remote.hs
index 6d55cee243..82af21bde4 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -130,10 +130,10 @@ nameToUUID n = byName' n >>= go
  - of the UUIDs. -}
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
-	here <- getUUID
+	hereu <- getUUID
 	m <- M.unionWith addname <$> uuidMap <*> remoteMap
-	maybeShowJSON [(desc, map (jsonify m here) uuids)]
-	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
+	maybeShowJSON [(desc, map (jsonify m hereu) uuids)]
+	return $ unwords $ map (\u -> "\t" ++ prettify m hereu u ++ "\n") uuids
 	where
 		addname d n
 			| d == n = d
@@ -141,20 +141,20 @@ prettyPrintUUIDs desc uuids = do
 			| otherwise = n ++ " (" ++ d ++ ")"
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
-		prettify m here u
+		prettify m hereu u
 			| not (null d) = fromUUID u ++ " -- " ++ d
 			| otherwise = fromUUID u
 			where
-				ishere = here == u
+				ishere = hereu == u
 				n = findlog m u
 				d
 					| null n && ishere = "here"
 					| ishere = addname n "here"
 					| otherwise = n
-		jsonify m here u = toJSObject
+		jsonify m hereu u = toJSObject
 			[ ("uuid", toJSON $ fromUUID u)
 			, ("description", toJSON $ findlog m u)
-			, ("here", toJSON $ here == u)
+			, ("here", toJSON $ hereu == u)
 			]
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 3e621ce569..b8d7cd3172 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -102,15 +102,13 @@ bupSplitParams r buprepo k src = do
 
 store :: Git.Repo -> BupRepo -> Key -> Annex Bool
 store r buprepo k = do
-	g <- gitRepo
-	let src = gitAnnexLocation g k
+	src <- fromRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo k (File src)
 	liftIO $ boolSystem "bup" params
 
 storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r buprepo (cipher, enck) k = do
-	g <- gitRepo
-	let src = gitAnnexLocation g k
+	src <- fromRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo enck (Param "-")
 	liftIO $ catchBool $
 		withEncryptedHandle cipher (L.readFile src) $ \h ->
@@ -147,7 +145,7 @@ checkPresent r bupr k
 		showAction $ "checking " ++ Git.repoDescribe r
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
-	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine bupr params
+	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine params bupr
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
@@ -165,9 +163,10 @@ storeBupUUID u buprepo = do
 					>>! error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
-			let olduuid = Git.configGet r' "annex.uuid" ""
-			when (olduuid == "") $ Git.run r' "config"
-				[Param "annex.uuid", Param v]
+			let olduuid = Git.configGet "annex.uuid" "" r'
+			when (olduuid == "") $
+				Git.run "config"
+					[Param "annex.uuid", Param v] r'
 	where
 		v = fromUUID u
 
@@ -194,7 +193,7 @@ getBupUUID r u
 	| otherwise = liftIO $ do
 		ret <- try $ Git.configRead r
 		case ret of
-			Right r' -> return (toUUID $ Git.configGet r' "annex.uuid" "", r')
+			Right r' -> return (toUUID $ Git.configGet "annex.uuid" "" r', r')
 			Left _ -> return (NoUUID, r)
 
 {- Converts a bup remote path spec into a Git.Repo. There are some
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index e8cf05a0e5..8e306e228b 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -70,15 +70,13 @@ dirKey d k = d  hashDirMixed k  f  f
 
 store :: FilePath -> Key -> Annex Bool
 store d k = do
-	g <- gitRepo
-	let src = gitAnnexLocation g k
+	src <- fromRepo $ gitAnnexLocation k
 	let dest = dirKey d k
 	liftIO $ catchBool $ storeHelper dest $ copyFileExternal src dest
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
-	g <- gitRepo
-	let src = gitAnnexLocation g k
+	src <- fromRepo $ gitAnnexLocation k
 	let dest = dirKey d enck
 	liftIO $ catchBool $ storeHelper dest $ encrypt src dest
 	where
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 4c76e8ce6d..75f0ac7576 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -35,19 +35,16 @@ remote = RemoteType {
 
 list :: Annex [Git.Repo]
 list = do
-	g <- gitRepo
-	let c = Git.configMap g
-	mapM (tweakurl c) $ Git.remotes g
+	c <- fromRepo Git.configMap
+	mapM (tweakurl c) =<< fromRepo Git.remotes
 	where
 		annexurl n = "remote." ++ n ++ ".annexurl"
 		tweakurl c r = do
 			let n = fromJust $ Git.repoRemoteName r
 			case M.lookup (annexurl n) c of
 				Nothing -> return r
-				Just url -> do
-					g <- gitRepo
-					r' <- liftIO $ Git.genRemote g url
-					return $ Git.repoRemoteNameSet r' n
+				Just url -> Git.repoRemoteNameSet n <$>
+					inRepo (Git.genRemote url)
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r u _ = do
@@ -178,7 +175,7 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = do
 		params <- rsyncParams r
-		rsyncOrCopyFile params (gitAnnexLocation r key) file
+		rsyncOrCopyFile params (gitAnnexLocation key r) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
 	| Git.repoIsHttp r = liftIO $ Url.download (keyUrl r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
@@ -187,8 +184,7 @@ copyFromRemote r key file
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
 	| not $ Git.repoIsUrl r = do
-		g <- gitRepo
-		let keysrc = gitAnnexLocation g key
+		keysrc <- fromRepo $ gitAnnexLocation key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
@@ -197,8 +193,7 @@ copyToRemote r key
 			Annex.Content.saveState
 			return ok
 	| Git.repoIsSsh r = do
-		g <- gitRepo
-		let keysrc = gitAnnexLocation g key
+		keysrc <- fromRepo $ gitAnnexLocation key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 38f24eb373..6cea170347 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -23,16 +23,16 @@ findSpecialRemotes s = do
 	return $ map construct $ remotepairs g
 	where
 		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
-		construct (k,_) = Git.repoRemoteNameFromKey Git.repoFromUnknown k
+		construct (k,_) = Git.repoRemoteNameFromKey k Git.repoFromUnknown
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
 gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex ()
 gitConfigSpecialRemote u c k v = do
-	g <- gitRepo
-	liftIO $ do
-		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param $ fromUUID u]
+	set ("annex-"++k) v
+	set ("annex-uuid") (fromUUID u)
 	where
+		set a b = inRepo $ Git.run "config"
+			[Param (configsetting a), Param b]
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 8b6a6cecfc..06568a3cb5 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -98,14 +98,13 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h
 
 store :: String -> Key -> Annex Bool
 store h k = do
-	g <- gitRepo
-	runHook h "store" k (Just $ gitAnnexLocation g k) $ return True
+	src <- fromRepo $ gitAnnexLocation k
+	runHook h "store" k (Just src) $ return True
 
 storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do
-	g <- gitRepo
-	let f = gitAnnexLocation g k
-	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
+	src <- fromRepo $ gitAnnexLocation k
+	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	runHook h "store" enck (Just tmp) $ return True
 
 retrieve :: String -> Key -> FilePath -> Annex Bool
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index e79762a382..0dfad72935 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -90,15 +90,12 @@ rsyncKeyDir :: RsyncOpts -> Key -> String
 rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  shellEscape (keyFile k)
 
 store :: RsyncOpts -> Key -> Annex Bool
-store o k = do
-	g <- gitRepo
-	rsyncSend o k (gitAnnexLocation g k)
+store o k = rsyncSend o k =<< fromRepo (gitAnnexLocation k)
 
 storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
-	g <- gitRepo
-	let f = gitAnnexLocation g k
-	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
+	src <- fromRepo $ gitAnnexLocation k
+	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	rsyncSend o enck tmp
 
 retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
@@ -151,9 +148,9 @@ partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial"
  - up trees for rsync. -}
 withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool
 withRsyncScratchDir a = do
-	g <- gitRepo
 	pid <- liftIO getProcessID
-	let tmp = gitAnnexTmpDir g  "rsynctmp"  show pid
+	t <- fromRepo gitAnnexTmpDir
+	let tmp = t  "rsynctmp"  show pid
 	nuke tmp
 	liftIO $ createDirectoryIfMissing True tmp
 	res <- a tmp
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 1281c27861..b201b5aadb 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -112,8 +112,8 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 store :: Remote Annex -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
-	g <- gitRepo
-	res <- liftIO $ storeHelper (conn, bucket) r k $ gitAnnexLocation g k
+	dest <- fromRepo $ gitAnnexLocation k
+	res <- liftIO $ storeHelper (conn, bucket) r k dest
 	s3Bool res
 
 storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool
@@ -121,8 +121,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) ->
 	-- To get file size of the encrypted content, have to use a temp file.
 	-- (An alternative would be chunking to to a constant size.)
 	withTmp enck $ \tmp -> do
-		g <- gitRepo
-		let f = gitAnnexLocation g k
+		f <- fromRepo $ gitAnnexLocation k
 		liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 		res <- liftIO $ storeHelper (conn, bucket) r enck tmp
 		s3Bool res
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 393932d478..da7f384727 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -27,7 +27,7 @@ remote = RemoteType {
 -- (If the web should cease to exist, remove this module and redistribute
 -- a new release to the survivors by carrier pigeon.)
 list :: Annex [Git.Repo]
-list = return [Git.repoRemoteNameSet Git.repoFromUnknown "web"]
+list = return [Git.repoRemoteNameSet "web" Git.repoFromUnknown]
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
diff --git a/Seek.hs b/Seek.hs
index 4ae943157b..3c83ebc35d 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -20,16 +20,18 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 import qualified Limit
 
+seekHelper :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> [FilePath] -> Annex [FilePath]
+seekHelper a params = do
+	g <- gitRepo
+	liftIO $ runPreserveOrder (\p -> a p g) params
+
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
-withFilesInGit a params = do
-	repo <- gitRepo
-	prepFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
 
 withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
-	repo <- gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	prepFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
+	files <- seekHelper LsFiles.inRepo params
+	prepFilteredGen a fst $ inRepo $ Git.checkAttr attr files
 
 withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
@@ -38,8 +40,7 @@ withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
-	repo <- gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+	files <- seekHelper LsFiles.inRepo params
 	prepBackendPairs a files
 
 withFilesMissing :: (String -> CommandStart) -> CommandSeek
@@ -49,9 +50,8 @@ withFilesMissing a params = prepFiltered a $ liftIO $ filterM missing params
 
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
-	repo <- gitRepo
 	force <- Annex.getState Annex.force
-	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
+	newfiles <- seekHelper (LsFiles.notInRepo force) params
 	prepBackendPairs a newfiles
 
 withWords :: ([String] -> CommandStart) -> CommandSeek
@@ -61,10 +61,8 @@ withStrings :: (String -> CommandStart) -> CommandSeek
 withStrings a params = return $ map a params
 
 withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
-withFilesToBeCommitted a params = do
-	repo <- gitRepo
-	prepFiltered a $
-		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
+withFilesToBeCommitted a params = prepFiltered a $
+	seekHelper LsFiles.stagedNotDeleted params
 
 withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
@@ -72,13 +70,13 @@ withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
 withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 
-withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
-	repo <- gitRepo
-	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
+	top <- fromRepo $ Git.workTree
+	typechangedfiles <- seekHelper typechanged params
 	unlockedfiles <- liftIO $ filterM notSymlink $
-		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
+		map (\f -> top ++ "/" ++ f) typechangedfiles
 	prepBackendPairs a unlockedfiles
 
 withKeys :: (Key -> CommandStart) -> CommandSeek
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index b1443fa460..eae5c87ce4 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -16,10 +16,9 @@ import qualified Upgrade.V1
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v0 to v1"
-	g <- gitRepo
 
 	-- do the reorganisation of the key files
-	let olddir = gitAnnexDir g
+	olddir <- fromRepo gitAnnexDir
 	keys <- getKeysPresent0 olddir
 	forM_ keys $ \k -> moveAnnex k $ olddir  keyFile0 k
 
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index be9a977ad7..fe59ad3da8 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -50,9 +50,9 @@ import qualified Upgrade.V2
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v1 to v2"
-
-	g <- gitRepo
-	if Git.repoIsLocalBare g
+	
+	bare <- fromRepo $ Git.repoIsLocalBare
+	if bare
 		then do
 			moveContent
 			setVersion
@@ -83,8 +83,8 @@ moveContent = do
 updateSymlinks :: Annex ()
 updateSymlinks = do
 	showAction "updating symlinks"
-	g <- gitRepo
-	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
+	top <- fromRepo Git.workTree
+	files <- inRepo $ LsFiles.inRepo [top]
 	forM_ files fixlink
 	where
 		fixlink f = do
@@ -104,8 +104,7 @@ moveLocationLogs = do
 	forM_ logkeys move
 		where
 			oldlocationlogs = do
-				g <- gitRepo
-				let dir = Upgrade.V2.gitStateDir g
+				dir <- fromRepo Upgrade.V2.gitStateDir
 				exists <- liftIO $ doesDirectoryExist dir
 				if exists
 					then do
@@ -113,9 +112,8 @@ moveLocationLogs = do
 						return $ mapMaybe oldlog2key contents
 					else return []
 			move (l, k) = do
-				g <- gitRepo
-				let dest = logFile2 g k
-				let dir = Upgrade.V2.gitStateDir g
+				dest <- fromRepo $ logFile2 k
+				dir <- fromRepo $ Upgrade.V2.gitStateDir
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
 				-- could just git mv, but this way deals with
@@ -206,9 +204,7 @@ lookupFile1 file = do
 					" (unknown backend " ++ bname ++ ")"
 
 getKeyFilesPresent1 :: Annex [FilePath]
-getKeyFilesPresent1  = do
-	g <- gitRepo
-	getKeyFilesPresent1' $ gitAnnexObjectDir g
+getKeyFilesPresent1  = getKeyFilesPresent1' =<< fromRepo gitAnnexObjectDir
 getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
 getKeyFilesPresent1' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
@@ -228,11 +224,11 @@ getKeyFilesPresent1' dir = do
 logFile1 :: Git.Repo -> Key -> String
 logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
 
-logFile2 :: Git.Repo -> Key -> String
+logFile2 :: Key -> Git.Repo -> String
 logFile2 = logFile' hashDirLower
 
-logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
-logFile' hasher repo key =
+logFile' :: (Key -> FilePath) -> Key -> Git.Repo -> String
+logFile' hasher key repo =
 	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
 
 stateDir :: FilePath
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 67f0205c14..7ef2a4d18e 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -37,45 +37,46 @@ olddir g
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v2 to v3"
-	g <- gitRepo
-	let bare = Git.repoIsLocalBare g
+	bare <- fromRepo Git.repoIsLocalBare
+	old <- fromRepo olddir
 
 	Annex.Branch.create
 	showProgress
 
-	e <- liftIO $ doesDirectoryExist (olddir g)
+	e <- liftIO $ doesDirectoryExist old
 	when e $ do
-		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
-		mapM_ (\f -> inject f f) =<< logFiles (olddir g)
+		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs
+		mapM_ (\f -> inject f f) =<< logFiles old
 
 	saveState
 	showProgress
 
-	when e $ liftIO $ do
-		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
-		unless bare $ gitAttributesUnWrite g
+	when e $ do
+		inRepo $ Git.run "rm" [Param "-r", Param "-f", Param "-q", File old]
+		unless bare $ inRepo $ gitAttributesUnWrite
 	showProgress
 
 	unless bare push
 
 	return True
 
-locationLogs :: Git.Repo -> Annex [(Key, FilePath)]
-locationLogs repo = liftIO $ do
-	levela <- dirContents dir
-	levelb <- mapM tryDirContents levela
-	files <- mapM tryDirContents (concat levelb)
-	return $ mapMaybe islogfile (concat files)
+locationLogs :: Annex [(Key, FilePath)]
+locationLogs = do
+	dir <- fromRepo gitStateDir
+	liftIO $ do
+		levela <- dirContents dir
+		levelb <- mapM tryDirContents levela
+		files <- mapM tryDirContents (concat levelb)
+		return $ mapMaybe islogfile (concat files)
 	where
 		tryDirContents d = catch (dirContents d) (return . const [])
-		dir = gitStateDir repo
 		islogfile f = maybe Nothing (\k -> Just (k, f)) $
 				logFileKey $ takeFileName f
 
 inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
-	g <- gitRepo
-	new <- liftIO (readFile $ olddir g  source)
+	old <- fromRepo olddir
+	new <- liftIO (readFile $ old  source)
 	Annex.Branch.change dest $ \prev -> 
 		unlines $ nub $ lines prev ++ lines new
 
@@ -102,8 +103,7 @@ push = do
 			Annex.Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
-			g <- gitRepo
-			liftIO $ Git.run g "push" [Param "origin", Param Annex.Branch.name]
+			inRepo $ Git.run "push" [Param "origin", Param Annex.Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
 			-- know about the new branch
@@ -126,7 +126,7 @@ gitAttributesUnWrite repo = do
 		c <- readFileStrict attributes
 		liftIO $ viaTmp writeFile attributes $ unlines $
 			filter (`notElem` attrLines) $ lines c
-		Git.run repo "add" [File attributes]
+		Git.run "add" [File attributes] repo
 
 stateDir :: FilePath
 stateDir = addTrailingPathSeparator ".git-annex"
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 0d1d0819d2..10ae842177 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -41,6 +41,6 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
-	Git.UnionMerge.merge g aref bref
-	Git.commit g "union merge" newref [aref, bref]
+	Git.UnionMerge.merge aref bref g
+	Git.commit "union merge" newref [aref, bref] g
 	cleanup g

From 56b8194470f0ddc1b3e1c51da8ca55a5c9c2eb05 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 01:15:51 -0400
Subject: [PATCH 2474/2835] cleanup

---
 Annex.hs                 | 6 +++---
 Annex/Content.hs         | 3 +--
 Command/Fsck.hs          | 3 +--
 Logs/Location.hs         | 9 +++------
 Logs/Web.hs              | 3 +--
 Remote.hs                | 5 +----
 Remote/Helper/Special.hs | 6 +++---
 7 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 501e54c662..1c1091799f 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -1,6 +1,6 @@
 {- git-annex monad
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -48,6 +48,8 @@ newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 		Applicative
 	)
 
+data OutputType = NormalOutput | QuietOutput | JSONOutput
+
 -- internal state storage
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
@@ -70,8 +72,6 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-data OutputType = NormalOutput | QuietOutput | JSONOutput
-
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
 	{ repo = gitrepo
diff --git a/Annex/Content.hs b/Annex/Content.hs
index fc2c2d0921..ecfec66aab 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -56,9 +56,8 @@ calcGitLink file key = do
  - repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
-	g <- gitRepo
 	u <- getUUID
-	logChange g key u status
+	logChange key u status
 
 {- Runs an action, passing it a temporary filename to download,
  - and if the action succeeds, moves the temp file into 
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 3feabeb9ee..89485b7788 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -109,8 +109,7 @@ verifyLocationLog key desc = do
 	where
 		fix u s = do
 			showNote "fixing location log"
-			g <- gitRepo
-			logChange g key u s
+			logChange key u s
 
 {- The size of the data for a key is checked against the size encoded in
  - the key's metadata, if available. -}
diff --git a/Logs/Location.hs b/Logs/Location.hs
index ff874a5964..ab29ffcadd 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -23,16 +23,13 @@ module Logs.Location (
 ) where
 
 import Common.Annex
-import qualified Git
 import qualified Annex.Branch
 import Logs.Presence
 
 {- Log a change in the presence of a key's value in a repository. -}
-logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
-logChange _ key (UUID u) s = addLog (logFile key) =<< logNow s u
-logChange repo _ NoUUID _ = error $
-	"unknown UUID for " ++ Git.repoDescribe repo ++ 
-	" (have you run git annex init there?)"
+logChange :: Key -> UUID -> LogStatus -> Annex ()
+logChange key (UUID u) s = addLog (logFile key) =<< logNow s u
+logChange _ NoUUID _ = return ()
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
diff --git a/Logs/Web.hs b/Logs/Web.hs
index b52e347e5f..62656b7ed8 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -42,12 +42,11 @@ getUrls key = do
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()
 setUrl key url status = do
-	g <- gitRepo
 	addLog (urlLog key) =<< logNow status url
 
 	-- update location log to indicate that the web has the key, or not
 	us <- getUrls key
-	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
+	logChange key webUUID (if null us then InfoMissing else InfoPresent)
 
 setUrlPresent :: Key -> URLString -> Annex ()
 setUrlPresent key url = setUrl key url InfoPresent
diff --git a/Remote.hs b/Remote.hs
index 82af21bde4..f820b62a18 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -231,9 +231,6 @@ forceTrust level remotename = do
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot always be relied on. -}
 remoteHasKey :: Remote Annex -> Key -> Bool -> Annex ()
-remoteHasKey remote key present	= do
-	let remoteuuid = uuid remote
-	g <- gitRepo
-	logChange g key remoteuuid status
+remoteHasKey remote key present	= logChange key (uuid remote) status
 	where
 		status = if present then InfoPresent else InfoMissing
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 6cea170347..5bbf4169d7 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -19,10 +19,10 @@ import qualified Git
  -}
 findSpecialRemotes :: String -> Annex [Git.Repo]
 findSpecialRemotes s = do
-	g <- gitRepo
-	return $ map construct $ remotepairs g
+	m <- fromRepo $ Git.configMap
+	return $ map construct $ remotepairs m
 	where
-		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
+		remotepairs = M.toList . M.filterWithKey match
 		construct (k,_) = Git.repoRemoteNameFromKey k Git.repoFromUnknown
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 

From 393b6b1bde7e049845ff125f8a32e566d6bd7adb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 13:34:17 -0400
Subject: [PATCH 2475/2835] problem that came to me at 2 am

---
 doc/bugs/cyclic_drop.mdwn | 53 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 doc/bugs/cyclic_drop.mdwn

diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
new file mode 100644
index 0000000000..b43762e78f
--- /dev/null
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -0,0 +1,53 @@
+drop's verification that a remote still has content can fail
+if the remote is also dropping the content at the same time. Each
+repository checks that the other still has the content, and then both
+drop it. Could also happen with larger cycles of repositories.
+
+---
+
+Fixing this requires locking. (Well, there are other ways, like moving the
+content to a holding area when checking if it's safe to drop, but they
+seem complicated, and would be hard to implement for move --from.)
+
+Add per-content lock files. An exclusive lock is held on content when
+it's in the process of being dropped, or moved. The lock is taken
+nonblocking; if it cannot be obtained, something else is acting on the
+content and git-annex should refuse to do anything.
+
+Then when checking inannex, try to take a shared lock. Note that to avoid
+deadlock, this must be a nonblocking lock. If it fails, the status of
+the content is unknown, so inannex should fail. Note that this needs to be
+distinguishable from "not in annex".
+
+---
+
+move --to can also be included in the cycle, since it can drop data. 
+
+Consider move to a remote that already has the content and 
+is at the same time doing a drop (or a move). The remote
+verifies the content is present on the movee, and removes its copy.
+The movee removes its copy.
+
+So move --to needs to take the content lock on start. Then the inannex
+will fail.
+
+-- 
+
+move --from is similar. Consider a case where both the local and the remote
+are doing a move --from. Both have the content, and confirm the other does,
+via inannex checks. Then both run git-annex-shell dropkey, removing both
+copies.
+
+So move --from needs to take the content lock on start, so the inannex will
+fail.
+
+---
+
+Another cycle might be running move --to and move --from on the same file,
+locally. The exclusivity of the content lock solves this, as only one can
+run at a time.
+
+---
+
+Another cycle might involve move --from and drop, both run on the same
+file, locally. Again, the exclusive lock solves this.

From a243d6e6e9dcc657b5620244c3157da5c86406ec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 14:32:31 -0400
Subject: [PATCH 2476/2835] directly lock content?

---
 doc/bugs/cyclic_drop.mdwn | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index b43762e78f..cc2943b7ed 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -19,6 +19,23 @@ deadlock, this must be a nonblocking lock. If it fails, the status of
 the content is unknown, so inannex should fail. Note that this needs to be
 distinguishable from "not in annex".
 
+> Thinking about these lock files, this would be a lot more files, 
+> and would possibly break some assumptions that everything in
+> `.git/annex/objects` is a key's content. (Or would need lots more
+> directories to put the lock files elsewhere.) There would be more
+> overhead to manage these and have them on disk.
+> 
+> What if it just locked the actual content file? The obvious limitation
+> is only content that was already inannex could be locked, but that
+> happens to be exactly what's needed here; if content is not present,
+> it's not going to get dropped or moved.
+>
+> Of course, if some consumer of a file locked it, then it could prevent it
+> from being dropped or moved. This could be considered a bug, or a feature. :)
+> 
+> However, this would mean that such a hypothetical consumer could also
+> make inannex checks fail.
+
 ---
 
 move --to can also be included in the cycle, since it can drop data. 
@@ -39,7 +56,7 @@ via inannex checks. Then both run git-annex-shell dropkey, removing both
 copies.
 
 So move --from needs to take the content lock on start, so the inannex will
-fail.
+fail.  NB: If the content is not locally present, don't take the lock.
 
 ---
 

From 58563c5b1aa995ea3ce72cddaa1f02d2ea792c2d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 14:37:14 -0400
Subject: [PATCH 2477/2835] warning about version of git-annex shipped in
 recent Ubuntu release

---
 doc/install/Ubuntu.mdwn | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/install/Ubuntu.mdwn b/doc/install/Ubuntu.mdwn
index afcf22b6d4..8d5341e19a 100644
--- a/doc/install/Ubuntu.mdwn
+++ b/doc/install/Ubuntu.mdwn
@@ -1,5 +1,11 @@
-If using Ubuntu natty or newer:
+If using Ubuntu Oneiric or newer:
 
 	sudo apt-get install git-annex
 
 Otherwise, see [[manual_installation_instructions|install]].
+
+---
+
+Warning: The version of git-annex shipped in Ubuntu Oneiric
+has [a bug that prevents upgrades from v1 git-annex repositories](https://bugs.launchpad.net/ubuntu/+source/git-annex/+bug/875958).
+If you need to upgrade such a repository, get a newer version of git-annex.

From 8ce7e73f74e95276472d18816b7c6a60bab25abb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 16:54:18 -0400
Subject: [PATCH 2478/2835] reorg to allow taking content lock

The lock will only persist during the perform stage, so the content must
be removed from the annex then, rather than in the cleanup stage.

(No lock is actually taken yet.)
---
 Annex/Content.hs          | 12 +++++++++++
 Command/Drop.hs           | 18 ++++++++--------
 Command/DropUnused.hs     |  3 ++-
 Command/Move.hs           | 43 +++++++++++++++++++++++----------------
 doc/bugs/cyclic_drop.mdwn |  4 ++++
 5 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index ecfec66aab..dc714276df 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -7,6 +7,8 @@
 
 module Annex.Content (
 	inAnnex,
+	lockExclusive,
+	lockShared,
 	calcGitLink,
 	logStatus,
 	getViaTmp,
@@ -41,6 +43,16 @@ inAnnex key = do
 		error "inAnnex cannot check remote repo"
 	inRepo $ doesFileExist . gitAnnexLocation key
 
+{- Content is exclusively locked to indicate that it's in the process of
+ - being removed. -}
+lockExclusive :: Key -> Annex a -> Annex a
+lockExclusive key a = a -- TODO
+
+{- Things that rely on content being present can take a shared lock to
+ - avoid it vanishing from under them. -}
+lockShared :: Key -> Annex a -> Annex a
+lockShared key a = a -- TODO
+
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 2267bd9416..e81bd9d7d6 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -52,17 +52,19 @@ startRemote file numcopies key remote = do
 	next $ performRemote key numcopies remote
 
 performLocal :: Key -> Maybe Int -> CommandPerform
-performLocal key numcopies = do
+performLocal key numcopies = lockExclusive key $ do
 	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
 	untrusteduuids <- trustGet UnTrusted
 	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 	success <- canDropKey key numcopies trusteduuids tocheck []
 	if success
-		then next $ cleanupLocal key
+		then do
+			whenM (inAnnex key) $ removeAnnex key
+			next $ cleanupLocal key
 		else stop
 
 performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
-performRemote key numcopies remote = do
+performRemote key numcopies remote = lockExclusive key $ do
 	-- Filter the remote it's being dropped from out of the lists of
 	-- places assumed to have the key, and places to check.
 	-- When the local repo has the key, that's one additional copy.
@@ -76,20 +78,20 @@ performRemote key numcopies remote = do
 		Remote.remotesWithoutUUID remotes (have++untrusteduuids)
 	success <- canDropKey key numcopies have tocheck [uuid]
 	if success
-		then next $ cleanupRemote key remote
+		then do
+			ok <- Remote.removeKey remote key
+			next $ cleanupRemote key remote ok
 		else stop
 	where
 		uuid = Remote.uuid remote
 
 cleanupLocal :: Key -> CommandCleanup
 cleanupLocal key = do
-	whenM (inAnnex key) $ removeAnnex key
 	logStatus key InfoMissing
 	return True
 
-cleanupRemote :: Key -> Remote.Remote Annex -> CommandCleanup
-cleanupRemote key remote = do
-	ok <- Remote.removeKey remote key
+cleanupRemote :: Key -> Remote.Remote Annex -> Bool -> CommandCleanup
+cleanupRemote key remote ok = do
 	-- better safe than sorry: assume the remote dropped the key
 	-- even if it seemed to fail; the failure could have occurred
 	-- after it really dropped it
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 55c21f83bf..2c3bb296ad 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -55,7 +55,8 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 		dropremote name = do
 			r <- Remote.byName name
 			showAction $ "from " ++ Remote.name r
-			next $ Command.Drop.cleanupRemote key r
+			ok <- Remote.removeKey r key
+			next $ Command.Drop.cleanupRemote key r ok
 		droplocal = Command.Drop.performLocal key (Just 0) -- force drop
 
 performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform
diff --git a/Command/Move.hs b/Command/Move.hs
index 5a3ea7172a..e955de827b 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -68,7 +68,7 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 			showMoveAction move file
 			next $ toPerform dest move key
 toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
-toPerform dest move key = do
+toPerform dest move key = moveLock move key $ do
 	-- Checking the remote is expensive, so not done in the start step.
 	-- In fast mode, location tracking is assumed to be correct,
 	-- and an explicit check is not done, when copying. When moving,
@@ -88,18 +88,20 @@ toPerform dest move key = do
 			showAction $ "to " ++ Remote.name dest
 			ok <- Remote.storeKey dest key
 			if ok
-				then next $ toCleanup dest move key
+				then finish
 				else do
 					when fastcheck $
 						warning "This could have failed because --fast is enabled."
 					stop
-		Right True -> next $ toCleanup dest move key
-toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
-toCleanup dest move key = do
-	Remote.remoteHasKey dest key True
-	if move
-		then Command.Drop.cleanupLocal key
-		else return True
+		Right True -> finish
+	where
+		finish = do
+			Remote.remoteHasKey dest key True
+			if move
+				then do
+					whenM (inAnnex key) $ removeAnnex key
+					next $ Command.Drop.cleanupLocal key
+				else next $ return True
 
 {- Moves (or copies) the content of an annexed file from a remote
  - to the current repository.
@@ -117,16 +119,23 @@ fromStart src move file = isAnnexed file $ \(key, _) -> do
 			showMoveAction move file
 			next $ fromPerform src move key
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
-fromPerform src move key = do
+fromPerform src move key = moveLock move key $ do
 	ishere <- inAnnex key
 	if ishere
-		then next $ fromCleanup src move key
+		then handle move True
 		else do
 			showAction $ "from " ++ Remote.name src
 			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
-			if ok
-				then next $ fromCleanup src move key
-				else stop -- fail
-fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
-fromCleanup src True key = Command.Drop.cleanupRemote key src
-fromCleanup _ False _ = return True
+			handle move ok
+	where
+		handle _ False = stop -- failed
+		handle False True = next $ return True -- copy complete
+		handle True True = do -- finish moving
+			ok <- Remote.removeKey src key
+			next $ Command.Drop.cleanupRemote key src ok
+
+{- Locks a key in order for it to be moved.
+ - No lock is needed when a key is being copied. -}
+moveLock :: Bool -> Key -> Annex a -> Annex a
+moveLock True key a = lockExclusive key a
+moveLock False _ a = a
diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index cc2943b7ed..d3264c7caa 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -38,6 +38,10 @@ distinguishable from "not in annex".
 
 ---
 
+drop --from could also cycle. Locking should fix.
+
+---
+
 move --to can also be included in the cycle, since it can drop data. 
 
 Consider move to a remote that already has the content and 

From 2934a65ac5bbab5ac127c495c8c2492e729c2b67 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 17:28:35 -0400
Subject: [PATCH 2479/2835] add safeSystem

This is more safe than System.Cmd.Utils.safeSystem, since it does not throw
an error on nonzero exit status.
---
 Common.hs              |  2 +-
 Utility/SafeCommand.hs | 20 +++++++++++++++-----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/Common.hs b/Common.hs
index 2e1e4d996c..ef068ac105 100644
--- a/Common.hs
+++ b/Common.hs
@@ -33,7 +33,7 @@ import Data.String.Utils
 import System.Path
 import System.FilePath
 import System.Directory
-import System.Cmd.Utils
+import System.Cmd.Utils hiding (safeSystem)
 import System.IO hiding (FilePath)
 import System.Posix.Files
 import System.Posix.IO
diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs
index ba9362603e..aedf271373 100644
--- a/Utility/SafeCommand.hs
+++ b/Utility/SafeCommand.hs
@@ -13,6 +13,7 @@ import System.Posix.Process hiding (executeFile)
 import System.Posix.Signals
 import Data.String.Utils
 import System.Log.Logger
+import Control.Applicative
 
 {- A type for parameters passed to a shell command. A command can
  - be passed either some Params (multiple parameters can be included,
@@ -36,14 +37,23 @@ toCommand = (>>= unwrap)
 
 {- Run a system command, and returns True or False
  - if it succeeded or failed.
- -
- - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
  -}
 boolSystem :: FilePath -> [CommandParam] -> IO Bool
 boolSystem command params = boolSystemEnv command params Nothing
 
 boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
-boolSystemEnv command params env = do
+boolSystemEnv command params env = dispatch <$> safeSystemEnv command params env
+	where
+		dispatch ExitSuccess = True
+		dispatch _ = False
+
+{- Runs a system command, returning the exit status. -}
+safeSystem :: FilePath -> [CommandParam] -> IO ExitCode
+safeSystem command params = safeSystemEnv command params Nothing
+
+{- SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -}
+safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode
+safeSystemEnv command params env = do
 	-- Going low-level because all the high-level system functions
 	-- block SIGINT etc. We need to block SIGCHLD, but allow
 	-- SIGINT to do its default program termination.
@@ -55,8 +65,8 @@ boolSystemEnv command params env = do
 	mps <- getProcessStatus True False childpid
 	restoresignals oldint oldset
 	case mps of
-		Just (Exited ExitSuccess) -> return True
-		_ -> return False
+		Just (Exited code) -> return code
+		_ -> error $ "unknown error running " ++ command
 	where
 		restoresignals oldint oldset = do
 			_ <- installHandler sigINT oldint Nothing

From d3e1a3619ff6939367f43cbd46131b7f60ef6bd0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 18:33:15 -0400
Subject: [PATCH 2480/2835] safer inannex checking

git-annex-shell inannex now returns always 0, 1, or 100 (the last when
it's unclear if content is currently in the index due to it currently being
moved or dropped).

(Actual locking code still not yet written.)
---
 Annex/Content.hs          | 36 ++++++++++++++++++++++++------------
 Command/Drop.hs           |  6 +++---
 Command/InAnnex.hs        | 11 ++++++-----
 Command/Move.hs           |  6 +++---
 Remote.hs                 |  6 +++---
 Remote/Bup.hs             |  8 ++++++--
 Remote/Directory.hs       |  8 ++++++--
 Remote/Git.hs             | 36 ++++++++++++++++++++++++++----------
 Remote/Hook.hs            |  6 ++++--
 Remote/Rsync.hs           |  2 +-
 Remote/S3real.hs          |  2 +-
 Remote/Web.hs             |  2 +-
 Types/Remote.hs           |  5 ++---
 doc/bugs/cyclic_drop.mdwn |  9 +++++++--
 14 files changed, 93 insertions(+), 50 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index dc714276df..efe12bb5d7 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -7,8 +7,8 @@
 
 module Annex.Content (
 	inAnnex,
-	lockExclusive,
-	lockShared,
+	inAnnexSafe,
+	lockContent,
 	calcGitLink,
 	logStatus,
 	getViaTmp,
@@ -36,22 +36,34 @@ import Types.Key
 import Utility.DataUnits
 import Config
 
-{- Checks if a given key is currently present in the gitAnnexLocation. -}
+{- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
-inAnnex key = do
+inAnnex = inAnnex' doesFileExist
+inAnnex' :: (FilePath -> IO a) -> Key -> Annex a
+inAnnex' a key = do
 	whenM (fromRepo Git.repoIsUrl) $
 		error "inAnnex cannot check remote repo"
-	inRepo $ doesFileExist . gitAnnexLocation key
+	inRepo $ a . gitAnnexLocation key
+
+{- A safer check; the key's content must not only be present, but
+ - is not in the process of being removed. -}
+inAnnexSafe :: Key -> Annex (Maybe Bool)
+inAnnexSafe = inAnnex' $ \f -> do
+	e <- doesFileExist f
+	if e
+		then do
+			locked <- testlock f
+			if locked
+				then return Nothing
+				else return $ Just True
+		else return $ Just False
+	where
+		testlock f = return False -- TODO
 
 {- Content is exclusively locked to indicate that it's in the process of
  - being removed. -}
-lockExclusive :: Key -> Annex a -> Annex a
-lockExclusive key a = a -- TODO
-
-{- Things that rely on content being present can take a shared lock to
- - avoid it vanishing from under them. -}
-lockShared :: Key -> Annex a -> Annex a
-lockShared key a = a -- TODO
+lockContent :: Key -> Annex a -> Annex a
+lockContent key a = a -- TODO
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
diff --git a/Command/Drop.hs b/Command/Drop.hs
index e81bd9d7d6..44685ffcd6 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -52,7 +52,7 @@ startRemote file numcopies key remote = do
 	next $ performRemote key numcopies remote
 
 performLocal :: Key -> Maybe Int -> CommandPerform
-performLocal key numcopies = lockExclusive key $ do
+performLocal key numcopies = lockContent key $ do
 	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
 	untrusteduuids <- trustGet UnTrusted
 	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
@@ -64,7 +64,7 @@ performLocal key numcopies = lockExclusive key $ do
 		else stop
 
 performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
-performRemote key numcopies remote = lockExclusive key $ do
+performRemote key numcopies remote = lockContent key $ do
 	-- Filter the remote it's being dropped from out of the lists of
 	-- places assumed to have the key, and places to check.
 	-- When the local repo has the key, that's one additional copy.
@@ -95,7 +95,7 @@ cleanupRemote key remote ok = do
 	-- better safe than sorry: assume the remote dropped the key
 	-- even if it seemed to fail; the failure could have occurred
 	-- after it really dropped it
-	Remote.remoteHasKey remote key False
+	Remote.logStatus remote key False
 	return ok
 
 {- Checks specified remotes to verify that enough copies of a key exist to
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 9c169d0d78..c41f9a92c1 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -19,8 +19,9 @@ seek :: [CommandSeek]
 seek = [withKeys start]
 
 start :: Key -> CommandStart
-start key = do
-	present <- inAnnex key
-	if present
-		then stop
-		else liftIO exitFailure
+start key = inAnnexSafe key >>= dispatch
+	where
+		dispatch (Just True) = stop
+		dispatch (Just False) = exit 1
+		dispatch Nothing = exit 100
+		exit n = liftIO $ exitWith $ ExitFailure n
diff --git a/Command/Move.hs b/Command/Move.hs
index e955de827b..f02f325580 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -82,7 +82,7 @@ toPerform dest move key = moveLock move key $ do
 		else Remote.hasKey dest key
 	case isthere of
 		Left err -> do
-			showNote $ show err
+			showNote $ err
 			stop
 		Right False -> do
 			showAction $ "to " ++ Remote.name dest
@@ -96,7 +96,7 @@ toPerform dest move key = moveLock move key $ do
 		Right True -> finish
 	where
 		finish = do
-			Remote.remoteHasKey dest key True
+			Remote.logStatus dest key True
 			if move
 				then do
 					whenM (inAnnex key) $ removeAnnex key
@@ -137,5 +137,5 @@ fromPerform src move key = moveLock move key $ do
 {- Locks a key in order for it to be moved.
  - No lock is needed when a key is being copied. -}
 moveLock :: Bool -> Key -> Annex a -> Annex a
-moveLock True key a = lockExclusive key a
+moveLock True key a = lockContent key a
 moveLock False _ a = a
diff --git a/Remote.hs b/Remote.hs
index f820b62a18..7c0362d2a7 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -26,7 +26,7 @@ module Remote (
 	showTriedRemotes,
 	showLocations,
 	forceTrust,
-	remoteHasKey
+	logStatus
 ) where
 
 import qualified Data.Map as M
@@ -230,7 +230,7 @@ forceTrust level remotename = do
  - in the local repo, not on the remote. The process of transferring the
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot always be relied on. -}
-remoteHasKey :: Remote Annex -> Key -> Bool -> Annex ()
-remoteHasKey remote key present	= logChange key (uuid remote) status
+logStatus :: Remote Annex -> Key -> Bool -> Annex ()
+logStatus remote key present = logChange key (uuid remote) status
 	where
 		status = if present then InfoPresent else InfoMissing
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index b8d7cd3172..866d4b42de 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -139,17 +139,21 @@ remove _ = do
  - in a bup repository. One way it to check if the git repository has
  - a branch matching the name (as created by bup split -n).
  -}
-checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either IOException Bool)
+checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either String Bool)
 checkPresent r bupr k
 	| Git.repoIsUrl bupr = do
 		showAction $ "checking " ++ Git.repoDescribe r
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
-	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine params bupr
+	| otherwise = dispatch <$> localcheck
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
 			, Param $ "refs/heads/" ++ show k]
+		localcheck = liftIO $ try $
+			boolSystem "git" $ Git.gitCommandLine params bupr
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
 
 {- Store UUID in the annex.uuid setting of the bup repository. -}
 storeBupUUID :: UUID -> BupRepo -> Annex ()
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 8e306e228b..6d3a5da7d0 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -114,5 +114,9 @@ remove d k = liftIO $ catchBool $ do
 		file = dirKey d k
 		dir = parentDir file
 
-checkPresent :: FilePath -> Key -> Annex (Either IOException Bool)
-checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k)
+checkPresent :: FilePath -> Key -> Annex (Either String Bool)
+checkPresent d k = dispatch <$> check
+	where
+		check = liftIO $ try $ doesFileExist (dirKey d k)
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 75f0ac7576..b63a8f124d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -125,22 +125,38 @@ tryGitConfigRead r
 				else old : exchange ls new
 
 {- Checks if a given remote has the content for a key inAnnex.
- - If the remote cannot be accessed, returns a Left error.
+ - If the remote cannot be accessed, or if it cannot determine
+ - whether it has the content, returns a Left error message.
  -}
-inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
+inAnnex :: Git.Repo -> Key -> Annex (Either String Bool)
 inAnnex r key
-	| Git.repoIsHttp r = safely checkhttp
+	| Git.repoIsHttp r = checkhttp
 	| Git.repoIsUrl r = checkremote
-	| otherwise = safely checklocal
+	| otherwise = checklocal
 	where
-		checklocal = onLocal r $ Annex.Content.inAnnex key
+		checkhttp = dispatch <$> check
+			where
+				check = safely $ Url.exists $ keyUrl r key
+				dispatch (Left e) = Left $ show e
+				dispatch (Right v) = Right v
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
-			inannex <- onRemote r (boolSystem, False) "inannex" 
-				[Param (show key)]
-			return $ Right inannex
-		checkhttp = Url.exists $ keyUrl r key
-		safely a = liftIO (try a ::IO (Either IOException Bool))
+			onRemote r (check, unknown) "inannex" [Param (show key)]
+			where
+				check c p = dispatch <$> safeSystem c p
+				dispatch ExitSuccess = Right True
+				dispatch (ExitFailure 1) = Right False
+				dispatch _ = unknown
+		checklocal = dispatch <$> check
+			where
+				check = safely $ onLocal r $
+					Annex.Content.inAnnexSafe key
+				dispatch (Left e) = Left $ show e
+				dispatch (Right (Just b)) = Right b
+				dispatch (Right Nothing) = unknown
+		safely :: IO a -> Annex (Either IOException a)
+		safely a = liftIO $ try a
+		unknown = Left $ "unable to check " ++ Git.repoDescribe r
 
 {- Runs an action on a local repository inexpensively, by making an annex
  - monad using that repository. -}
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 06568a3cb5..9f9250e41d 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -119,14 +119,16 @@ retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
 remove :: String -> Key -> Annex Bool
 remove h k = runHook h "remove" k Nothing $ return True
 
-checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool)
+checkPresent :: Git.Repo -> String -> Key -> Annex (Either String Bool)
 checkPresent r h k = do
 	showAction $ "checking " ++ Git.repoDescribe r
 	v <- lookupHook h "checkpresent"
-	liftIO (try (check v) ::IO (Either IOException Bool))
+	dispatch <$> liftIO (try (check v) ::IO (Either IOException Bool))
 	where
 		findkey s = show k `elem` lines s
 		env = hookEnv k Nothing
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
 		check Nothing = error "checkpresent hook misconfigured"
 		check (Just hook) = do
 			(frompipe, topipe) <- createPipe
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 0dfad72935..54834be135 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -128,7 +128,7 @@ remove o k = withRsyncScratchDir $ \tmp -> do
 		, Param $ rsyncKeyDir o k
 		]
 
-checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool)
+checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool)
 checkPresent r o k = do
 	showAction $ "checking " ++ Git.repoDescribe r
 	-- note: Does not currently differnetiate between rsync failing
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index b201b5aadb..29117b3a41 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -172,7 +172,7 @@ remove r k = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ deleteObject conn $ bucketKey r bucket k
 	s3Bool res
 
-checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
+checkPresent :: Remote Annex -> Key -> Annex (Either String Bool)
 checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
 	showAction $ "checking " ++ name r
 	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k
diff --git a/Remote/Web.hs b/Remote/Web.hs
index da7f384727..64fcd51aaa 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -64,7 +64,7 @@ dropKey _ = do
 	warning "removal from web not supported"
 	return False
 
-checkKey :: Key -> Annex (Either IOException Bool)
+checkKey :: Key -> Annex (Either String Bool)
 checkKey key = do
 	us <- getUrls key
 	if null us
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 0a4a0fa883..ec9b7a7a70 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -9,7 +9,6 @@
 
 module Types.Remote where
 
-import Control.Exception
 import Data.Map as M
 import Data.Ord
 
@@ -46,8 +45,8 @@ data Remote a = Remote {
 	-- removes a key's contents
 	removeKey :: Key -> a Bool,
 	-- Checks if a key is present in the remote; if the remote
-	-- cannot be accessed returns a Left error.
-	hasKey :: Key -> a (Either IOException Bool),
+	-- cannot be accessed returns a Left error message.
+	hasKey :: Key -> a (Either String Bool),
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
 	hasKeyCheap :: Bool,
diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index d3264c7caa..7804380ae0 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -16,8 +16,8 @@ content and git-annex should refuse to do anything.
 
 Then when checking inannex, try to take a shared lock. Note that to avoid
 deadlock, this must be a nonblocking lock. If it fails, the status of
-the content is unknown, so inannex should fail. Note that this needs to be
-distinguishable from "not in annex".
+the content is unknown, so inannex should fail. Note that this failure 
+needs to be distinguishable from "not in annex".
 
 > Thinking about these lock files, this would be a lot more files, 
 > and would possibly break some assumptions that everything in
@@ -52,6 +52,11 @@ The movee removes its copy.
 So move --to needs to take the content lock on start. Then the inannex
 will fail.
 
+This is why it's important for inannex to fail in a way that is
+distinguishable from "not in annex". Otherwise, move --to
+would see the cycle as the remote not having content, and try to
+redundantly send it, drop it locally, and still race.
+
 -- 
 
 move --from is similar. Consider a case where both the local and the remote

From 992bf133823b6884e9fb1b320fd33d3fb337ee30 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 19:47:04 -0400
Subject: [PATCH 2481/2835] lockContent in dropkey

This is needed for drop --from and move --from to check the lock,
as they do not use git-annex-shell inannex.
---
 Command/DropKey.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index cac955b4ac..b63d481bf3 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -35,7 +35,7 @@ start key = do
 				error "dropkey can cause data loss; use --force if you're sure you want to do this"
 
 perform :: Key -> CommandPerform
-perform key = do
+perform key = lockContent key $ do
 	removeAnnex key
 	next $ cleanup key
 

From 737f043c55b13bf8dbd6887d3e78d32d13a8682a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 21:36:11 -0400
Subject: [PATCH 2482/2835] fast build mode for vim

---
 Makefile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 73e4260008..f0f5360b42 100644
--- a/Makefile
+++ b/Makefile
@@ -1,16 +1,26 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd
 GHCFLAGS=-O2 -Wall $(IGNORE) -fspec-constr-count=5
+
 ifdef PROFILE
 GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE)
 endif
+
 GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell git-union-merge
 mans=git-annex.1 git-annex-shell.1 git-union-merge.1
 sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
 
-all: $(bins) $(mans) docs
+all=$(bins) $(mans) docs
+
+# Am I typing :make in vim? Do a fast build without optimisation.
+ifdef VIM
+GHCFLAGS=-Wall $(IGNORE)
+all=$(bins)
+endif
+
+all: $(all)
 
 Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	$(GHCMAKE) configure

From cf0174c922e4a4f473a846ec0488ea4011ab500c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 21:45:03 -0400
Subject: [PATCH 2483/2835] content locking

I've tested that this solves the cyclic drop problem.
Have not looked at cyclic move, etc.
---
 Annex/Content.hs | 52 ++++++++++++++++++++++++++++++++++++------------
 debian/changelog |  1 +
 2 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index efe12bb5d7..65dbe43f6d 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -23,6 +23,9 @@ module Annex.Content (
 	saveState
 ) where
 
+import Control.Exception (bracket_)
+import System.Posix.Types
+
 import Common.Annex
 import Logs.Location
 import Annex.UUID
@@ -35,6 +38,7 @@ import Utility.FileMode
 import Types.Key
 import Utility.DataUnits
 import Config
+import Annex.Exception
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
@@ -48,22 +52,44 @@ inAnnex' a key = do
 {- A safer check; the key's content must not only be present, but
  - is not in the process of being removed. -}
 inAnnexSafe :: Key -> Annex (Maybe Bool)
-inAnnexSafe = inAnnex' $ \f -> do
-	e <- doesFileExist f
-	if e
-		then do
-			locked <- testlock f
-			if locked
-				then return Nothing
-				else return $ Just True
-		else return $ Just False
+inAnnexSafe = inAnnex' $ \f -> openForLock f False >>= check
 	where
-		testlock f = return False -- TODO
+		check Nothing = return is_missing
+		check (Just h) = do
+			v <- getLock h (ReadLock, AbsoluteSeek, 0, 0)
+			closeFd h
+			return $ case v of
+				Just _ -> is_locked
+				Nothing -> is_unlocked
+		is_locked = Nothing
+		is_unlocked = Just True
+		is_missing = Just False
 
-{- Content is exclusively locked to indicate that it's in the process of
- - being removed. -}
+{- Content is exclusively locked to indicate that it's in the process
+ - of being removed. (If the content is not present, no locking is done.) -}
 lockContent :: Key -> Annex a -> Annex a
-lockContent key a = a -- TODO
+lockContent key a = do
+	file <- fromRepo $ gitAnnexLocation key
+	bracketIO (openForLock file True >>= lock) unlock a
+	where
+		lock Nothing = return Nothing
+		lock (Just l) = do
+			setLock l (WriteLock, AbsoluteSeek, 0, 0)
+			return $ Just l
+		unlock Nothing = return ()
+		unlock (Just l) = closeFd l
+
+openForLock :: FilePath -> Bool -> IO (Maybe Fd)
+openForLock file writelock = bracket_ prep cleanup $
+	catch (Just <$> openFd file mode Nothing defaultFileFlags)
+		(const $ return Nothing)
+	where
+		mode = if writelock then ReadWrite else ReadOnly
+		-- Since files are stored with the write bit disabled,
+		-- have to fiddle with permissions to open for an
+		-- exclusive lock.
+		prep = when writelock $ allowWrite file
+		cleanup = when writelock $ preventWrite file
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
diff --git a/debian/changelog b/debian/changelog
index 265a010c45..9fa96e06ab 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20111108) UNRELEASED; urgency=low
 
   * Handle a case where an annexed file is moved into a gitignored directory,
     by having fix --force add its change.
+  * Avoid cyclic drop problems.
 
  -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
 

From a218ce41cfef19f306ca462fb5d57c6647a680e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 22:15:33 -0400
Subject: [PATCH 2484/2835] exclusive locks, ugh

---
 Annex/Content.hs          | 17 ++++++++++-------
 doc/bugs/cyclic_drop.mdwn |  9 +++++++++
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 65dbe43f6d..b111bfabc6 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -65,8 +65,8 @@ inAnnexSafe = inAnnex' $ \f -> openForLock f False >>= check
 		is_unlocked = Just True
 		is_missing = Just False
 
-{- Content is exclusively locked to indicate that it's in the process
- - of being removed. (If the content is not present, no locking is done.) -}
+{- Content is exclusively locked while running an action that might remove
+ - it. (If the content is not present, no locking is done.) -}
 lockContent :: Key -> Annex a -> Annex a
 lockContent key a = do
 	file <- fromRepo $ gitAnnexLocation key
@@ -85,11 +85,14 @@ openForLock file writelock = bracket_ prep cleanup $
 		(const $ return Nothing)
 	where
 		mode = if writelock then ReadWrite else ReadOnly
-		-- Since files are stored with the write bit disabled,
-		-- have to fiddle with permissions to open for an
-		-- exclusive lock.
-		prep = when writelock $ allowWrite file
-		cleanup = when writelock $ preventWrite file
+		{- Since files are stored with the write bit disabled,
+		 - have to fiddle with permissions to open for an
+		 - exclusive lock. flock locking would avoid this,
+		 - but -}
+		prep = forwritelock $ allowWrite file
+		cleanup = forwritelock $ preventWrite file
+		forwritelock a = 
+			when writelock $ whenM (doesFileExist file) $ a
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index 7804380ae0..beaaa6e35b 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -73,6 +73,15 @@ Another cycle might be running move --to and move --from on the same file,
 locally. The exclusivity of the content lock solves this, as only one can
 run at a time.
 
+Would it work with a shared lock? The --to would run git-annex-shell
+inannex. The --from would also be running, and would run git-annex-shell
+dropkey. So inannex and dropkey would end up running on the remote
+at the same time. Dropkey takes the content lock, and inannex checks it,
+but what if inannex runs first? Then it returns true, and then the content
+is removed, so both the --to and --from see success and the --to proceeds
+to remove the local content that the --from already caused to be removed
+from the remote. So, no, the nasty exclusive lock is needed.
+
 ---
 
 Another cycle might involve move --from and drop, both run on the same

From 95704212510b525bf4fb3c76fc37043da26cb825 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 02:59:13 -0400
Subject: [PATCH 2485/2835] better message when content is locked

---
 Annex/Content.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index b111bfabc6..f50616af93 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -23,6 +23,7 @@ module Annex.Content (
 	saveState
 ) where
 
+import System.IO.Error (try)
 import Control.Exception (bracket_)
 import System.Posix.Types
 
@@ -74,8 +75,10 @@ lockContent key a = do
 	where
 		lock Nothing = return Nothing
 		lock (Just l) = do
-			setLock l (WriteLock, AbsoluteSeek, 0, 0)
-			return $ Just l
+			v <- try $ setLock l (WriteLock, AbsoluteSeek, 0, 0)
+			case v of
+				Left _ -> error "content is locked"
+				Right _ -> return $ Just l
 		unlock Nothing = return ()
 		unlock (Just l) = closeFd l
 

From e4105df78a9a12eaee7d1c00758854186c818931 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 02:59:41 -0400
Subject: [PATCH 2486/2835] tested all known types of cycles, all are fixed

My testing involved widening the race by adding sleeps, and making sure
something sane happens in each case.
---
 doc/bugs/cyclic_drop.mdwn | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index beaaa6e35b..296d61aac5 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -3,6 +3,8 @@ if the remote is also dropping the content at the same time. Each
 repository checks that the other still has the content, and then both
 drop it. Could also happen with larger cycles of repositories.
 
+> Confirmed fixed now. All cases tested. [[done]]
+
 ---
 
 Fixing this requires locking. (Well, there are other ways, like moving the
@@ -15,9 +17,10 @@ nonblocking; if it cannot be obtained, something else is acting on the
 content and git-annex should refuse to do anything.
 
 Then when checking inannex, try to take a shared lock. Note that to avoid
-deadlock, this must be a nonblocking lock. If it fails, the status of
-the content is unknown, so inannex should fail. Note that this failure 
-needs to be distinguishable from "not in annex".
+deadlock, this must be a nonblocking lock. (Actually, with fcntl locking,
+can just check if there is a lock, without needing to take one.) 
+If it fails, the status of the content is unknown, so inannex should fail.
+Note that this failure needs to be distinguishable from "not in annex".
 
 > Thinking about these lock files, this would be a lot more files, 
 > and would possibly break some assumptions that everything in
@@ -35,11 +38,18 @@ needs to be distinguishable from "not in annex".
 > 
 > However, this would mean that such a hypothetical consumer could also
 > make inannex checks fail.
+>
+> The other downside is that, for fcntl exclusive locking, the file has to
+> be opened for write. Normally the modes of content files are locked down
+> to prevent modifcation. Dealt with, but oh so nasty. Almost makes flock
+> locking seem worth using.
 
 ---
 
 drop --from could also cycle. Locking should fix.
 
+> Confirmed fixed now. 
+
 ---
 
 move --to can also be included in the cycle, since it can drop data. 
@@ -57,6 +67,8 @@ distinguishable from "not in annex". Otherwise, move --to
 would see the cycle as the remote not having content, and try to
 redundantly send it, drop it locally, and still race.
 
+> Confirmed fixed now.
+
 -- 
 
 move --from is similar. Consider a case where both the local and the remote
@@ -67,6 +79,8 @@ copies.
 So move --from needs to take the content lock on start, so the inannex will
 fail.  NB: If the content is not locally present, don't take the lock.
 
+> Confirmed fixed now.
+
 ---
 
 Another cycle might be running move --to and move --from on the same file,
@@ -82,6 +96,8 @@ is removed, so both the --to and --from see success and the --to proceeds
 to remove the local content that the --from already caused to be removed
 from the remote. So, no, the nasty exclusive lock is needed.
 
+> Confirmed fixed now.
+
 ---
 
 Another cycle might involve move --from and drop, both run on the same

From a71c03bc5162916853ff520d5c7c89e849c6a047 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 03:10:17 -0400
Subject: [PATCH 2487/2835] add make fast target

---
 Makefile | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index f0f5360b42..cef79be88c 100644
--- a/Makefile
+++ b/Makefile
@@ -14,14 +14,17 @@ sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
 
 all=$(bins) $(mans) docs
 
-# Am I typing :make in vim? Do a fast build without optimisation.
+# Am I typing :make in vim? Do a fast build.
 ifdef VIM
-GHCFLAGS=-Wall $(IGNORE)
-all=$(bins)
+all=fast
 endif
 
 all: $(all)
 
+# Disables optimisation. Not for production use.
+fast: GHCFLAGS=-Wall $(IGNORE)
+fast: $(bins)
+
 Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	$(GHCMAKE) configure
 	./configure

From 49d2177d51b95b4a01c05ee07e166e93751b4c51 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 20:24:24 -0400
Subject: [PATCH 2488/2835] factored out some useful error catching methods

---
 Annex/Branch.hs     | 10 ++++------
 Annex/Content.hs    | 12 +++++-------
 Crypto.hs           |  2 +-
 Git.hs              |  2 +-
 Remote/Bup.hs       | 13 +++++--------
 Remote/Directory.hs | 15 +++++----------
 Remote/Git.hs       | 12 +++---------
 Remote/Hook.hs      |  7 ++-----
 Remote/Rsync.hs     |  2 +-
 Remote/S3real.hs    |  2 +-
 Upgrade/V1.hs       |  2 +-
 Upgrade/V2.hs       |  2 +-
 Utility/Misc.hs     | 21 +++++++++++++++++++--
 Utility/TempFile.hs |  4 ++--
 git-annex-shell.hs  |  9 +++------
 15 files changed, 54 insertions(+), 61 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 189289ad39..6c28a0c84a 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -295,10 +295,8 @@ setJournalFile file content = do
 
 {- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
-getJournalFile file = do
-	g <- gitRepo
-	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)
-		(const $ return Nothing)
+getJournalFile file = inRepo $ \g -> catchMaybeIO $
+	readFileStrict $ journalFile g file
 
 {- List of files that have updated content in the journal. -}
 getJournalledFiles :: Annex [FilePath]
@@ -308,8 +306,8 @@ getJournalledFiles = map fileJournal <$> getJournalFiles
 getJournalFiles :: Annex [FilePath]
 getJournalFiles = do
 	g <- gitRepo
-	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
-		(const $ return [])
+	fs <- liftIO $
+		catchDefaultIO (getDirectoryContents $ gitAnnexJournalDir g) []
 	return $ filter (`notElem` [".", ".."]) fs
 
 {- Stages the specified journalfiles. -}
diff --git a/Annex/Content.hs b/Annex/Content.hs
index f50616af93..7586bb96f6 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -83,19 +83,17 @@ lockContent key a = do
 		unlock (Just l) = closeFd l
 
 openForLock :: FilePath -> Bool -> IO (Maybe Fd)
-openForLock file writelock = bracket_ prep cleanup $
-	catch (Just <$> openFd file mode Nothing defaultFileFlags)
-		(const $ return Nothing)
+openForLock file writelock = bracket_ prep cleanup go
 	where
+		go = catchMaybeIO $ openFd file mode Nothing defaultFileFlags
 		mode = if writelock then ReadWrite else ReadOnly
 		{- Since files are stored with the write bit disabled,
 		 - have to fiddle with permissions to open for an
-		 - exclusive lock. flock locking would avoid this,
-		 - but -}
-		prep = forwritelock $ allowWrite file
-		cleanup = forwritelock $ preventWrite file
+		 - exclusive lock. -}
 		forwritelock a = 
 			when writelock $ whenM (doesFileExist file) $ a
+		prep = forwritelock $ allowWrite file
+		cleanup = forwritelock $ preventWrite file
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
diff --git a/Crypto.hs b/Crypto.hs
index b3acb30a6e..24bb79ba0d 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -173,7 +173,7 @@ gpgParams :: [CommandParam] -> IO [String]
 gpgParams params = do
 	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
 	-- gpg output about password prompts.
-	e <- catch (getEnv "GPG_AGENT_INFO") (const $ return "")
+	e <- catchDefaultIO (getEnv "GPG_AGENT_INFO") ""
 	let batch = if null e then [] else ["--batch"]
 	return $ batch ++ defaults ++ toCommand params
 	where
diff --git a/Git.hs b/Git.hs
index 6fb6e8361c..5ceaa67f76 100644
--- a/Git.hs
+++ b/Git.hs
@@ -414,7 +414,7 @@ pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
 reap :: IO ()
 reap = do
 	-- throws an exception when there are no child processes
-	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
+	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
 	maybe (return ()) (const reap) r
 
 {- Forces git to use the specified index file.
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 866d4b42de..4c826498da 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -110,21 +110,21 @@ storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r buprepo (cipher, enck) k = do
 	src <- fromRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo enck (Param "-")
-	liftIO $ catchBool $
+	liftIO $ catchBoolIO $
 		withEncryptedHandle cipher (L.readFile src) $ \h ->
 			pipeBup params (Just h) Nothing
 
 retrieve :: BupRepo -> Key -> FilePath -> Annex Bool
 retrieve buprepo k f = do
 	let params = bupParams "join" buprepo [Param $ show k]
-	liftIO $ catchBool $ do
+	liftIO $ catchBoolIO $ do
 		tofile <- openFile f WriteMode
 		pipeBup params Nothing (Just tofile)
 
 retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted buprepo (cipher, enck) f = do
 	let params = bupParams "join" buprepo [Param $ show enck]
-	liftIO $ catchBool $ do
+	liftIO $ catchBoolIO $ do
 		(pid, h) <- hPipeFrom "bup" $ toCommand params
 		withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f
 		forceSuccess pid
@@ -145,15 +145,12 @@ checkPresent r bupr k
 		showAction $ "checking " ++ Git.repoDescribe r
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
-	| otherwise = dispatch <$> localcheck
+	| otherwise = liftIO $ catchMsgIO $
+		boolSystem "git" $ Git.gitCommandLine params bupr
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
 			, Param $ "refs/heads/" ++ show k]
-		localcheck = liftIO $ try $
-			boolSystem "git" $ Git.gitCommandLine params bupr
-		dispatch (Left e) = Left $ show e
-		dispatch (Right v) = Right v
 
 {- Store UUID in the annex.uuid setting of the bup repository. -}
 storeBupUUID :: UUID -> BupRepo -> Annex ()
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 6d3a5da7d0..b592f41ff0 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -8,7 +8,6 @@
 module Remote.Directory (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import System.IO.Error
 import qualified Data.Map as M
 
 import Common.Annex
@@ -72,13 +71,13 @@ store :: FilePath -> Key -> Annex Bool
 store d k = do
 	src <- fromRepo $ gitAnnexLocation k
 	let dest = dirKey d k
-	liftIO $ catchBool $ storeHelper dest $ copyFileExternal src dest
+	liftIO $ catchBoolIO $ storeHelper dest $ copyFileExternal src dest
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
 	src <- fromRepo $ gitAnnexLocation k
 	let dest = dirKey d enck
-	liftIO $ catchBool $ storeHelper dest $ encrypt src dest
+	liftIO $ catchBoolIO $ storeHelper dest $ encrypt src dest
 	where
 		encrypt src dest = do
 			withEncryptedContent cipher (L.readFile src) $ L.writeFile dest
@@ -100,12 +99,12 @@ retrieve d k f = liftIO $ copyFileExternal (dirKey d k) f
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =
-	liftIO $ catchBool $ do
+	liftIO $ catchBoolIO $ do
 		withDecryptedContent cipher (L.readFile (dirKey d enck)) $ L.writeFile f
 		return True
 
 remove :: FilePath -> Key -> Annex Bool
-remove d k = liftIO $ catchBool $ do
+remove d k = liftIO $ catchBoolIO $ do
 	allowWrite dir
 	removeFile file
 	removeDirectory dir
@@ -115,8 +114,4 @@ remove d k = liftIO $ catchBool $ do
 		dir = parentDir file
 
 checkPresent :: FilePath -> Key -> Annex (Either String Bool)
-checkPresent d k = dispatch <$> check
-	where
-		check = liftIO $ try $ doesFileExist (dirKey d k)
-		dispatch (Left e) = Left $ show e
-		dispatch (Right v) = Right v
+checkPresent d k = liftIO $ catchMsgIO $ doesFileExist (dirKey d k)
diff --git a/Remote/Git.hs b/Remote/Git.hs
index b63a8f124d..30d992e8ca 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -134,11 +134,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = checklocal
 	where
-		checkhttp = dispatch <$> check
-			where
-				check = safely $ Url.exists $ keyUrl r key
-				dispatch (Left e) = Left $ show e
-				dispatch (Right v) = Right v
+		checkhttp = liftIO $ catchMsgIO $ Url.exists $ keyUrl r key
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			onRemote r (check, unknown) "inannex" [Param (show key)]
@@ -149,13 +145,11 @@ inAnnex r key
 				dispatch _ = unknown
 		checklocal = dispatch <$> check
 			where
-				check = safely $ onLocal r $
+				check = liftIO $ catchMsgIO $ onLocal r $
 					Annex.Content.inAnnexSafe key
-				dispatch (Left e) = Left $ show e
+				dispatch (Left e) = Left e
 				dispatch (Right (Just b)) = Right b
 				dispatch (Right Nothing) = unknown
-		safely :: IO a -> Annex (Either IOException a)
-		safely a = liftIO $ try a
 		unknown = Left $ "unable to check " ++ Git.repoDescribe r
 
 {- Runs an action on a local repository inexpensively, by making an annex
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 9f9250e41d..03976fc709 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -9,7 +9,6 @@ module Remote.Hook (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
-import System.IO.Error (try)
 import System.Exit
 
 import Common.Annex
@@ -112,7 +111,7 @@ retrieve h k f = runHook h "retrieve" k (Just f) $ return True
 
 retrieveEncrypted :: String -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
-	runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBool $ do
+	runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBoolIO $ do
 		withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
 		return True
 
@@ -123,12 +122,10 @@ checkPresent :: Git.Repo -> String -> Key -> Annex (Either String Bool)
 checkPresent r h k = do
 	showAction $ "checking " ++ Git.repoDescribe r
 	v <- lookupHook h "checkpresent"
-	dispatch <$> liftIO (try (check v) ::IO (Either IOException Bool))
+	liftIO $ catchMsgIO $ check v
 	where
 		findkey s = show k `elem` lines s
 		env = hookEnv k Nothing
-		dispatch (Left e) = Left $ show e
-		dispatch (Right v) = Right v
 		check Nothing = error "checkpresent hook misconfigured"
 		check (Just hook) = do
 			(frompipe, topipe) <- createPipe
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 54834be135..86ff2ea5b1 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -110,7 +110,7 @@ retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
 	res <- retrieve o enck tmp
 	if res
-		then liftIO $ catchBool $ do
+		then liftIO $ catchBoolIO $ do
 			withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
 			return True
 		else return res
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 29117b3a41..97ac648218 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -286,7 +286,7 @@ s3GetCreds c = do
 				_ -> return Nothing
 		else return $ Just (ak, sk)
 	where
-		getEnvKey s = liftIO $ catch (getEnv s) (const $ return "")
+		getEnvKey s = liftIO $ catchDefaultIO (getEnv s) ""
 
 {- Stores S3 creds encrypted in the remote's config if possible. -}
 s3SetCreds :: RemoteConfig -> Annex RemoteConfig
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index fe59ad3da8..377e4b21b3 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -179,7 +179,7 @@ writeLog1 :: FilePath -> [LogLine] -> IO ()
 writeLog1 file ls = viaTmp writeFile file (showLog ls)
 
 readLog1 :: FilePath -> IO [LogLine]
-readLog1 file = catch (parseLog <$> readFileStrict file) (const $ return [])
+readLog1 file = catchDefaultIO (parseLog <$> readFileStrict file) []
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 7ef2a4d18e..6a46ad8a16 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -69,7 +69,7 @@ locationLogs = do
 		files <- mapM tryDirContents (concat levelb)
 		return $ mapMaybe islogfile (concat files)
 	where
-		tryDirContents d = catch (dirContents d) (return . const [])
+		tryDirContents d = catchDefaultIO (dirContents d) []
 		islogfile f = maybe Nothing (\k -> Just (k, f)) $
 				logFileKey $ takeFileName f
 
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index 4c4aa4c935..7285987231 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -8,7 +8,9 @@
 module Utility.Misc where
 
 import System.IO
+import System.IO.Error (try)
 import Control.Monad
+import Control.Applicative
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -26,5 +28,20 @@ readMaybe s = case reads s of
 	_ -> Nothing
 
 {- Catches IO errors and returns a Bool -}
-catchBool :: IO Bool -> IO Bool
-catchBool = flip catch (const $ return False)
+catchBoolIO :: IO Bool -> IO Bool
+catchBoolIO a = catchDefaultIO a False
+
+{- Catches IO errors and returns a Maybe -}
+catchMaybeIO :: IO a -> IO (Maybe a)
+catchMaybeIO a = catchDefaultIO (Just <$> a) Nothing
+
+{- Catches IO errors and returns a default value. -}
+catchDefaultIO :: IO a -> a -> IO a
+catchDefaultIO a def = catch a (const $ return def)
+
+{- Catches IO errors and returns the error message. -}
+catchMsgIO :: IO a -> IO (Either String a)
+catchMsgIO a = dispatch <$> try a
+	where
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs
index 1e823c10ef..8d50dd8b2c 100644
--- a/Utility/TempFile.hs
+++ b/Utility/TempFile.hs
@@ -31,9 +31,9 @@ withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
 withTempFile template a = bracket create remove use
 	where
 		create = do
-			tmpdir <- catch getTemporaryDirectory (const $ return ".")
+			tmpdir <- catchDefaultIO getTemporaryDirectory "."
 			openTempFile tmpdir template
 		remove (name, handle) = do
 			hClose handle
-			catchBool (removeFile name >> return True)
+			catchBoolIO (removeFile name >> return True)
 		use (name, handle) = a name handle
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 12cc65e4dd..57f6b29162 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -104,9 +104,6 @@ checkNotReadOnly cmd
 	| otherwise = checkEnv "GIT_ANNEX_SHELL_READONLY"
 
 checkEnv :: String -> IO ()
-checkEnv var = catch check (const $ return ())
-	where
-		check = do
-			val <- getEnv var
-			when (not $ null val) $
-				error $ "Action blocked by " ++ var
+checkEnv var =
+	whenM (not . null <$> catchDefaultIO (getEnv var) "") $
+		error $ "Action blocked by " ++ var

From 2de1e2c2cee4ea75b971000fe480c48b6cbaab29 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 21:32:42 -0400
Subject: [PATCH 2489/2835] Optimized copy --from and get --from to avoid
 checking the location log for files that are already present.

This can be a significant speedup when running in large trees that are
only missing a few files; it makes copy --from just as fast as get.
---
 Command/Move.hs  | 22 ++++++++++++++--------
 debian/changelog |  2 ++
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index f02f325580..9a6b0190e6 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -110,14 +110,20 @@ toPerform dest move key = moveLock move key $ do
  - from the remote.
  -}
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
-fromStart src move file = isAnnexed file $ \(key, _) -> do
-	u <- getUUID
-	remotes <- Remote.keyPossibilities key
-	if u == Remote.uuid src || not (any (== src) remotes)
-		then stop
-		else do
-			showMoveAction move file
-			next $ fromPerform src move key
+fromStart src move file
+	| move == True = isAnnexed file $ \(key, _) -> go key
+	| otherwise = isAnnexed file $ \(key, _) -> do
+		ishere <- inAnnex key
+		if ishere then stop else go key
+	where
+		go key = do
+			u <- getUUID
+			remotes <- Remote.keyPossibilities key
+			if u == Remote.uuid src || not (any (== src) remotes)
+				then stop
+				else do
+					showMoveAction move file
+					next $ fromPerform src move key
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = moveLock move key $ do
 	ishere <- inAnnex key
diff --git a/debian/changelog b/debian/changelog
index 9fa96e06ab..72110785bb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20111108) UNRELEASED; urgency=low
   * Handle a case where an annexed file is moved into a gitignored directory,
     by having fix --force add its change.
   * Avoid cyclic drop problems.
+  * Optimized copy --from and get --from to avoid checking the location log
+    for files that are already present.
 
  -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
 

From 4389782628a1cc683ef238e848b6311fc4bd82c3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 22:37:52 -0400
Subject: [PATCH 2490/2835] tweak

---
 Command/Move.hs | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 9a6b0190e6..ffa246ab6f 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -28,7 +28,7 @@ seek = [withFilesInGit $ start True]
  - This only operates on the cached file content; it does not involve
  - moving data in the key-value backend. -}
 start :: Bool -> FilePath -> CommandStart
-start move file = do
+start move file = isAnnexed file $ \(key, _) -> do
 	noAuto
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
@@ -36,10 +36,10 @@ start move file = do
 		(Nothing, Nothing) -> error "specify either --from or --to"
 		(Nothing, Just name) -> do
 			dest <- Remote.byName name
-			toStart dest move file
+			toStart dest move file key
 		(Just name, Nothing) -> do
 			src <- Remote.byName name
-			fromStart src move file
+			fromStart src move file key
 		(_ ,  _) -> error "only one of --from or --to can be specified"
 	where
 		noAuto = when move $ whenM (Annex.getState Annex.auto) $ error
@@ -58,8 +58,8 @@ showMoveAction False file = showStart "copy" file
  - A file's content can be moved even if there are insufficient copies to
  - allow it to be dropped.
  -}
-toStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
-toStart dest move file = isAnnexed file $ \(key, _) -> do
+toStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
+toStart dest move file key = do
 	u <- getUUID
 	ishere <- inAnnex key
 	if not ishere || u == Remote.uuid dest
@@ -109,14 +109,14 @@ toPerform dest move key = moveLock move key $ do
  - If the current repository already has the content, it is still removed
  - from the remote.
  -}
-fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
-fromStart src move file
-	| move == True = isAnnexed file $ \(key, _) -> go key
-	| otherwise = isAnnexed file $ \(key, _) -> do
+fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
+fromStart src move file key
+	| move == True = go
+	| otherwise = do
 		ishere <- inAnnex key
-		if ishere then stop else go key
+		if ishere then stop else go
 	where
-		go key = do
+		go = do
 			u <- getUUID
 			remotes <- Remote.keyPossibilities key
 			if u == Remote.uuid src || not (any (== src) remotes)

From b327227ba596d4fc5012138d03390c3eb861b808 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 23:35:08 -0400
Subject: [PATCH 2491/2835] better limiting of start actions to only run
 whenAnnexed

Mostly only refactoring, but this does remove one redundant stat of the
symlink by copy.
---
 Command.hs           | 10 ++++++----
 Command/Copy.hs      | 10 +++++-----
 Command/Drop.hs      | 25 ++++++++++++-------------
 Command/Find.hs      |  6 +++---
 Command/Fix.hs       |  6 +++---
 Command/Fsck.hs      |  9 ++++++---
 Command/Get.hs       |  6 +++---
 Command/Migrate.hs   |  7 +++----
 Command/Move.hs      |  6 +++---
 Command/PreCommit.hs |  5 +++--
 Command/Reinject.hs  | 10 +++++-----
 Command/Unannex.hs   |  6 +++---
 Command/Uninit.hs    |  8 ++++----
 Command/Unlock.hs    |  6 +++---
 Command/Whereis.hs   |  6 +++---
 Seek.hs              |  4 ++--
 16 files changed, 67 insertions(+), 63 deletions(-)

diff --git a/Command.hs b/Command.hs
index c436c5b628..083be37f22 100644
--- a/Command.hs
+++ b/Command.hs
@@ -15,8 +15,8 @@ module Command (
 	stop,
 	prepCommand,
 	doCommand,
+	whenAnnexed,
 	notAnnexed,
-	isAnnexed,
 	notBareRepo,
 	isBareRepo,
 	autoCopies
@@ -65,12 +65,14 @@ doCommand = start
 		failure = showEndFail >> return False
 		status r = showEndResult r >> return r
 
+{- Modifies an action to only act on files that are already annexed,
+ - and passes the key and backend on to it. -}
+whenAnnexed :: (FilePath -> (Key, Backend Annex) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a)
+whenAnnexed a file = maybe (return Nothing) (a file) =<< Backend.lookupFile file
+
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
 
-isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
-isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
-
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
 	whenM isBareRepo $
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 8316b7cabf..16de423acb 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -7,6 +7,7 @@
 
 module Command.Copy where
 
+import Common.Annex
 import Command
 import qualified Command.Move
 
@@ -16,11 +17,10 @@ def = [dontCheck toOpt $ dontCheck fromOpt $
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start]
+seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
 -- A copy is just a move that does not delete the source file.
 -- However, --auto mode avoids unnecessary copies.
-start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, _) ->
-	autoCopies key (<) numcopies $
-		Command.Move.start False file
+start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start numcopies file (key, backend) = autoCopies key (<) numcopies $
+	Command.Move.start False file (key, backend)
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 44685ffcd6..ee3583869b 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -22,20 +22,19 @@ def = [dontCheck fromOpt $ command "drop" paramPaths seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start]
+seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
-start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, _) ->
-	autoCopies key (>) numcopies $ do
-		from <- Annex.getState Annex.fromremote
-		case from of
-			Nothing -> startLocal file numcopies key
-			Just name -> do
-				remote <- Remote.byName name
-				u <- getUUID
-				if Remote.uuid remote == u
-					then startLocal file numcopies key
-					else startRemote file numcopies key remote
+start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start numcopies file (key, _) = autoCopies key (>) numcopies $ do
+	from <- Annex.getState Annex.fromremote
+	case from of
+		Nothing -> startLocal file numcopies key
+		Just name -> do
+			remote <- Remote.byName name
+			u <- getUUID
+			if Remote.uuid remote == u
+				then startLocal file numcopies key
+				else startRemote file numcopies key remote
 
 startLocal :: FilePath -> Maybe Int -> Key -> CommandStart
 startLocal file numcopies key = do
diff --git a/Command/Find.hs b/Command/Find.hs
index 46364c9876..c816ff0712 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -16,10 +16,10 @@ def :: [Command]
 def = [command "find" paramPaths seek "lists available files"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
 	whenM (liftM2 (||) (inAnnex key) limited) $
diff --git a/Command/Fix.hs b/Command/Fix.hs
index b46d6e8ecd..27c4b167e5 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -17,11 +17,11 @@ def = [command "fix" paramPaths seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
 {- Fixes the symlink to an annexed file. -}
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	link <- calcGitLink file key
 	l <- liftIO $ readSymbolicLink file
 	if link == l
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 89485b7788..bdc5099410 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -25,10 +25,13 @@ def :: [Command]
 def = [command "fsck" paramPaths seek "check for problems"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start, withBarePresentKeys startBare]
+seek =
+	[ withNumCopies $ \n -> whenAnnexed $ start n
+	, withBarePresentKeys startBare
+	]
 
-start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, backend) -> do
+start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start numcopies file (key, backend) = do
 	showStart "fsck" file
 	next $ perform key file backend numcopies
 
diff --git a/Command/Get.hs b/Command/Get.hs
index 4a0908bdc8..f7d953bb65 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -19,10 +19,10 @@ def = [dontCheck fromOpt $ command "get" paramPaths seek
 	"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start]
+seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
-start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, _) -> do
+start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start numcopies file (key, _) = do
 	inannex <- inAnnex key
 	if inannex
 		then stop
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index a823466dc7..3c87f41363 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -13,17 +13,16 @@ import qualified Backend
 import qualified Types.Key
 import Annex.Content
 import qualified Command.Add
-import Backend
 import Logs.Web
 
 def :: [Command]
 def = [command "migrate" paramPaths seek "switch data to different backend"]
 
 seek :: [CommandSeek]
-seek = [withBackendFilesInGit start]
+seek = [withBackendFilesInGit $ \(b, f) -> whenAnnexed (start b) f]
 
-start :: BackendFile -> CommandStart
-start (b, file) = isAnnexed file $ \(key, oldbackend) -> do
+start :: Maybe (Backend Annex) -> FilePath -> (Key, Backend Annex) -> CommandStart
+start b file (key, oldbackend) = do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
 	if (newbackend /= oldbackend || upgradableKey key) && exists
diff --git a/Command/Move.hs b/Command/Move.hs
index ffa246ab6f..9553d1639b 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -21,14 +21,14 @@ def = [dontCheck toOpt $ dontCheck fromOpt $
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit $ start True]
+seek = [withFilesInGit $ whenAnnexed $ start True]
 
 {- Move (or copy) a file either --to or --from a repository.
  -
  - This only operates on the cached file content; it does not involve
  - moving data in the key-value backend. -}
-start :: Bool -> FilePath -> CommandStart
-start move file = isAnnexed file $ \(key, _) -> do
+start :: Bool -> FilePath -> (Key, Backend Annex) -> CommandStart
+start move file (key, _) = do
 	noAuto
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 1949de113f..57bc7ac138 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -18,8 +18,9 @@ def = [command "pre-commit" paramPaths seek "run by git pre-commit hook"]
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
 seek :: [CommandSeek]
-seek = [withFilesToBeCommitted Command.Fix.start,
-	withFilesUnlockedToBeCommitted start]
+seek =
+	[ withFilesToBeCommitted $ whenAnnexed Command.Fix.start
+	, withFilesUnlockedToBeCommitted start]
 
 start :: BackendFile -> CommandStart
 start p = next $ perform p
diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 1277edf906..cfa0655ef1 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -25,19 +25,19 @@ start (src:dest:[])
 	| src == dest = stop
 	| otherwise = do
 		showStart "reinject" dest
-		next $ perform src dest
+		next $ whenAnnexed (perform src) dest
 start _ = error "specify a src file and a dest file"
 
-perform :: FilePath -> FilePath -> CommandPerform
-perform src dest = isAnnexed dest $ \(key, backend) -> do
-	unlessM (move key) $ error "mv failed!"
+perform :: FilePath -> FilePath -> (Key, Backend Annex) -> CommandPerform
+perform src _dest (key, backend) = do
+	unlessM move $ error "mv failed!"
 	next $ cleanup key backend
 	where
 		-- the file might be on a different filesystem,
 		-- so mv is used rather than simply calling
 		-- moveToObjectDir; disk space is also
 		-- checked this way.
-		move key = getViaTmp key $ \tmp ->
+		move = getViaTmp key $ \tmp ->
 			liftIO $ boolSystem "mv" [File src, File tmp]
 
 cleanup :: Key -> Backend Annex -> CommandCleanup
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index d24f921a93..b9190ce044 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -21,11 +21,11 @@ def :: [Command]
 def = [command "unannex" paramPaths seek "undo accidential add command"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
 {- The unannex subcommand undoes an add. -}
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	ishere <- inAnnex key
 	if ishere
 		then do
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index f317b7620d..8987240bee 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -33,15 +33,15 @@ check = do
 			[Params "rev-parse --abbrev-ref HEAD"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit startUnannex, withNothing start]
+seek = [withFilesInGit $ whenAnnexed startUnannex, withNothing start]
 
-startUnannex :: FilePath -> CommandStart
-startUnannex file = do
+startUnannex :: FilePath -> (Key, Backend Annex) -> CommandStart
+startUnannex file info = do
 	-- Force fast mode before running unannex. This way, if multiple
 	-- files link to a key, it will be left in the annex and hardlinked
 	-- to by each.
 	Annex.changeState $ \s -> s { Annex.fast = True }
-	Command.Unannex.start file
+	Command.Unannex.start file info
 
 start :: CommandStart
 start = next perform
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 590b753111..22f9ce7108 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -22,12 +22,12 @@ def =
 		c n = command n paramPaths seek
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
 {- The unlock subcommand replaces the symlink with a copy of the file's
  - content. -}
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	showStart "unlock" file
 	next $ perform file key
 
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 0681bfba1e..eb2ae3d4e7 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -18,10 +18,10 @@ def = [command "whereis" paramPaths seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	showStart "whereis" file
 	next $ perform key
 
diff --git a/Seek.hs b/Seek.hs
index 3c83ebc35d..9863b33c40 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -33,10 +33,10 @@ withAttrFilesInGit attr a params = do
 	files <- seekHelper LsFiles.inRepo params
 	prepFilteredGen a fst $ inRepo $ Git.checkAttr attr files
 
-withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
+withNumCopies :: (Maybe Int -> FilePath -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 	where
-		go (file, v) = a file (readMaybe v)
+		go (file, v) = a (readMaybe v) file
 
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do

From 637b5feb45013f69f3aacbafeb796de666d3faa3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 01:52:58 -0400
Subject: [PATCH 2492/2835] lint

---
 Annex/Branch.hs          |  8 ++---
 Annex/CatFile.hs         |  2 +-
 Annex/Content.hs         |  4 +--
 Checks.hs                |  4 +--
 Command.hs               | 17 +++++------
 Command/Describe.hs      |  2 +-
 Command/FromKey.hs       |  2 +-
 Command/Fsck.hs          |  4 +--
 Command/Migrate.hs       |  4 +--
 Command/Move.hs          |  4 +--
 Command/Uninit.hs        |  4 +--
 Common.hs                | 63 ++++++++++++++--------------------------
 Common/Annex.hs          | 21 +++++---------
 Config.hs                |  2 +-
 Git/LsFiles.hs           |  2 +-
 Git/UnionMerge.hs        |  2 +-
 Logs/Remote.hs           |  2 +-
 Logs/Trust.hs            |  4 +--
 Logs/UUID.hs             |  2 +-
 Logs/UUIDBased.hs        |  2 +-
 Remote/Git.hs            |  2 +-
 Remote/Helper/Special.hs |  2 +-
 Seek.hs                  |  6 ++--
 Upgrade/V1.hs            |  4 +--
 configure.hs             |  2 +-
 git-annex.cabal          |  2 +-
 26 files changed, 71 insertions(+), 102 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 6c28a0c84a..05c89ed97e 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -63,7 +63,7 @@ withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
-	f <- fromRepo $ index
+	f <- fromRepo index
 	bracketIO (Git.useIndex f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
@@ -336,8 +336,8 @@ stageJournalFiles = do
 	where
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
-		git_hash_object g = Git.gitCommandLine
-			[Param "hash-object", Param "-w", Param "--stdin-paths"] g
+		git_hash_object = Git.gitCommandLine
+			[Param "hash-object", Param "-w", Param "--stdin-paths"]
 
 
 {- Checks if there are changes in the journal. -}
@@ -366,7 +366,7 @@ fileJournal = replace "//" "_" . replace "_" "/"
  - contention with other git-annex processes. -}
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
-	file <- fromRepo $ gitAnnexJournalLock
+	file <- fromRepo gitAnnexJournalLock
 	bracketIO (lock file) unlock a
 	where
 		lock file = do
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index a043e1ae3d..99cc519f51 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -17,7 +17,7 @@ catFile :: String -> FilePath -> Annex String
 catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
 	where
 		startup = do
-			h <- inRepo $ Git.CatFile.catFileStart
+			h <- inRepo Git.CatFile.catFileStart
 			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
 			go h
 		go h = liftIO $ Git.CatFile.catFile h branch file
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 7586bb96f6..83839ea135 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -91,7 +91,7 @@ openForLock file writelock = bracket_ prep cleanup go
 		 - have to fiddle with permissions to open for an
 		 - exclusive lock. -}
 		forwritelock a = 
-			when writelock $ whenM (doesFileExist file) $ a
+			when writelock $ whenM (doesFileExist file) a
 		prep = forwritelock $ allowWrite file
 		cleanup = forwritelock $ preventWrite file
 
@@ -251,7 +251,7 @@ fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
 moveBad :: Key -> Annex FilePath
 moveBad key = do
 	src <- fromRepo $ gitAnnexLocation key
-	bad <- fromRepo $ gitAnnexBadDir
+	bad <- fromRepo gitAnnexBadDir
 	let dest = bad  takeFileName src
 	liftIO $ do
 		createDirectoryIfMissing True (parentDir dest)
diff --git a/Checks.hs b/Checks.hs
index 6a70fc52db..e443811cdc 100644
--- a/Checks.hs
+++ b/Checks.hs
@@ -24,12 +24,12 @@ repoExists = CommandCheck 0 ensureInitialized
 fromOpt :: CommandCheck
 fromOpt = CommandCheck 1 $ do
 	v <- Annex.getState Annex.fromremote
-	unless (v == Nothing) $ error "cannot use --from with this command"
+	unless (isNothing v) $ error "cannot use --from with this command"
 
 toOpt :: CommandCheck
 toOpt = CommandCheck 2 $ do
 	v <- Annex.getState Annex.toremote
-	unless (v == Nothing) $ error "cannot use --to with this command"
+	unless (isNothing v) $ error "cannot use --to with this command"
 
 dontCheck :: CommandCheck -> Command -> Command
 dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
diff --git a/Command.hs b/Command.hs
index 083be37f22..d22c2d12f2 100644
--- a/Command.hs
+++ b/Command.hs
@@ -6,10 +6,6 @@
  -}
 
 module Command (
-	module Types.Command,
-	module Seek,
-	module Checks,
-	module Options,
 	command,
 	next,
 	stop,
@@ -19,20 +15,21 @@ module Command (
 	notAnnexed,
 	notBareRepo,
 	isBareRepo,
-	autoCopies
+	autoCopies,
+	module ReExported
 ) where
 
 import Common.Annex
 import qualified Backend
 import qualified Annex
 import qualified Git
-import Types.Command
+import Types.Command as ReExported
+import Seek as ReExported
+import Checks as ReExported
+import Options as ReExported
 import Logs.Trust
 import Logs.Location
 import Config
-import Seek
-import Checks
-import Options
 
 {- Generates a command with the common checks. -}
 command :: String -> String -> [CommandSeek] -> String -> Command
@@ -50,7 +47,7 @@ stop = return Nothing
  - list of actions to perform to run the command. -}
 prepCommand :: Command -> [String] -> Annex [CommandCleanup]
 prepCommand Command { cmdseek = seek, cmdcheck = c } params = do
-	sequence_ $ map runCheck c
+	mapM_ runCheck c
 	map doCommand . concat <$> mapM (\s -> s params) seek
 
 {- Runs a command through the start, perform and cleanup stages -}
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 1cc81bcbd5..61297e77c7 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -24,7 +24,7 @@ start (name:description) = do
 	showStart "describe" name
 	u <- Remote.nameToUUID name
 	next $ perform u $ unwords description
-start _ = do error "Specify a repository and a description."	
+start _ = error "Specify a repository and a description."	
 
 perform :: UUID -> String -> CommandPerform
 perform u description = do
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index b910dd1f09..ec194e06e8 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -22,7 +22,7 @@ seek = [withWords start]
 
 start :: [String] -> CommandStart
 start (keyname:file:[]) = notBareRepo $ do
-	let key = maybe (error "bad key") id $ readKey keyname
+	let key = fromMaybe (error "bad key") $ readKey keyname
 	inbackend <- inAnnex key
 	unless inbackend $ error $
 		"key ("++ keyname ++") is not present in backend"
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index bdc5099410..99dda99e5f 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -50,7 +50,7 @@ withBarePresentKeys a params = isBareRepo >>= go
 	where
 		go False = return []
 		go True = do
-			unless (null params) $ do
+			unless (null params) $
 				error "fsck should be run without parameters in a bare repository"
 			prepStart a loggedKeys
 
@@ -137,7 +137,7 @@ checkKeySize key = do
 
 
 checkBackend :: Backend Annex -> Key -> Annex Bool
-checkBackend backend key =  (Types.Backend.fsckKey backend) key
+checkBackend = Types.Backend.fsckKey
 
 checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 3c87f41363..860c4bd470 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -46,7 +46,7 @@ perform file oldkey newbackend = do
 	-- The old backend's key is not dropped from it, because there may
 	-- be other files still pointing at that key.
 	src <- fromRepo $ gitAnnexLocation oldkey
-	tmp <- fromRepo $ gitAnnexTmpDir
+	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp  takeFileName file
 	liftIO $ createLink src tmpfile
 	k <- Backend.genKey tmpfile $ Just newbackend
@@ -64,7 +64,7 @@ perform file oldkey newbackend = do
 					-- associated urls, record them for
 					-- the new key as well.
 					urls <- getUrls oldkey
-					when (not $ null urls) $
+					unless (null urls) $
 						mapM_ (setUrlPresent newkey) urls
 
 					next $ Command.Add.cleanup file newkey True
diff --git a/Command/Move.hs b/Command/Move.hs
index 9553d1639b..155f4d605f 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -82,7 +82,7 @@ toPerform dest move key = moveLock move key $ do
 		else Remote.hasKey dest key
 	case isthere of
 		Left err -> do
-			showNote $ err
+			showNote err
 			stop
 		Right False -> do
 			showAction $ "to " ++ Remote.name dest
@@ -111,7 +111,7 @@ toPerform dest move key = moveLock move key $ do
  -}
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
 fromStart src move file key
-	| move == True = go
+	| move = go
 	| otherwise = do
 		ishere <- inAnnex key
 		if ishere then stop else go
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 8987240bee..ca18c478c5 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -51,11 +51,11 @@ perform = next cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
-	annexdir <- fromRepo $ gitAnnexDir
+	annexdir <- fromRepo gitAnnexDir
 	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
 	saveState
 	inRepo $ Git.run "branch" [Param "-D", Param Annex.Branch.name]
-	liftIO $ exitSuccess
+	liftIO exitSuccess
diff --git a/Common.hs b/Common.hs
index ef068ac105..e0132d9e96 100644
--- a/Common.hs
+++ b/Common.hs
@@ -1,46 +1,25 @@
-module Common (
-	module Control.Monad,
-	module Control.Applicative,
-	module Control.Monad.State,
-	module Control.Exception.Extensible,
-	module Data.Maybe,
-	module Data.List,
-	module Data.String.Utils,
-	module System.Path,
-	module System.FilePath,
-	module System.Directory,
-	module System.Cmd.Utils,
-	module System.IO,
-	module System.Posix.Files,
-	module System.Posix.IO,
-	module System.Posix.Process,
-	module System.Exit,
-	module Utility.Misc,
-	module Utility.Conditional,
-	module Utility.SafeCommand,
-	module Utility.Path,
-) where
+module Common (module X) where
 
-import Control.Monad hiding (join)
-import Control.Applicative
-import Control.Monad.State (liftIO)
-import Control.Exception.Extensible (IOException)
+import Control.Monad as X hiding (join)
+import Control.Applicative as X
+import Control.Monad.State as X (liftIO)
+import Control.Exception.Extensible as X (IOException)
 
-import Data.Maybe
-import Data.List
-import Data.String.Utils
+import Data.Maybe as X
+import Data.List as X
+import Data.String.Utils as X
 
-import System.Path
-import System.FilePath
-import System.Directory
-import System.Cmd.Utils hiding (safeSystem)
-import System.IO hiding (FilePath)
-import System.Posix.Files
-import System.Posix.IO
-import System.Posix.Process hiding (executeFile)
-import System.Exit
+import System.Path as X
+import System.FilePath as X
+import System.Directory as X
+import System.Cmd.Utils as X hiding (safeSystem)
+import System.IO as X hiding (FilePath)
+import System.Posix.Files as X
+import System.Posix.IO as X
+import System.Posix.Process as X hiding (executeFile)
+import System.Exit as X
 
-import Utility.Misc
-import Utility.Conditional
-import Utility.SafeCommand
-import Utility.Path
+import Utility.Misc as X
+import Utility.Conditional as X
+import Utility.SafeCommand as X
+import Utility.Path as X
diff --git a/Common/Annex.hs b/Common/Annex.hs
index 6b5bc31de2..e90825f0e9 100644
--- a/Common/Annex.hs
+++ b/Common/Annex.hs
@@ -1,15 +1,8 @@
-module Common.Annex (
-	module Common,
-	module Types,
-	module Types.UUID,
-	module Annex,
-	module Locations,
-	module Messages,
-) where
+module Common.Annex (module X) where
 
-import Common
-import Types
-import Types.UUID (toUUID, fromUUID)
-import Annex (gitRepo, inRepo, fromRepo)
-import Locations
-import Messages
+import Common as X
+import Types as X
+import Types.UUID as X (toUUID, fromUUID)
+import Annex as X (gitRepo, inRepo, fromRepo)
+import Locations as X
+import Messages as X
diff --git a/Config.hs b/Config.hs
index c24ab9d04d..dbd13ad3fd 100644
--- a/Config.hs
+++ b/Config.hs
@@ -18,7 +18,7 @@ setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
 	inRepo $ Git.run "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
-	newg <- inRepo $ Git.configRead
+	newg <- inRepo Git.configRead
 	Annex.changeState $ \s -> s { Annex.repo = newg }
 
 {- Looks up a per-remote config setting in git config.
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index bceee26fcd..85215fe040 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -20,7 +20,7 @@ import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: [FilePath] -> Repo -> IO [FilePath]
-inRepo l repo = pipeNullSplit (Params "ls-files --cached -z --" : map File l) repo
+inRepo l = pipeNullSplit $ Params "ls-files --cached -z --" : map File l
 
 {- Scans for files at the specified locations that are not checked into git. -}
 notInRepo :: Bool -> [FilePath] -> Repo -> IO [FilePath]
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 32966c846d..30778d0344 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -37,7 +37,7 @@ merge x y repo = do
  - the index are preserved (and participate in the merge). -}
 merge_index :: Repo -> [String] -> IO ()
 merge_index repo bs =
-	update_index repo =<< concat <$> mapM (\b -> merge_tree_index b repo) bs
+	update_index repo =<< concat <$> mapM (`merge_tree_index` repo) bs
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index e2b04bf471..8d15f3151e 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -30,7 +30,7 @@ remoteLog = "remote.log"
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
 configSet u c = do
-	ts <- liftIO $ getPOSIXTime
+	ts <- liftIO getPOSIXTime
 	Annex.Branch.change remoteLog $
 		showLog showConfig . changeLog ts u c . parseLog parseConfig
 
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 8c4507dcb8..cb91861fd9 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -47,7 +47,7 @@ parseTrust :: String -> Maybe TrustLevel
 parseTrust s
 	| length w > 0 = Just $ parse $ head w
 	-- back-compat; the trust.log used to only list trusted repos
-	| otherwise = Just $ Trusted
+	| otherwise = Just Trusted
 	where
 		w = words s
 		parse "1" = Trusted
@@ -62,7 +62,7 @@ showTrust Trusted = "1"
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid@(UUID _) level = do
-	ts <- liftIO $ getPOSIXTime
+	ts <- liftIO getPOSIXTime
 	Annex.Branch.change trustLog $
 		showLog showTrust . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 77cfb5ce0f..da611d7bf5 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -34,7 +34,7 @@ logfile = "uuid.log"
 {- Records a description for a uuid in the log. -}
 describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
-	ts <- liftIO $ getPOSIXTime
+	ts <- liftIO getPOSIXTime
 	Annex.Branch.change logfile $
 		showLog id . changeLog ts uuid desc . parseLog Just
 
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 9609d73213..42908ab1d3 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -55,7 +55,7 @@ showLog shower = unlines . map showpair . M.toList
 			unwords [fromUUID k, shower v]
 
 parseLog :: (String -> Maybe a) -> String -> Log a
-parseLog parser = M.fromListWith best . catMaybes . map parse . lines
+parseLog parser = M.fromListWith best . mapMaybe parse . lines
 	where
 		parse line
 			| null ws = Nothing
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 30d992e8ca..541d8e5f65 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -166,7 +166,7 @@ onLocal r a = do
 		-- for anything onLocal is used to do.
 		Annex.Branch.disableUpdate
 		ret <- a
-		liftIO $ Git.reap
+		liftIO Git.reap
 		return ret
 
 keyUrl :: Git.Repo -> Key -> String
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 5bbf4169d7..77478eb1d5 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -19,7 +19,7 @@ import qualified Git
  -}
 findSpecialRemotes :: String -> Annex [Git.Repo]
 findSpecialRemotes s = do
-	m <- fromRepo $ Git.configMap
+	m <- fromRepo Git.configMap
 	return $ map construct $ remotepairs m
 	where
 		remotepairs = M.toList . M.filterWithKey match
diff --git a/Seek.hs b/Seek.hs
index 9863b33c40..1430ebabd1 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -23,7 +23,7 @@ import qualified Limit
 seekHelper :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> [FilePath] -> Annex [FilePath]
 seekHelper a params = do
 	g <- gitRepo
-	liftIO $ runPreserveOrder (\p -> a p g) params
+	liftIO $ runPreserveOrder (`a` g) params
 
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
@@ -73,7 +73,7 @@ withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
-	top <- fromRepo $ Git.workTree
+	top <- fromRepo Git.workTree
 	typechangedfiles <- seekHelper typechanged params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> top ++ "/" ++ f) typechangedfiles
@@ -109,7 +109,7 @@ prepFilteredGen a d fs = do
  - command, using a list (ie of files) coming from an action. The list
  - will be produced and consumed lazily. -}
 prepStart :: (b -> CommandStart) -> Annex [b] -> Annex [CommandStart]
-prepStart a fs = liftM (map a) fs
+prepStart a = liftM (map a)
 
 notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 377e4b21b3..567cf8e5bf 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -51,7 +51,7 @@ upgrade :: Annex Bool
 upgrade = do
 	showAction "v1 to v2"
 	
-	bare <- fromRepo $ Git.repoIsLocalBare
+	bare <- fromRepo Git.repoIsLocalBare
 	if bare
 		then do
 			moveContent
@@ -113,7 +113,7 @@ moveLocationLogs = do
 					else return []
 			move (l, k) = do
 				dest <- fromRepo $ logFile2 k
-				dir <- fromRepo $ Upgrade.V2.gitStateDir
+				dir <- fromRepo Upgrade.V2.gitStateDir
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
 				-- could just git mv, but this way deals with
diff --git a/configure.hs b/configure.hs
index 1d1c023356..cb73af2a99 100644
--- a/configure.hs
+++ b/configure.hs
@@ -69,7 +69,7 @@ checkGitVersion = do
 		-- for git-check-attr behavior change
 		need = "1.7.7"
 		dotted = sum . mult 1 . reverse . extend 10 . map readi . split "." 
-		extend n l = l ++ take (n - length l) (repeat 0)
+		extend n l = l ++ replicate (n - length l) 0
 		mult _ [] = []
 		mult n (x:xs) = (n*x) : (mult (n*100) xs)
 		readi :: String -> Integer
diff --git a/git-annex.cabal b/git-annex.cabal
index 4b3a708a4e..7b9459b46b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111107
+Version: 3.20111108
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 826d5887b2c31c9dca1415997d7704d9442077b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 13:42:31 -0400
Subject: [PATCH 2493/2835] Automatically fix up badly formatted uuid.log
 entries produced by 3.20111105, whenever the uuid.log is changed (ie, by init
 or describe).

---
 Logs/UUID.hs      | 31 ++++++++++++++++++++++++++++++-
 Logs/UUIDBased.hs |  1 +
 debian/changelog  |  2 ++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index da611d7bf5..20f43d15ca 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -36,7 +36,36 @@ describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
 	ts <- liftIO getPOSIXTime
 	Annex.Branch.change logfile $
-		showLog id . changeLog ts uuid desc . parseLog Just
+		showLog id . changeLog ts uuid desc . fixBadUUID . parseLog Just
+
+{- Temporarily here to fix badly formatted uuid logs generated by
+ - versions 3.20111105 and 3.20111025. 
+ -
+ - Those logs contain entries with the UUID and description flipped.
+ - Due to parsing, if the description is multiword, only the first
+ - will be taken to be the UUID. So, if the UUID of an entry does
+ - not look like a UUID, and the last word of the description does,
+ - flip them back.
+ -}
+fixBadUUID :: Log String -> Log String
+fixBadUUID = M.fromList . map fixup . M.toList
+	where
+		fixup (k, v)
+			| isbad = (fixeduuid, LogEntry (Date $ newertime v) fixedvalue)
+			| otherwise = (k, v)
+			where
+				kuuid = fromUUID k
+				isbad = (not $ isuuid kuuid) && isuuid lastword
+				ws = words $ value v
+				lastword = last ws
+				fixeduuid = toUUID lastword
+				fixedvalue = unwords $ kuuid:(take (length ws - 1) ws)
+		-- For the fixed line to take precidence, it should be
+		-- slightly newer, but only slightly.
+		newertime (LogEntry (Date d) _) = d + minimumPOSIXTimeSlice
+		newertime (LogEntry (Unknown) _) = minimumPOSIXTimeSlice
+		minimumPOSIXTimeSlice = 0.000001
+		isuuid s = length s == 36 && length (split "-" s) == 5
 
 {- Records the uuid in the log, if it's not already there. -}
 recordUUID :: UUID -> Annex ()
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 42908ab1d3..04b12887d7 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -15,6 +15,7 @@
 module Logs.UUIDBased (
 	Log,
 	LogEntry(..),
+	TimeStamp(..),
 	parseLog,
 	showLog,
 	changeLog,
diff --git a/debian/changelog b/debian/changelog
index 72110785bb..e1d08ecdb8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (3.20111108) UNRELEASED; urgency=low
   * Avoid cyclic drop problems.
   * Optimized copy --from and get --from to avoid checking the location log
     for files that are already present.
+  * Automatically fix up badly formatted uuid.log entries produced by
+    3.20111105, whenever the uuid.log is changed (ie, by init or describe).
 
  -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
 

From 71b216d1fb2d2308c75ba0da62e6e6a85c5adb19 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 16:18:41 -0400
Subject: [PATCH 2494/2835] map: Support remotes with /~/ and /~user/

More accurately, it was supported already when map uses git-annex-shell,
but not when it does not.

Note that the user name cannot be shell escaped using git-annex's current
approach for shell escaping. I tried and some shells like dash cannot
cd ~'joey'. Rest of directory is still shell escaped, not for security but
in case a directory has a space or other weird character.
---
 Command/Map.hs                                       | 12 +++++++++---
 debian/changelog                                     |  5 +++--
 ...ap_has_problems_with_urls_containing___126__.mdwn |  2 ++
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index f72cb107ad..8755bc7c2d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -195,11 +195,17 @@ tryScan r
 		configlist =
 			onRemote r (pipedconfig, Nothing) "configlist" []
 		manualconfiglist = do
-			let sshcmd =
-				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
-				"git config --list"
 			sshparams <- sshToRepo r [Param sshcmd]
 			liftIO $ pipedconfig "ssh" sshparams
+			where
+				sshcmd = cddir ++ " && " ++
+					"git config --list"
+				dir = Git.workTree r
+				cddir
+					| take 2 dir == "/~" =
+						let (userhome, reldir) = span (/= '/') (drop 1 dir)
+						in "cd " ++ userhome ++ " && cd " ++ shellEscape (drop 1 reldir)
+					| otherwise = "cd " ++ shellEscape dir
 
 		-- First, try sshing and running git config manually,
 		-- only fall back to git-annex-shell configlist if that
diff --git a/debian/changelog b/debian/changelog
index e1d08ecdb8..9c2de7b083 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111108) UNRELEASED; urgency=low
+git-annex (3.20111111) unstable; urgency=low
 
   * Handle a case where an annexed file is moved into a gitignored directory,
     by having fix --force add its change.
@@ -7,8 +7,9 @@ git-annex (3.20111108) UNRELEASED; urgency=low
     for files that are already present.
   * Automatically fix up badly formatted uuid.log entries produced by
     3.20111105, whenever the uuid.log is changed (ie, by init or describe).
+  * map: Support remotes with /~/ and /~user/
 
- -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
+ -- Joey Hess   Fri, 11 Nov 2011 13:44:18 -0400
 
 git-annex (3.20111107) unstable; urgency=low
 
diff --git a/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
index c6eb7bc73a..24f04eeb54 100644
--- a/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
+++ b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
@@ -42,3 +42,5 @@ map` works too:
 
     ok
 
+
+> [[fixed|done]] --[[Joey]] 

From baa07971ff8e52cddc4ad31e75d7b3a6c4c6ba07 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 16:29:19 -0400
Subject: [PATCH 2495/2835] releasing version 3.20111111

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 7b9459b46b..b2638c3ac7 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111108
+Version: 3.20111111
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 15b92ad6a19a7c486fb30fb3552be4a4ef9f9f3a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 16:29:57 -0400
Subject: [PATCH 2496/2835] add news item for git-annex 3.20111111

---
 doc/news/version_3.20110928.mdwn | 19 -------------------
 doc/news/version_3.20111111.mdwn | 10 ++++++++++
 2 files changed, 10 insertions(+), 19 deletions(-)
 delete mode 100644 doc/news/version_3.20110928.mdwn
 create mode 100644 doc/news/version_3.20111111.mdwn

diff --git a/doc/news/version_3.20110928.mdwn b/doc/news/version_3.20110928.mdwn
deleted file mode 100644
index 3b11e18f5c..0000000000
--- a/doc/news/version_3.20110928.mdwn
+++ /dev/null
@@ -1,19 +0,0 @@
-git-annex 3.20110928 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * --in can be used to make git-annex only operate on files
-     believed to be present in a given repository.
-   * Arbitrarily complex expressions can be built to limit the files git-annex
-     operates on, by combining the options --not --and --or -( and -)
-     Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
-   * --copies=N can be used to make git-annex only operate on files with
-     the specified number of copies. (And --not --copies=N for the inverse.)
-   * find: Rather than only showing files whose contents are present,
-     when used with --exclude --copies or --in, displays all files that
-     match the specified conditions.
-   * Note that this is a behavior change for git-annex find! Old behavior
-     can be gotten by using: git-annex find --in .
-   * status: Massively sped up; remove --fast mode.
-   * unused: File contents used by branches and tags are no longer
-     considered unused, even when not used by the current branch. This is
-     the final piece of the puzzle needed for git-annex to to play nicely
-     with branches."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111111.mdwn b/doc/news/version_3.20111111.mdwn
new file mode 100644
index 0000000000..2173400154
--- /dev/null
+++ b/doc/news/version_3.20111111.mdwn
@@ -0,0 +1,10 @@
+git-annex 3.20111111 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Handle a case where an annexed file is moved into a gitignored directory,
+     by having fix --force add its change.
+   * Avoid cyclic drop problems.
+   * Optimized copy --from and get --from to avoid checking the location log
+     for files that are already present.
+   * Automatically fix up badly formatted uuid.log entries produced by
+     3.20111105, whenever the uuid.log is changed (ie, by init or describe).
+   * map: Support remotes with /~/ and /~user/"""]]
\ No newline at end of file

From 30f1aeb13c6627e8264f222a799b2702c5f13714 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 14:24:07 -0400
Subject: [PATCH 2497/2835] golf

---
 Annex.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 1c1091799f..24cc78a647 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -122,9 +122,7 @@ gitRepo = getState repo
 
 {- Runs an IO action in the annex's git repository. -}
 inRepo :: (Git.Repo -> IO a) -> Annex a
-inRepo a = do
-	g <- gitRepo
-	liftIO $ a g
+inRepo a = liftIO . a =<< gitRepo
 
 {- Extracts a value from the annex's git repisitory. -}
 fromRepo :: (Git.Repo -> a) -> Annex a

From 6e946b9a39f8e5ba55651accbc1307e8cce5c4e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 14:24:14 -0400
Subject: [PATCH 2498/2835] add

---
 doc/todo/optimise_git-annex_merge.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/todo/optimise_git-annex_merge.mdwn

diff --git a/doc/todo/optimise_git-annex_merge.mdwn b/doc/todo/optimise_git-annex_merge.mdwn
new file mode 100644
index 0000000000..a2cdfb15fe
--- /dev/null
+++ b/doc/todo/optimise_git-annex_merge.mdwn
@@ -0,0 +1,13 @@
+Typically `git-annex merge` is fast, but it could still be sped up.
+
+`git-annex merge` runs `git-hash-object` once per file that needs to be
+merged. Elsewhere in git-annex, `git-hash-object` is used in a faster mode,
+reading files from disk via `--stdin-paths`. But here, the data is not
+in raw files on disk, and I doubt writing them is the best approach.
+Instead, I'd like a way to stream multiple objects into git using stdin.
+Sometime, should look at either extending git-hash-object to support that,
+or possibly look at using git-fast-import instead.
+
+`git-annex merge` also runs `git show` once per file that needs to be
+merged. This could be reduced to a single call to `git-cat-file --batch`,
+There is already a Git.CatFile library that can do this easily. --[[Joey]]

From fe4ad93e4a82eabc2300705a8bfc59caaf4814b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 14:46:32 -0400
Subject: [PATCH 2499/2835] add

---
 doc/todo/avoid_unnecessary_union_merges.mdwn | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/todo/avoid_unnecessary_union_merges.mdwn

diff --git a/doc/todo/avoid_unnecessary_union_merges.mdwn b/doc/todo/avoid_unnecessary_union_merges.mdwn
new file mode 100644
index 0000000000..67aa285799
--- /dev/null
+++ b/doc/todo/avoid_unnecessary_union_merges.mdwn
@@ -0,0 +1,18 @@
+Some commands cause a union merge unnecessarily. For example, `git annex add`
+modifies the location log, which first requires reading the current log (if
+any), which triggers a merge.
+
+Would be good to avoid these unnecessary union merges. First because it's
+faster and second because it avoids a possible delay when a user might
+ctrl-c and leave the repo in an inconsistent state. In the case of an add,
+the file will be in the annex, but no location log will exist for it (fsck
+fixes that).
+
+It may be that all that's needed is to modify Annex.Branch.change
+to read the current value, without merging. Then commands like `get`, that
+query the branch, will still cause merges, and commands like `add` that
+only modify it, will not. Note that for a command like `get`, the merge
+occurs before it has done anything, so ctrl-c should not be a problem
+there.
+
+This is a delicate change, I need to take care.. --[[Joey]]

From 897bf938f687c1bd9fcfa0097b65708cf0891041 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 14:51:19 -0400
Subject: [PATCH 2500/2835] merge: Improve commit messages to mention what was
 merged.

---
 Annex/Branch.hs  | 15 +++++++++------
 debian/changelog |  6 ++++++
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 05c89ed97e..e86a3d7d18 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -120,8 +120,8 @@ commit message = whenM journalDirty $ lockJournal $ do
  - journal into the index. Otherwise, any changes in the journal would
  - later get staged, and might overwrite changes made during the merge.
  -
- - It would be cleaner to handle the merge by updating the journal, not the
- - index, with changes from the branches.
+ - (It would be cleaner to handle the merge by updating the journal, not the
+ - index, with changes from the branches.)
  -
  - The index is always updated using a union merge, as that's the most
  - efficient way to update it. However, if the branch can be
@@ -136,10 +136,13 @@ update = onceonly $ do
 	let (refs, branches) = unzip c
 	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
 		when dirty stageJournalFiles
-		unless (null branches) $ do
-			showSideAction $ "merging " ++
-				(unwords $ map Git.refDescribe branches) ++
+		let merge_desc = if null branches
+			then "update" 
+			else "merging " ++
+				(unwords $ map Git.refDescribe branches) ++ 
 				" into " ++ name
+		unless (null branches) $ do
+			showSideAction merge_desc
 			{- Note: This merges the branches into the index.
 			 - Any unstaged changes in the git-annex branch
 			 - (if it's checked out) will be removed. So,
@@ -149,7 +152,7 @@ update = onceonly $ do
 			inRepo $ \g -> Git.UnionMerge.merge_index g branches
 		ff <- if dirty then return False else tryFastForwardTo refs
 		unless ff $ inRepo $
-			Git.commit "update" fullname (nub $ fullname:refs)
+			Git.commit merge_desc fullname (nub $ fullname:refs)
 		invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
diff --git a/debian/changelog b/debian/changelog
index 9c2de7b083..ff735a79d3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20111112) UNRELEASED; urgency=low
+
+  * merge: Improve commit messages to mention what was merged.
+
+ -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
+
 git-annex (3.20111111) unstable; urgency=low
 
   * Handle a case where an annexed file is moved into a gitignored directory,

From e9bfa8eaed3ff59a4c0bc8d4d677bc493177807c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 15:15:57 -0400
Subject: [PATCH 2501/2835] avoid unnecessary auto-merge when only changing a
 file in the branch.

Avoids doing auto-merging in commands that don't need fully current
information from the git-annex branch. In particular, git annex add no
longer needs to auto-merge. Affected commands: Anything that doesn't
look up data from the branch, but does write a change to it.

It might seem counterintuitive that we can change a value without first
making sure we have the current value. This optimisation works because
these two sequences are equivilant:

1. pull from remote
2. union merge
3. read file from branch
4. modify file and write to branch

vs.

1. read file from branch
2. modify file and write to branch
3. pull from remote
4. union merge

After either sequence, the git-annex branch contains the same logical content
for the modified file. (Possibly with lines in a different order or
additional old lines of course).
---
 Annex/Branch.hs                              | 25 ++++++++++++++++----
 debian/changelog                             |  3 +++
 doc/todo/avoid_unnecessary_union_merges.mdwn |  2 ++
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index e86a3d7d18..fad818fb0a 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -244,9 +244,13 @@ siblingBranches = do
 		pair l = (head l, last l)
 		uref (a, _) (b, _) = a == b
 
-{- Applies a function to modifiy the content of a file. -}
+{- Applies a function to modifiy the content of a file.
+ -
+ - Note that this does not cause the branch to be merged, it only
+ - modifes the current content of the file on the branch.
+ -}
 change :: FilePath -> (String -> String) -> Annex ()
-change file a = lockJournal $ get file >>= return . a >>= set file
+change file a = lockJournal $ getStale file >>= return . a >>= set file
 
 {- Records new content of a file into the journal. -}
 set :: FilePath -> String -> Annex ()
@@ -259,13 +263,24 @@ set file content = do
  -
  - Returns an empty string if the file doesn't exist yet. -}
 get :: FilePath -> Annex String
-get file = fromcache =<< getCache file
+get = get' False
+
+{- Like get, but does not merge the branch, so the info returned may not
+ - reflect changes in remotes. (Changing the value this returns, and then
+ - merging is always the same as using get, and then changing its value.) -}
+getStale :: FilePath -> Annex String
+getStale = get' True
+
+get' :: Bool -> FilePath -> Annex String
+get' staleok file = fromcache =<< getCache file
 	where
 		fromcache (Just content) = return content
 		fromcache Nothing = fromjournal =<< getJournalFile file
 		fromjournal (Just content) = cache content
-		fromjournal Nothing = withIndexUpdate $
-			cache =<< catFile fullname file
+		fromjournal Nothing
+			| staleok = withIndex frombranch
+			| otherwise = withIndexUpdate $ frombranch >>= cache
+		frombranch = catFile fullname file
 		cache content = do
 			setCache file content
 			return content
diff --git a/debian/changelog b/debian/changelog
index ff735a79d3..76a29131db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,9 @@
 git-annex (3.20111112) UNRELEASED; urgency=low
 
   * merge: Improve commit messages to mention what was merged.
+  * Avoid doing auto-merging in commands that don't need fully current
+    information from the git-annex branch. In particular, git annex add
+    no longer needs to auto-merge.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/todo/avoid_unnecessary_union_merges.mdwn b/doc/todo/avoid_unnecessary_union_merges.mdwn
index 67aa285799..5cd4b64373 100644
--- a/doc/todo/avoid_unnecessary_union_merges.mdwn
+++ b/doc/todo/avoid_unnecessary_union_merges.mdwn
@@ -16,3 +16,5 @@ occurs before it has done anything, so ctrl-c should not be a problem
 there.
 
 This is a delicate change, I need to take care.. --[[Joey]]
+
+> [[done]] (assuming I didn't miss any cases where this is not safe!) --[[Joey]] 

From cea65b9e5bf6bcc9a9350703dbbb0951c6e00c82 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 15:42:52 -0400
Subject: [PATCH 2502/2835] init: When run in an already initalized repository,
 and without a description specified, don't delete the old description.

---
 Command/Init.hs  | 2 +-
 debian/changelog | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index a6d72e4226..bbabdc4c25 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -27,5 +27,5 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
-	initialize (Just description)
+	initialize $ if null description then Nothing else Just description
 	next $ return True
diff --git a/debian/changelog b/debian/changelog
index 76a29131db..a8ce33435c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * Avoid doing auto-merging in commands that don't need fully current
     information from the git-annex branch. In particular, git annex add
     no longer needs to auto-merge.
+  * init: When run in an already initalized repository, and without
+    a description specified, don't delete the old description. 
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 

From 04edae6791b4eddaa77dda2407264dc4434d74b7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 17:45:12 -0400
Subject: [PATCH 2503/2835] Optimised union merging; now only runs git cat-file
 once.

---
 Annex/Branch.hs                        |  5 ++--
 Annex/CatFile.hs                       | 17 ++++++++----
 Command/Unused.hs                      |  2 +-
 Git/CatFile.hs                         | 38 ++++++++++++++++----------
 Git/UnionMerge.hs                      | 31 +++++++++++----------
 debian/changelog                       |  1 +
 doc/todo/optimise_git-annex_merge.mdwn |  4 +++
 7 files changed, 62 insertions(+), 36 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index fad818fb0a..20134003d3 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -149,7 +149,8 @@ update = onceonly $ do
 			 - documentation advises users not to directly
 			 - modify the branch.
 			 -}
-			inRepo $ \g -> Git.UnionMerge.merge_index g branches
+			h <- catFileHandle
+			inRepo $ \g -> Git.UnionMerge.merge_index h g branches
 		ff <- if dirty then return False else tryFastForwardTo refs
 		unless ff $ inRepo $
 			Git.commit merge_desc fullname (nub $ fullname:refs)
@@ -280,7 +281,7 @@ get' staleok file = fromcache =<< getCache file
 		fromjournal Nothing
 			| staleok = withIndex frombranch
 			| otherwise = withIndexUpdate $ frombranch >>= cache
-		frombranch = catFile fullname file
+		frombranch = L.unpack <$> catFile fullname file
 		cache content = do
 			setCache file content
 			return content
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 99cc519f51..0541f7269d 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -6,18 +6,25 @@
  -}
 
 module Annex.CatFile (
-	catFile
+	catFile,
+	catFileHandle
 ) where
 
+import qualified Data.ByteString.Lazy.Char8 as L
+
 import Common.Annex
 import qualified Git.CatFile
 import qualified Annex
 
-catFile :: String -> FilePath -> Annex String
-catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
+catFile :: String -> FilePath -> Annex L.ByteString
+catFile branch file = do
+	h <- catFileHandle
+	liftIO $ Git.CatFile.catFile h branch file
+
+catFileHandle :: Annex Git.CatFile.CatFileHandle
+catFileHandle = maybe startup return =<< Annex.getState Annex.catfilehandle
 	where
 		startup = do
 			h <- inRepo Git.CatFile.catFileStart
 			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
-			go h
-		go h = liftIO $ Git.CatFile.catFile h branch file
+			return h
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 9d56d1ff11..34d8ac232a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -197,7 +197,7 @@ getKeysReferencedInGit ref = do
 		findkeys c (l:ls)
 			| isSymLink (LsTree.mode l) = do
 				content <- catFile ref $ LsTree.file l
-				case fileKey (takeFileName content) of
+				case fileKey (takeFileName $ L.unpack content) of
 					Nothing -> findkeys c ls
 					Just k -> findkeys (k:c) ls
 			| otherwise = findkeys c ls
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 51fa585a82..83c1235086 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -9,13 +9,15 @@ module Git.CatFile (
 	CatFileHandle,
 	catFileStart,
 	catFileStop,
-	catFile
+	catFile,
+	catObject
 ) where
 
 import Control.Monad.State
 import System.Cmd.Utils
 import System.IO
-import qualified Data.ByteString.Char8 as B
+import qualified Data.ByteString.Char8 as S
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
 import Utility.SafeCommand
@@ -34,30 +36,38 @@ catFileStop (pid, from, to) = do
 	hClose from
 	forceSuccess pid
 
-{- Uses a running git cat-file read the content of a file from a branch.
- - Files that do not exist on the branch will have "" returned. -}
-catFile :: CatFileHandle -> String -> FilePath -> IO String
-catFile (_, from, to) branch file = do
-	hPutStrLn to want
+{- Reads a file from a specified branch. -}
+catFile :: CatFileHandle -> String -> FilePath -> IO L.ByteString
+catFile h branch file = catObject h (branch ++ ":" ++ file)
+
+{- Uses a running git cat-file read the content of an object.
+ - Objects that do not exist will have "" returned. -}
+catObject :: CatFileHandle -> String -> IO L.ByteString
+catObject (_, from, to) object = do
+	hPutStrLn to object
 	hFlush to
 	header <- hGetLine from
 	case words header of
-		[sha, blob, size]
+		[sha, objtype, size]
 			| length sha == Git.shaSize &&
-			  blob == "blob" -> handle size
+			  validobjtype objtype -> handle size
 			| otherwise -> empty
 		_
-			| header == want ++ " missing" -> empty
+			| header == object ++ " missing" -> empty
 			| otherwise -> error $ "unknown response from git cat-file " ++ header
 	where
-		want = branch ++ ":" ++ file
 		handle size = case reads size of
 			[(bytes, "")] -> readcontent bytes
 			_ -> empty
 		readcontent bytes = do
-			content <- B.hGet from bytes
+			content <- S.hGet from bytes
 			c <- hGetChar from
 			when (c /= '\n') $
 				error "missing newline from git cat-file"
-			return $ B.unpack content
-		empty = return ""
+			return $ L.fromChunks [content]
+		empty = return L.empty
+		validobjtype t
+			| t == "blob" = True
+			| t == "commit" = True
+			| t == "tree" = True
+			| otherwise = False
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 30778d0344..67e6fd951a 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -21,6 +21,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
 import Git
+import Git.CatFile
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
@@ -30,14 +31,16 @@ import Git
 merge :: String -> String -> Repo -> IO ()
 merge x y repo = do
 	a <- ls_tree x repo
-	b <- merge_trees x y repo
+	h <- catFileStart repo
+	b <- merge_trees x y h repo
+	catFileStop h
 	update_index repo (a++b)
 
 {- Merges a list of branches into the index. Previously staged changed in
  - the index are preserved (and participate in the merge). -}
-merge_index :: Repo -> [String] -> IO ()
-merge_index repo bs =
-	update_index repo =<< concat <$> mapM (`merge_tree_index` repo) bs
+merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
+merge_index h repo bs =
+	update_index repo =<< concat <$> mapM (\b -> merge_tree_index b h repo) bs
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
@@ -60,22 +63,22 @@ ls_tree x = pipeNullSplit params
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: String -> String -> Repo -> IO [String]
-merge_trees x y = calc_merge $ "diff-tree":diff_opts ++ [x, y]
+merge_trees :: String -> String -> CatFileHandle -> Repo -> IO [String]
+merge_trees x y h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: String -> Repo -> IO [String]
-merge_tree_index x = calc_merge $ "diff-index":diff_opts ++ ["--cached", x]
+merge_tree_index :: String -> CatFileHandle -> Repo -> IO [String]
+merge_tree_index x h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
 diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: [String] -> Repo -> IO [String]
-calc_merge differ repo = do
+calc_merge :: CatFileHandle -> [String] -> Repo -> IO [String]
+calc_merge h differ repo = do
 	diff <- pipeNullSplit (map Param differ) repo
-	l <- mapM (\p -> mergeFile p repo) (pairs diff)
+	l <- mapM (\p -> mergeFile p h repo) (pairs diff)
 	return $ catMaybes l
 	where
 		pairs [] = []
@@ -97,12 +100,12 @@ hashObject content repo = getSha subcmd $ do
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
-mergeFile :: (String, FilePath) -> Repo -> IO (Maybe String)
-mergeFile (info, file) repo = case filter (/= nullsha) [asha, bsha] of
+mergeFile :: (String, FilePath) -> CatFileHandle -> Repo -> IO (Maybe String)
+mergeFile (info, file) h repo = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- pipeRead (map Param ("show":shas)) repo
+		content <- L.concat <$> mapM (catObject h) shas
 		sha <- hashObject (unionmerge content) repo
 		return $ Just $ update_index_line sha file
 	where
diff --git a/debian/changelog b/debian/changelog
index a8ce33435c..7ff819a14b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     no longer needs to auto-merge.
   * init: When run in an already initalized repository, and without
     a description specified, don't delete the old description. 
+  * Optimised union merging; now only runs git cat-file once.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/todo/optimise_git-annex_merge.mdwn b/doc/todo/optimise_git-annex_merge.mdwn
index a2cdfb15fe..2df196cfd4 100644
--- a/doc/todo/optimise_git-annex_merge.mdwn
+++ b/doc/todo/optimise_git-annex_merge.mdwn
@@ -8,6 +8,10 @@ Instead, I'd like a way to stream multiple objects into git using stdin.
 Sometime, should look at either extending git-hash-object to support that,
 or possibly look at using git-fast-import instead.
 
+--- 
+
 `git-annex merge` also runs `git show` once per file that needs to be
 merged. This could be reduced to a single call to `git-cat-file --batch`,
 There is already a Git.CatFile library that can do this easily. --[[Joey]]
+
+> This is now done, part above remains todo. --[[Joey]] 

From aa4fbbdd33c4d584b734476a341a0c38980281a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 14 Nov 2011 16:14:17 -0400
Subject: [PATCH 2504/2835] status: Now displays trusted, untrusted, and
 semitrusted repositories separately.

---
 Command/Status.hs                                | 16 +++++++++++-----
 debian/changelog                                 |  2 ++
 ...hings_like_description__44___trust_level.mdwn |  3 +++
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 53d64d0428..e66f59858d 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -24,6 +24,7 @@ import Annex.Content
 import Types.Key
 import Backend
 import Logs.UUID
+import Logs.Trust
 import Remote
 
 -- a named computation that produces a statistic
@@ -52,7 +53,9 @@ stats :: [Stat]
 stats = 
 	[ supported_backends
 	, supported_remote_types
-	, remote_list
+	, remote_list Trusted "trusted"
+	, remote_list UnTrusted "untrusted"
+	, remote_list SemiTrusted "semitrusted"
 	, tmp_size
 	, bad_data_size
 	, local_annex_keys
@@ -90,10 +93,13 @@ supported_remote_types :: Stat
 supported_remote_types = stat "supported remote types" $
 	return $ unwords $ map R.typename Remote.remoteTypes
 
-remote_list :: Stat
-remote_list = stat "known repositories" $ lift $ do
-	s <- prettyPrintUUIDs "repos" =<< M.keys <$> uuidMap
-	return $ '\n':init s
+remote_list :: TrustLevel -> String -> Stat
+remote_list level desc = stat n $ lift $ do
+	us <- M.keys <$> uuidMap
+	s <- prettyPrintUUIDs n =<< fst <$> trustPartition level us
+	return $ if null s then "none" else '\n':init s
+	where
+		n = desc ++ " repositories"
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
diff --git a/debian/changelog b/debian/changelog
index 7ff819a14b..b49967fe9f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,8 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * init: When run in an already initalized repository, and without
     a description specified, don't delete the old description. 
   * Optimised union merging; now only runs git cat-file once.
+  * status: Now displays trusted, untrusted, and semitrusted repositories
+    separately.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn b/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
index 6404393a89..d158850cd6 100644
--- a/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
+++ b/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
@@ -1 +1,4 @@
 It would be helpful to have a way to query things like a repository's description and trust level, without having to poke in the git-annex branch.  For example, "git annex describe ." currently clears the description but could print the current one instead.
+
+> `git annex status` now breaks down the repository list by type. [[done]]
+> --[[Joey]] 

From 364981ad924e7f4fdb92121dba52318bce3644d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 14 Nov 2011 16:15:48 -0400
Subject: [PATCH 2505/2835] probably makes sense to list semitrusted before
 untrusted

---
 Command/Status.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index e66f59858d..3fddb599d5 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -54,8 +54,8 @@ stats =
 	[ supported_backends
 	, supported_remote_types
 	, remote_list Trusted "trusted"
-	, remote_list UnTrusted "untrusted"
 	, remote_list SemiTrusted "semitrusted"
+	, remote_list UnTrusted "untrusted"
 	, tmp_size
 	, bad_data_size
 	, local_annex_keys

From 522df3da5800f821f1d7e7ddb6ddf70befa8f8ac Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Mon, 14 Nov 2011 22:30:02 +0000
Subject: [PATCH 2506/2835]

---
 doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn

diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn
new file mode 100644
index 0000000000..f1aa5cc06d
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn
@@ -0,0 +1 @@
+I just started experimenting with git annex, and I found that I would like to have a way to figure out metadata (well, size. Maybe modification date) of a non-local file. I first checked if there is "git annex ls" (which could list known files in an ls-like way) and found "git annex whereis" as somewhat a replacement, but it does not give metadata information.

From 4d72c1b69c4423bd13c8d407cbedca3b985c3585 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 14 Nov 2011 22:46:35 +0000
Subject: [PATCH 2507/2835] Added a comment

---
 .../comment_1_7fba10b85f4d9289c7782eccef46949e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment

diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment
new file mode 100644
index 0000000000..379b9f976b
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-11-14T22:46:35Z"
+ content="""
+When  I want that, I ls -l the file and look at the symlink to the key. Ie, in SHA1-s10481423--efc7eec0d711212842cd6bb8f957e1628146d6ed the size is 10481423 bytes. 
+"""]]

From 02f1d5467ad05315905b9cb90307049ababb4a63 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 14 Nov 2011 22:48:03 +0000
Subject: [PATCH 2508/2835] Added a comment

---
 .../comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment

diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment
new file mode 100644
index 0000000000..3dd14bf010
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-11-14T22:48:03Z"
+ content="""
+It might make sense to put this functionality in git annex find. Perhaps a format string with a %s for example. 
+"""]]

From bfe38f8ff16dc6ccc32d545e6fed87817caa26cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 14 Nov 2011 19:27:00 -0400
Subject: [PATCH 2509/2835] status --json --fast for esc

* status: Fix --json mode (only the repository lists are currently
  displayed)
* status: --fast is back
---
 Command/Status.hs  | 22 ++++++++++++++++------
 Messages.hs        | 14 +++++++++++++-
 Messages/JSON.hs   |  8 ++++++--
 debian/changelog   |  3 +++
 doc/git-annex.mdwn |  4 +++-
 5 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 3fddb599d5..39d9400fbd 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -18,6 +18,7 @@ import qualified Types.Remote as R
 import qualified Remote
 import qualified Command.Unused
 import qualified Git
+import qualified Annex
 import Command
 import Utility.DataUnits
 import Annex.Content
@@ -49,14 +50,17 @@ seek = [withNothing start]
 {- Order is significant. Less expensive operations, and operations
  - that share data go together.
  -}
-stats :: [Stat]
-stats = 
+fast_stats :: [Stat]
+fast_stats = 
 	[ supported_backends
 	, supported_remote_types
 	, remote_list Trusted "trusted"
 	, remote_list SemiTrusted "semitrusted"
 	, remote_list UnTrusted "untrusted"
-	, tmp_size
+	]
+slow_stats :: [Stat]
+slow_stats = 
+	[ tmp_size
 	, bad_data_size
 	, local_annex_keys
 	, local_annex_size
@@ -67,6 +71,10 @@ stats =
 
 start :: CommandStart
 start = do
+	showStart "status" "."
+	showWith $ liftIO $ putStrLn ""
+	fast <- Annex.getState Annex.fast
+	let stats = if fast then fast_stats else fast_stats ++ slow_stats
 	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
 	stop
 
@@ -80,9 +88,11 @@ showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
 		calc (Just (desc, a)) = do
-			liftIO $ putStr $ desc ++ ": "
-			liftIO $ hFlush stdout
-			liftIO . putStrLn =<< a
+			r <- a -- run first, it may produce JSON
+			lift . showWith $ do
+				liftIO $ putStr $ desc ++ ": "
+				liftIO $ hFlush stdout
+				liftIO $ putStrLn r
 		calc Nothing = return ()
 
 supported_backends :: Stat
diff --git a/Messages.hs b/Messages.hs
index 6f4880e2d5..d7eabccbb2 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -20,6 +20,8 @@ module Messages (
 	warning,
 	indent,
 	maybeShowJSON,
+	showWith,
+	
 	setupConsole
 ) where
 
@@ -31,7 +33,7 @@ import qualified Annex
 import qualified Messages.JSON as JSON
 
 showStart :: String -> String -> Annex ()
-showStart command file = handle (JSON.start command file) $
+showStart command file = handle (JSON.start command $ Just file) $
 	flushed $ putStr $ command ++ " " ++ file ++ " "
 
 showNote :: String -> Annex ()
@@ -111,6 +113,16 @@ handle json normal = do
 maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
 maybeShowJSON v = handle (JSON.add v) q
 
+{- Performs an a action (such as displaying something) only when
+ - not in json mode, and not quiet. -}
+showWith :: Annex () -> Annex ()
+showWith a = do
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> a
+		Annex.QuietOutput -> q
+		Annex.JSONOutput -> q
+
 q :: Monad m => m ()
 q = return ()
 
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
index fb95f550e8..a325ef130b 100644
--- a/Messages/JSON.hs
+++ b/Messages/JSON.hs
@@ -16,8 +16,12 @@ import Text.JSON
 
 import qualified Utility.JSONStream as Stream
 
-start :: String -> String -> IO ()
-start command file = putStr $ Stream.start [("command", command), ("file", file)]
+start :: String -> Maybe String -> IO ()
+start command file =
+	putStr $ Stream.start $ ("command", command) : filepart file
+	where
+		filepart Nothing = []
+		filepart (Just f) = [("file", f)]
 
 end :: Bool -> IO ()
 end b = putStr $ Stream.add [("success", b)] ++ Stream.end
diff --git a/debian/changelog b/debian/changelog
index b49967fe9f..aa52730bca 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,9 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * Optimised union merging; now only runs git cat-file once.
   * status: Now displays trusted, untrusted, and semitrusted repositories
     separately.
+  * status: Fix --json mode (only the repository lists are currently
+    displayed)
+  * status: --fast is back
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index fdd8dd1c19..35ba25115d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -240,7 +240,9 @@ subdirectories).
 * status
 
   Displays some statistics and other information, including how much data
-  is in the annex.
+  is in the annex and a list of all known repositories.
+
+  To only show the data that can be gathered quickly, use --fast.
 
 * map
 

From 2412b7e6896b967bb3f8ec8cf5e6ccf3094a3ad1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 14 Nov 2011 19:29:35 -0400
Subject: [PATCH 2510/2835] fix exit status so json gets terminated properly

---
 Command/Status.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 39d9400fbd..2d59965072 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -76,7 +76,7 @@ start = do
 	fast <- Annex.getState Annex.fast
 	let stats = if fast then fast_stats else fast_stats ++ slow_stats
 	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
-	stop
+	next $ next $ return True
 
 stat :: String -> StatState String -> Stat
 stat desc a = return $ Just (desc, a)

From c093839a40034a6e3fa967a18459a52ea326b867 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 00:41:08 +0000
Subject: [PATCH 2511/2835]

---
 .../Trouble_initializing_git_annex_on_NFS.mdwn     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
new file mode 100644
index 0000000000..23977af34e
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
@@ -0,0 +1,14 @@
+The following occurs in a directory that is shared on an NFS server:
+
+    /media/mybook/movies $ git init
+    Initialized empty Git repository in /media/mybook/movies/.git/
+    /media/mybook/movies $ git annex init mybook-movies
+    init mybook-movies 
+    git-annex: waitToSetLock: resource exhausted (No locks available)
+    failed
+    git-annex: init: 1 failed
+    /media/mybook/movies $
+
+This happens reliably.  Is there any way around it?  I have shell
+access on the NFS server, but it is a NAS, so I don't think it is
+capable of running git-annex.

From 6368c79fe41abc195e809340d10d2b1714188bd4 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 00:47:57 +0000
Subject: [PATCH 2512/2835] Fix typo

---
 doc/tips/centralized_git_repository_tutorial.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
index 977a583cca..00283829fd 100644
--- a/doc/tips/centralized_git_repository_tutorial.mdwn
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -131,7 +131,7 @@ which will broadcast its updated location information.
 ## take it farther
 
 Of course you can create as many checkouts as you desire. If you have a
-desktop matchine too, you can make a checkout there, and use `git remote
+desktop machine too, you can make a checkout there, and use `git remote
 add` to also let your desktop access the backup repository. 
 
 You can add remotes for each direct connection between machines you find you

From 019373f827309e4f4a1cf694a50270142e26aa6e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 00:30:00 -0400
Subject: [PATCH 2513/2835] better status output

---
 Command/Status.hs | 15 ++++++---------
 Messages.hs       | 42 ++++++++++++++++++++++++++----------------
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 2d59965072..4f7529732d 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -71,12 +71,12 @@ slow_stats =
 
 start :: CommandStart
 start = do
-	showStart "status" "."
-	showWith $ liftIO $ putStrLn ""
 	fast <- Annex.getState Annex.fast
 	let stats = if fast then fast_stats else fast_stats ++ slow_stats
-	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
-	next $ next $ return True
+	showCustom "status" $ do
+		evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
+		return True
+	stop
 
 stat :: String -> StatState String -> Stat
 stat desc a = return $ Just (desc, a)
@@ -88,11 +88,8 @@ showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
 		calc (Just (desc, a)) = do
-			r <- a -- run first, it may produce JSON
-			lift . showWith $ do
-				liftIO $ putStr $ desc ++ ": "
-				liftIO $ hFlush stdout
-				liftIO $ putStrLn r
+			(lift . showHeader) desc
+			lift . showRaw =<< a
 		calc Nothing = return ()
 
 supported_backends :: Stat
diff --git a/Messages.hs b/Messages.hs
index d7eabccbb2..57b7068041 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -1,6 +1,6 @@
 {- git-annex output messages
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -20,7 +20,9 @@ module Messages (
 	warning,
 	indent,
 	maybeShowJSON,
-	showWith,
+	showCustom,
+	showHeader,
+	showRaw,
 	
 	setupConsole
 ) where
@@ -88,6 +90,28 @@ warning' w = do
 indent :: String -> String
 indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
 
+{- Shows a JSON value only when in json mode. -}
+maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
+maybeShowJSON v = handle (JSON.add v) q
+
+{- Performs an action that outputs nonstandard/customized output, and
+ - in JSON mode wraps its output in JSON.start and JSON.end, so it's
+ - a complete JSON document.
+ - This is only needed when showStart and showEndOk is not used. -}
+showCustom :: String -> Annex Bool -> Annex ()
+showCustom command a = do
+	handle (JSON.start command Nothing) q
+	r <- a
+	handle (JSON.end r) q
+
+showHeader :: String -> Annex ()
+showHeader h = handle q $ do
+	putStr $ h ++ ": "
+	hFlush stdout
+
+showRaw :: String -> Annex ()
+showRaw s = handle q $ putStrLn s
+
 {- By default, haskell honors the user's locale in its output to stdout
  - and stderr. While that's great for proper unicode support, for git-annex
  - all that's really needed is the ability to display simple messages
@@ -109,20 +133,6 @@ handle json normal = do
 		Annex.QuietOutput -> q
 		Annex.JSONOutput -> liftIO json
 
-{- Shows a JSON value only when in json mode. -}
-maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
-maybeShowJSON v = handle (JSON.add v) q
-
-{- Performs an a action (such as displaying something) only when
- - not in json mode, and not quiet. -}
-showWith :: Annex () -> Annex ()
-showWith a = do
-	output <- Annex.getState Annex.output
-	case output of
-		Annex.NormalOutput -> a
-		Annex.QuietOutput -> q
-		Annex.JSONOutput -> q
-
 q :: Monad m => m ()
 q = return ()
 

From def0788698f9ed29856aa5b608356ae7b298e3f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 00:33:54 -0400
Subject: [PATCH 2514/2835] show number of repos

---
 Command/Status.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 4f7529732d..2c0afa62c4 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -103,8 +103,9 @@ supported_remote_types = stat "supported remote types" $
 remote_list :: TrustLevel -> String -> Stat
 remote_list level desc = stat n $ lift $ do
 	us <- M.keys <$> uuidMap
-	s <- prettyPrintUUIDs n =<< fst <$> trustPartition level us
-	return $ if null s then "none" else '\n':init s
+	rs <- fst <$> trustPartition level us
+	s <- prettyPrintUUIDs n rs
+	return $ if null s then "0" else show (length rs) ++ "\n" ++ init s
 	where
 		n = desc ++ " repositories"
 

From a6091dc27180f0db9df647fa08460a2c0f03d3e5 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 15 Nov 2011 04:40:35 +0000
Subject: [PATCH 2515/2835] Added a comment

---
 ...comment_1_e26952373150d63b8a5d3643a2762de1._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment
new file mode 100644
index 0000000000..8e951ab7c9
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-11-15T04:40:35Z"
+ content="""
+git-annex uses locking to avoid problems if multiple processes are run at the same time.
+
+I just tested on NFS, with Linux on the server and client, and it works ok. It seems your NFS client (or server) must not support fncl locking. What OS is your NAS running?
+"""]]

From cfc518190c08f88ea7cc4480d6d5f1cf9fab351b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 15 Nov 2011 04:46:13 +0000
Subject: [PATCH 2516/2835] Added a comment

---
 ...comment_2_f80b10ed395738e50e345fc22c708ae5._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment
new file mode 100644
index 0000000000..bd302e6bef
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-11-15T04:46:13Z"
+ content="""
+You might try mounting your NAS with the mount option `local_lock=all`
+
+This will keep the lock files on your (I assume linux) client. If you do this make sure you don't have another client using git-annex in the same NFS directory.
+"""]]

From eb214f719cae6c322f1fc57ad3a4d8a710c94820 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 05:14:05 +0000
Subject: [PATCH 2517/2835] Added a comment

---
 .../comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment
new file mode 100644
index 0000000000..b95c795eab
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 3"
+ date="2011-11-15T05:14:03Z"
+ content="""
+I did a bit of research and my NAS had ancient NFS software on it.  I upgraded that and things are now working as expected. Sorry for the noise.
+"""]]

From 8d9d94f90c26117e65f949cc8d087d1045b7d5b6 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 05:15:47 +0000
Subject: [PATCH 2518/2835] Added a comment

---
 .../comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment
new file mode 100644
index 0000000000..5a741b8c95
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 4"
+ date="2011-11-15T05:15:47Z"
+ content="""
+I did a bit of research and my NAS had ancient NFS software on it.  I upgraded that and things are now working as expected. Sorry for the noise.
+"""]]

From d99fa3ec4ec647a62177821f666ed5398fe95b0d Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 05:22:09 +0000
Subject: [PATCH 2519/2835] removed

---
 .../comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment   | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment
deleted file mode 100644
index 5a741b8c95..0000000000
--- a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://cgray.myopenid.com/"
- nickname="cgray"
- subject="comment 4"
- date="2011-11-15T05:15:47Z"
- content="""
-I did a bit of research and my NAS had ancient NFS software on it.  I upgraded that and things are now working as expected. Sorry for the noise.
-"""]]

From 3c4537111509ea61138592a254a3a55a185cee23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 01:53:28 -0400
Subject: [PATCH 2520/2835] close as resolved

---
 doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
index 23977af34e..8eb20baf97 100644
--- a/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
@@ -12,3 +12,5 @@ The following occurs in a directory that is shared on an NFS server:
 This happens reliably.  Is there any way around it?  I have shell
 access on the NFS server, but it is a NAS, so I don't think it is
 capable of running git-annex.
+
+[[done]]

From 7d05ca1d6d01f7e9f8b2acdebac2b656e178a32c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 14:06:38 -0400
Subject: [PATCH 2521/2835] Fix support for insteadOf url remapping. Closes:
 #644278

---
 Git.hs           | 7 ++++---
 debian/changelog | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Git.hs b/Git.hs
index 5ceaa67f76..3ff42d28f5 100644
--- a/Git.hs
+++ b/Git.hs
@@ -523,13 +523,14 @@ genRemote s repo = gen $ calcloc s
 			| null insteadofs = l
 			| otherwise = replacement ++ drop (length replacement) l
 			where
-				replacement = take (length bestkey - length prefix) bestkey
+				replacement = drop (length "url.") $
+					take (length bestkey - length suffix) bestkey
 				bestkey = fst $ maximumBy longestvalue insteadofs
 				longestvalue (_, a) (_, b) = compare b a
 				insteadofs = filterconfig $ \(k, v) -> 
-					endswith prefix k &&
+					endswith suffix k &&
 					startswith v l
-				prefix = ".insteadof"
+				suffix = ".insteadof"
 		-- git remotes can be written scp style -- [user@]host:dir
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
diff --git a/debian/changelog b/debian/changelog
index aa52730bca..c145b14f9a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * status: Fix --json mode (only the repository lists are currently
     displayed)
   * status: --fast is back
+  * Fix support for insteadOf url remapping. Closes: #644278
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 

From 80a0be5116ada2c718fa6db0a90100d24660beec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 19:03:25 -0400
Subject: [PATCH 2522/2835] further insteadOf fix

---
 Git.hs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Git.hs b/Git.hs
index 3ff42d28f5..3f3f746324 100644
--- a/Git.hs
+++ b/Git.hs
@@ -521,16 +521,17 @@ genRemote s repo = gen $ calcloc s
 		-- insteadof config can rewrite remote location
 		calcloc l
 			| null insteadofs = l
-			| otherwise = replacement ++ drop (length replacement) l
+			| otherwise = replacement ++ drop (length bestvalue) l
 			where
-				replacement = drop (length "url.") $
+				replacement = drop (length prefix) $
 					take (length bestkey - length suffix) bestkey
-				bestkey = fst $ maximumBy longestvalue insteadofs
+				(bestkey, bestvalue) = maximumBy longestvalue insteadofs
 				longestvalue (_, a) (_, b) = compare b a
 				insteadofs = filterconfig $ \(k, v) -> 
+					startswith prefix k &&
 					endswith suffix k &&
 					startswith v l
-				suffix = ".insteadof"
+				(prefix, suffix) = ("url." , ".insteadof")
 		-- git remotes can be written scp style -- [user@]host:dir
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir

From 5c5ff2cb78230ca6311d49dc2573579af49d8c37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 19:17:07 -0400
Subject: [PATCH 2523/2835] better error message

---
 git-annex-shell.hs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 57f6b29162..658eddd77f 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -40,14 +40,17 @@ cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly
 
 options :: [OptDescr (Annex ())]
 options = commonOptions ++
-	[ Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
+	[ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "repository uuid"
 	]
 	where
-		check expected = do
-			u <- getUUID
-			when (u /= toUUID expected) $ error $
-				"expected repository UUID " ++ expected
-					++ " but found UUID " ++ fromUUID u
+		checkuuid expected = getUUID >>= check
+			where
+				check u | u == toUUID expected = return ()
+				check NoUUID = unexpected "uninitialized repository"
+				check u = unexpected $ "UUID " ++ fromUUID u
+				unexpected s = error $
+					"expected repository UUID " ++
+					expected ++ " but found " ++ s
 
 header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"

From b76dc2d210889a77160a834e3849e3040ef69b12 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 22:01:32 -0400
Subject: [PATCH 2524/2835] avoid space leak writing merge

This reduces the memory use of a merge by 1/3rd. The space leak was
apparently because the whole update-index input was generated strictly, not
lazily.

I wondered if the change to ByteStrings contributed to this, due to the
need to convert with L.pack here. But going back to the old code, I still
see a much similar leak, and worse performance besides due to it not using
ByteStrings.

The fix is to just hPutStr the lines repeatedly. (Note the \0 is written
separately, to avoid allocation overheads in adding it to the string.)
The Git.pipeWrite interface is probably just wrong for any large inputs to
git. This was the only place using it for input of any size.

There is still at least one other space leak in the merge code.
---
 Git/UnionMerge.hs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 67e6fd951a..460c767168 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -46,10 +46,13 @@ merge_index h repo bs =
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Repo -> [String] -> IO ()
-update_index repo l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
+update_index repo l = do
+	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
+	mapM_ (\s -> hPutStr h s >> hPutStr h "\0") l
+	hClose h
+	forceSuccess p
 	where
-		togit ps content = pipeWrite (map Param ps) (L.pack content) repo
-			>>= forceSuccess
+		params = map Param ["update-index", "-z", "--index-info"]
 
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}

From 922e9af5281b01709e5fa631ebe048c7da7c4d71 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 22:40:40 -0400
Subject: [PATCH 2525/2835] cleanup

---
 Git/UnionMerge.hs | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 460c767168..493411a069 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -16,7 +16,6 @@ module Git.UnionMerge (
 import System.Cmd.Utils
 import Data.List
 import Data.Maybe
-import Data.String.Utils
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
@@ -88,18 +87,6 @@ calc_merge h differ repo = do
 		pairs (_:[]) = error "calc_merge parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
 
-{- Injects some content into git, returning its hash. -}
-hashObject :: L.ByteString -> Repo -> IO String
-hashObject content repo = getSha subcmd $ do
-	(h, s) <- pipeWriteRead (map Param params) content repo
-	L.length s `seq` do
-		forceSuccess h
-		reap -- XXX unsure why this is needed
-		return $ L.unpack s
-	where
-		subcmd = "hash-object"
-		params = [subcmd, "-w", "--stdin"]
-
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
@@ -115,3 +102,15 @@ mergeFile (info, file) h repo = case filter (/= nullsha) [asha, bsha] of
 		[_colonamode, _bmode, asha, bsha, _status] = words info
 		nullsha = replicate shaSize '0'
 		unionmerge = L.unlines . nub . L.lines
+
+{- Injects some content into git, returning its hash. -}
+hashObject :: L.ByteString -> Repo -> IO String
+hashObject content repo = getSha subcmd $ do
+	(h, s) <- pipeWriteRead (map Param params) content repo
+	L.length s `seq` do
+		forceSuccess h
+		reap -- XXX unsure why this is needed
+		return $ L.unpack s
+	where
+		subcmd = "hash-object"
+		params = [subcmd, "-w", "--stdin"]

From eb0c8c955c57d4161b86dd9ed48c93036f9b83a0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Wed, 16 Nov 2011 03:22:29 +0000
Subject: [PATCH 2526/2835]

---
 .../git_annex_version_should_without_being_in_a_repo_.mdwn   | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn

diff --git a/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
new file mode 100644
index 0000000000..0bae8bdb0e
--- /dev/null
+++ b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
@@ -0,0 +1,5 @@
+was checking the version of git-annex on a machine before cloning a repo...
+
+    $ git annex version
+    git-annex: Not in a git repository.
+

From e92534e5b5514a1cf4ce536974aeac807d849e7d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Wed, 16 Nov 2011 03:24:31 +0000
Subject: [PATCH 2527/2835] Added a comment

---
 .../comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment

diff --git a/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment b/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment
new file mode 100644
index 0000000000..ab30d8a452
--- /dev/null
+++ b/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2011-11-16T03:24:30Z"
+ content="""
+oh, and that probably goes for 'help' and other subcommands as well.
+"""]]

From 21a925dcf1ebe088b5c64da0ce159ffb6d535f04 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 23:28:01 -0400
Subject: [PATCH 2528/2835] merge: Now runs in constant space.

Before, a merge was first calculated, by running various actions that
called git and built up a list of lines, which were at the end sent
to git update-index. This necessarily used space proportional to the size
of the diff between the trees being merged.

Now, lines are streamed into git update-index from each of the actions in
turn.

Runtime size of git-annex merge when merging 50000 location log files
drops from around 100 mb to a constant 4 mb.

Presumably it runs quite a lot faster, too.
---
 Annex/Branch.hs   |  3 ++-
 Git/UnionMerge.hs | 55 ++++++++++++++++++++++++++---------------------
 debian/changelog  |  1 +
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 20134003d3..ae33f66cf0 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -56,7 +56,8 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.ls_tree fullname g >>= Git.UnionMerge.update_index g
+genIndex g = Git.UnionMerge.update_index_via g
+	[Git.UnionMerge.ls_tree fullname g]
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 493411a069..c86d699795 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -9,13 +9,13 @@ module Git.UnionMerge (
 	merge,
 	merge_index,
 	update_index,
+	update_index_via,
 	update_index_line,
 	ls_tree
 ) where
 
 import System.Cmd.Utils
 import Data.List
-import Data.Maybe
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
@@ -29,47 +29,56 @@ import Git.CatFile
  -}
 merge :: String -> String -> Repo -> IO ()
 merge x y repo = do
-	a <- ls_tree x repo
 	h <- catFileStart repo
-	b <- merge_trees x y h repo
+	update_index_via repo
+		[ ls_tree x repo
+		, merge_trees x y h repo
+		]
 	catFileStop h
-	update_index repo (a++b)
 
 {- Merges a list of branches into the index. Previously staged changed in
  - the index are preserved (and participate in the merge). -}
 merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
 merge_index h repo bs =
-	update_index repo =<< concat <$> mapM (\b -> merge_tree_index b h repo) bs
+	update_index_via repo $ map (\b -> merge_tree_index b h repo) bs
 
-{- Feeds a list into update-index. Later items in the list can override
+update_index :: Repo -> [String] -> IO ()
+update_index repo ls = update_index_via repo [\h -> mapM_ (sendContent h) ls]
+
+{- Feeds content into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
-update_index :: Repo -> [String] -> IO ()
-update_index repo l = do
+update_index_via :: Repo -> [Handle -> IO ()] -> IO ()
+update_index_via repo ls = do
 	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
-	mapM_ (\s -> hPutStr h s >> hPutStr h "\0") l
+	forM_ ls $ \l -> l h
 	hClose h
 	forceSuccess p
 	where
 		params = map Param ["update-index", "-z", "--index-info"]
 
+sendContent :: Handle -> String -> IO ()
+sendContent h s = do
+	hPutStr h s
+	hPutStr h "\0"
+
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}
 update_index_line :: String -> FilePath -> String
 update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 
-{- Gets the contents of a tree in a format suitable for update_index. -}
-ls_tree :: String -> Repo -> IO [String]
-ls_tree x = pipeNullSplit params
+{- Gets the contents of a tree. -}
+ls_tree :: String -> Repo -> Handle -> IO ()
+ls_tree x repo h = mapM_ (sendContent h) =<< pipeNullSplit params repo
 	where
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: String -> String -> CatFileHandle -> Repo -> IO [String]
+merge_trees :: String -> String -> CatFileHandle -> Repo -> Handle -> IO ()
 merge_trees x y h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: String -> CatFileHandle -> Repo -> IO [String]
+merge_tree_index :: String -> CatFileHandle -> Repo -> Handle -> IO ()
 merge_tree_index x h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
@@ -77,21 +86,19 @@ diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: CatFileHandle -> [String] -> Repo -> IO [String]
-calc_merge h differ repo = do
-	diff <- pipeNullSplit (map Param differ) repo
-	l <- mapM (\p -> mergeFile p h repo) (pairs diff)
-	return $ catMaybes l
+calc_merge :: CatFileHandle -> [String] -> Repo -> Handle -> IO ()
+calc_merge ch differ repo ih = pipeNullSplit (map Param differ) repo >>= go
 	where
-		pairs [] = []
-		pairs (_:[]) = error "calc_merge parse error"
-		pairs (a:b:rest) = (a,b):pairs rest
+		go [] = return ()
+		go (info:file:rest) = mergeFile info file ch repo >>=
+			maybe (go rest) (\l -> sendContent ih l >> go rest)
+		go (_:[]) = error "calc_merge parse error"
 
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
-mergeFile :: (String, FilePath) -> CatFileHandle -> Repo -> IO (Maybe String)
-mergeFile (info, file) h repo = case filter (/= nullsha) [asha, bsha] of
+mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
+mergeFile info file h repo = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
diff --git a/debian/changelog b/debian/changelog
index c145b14f9a..37578f5975 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     displayed)
   * status: --fast is back
   * Fix support for insteadOf url remapping. Closes: #644278
+  * merge: Now runs in constant space.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 

From e83b966eb5197171a6af46595049d661c4229c58 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 23:48:50 -0400
Subject: [PATCH 2529/2835] cleanup

---
 Git/UnionMerge.hs                      | 39 ++++++++++++++------------
 doc/todo/optimise_git-annex_merge.mdwn |  6 ++++
 2 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index c86d699795..a2344e59db 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -42,25 +42,27 @@ merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
 merge_index h repo bs =
 	update_index_via repo $ map (\b -> merge_tree_index b h repo) bs
 
-update_index :: Repo -> [String] -> IO ()
-update_index repo ls = update_index_via repo [\h -> mapM_ (sendContent h) ls]
-
 {- Feeds content into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
-update_index_via :: Repo -> [Handle -> IO ()] -> IO ()
-update_index_via repo ls = do
+update_index :: Repo -> [String] -> IO ()
+update_index repo ls = update_index_via repo [\s -> mapM_ s ls]
+
+type Streamer = (String -> IO ()) -> IO ()
+
+{- Streams content into update-index. -}
+update_index_via :: Repo -> [Streamer] -> IO ()
+update_index_via repo as = do
 	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
-	forM_ ls $ \l -> l h
+	forM_ as (stream h)
 	hClose h
 	forceSuccess p
 	where
 		params = map Param ["update-index", "-z", "--index-info"]
-
-sendContent :: Handle -> String -> IO ()
-sendContent h s = do
-	hPutStr h s
-	hPutStr h "\0"
+		stream h a = a (streamer h)
+		streamer h s = do
+			hPutStr h s
+			hPutStr h "\0"
 
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}
@@ -68,17 +70,17 @@ update_index_line :: String -> FilePath -> String
 update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 
 {- Gets the contents of a tree. -}
-ls_tree :: String -> Repo -> Handle -> IO ()
-ls_tree x repo h = mapM_ (sendContent h) =<< pipeNullSplit params repo
+ls_tree :: String -> Repo -> Streamer
+ls_tree x repo streamer = mapM_ streamer =<< pipeNullSplit params repo
 	where
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: String -> String -> CatFileHandle -> Repo -> Handle -> IO ()
+merge_trees :: String -> String -> CatFileHandle -> Repo -> Streamer
 merge_trees x y h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: String -> CatFileHandle -> Repo -> Handle -> IO ()
+merge_tree_index :: String -> CatFileHandle -> Repo -> Streamer
 merge_tree_index x h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
@@ -86,12 +88,13 @@ diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: CatFileHandle -> [String] -> Repo -> Handle -> IO ()
-calc_merge ch differ repo ih = pipeNullSplit (map Param differ) repo >>= go
+calc_merge :: CatFileHandle -> [String] -> Repo -> Streamer
+calc_merge ch differ repo streamer = gendiff >>= go
 	where
+		gendiff = pipeNullSplit (map Param differ) repo
 		go [] = return ()
 		go (info:file:rest) = mergeFile info file ch repo >>=
-			maybe (go rest) (\l -> sendContent ih l >> go rest)
+			maybe (go rest) (\l -> streamer l >> go rest)
 		go (_:[]) = error "calc_merge parse error"
 
 {- Given an info line from a git raw diff, and the filename, generates
diff --git a/doc/todo/optimise_git-annex_merge.mdwn b/doc/todo/optimise_git-annex_merge.mdwn
index 2df196cfd4..91d18ebd77 100644
--- a/doc/todo/optimise_git-annex_merge.mdwn
+++ b/doc/todo/optimise_git-annex_merge.mdwn
@@ -15,3 +15,9 @@ merged. This could be reduced to a single call to `git-cat-file --batch`,
 There is already a Git.CatFile library that can do this easily. --[[Joey]]
 
 > This is now done, part above remains todo. --[[Joey]] 
+
+---
+
+Merging used to use memory proportional to the size of the diff. It now
+streams data, running in constant space. This probably sped it up a lot,
+as there's much less allocation and GC action. --[[Joey]] 

From 9b71b5f26c158973ca2d60dccde38290a1c9e6ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 00:01:07 -0400
Subject: [PATCH 2530/2835] fix display of semitrusted repos in status

semitrusted uuids rarely are listed in trust.log, so a special case
is needed to get a list of them. Take the difference of all known uuids
with non-semitrusted uuids.
---
 Command/Status.hs | 2 +-
 Logs/Trust.hs     | 5 +++++
 Logs/UUID.hs      | 6 +++++-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 2c0afa62c4..47b500b90a 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -102,7 +102,7 @@ supported_remote_types = stat "supported remote types" $
 
 remote_list :: TrustLevel -> String -> Stat
 remote_list level desc = stat n $ lift $ do
-	us <- M.keys <$> uuidMap
+	us <- uuidList
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
 	return $ if null s then "0" else show (length rs) ++ "\n" ++ init s
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index cb91861fd9..6305d281fc 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -20,6 +20,7 @@ import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
 import Logs.UUIDBased
+import Logs.UUID
 
 {- Filename of trust.log. -}
 trustLog :: FilePath
@@ -27,6 +28,10 @@ trustLog = "trust.log"
 
 {- Returns a list of UUIDs at the specified trust level. -}
 trustGet :: TrustLevel -> Annex [UUID]
+trustGet SemiTrusted = do -- special case; trustMap does not contain all these
+	others <- M.keys . M.filter (/= SemiTrusted) <$> trustMap
+	all <- uuidList
+	return $ all \\ others
 trustGet level = M.keys . M.filter (== level) <$> trustMap
 
 {- Read the trustLog into a map, overriding with any
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 20f43d15ca..17b0330c11 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -16,7 +16,8 @@
 module Logs.UUID (
 	describeUUID,
 	recordUUID,
-	uuidMap
+	uuidMap,
+	uuidList
 ) where
 
 import qualified Data.Map as M
@@ -87,3 +88,6 @@ uuidMap = do
 	return $ M.insertWith' preferold u "" m
 	where
 		preferold = flip const
+
+uuidList :: Annex [UUID]
+uuidList = M.keys <$> uuidMap

From 84784e2ca1ababf21342cba36f7e65b4c3cd303b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 00:07:06 -0400
Subject: [PATCH 2531/2835] cleanup

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 37578f5975..46518b2a80 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,14 +6,14 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     no longer needs to auto-merge.
   * init: When run in an already initalized repository, and without
     a description specified, don't delete the old description. 
-  * Optimised union merging; now only runs git cat-file once.
+  * Optimised union merging; now only runs git cat-file once, and runs
+    in constant space.
   * status: Now displays trusted, untrusted, and semitrusted repositories
     separately.
   * status: Fix --json mode (only the repository lists are currently
     displayed)
   * status: --fast is back
   * Fix support for insteadOf url remapping. Closes: #644278
-  * merge: Now runs in constant space.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 

From 2bb6b02948da8a33b2edcd911fcf3c2597b0ee58 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 00:49:09 -0400
Subject: [PATCH 2532/2835] When not run in a git repository, git-annex can
 still display a usage message, and "git annex version" even works.

Things that sound simple, but are made hard by the Annex monad being built
with the assumption that there will always be a git repo.
---
 CmdLine.hs                                    | 24 ++++++++++++-------
 Command.hs                                    | 10 ++++++--
 Command/Version.hs                            |  7 ++++--
 GitAnnex.hs                                   |  2 +-
 Logs/Trust.hs                                 |  4 ++--
 Types/Command.hs                              |  1 +
 debian/changelog                              |  2 ++
 ...rsion_should_without_being_in_a_repo_.mdwn |  2 ++
 git-annex-shell.hs                            |  4 ++--
 9 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index af53abc628..78f46a2e39 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -11,7 +11,9 @@ module CmdLine (
 	shutdown
 ) where
 
-import System.IO.Error (try)
+import qualified System.IO.Error as IO
+import qualified Control.Exception as E
+import Control.Exception (throw)
 import System.Console.GetOpt
 
 import Common.Annex
@@ -25,14 +27,18 @@ type Params = [String]
 type Flags = [Annex ()]
 
 {- Runs the passed command line. -}
-dispatch :: Params -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
-dispatch args cmds options header gitrepo = do
+dispatch :: Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
+dispatch args cmds options header getgitrepo = do
 	setupConsole
-	state <- Annex.new gitrepo
-	(actions, state') <- Annex.run state $ do
-		sequence_ flags
-		prepCommand cmd params
-	tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
+	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo)
+	case r of
+		Left e -> maybe (throw e) id (cmdnorepo cmd)
+		Right g -> do
+			state <- Annex.new g
+			(actions, state') <- Annex.run state $ do
+				sequence_ flags
+				prepCommand cmd params
+			tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
 	where
 		(flags, cmd, params) = parseCmd args cmds options header
 
@@ -77,7 +83,7 @@ tryRun' errnum _ cmd []
 	| otherwise = return ()
 tryRun' errnum state cmd (a:as) = run >>= handle
 	where
-		run = try $ Annex.run state $ do
+		run = IO.try $ Annex.run state $ do
 			Annex.Queue.flushWhenFull
 			a
 		handle (Left err) = showerr err >> cont False state
diff --git a/Command.hs b/Command.hs
index d22c2d12f2..b662171926 100644
--- a/Command.hs
+++ b/Command.hs
@@ -7,6 +7,7 @@
 
 module Command (
 	command,
+	noRepo,
 	next,
 	stop,
 	prepCommand,
@@ -31,9 +32,14 @@ import Logs.Trust
 import Logs.Location
 import Config
 
-{- Generates a command with the common checks. -}
+{- Generates a normal command -}
 command :: String -> String -> [CommandSeek] -> String -> Command
-command = Command commonChecks
+command = Command Nothing commonChecks
+
+{- Adds a fallback action to a command, that will be run if it's used
+ - outside a git repository. -}
+noRepo :: IO () -> Command -> Command
+noRepo a c = c { cmdnorepo = Just a }
 
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
diff --git a/Command/Version.hs b/Command/Version.hs
index a584264828..9fb7fe5bdb 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -13,7 +13,7 @@ import qualified Build.SysConfig as SysConfig
 import Annex.Version
 
 def :: [Command]
-def = [dontCheck repoExists $
+def = [noRepo showPackageVersion $ dontCheck repoExists $
 	command "version" paramNothing seek "show version info"]
 
 seek :: [CommandSeek]
@@ -23,7 +23,7 @@ start :: CommandStart
 start = do
 	v <- getVersion
 	liftIO $ do
-		putStrLn $ "git-annex version: " ++ SysConfig.packageversion
+		showPackageVersion
 		putStrLn $ "local repository version: " ++ fromMaybe "unknown" v
 		putStrLn $ "default repository version: " ++ defaultVersion
 		putStrLn $ "supported repository versions: " ++ vs supportedVersions
@@ -31,3 +31,6 @@ start = do
 	stop
 	where
 		vs = join " "
+
+showPackageVersion :: IO ()
+showPackageVersion = putStrLn $ "git-annex version: " ++ SysConfig.packageversion
diff --git a/GitAnnex.hs b/GitAnnex.hs
index f416b7bead..7b51602be6 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -123,4 +123,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch args cmds options header =<< Git.repoFromCwd
+run args = dispatch args cmds options header Git.repoFromCwd
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 6305d281fc..072ea41d6d 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -30,8 +30,8 @@ trustLog = "trust.log"
 trustGet :: TrustLevel -> Annex [UUID]
 trustGet SemiTrusted = do -- special case; trustMap does not contain all these
 	others <- M.keys . M.filter (/= SemiTrusted) <$> trustMap
-	all <- uuidList
-	return $ all \\ others
+	alluuids <- uuidList
+	return $ alluuids \\ others
 trustGet level = M.keys . M.filter (== level) <$> trustMap
 
 {- Read the trustLog into a map, overriding with any
diff --git a/Types/Command.hs b/Types/Command.hs
index d39876a7ae..5341a40545 100644
--- a/Types/Command.hs
+++ b/Types/Command.hs
@@ -33,6 +33,7 @@ type CommandCleanup = Annex Bool
 
 {- A command is defined by specifying these things. -}
 data Command = Command {
+	cmdnorepo :: Maybe (IO ()),
 	cmdcheck :: [CommandCheck],
 	cmdname :: String,
 	cmdparams :: String,
diff --git a/debian/changelog b/debian/changelog
index 46518b2a80..fd989546a3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -14,6 +14,8 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     displayed)
   * status: --fast is back
   * Fix support for insteadOf url remapping. Closes: #644278
+  * When not run in a git repository, git-annex can still display a usage
+    message, and "git annex version" even works.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
index 0bae8bdb0e..5c995852b1 100644
--- a/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
+++ b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
@@ -3,3 +3,5 @@ was checking the version of git-annex on a machine before cloning a repo...
     $ git annex version
     git-annex: Not in a git repository.
 
+> made difficult by the Annex monad, but I made it work! --[[Joey]]
+> [[done]]
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 658eddd77f..9a9d2f0925 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -79,8 +79,8 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
 	checkNotReadOnly cmd
-	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
-		dispatch (cmd : filterparams params) cmds options header
+	dispatch (cmd : filterparams params) cmds options header $
+		Git.repoAbsPath dir >>= Git.repoFromAbsPath
 
 external :: [String] -> IO ()
 external params = do

From 272a67921cdc6d8d40641a9cb71b744ef1f76128 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 01:46:46 -0400
Subject: [PATCH 2533/2835] better name

---
 Annex/Branch.hs   |  2 +-
 Git/UnionMerge.hs | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index ae33f66cf0..a75773e192 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -56,7 +56,7 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.update_index_via g
+genIndex g = Git.UnionMerge.stream_update_index g
 	[Git.UnionMerge.ls_tree fullname g]
 
 {- Runs an action using the branch's index file. -}
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a2344e59db..60ccd6dcdc 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -9,7 +9,7 @@ module Git.UnionMerge (
 	merge,
 	merge_index,
 	update_index,
-	update_index_via,
+	stream_update_index,
 	update_index_line,
 	ls_tree
 ) where
@@ -30,7 +30,7 @@ import Git.CatFile
 merge :: String -> String -> Repo -> IO ()
 merge x y repo = do
 	h <- catFileStart repo
-	update_index_via repo
+	stream_update_index repo
 		[ ls_tree x repo
 		, merge_trees x y h repo
 		]
@@ -40,19 +40,19 @@ merge x y repo = do
  - the index are preserved (and participate in the merge). -}
 merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
 merge_index h repo bs =
-	update_index_via repo $ map (\b -> merge_tree_index b h repo) bs
+	stream_update_index repo $ map (\b -> merge_tree_index b h repo) bs
 
 {- Feeds content into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Repo -> [String] -> IO ()
-update_index repo ls = update_index_via repo [\s -> mapM_ s ls]
+update_index repo ls = stream_update_index repo [\s -> mapM_ s ls]
 
 type Streamer = (String -> IO ()) -> IO ()
 
 {- Streams content into update-index. -}
-update_index_via :: Repo -> [Streamer] -> IO ()
-update_index_via repo as = do
+stream_update_index :: Repo -> [Streamer] -> IO ()
+stream_update_index repo as = do
 	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
 	forM_ as (stream h)
 	hClose h

From 9290095fc21953cd1fe0a71f7c8a454934194e3b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 02:23:34 -0400
Subject: [PATCH 2534/2835] improve type signatures with a Ref newtype

In git, a Ref can be a Sha, or a Branch, or a Tag. I added type aliases for
those. Note that this does not prevent mixing up of eg, refs and branches
at the type level. Since git really doesn't care, except rare cases like
git update-ref, or git tag -d, that seems ok for now.

There's also a tree-ish, but let's just use Ref for it. A given Sha or Ref
may or may not be a tree-ish, depending on the object type, so there seems
no point in trying to represent it at the type level.
---
 Annex/Branch.hs    | 47 +++++++++++++++++++++++-----------------------
 Annex/CatFile.hs   |  3 ++-
 Command/Uninit.hs  |  6 +++---
 Command/Unused.hs  |  7 ++++---
 Git.hs             | 36 +++++++++++++++++++++++++----------
 Git/CatFile.hs     | 10 +++++-----
 Git/LsTree.hs      |  8 +++-----
 Git/UnionMerge.hs  | 34 ++++++++++++++++-----------------
 Upgrade/V2.hs      |  4 ++--
 git-union-merge.hs |  2 +-
 10 files changed, 86 insertions(+), 71 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a75773e192..a62a1384c9 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -31,19 +31,17 @@ import qualified Git.UnionMerge
 import qualified Annex
 import Annex.CatFile
 
-type GitRef = String
-
 {- Name of the branch that is used to store git-annex's information. -}
-name :: GitRef
-name = "git-annex"
+name :: Git.Ref
+name = Git.Ref "git-annex"
 
 {- Fully qualified name of the branch. -}
-fullname :: GitRef
-fullname = "refs/heads/" ++ name
+fullname :: Git.Ref
+fullname = Git.Ref $ "refs/heads/" ++ show name
 
 {- Branch's name in origin. -}
-originname :: GitRef
-originname = "origin/" ++ name
+originname :: Git.Ref
+originname = Git.Ref $ "origin/" ++ show name
 
 {- A separate index file for the branch. -}
 index :: Git.Repo -> FilePath
@@ -104,7 +102,8 @@ create :: Annex ()
 create = unlessM hasBranch $ do
 	e <- hasOrigin
 	if e
-		then inRepo $ Git.run "branch" [Param name, Param originname]
+		then inRepo $ Git.run "branch"
+			[Param $ show name, Param $ show originname]
 		else withIndex' True $
 			inRepo $ Git.commit "branch created" fullname []
 
@@ -140,8 +139,8 @@ update = onceonly $ do
 		let merge_desc = if null branches
 			then "update" 
 			else "merging " ++
-				(unwords $ map Git.refDescribe branches) ++ 
-				" into " ++ name
+				(unwords $ map (show . Git.refDescribe) branches) ++ 
+				" into " ++ show name
 		unless (null branches) $ do
 			showSideAction merge_desc
 			{- Note: This merges the branches into the index.
@@ -164,12 +163,12 @@ update = onceonly $ do
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
-changedBranch :: String -> String -> Annex Bool
+changedBranch :: Git.Branch -> Git.Branch -> Annex Bool
 changedBranch origbranch newbranch = not . L.null <$> diffs
 	where
 		diffs = inRepo $ Git.pipeRead
 			[ Param "log"
-			, Param (origbranch ++ ".." ++ newbranch)
+			, Param (show origbranch ++ ".." ++ show newbranch)
 			, Params "--oneline -n1"
 			]
 
@@ -181,7 +180,7 @@ changedBranch origbranch newbranch = not . L.null <$> diffs
  - every commit present in all the other refs, as well as in the
  - git-annex branch.
  -}
-tryFastForwardTo :: [String] -> Annex Bool
+tryFastForwardTo :: [Git.Ref] -> Annex Bool
 tryFastForwardTo [] = return True
 tryFastForwardTo (first:rest) = do
 	-- First, check that the git-annex branch does not contain any
@@ -194,7 +193,7 @@ tryFastForwardTo (first:rest) = do
 	where
 		no_ff = return False
 		do_ff branch = do
-			inRepo $ Git.run "update-ref" [Param fullname, Param branch]
+			inRepo $ Git.run "update-ref" [Param $ show fullname, Param $ show branch]
 			return True
 		findbest c [] = return $ Just c
 		findbest c (r:rs)
@@ -220,9 +219,9 @@ disableUpdate = Annex.changeState setupdated
 				old = Annex.branchstate s
 
 {- Checks if a git ref exists. -}
-refExists :: GitRef -> Annex Bool
+refExists :: Git.Ref -> Annex Bool
 refExists ref = inRepo $ Git.runBool "show-ref"
-	[Param "--verify", Param "-q", Param ref]
+	[Param "--verify", Param "-q", Param $ show ref]
 
 {- Does the main git-annex branch exist? -}
 hasBranch :: Annex Bool
@@ -238,12 +237,12 @@ hasSomeBranch = not . null <$> siblingBranches
 
 {- List of git-annex (refs, branches), including the main one and any
  - from remotes. Duplicate refs are filtered out. -}
-siblingBranches :: Annex [(String, String)]
+siblingBranches :: Annex [(Git.Ref, Git.Branch)]
 siblingBranches = do
-	r <- inRepo $ Git.pipeRead [Param "show-ref", Param name]
-	return $ nubBy uref $ map (pair . words . L.unpack) (L.lines r)
+	r <- inRepo $ Git.pipeRead [Param "show-ref", Param $ show name]
+	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
 	where
-		pair l = (head l, last l)
+		gen l = (Git.Ref $ head l, Git.Ref $ last l)
 		uref (a, _) (b, _) = a == b
 
 {- Applies a function to modifiy the content of a file.
@@ -291,7 +290,7 @@ get' staleok file = fromcache =<< getCache file
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
 	bfiles <- inRepo $ Git.pipeNullSplit
-		[Params "ls-tree --name-only -r -z", Param fullname]
+		[Params "ls-tree --name-only -r -z", Param $ show fullname]
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
 
@@ -346,10 +345,10 @@ stageJournalFiles = do
 			hClose toh
 			exitSuccess
 		hClose toh
-		s <- hGetContents fromh
+		shas <- map Git.Ref . lines <$> hGetContents fromh
 		-- update the index, also in just one command
 		Git.UnionMerge.update_index g $
-			index_lines (lines s) $ map fileJournal fs
+			index_lines shas (map fileJournal fs)
 		hClose fromh
 		forceSuccess pid
 		mapM_ removeFile paths
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 0541f7269d..1d996edfd5 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -13,10 +13,11 @@ module Annex.CatFile (
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common.Annex
+import qualified Git
 import qualified Git.CatFile
 import qualified Annex
 
-catFile :: String -> FilePath -> Annex L.ByteString
+catFile :: Git.Branch -> FilePath -> Annex L.ByteString
 catFile branch file = do
 	h <- catFileHandle
 	liftIO $ Git.CatFile.catFile h branch file
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index ca18c478c5..48f5b1ac14 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -26,9 +26,9 @@ check :: Annex ()
 check = do
 	b <- current_branch	
 	when (b == Annex.Branch.name) $ error $
-		"cannot uninit when the " ++ b ++ " branch is checked out"
+		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	where
-		current_branch = head . lines . B.unpack <$> revhead
+		current_branch = Git.Ref . head . lines . B.unpack <$> revhead
 		revhead = inRepo $ Git.pipeRead 
 			[Params "rev-parse --abbrev-ref HEAD"]
 
@@ -57,5 +57,5 @@ cleanup = do
 	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
 	saveState
-	inRepo $ Git.run "branch" [Param "-D", Param Annex.Branch.name]
+	inRepo $ Git.run "branch" [Param "-D", Param $ show Annex.Branch.name]
 	liftIO exitSuccess
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 34d8ac232a..ccb9c48d81 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -152,12 +152,13 @@ excludeReferenced l = do
 		(S.fromList l)
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
-		refs = map last .
+		refs = map Git.Ref . 
+			last .
 			nubBy cmpheads .
 			filter ourbranches .
 			map words . lines . L.unpack
 		cmpheads a b = head a == head b
-		ourbranchend = '/' : Annex.Branch.name
+		ourbranchend = '/' : show (Annex.Branch.name)
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
 		removewith [] s = return $ S.toList s
 		removewith (a:as) s
@@ -188,7 +189,7 @@ getKeysReferenced = do
 	return $ map fst $ catMaybes keypairs
 
 {- List of keys referenced by symlinks in a git ref. -}
-getKeysReferencedInGit :: String -> Annex [Key]
+getKeysReferencedInGit :: Git.Ref -> Annex [Key]
 getKeysReferencedInGit ref = do
 	showAction $ "checking " ++ Git.refDescribe ref
 	findkeys [] =<< inRepo (LsTree.lsTree ref)
diff --git a/Git.hs b/Git.hs
index 3f3f746324..ba5d831fea 100644
--- a/Git.hs
+++ b/Git.hs
@@ -10,6 +10,10 @@
 
 module Git (
 	Repo,
+	Ref(..),
+	Branch,
+	Sha,
+	Tag,
 	repoFromCwd,
 	repoFromAbsPath,
 	repoFromUnknown,
@@ -94,6 +98,18 @@ data Repo = Repo {
 	remoteName :: Maybe String 
 } deriving (Show, Eq)
 
+{- A git ref. Can be a sha1, or a branch or tag name. -}
+newtype Ref = Ref String
+	deriving (Eq)
+
+instance Show Ref where
+	show (Ref v) = v
+
+{- Aliases for Ref. -}
+type Branch = Ref
+type Sha = Ref
+type Tag = Ref
+
 newFrom :: RepoLocation -> Repo
 newFrom l = 
 	Repo {
@@ -162,9 +178,9 @@ repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
 repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
-{- Converts a fully qualified git ref into a user-visible version -}
-refDescribe :: String -> String
-refDescribe = remove "refs/heads/" . remove "refs/remotes/"
+{- Converts a fully qualified git ref into a user-visible version. -}
+refDescribe :: Ref -> String
+refDescribe = remove "refs/heads/" . remove "refs/remotes/" . show
 	where
 		remove prefix s
 			| prefix `isPrefixOf` s = drop (length prefix) s
@@ -432,7 +448,7 @@ useIndex index = do
 
 {- Runs an action that causes a git subcommand to emit a sha, and strips
    any trailing newline, returning the sha. -}
-getSha :: String -> IO String -> IO String
+getSha :: String -> IO String -> IO Sha
 getSha subcommand a = do
 	t <- a
 	let t' = if last t == '\n'
@@ -440,27 +456,27 @@ getSha subcommand a = do
 		else t
 	when (length t' /= shaSize) $
 		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return t'
+	return $ Ref t'
 
 {- Size of a git sha. -}
 shaSize :: Int
 shaSize = 40
 
-{- Commits the index into the specified branch, 
+{- Commits the index into the specified branch (or other ref), 
  - with the specified parent refs. -}
-commit :: String -> String -> [String] -> Repo -> IO ()
+commit :: String -> Ref -> [Ref] -> Repo -> IO ()
 commit message newref parentrefs repo = do
 	tree <- getSha "write-tree" $ asString $
 		pipeRead [Param "write-tree"] repo
 	sha <- getSha "commit-tree" $ asString $
 		ignorehandle $ pipeWriteRead
-			(map Param $ ["commit-tree", tree] ++ ps)
+			(map Param $ ["commit-tree", show tree] ++ ps)
 			(L.pack message) repo
-	run "update-ref" [Param newref, Param sha] repo
+	run "update-ref" [Param $ show newref, Param $ show sha] repo
 	where
 		ignorehandle a = snd <$> a
 		asString a = L.unpack <$> a
-		ps = concatMap (\r -> ["-p", r]) parentrefs
+		ps = concatMap (\r -> ["-p", show r]) parentrefs
 
 {- Runs git config and populates a repo with its config. -}
 configRead :: Repo -> IO Repo
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 83c1235086..c1cafb8ba8 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -37,14 +37,14 @@ catFileStop (pid, from, to) = do
 	forceSuccess pid
 
 {- Reads a file from a specified branch. -}
-catFile :: CatFileHandle -> String -> FilePath -> IO L.ByteString
-catFile h branch file = catObject h (branch ++ ":" ++ file)
+catFile :: CatFileHandle -> Branch -> FilePath -> IO L.ByteString
+catFile h branch file = catObject h $ Ref $ show branch ++ ":" ++ file
 
 {- Uses a running git cat-file read the content of an object.
  - Objects that do not exist will have "" returned. -}
-catObject :: CatFileHandle -> String -> IO L.ByteString
+catObject :: CatFileHandle -> Ref -> IO L.ByteString
 catObject (_, from, to) object = do
-	hPutStrLn to object
+	hPutStrLn to $ show object
 	hFlush to
 	header <- hGetLine from
 	case words header of
@@ -53,7 +53,7 @@ catObject (_, from, to) object = do
 			  validobjtype objtype -> handle size
 			| otherwise -> empty
 		_
-			| header == object ++ " missing" -> empty
+			| header == show object ++ " missing" -> empty
 			| otherwise -> error $ "unknown response from git cat-file " ++ header
 	where
 		handle size = case reads size of
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 1fcdf13ed5..8aa16a308b 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -19,8 +19,6 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import Git
 import Utility.SafeCommand
 
-type Treeish = String
-
 data TreeItem = TreeItem
 	{ mode :: FileMode
 	, typeobj :: String
@@ -28,10 +26,10 @@ data TreeItem = TreeItem
 	, file :: FilePath
 	} deriving Show
 
-{- Lists the contents of a Treeish -}
-lsTree :: Treeish -> Repo -> IO [TreeItem]
+{- Lists the contents of a Ref -}
+lsTree :: Ref -> Repo -> IO [TreeItem]
 lsTree t repo = map parseLsTree <$>
-	pipeNullSplitB [Params "ls-tree --full-tree -z -r --", File t] repo
+	pipeNullSplitB [Params "ls-tree --full-tree -z -r --", File $ show t] repo
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 60ccd6dcdc..edc8cb20ba 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -22,12 +22,14 @@ import Common
 import Git
 import Git.CatFile
 
+type Streamer = (String -> IO ()) -> IO ()
+
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
  -
  - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: String -> String -> Repo -> IO ()
+merge :: Ref -> Ref -> Repo -> IO ()
 merge x y repo = do
 	h <- catFileStart repo
 	stream_update_index repo
@@ -38,7 +40,7 @@ merge x y repo = do
 
 {- Merges a list of branches into the index. Previously staged changed in
  - the index are preserved (and participate in the merge). -}
-merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
+merge_index :: CatFileHandle -> Repo -> [Ref] -> IO ()
 merge_index h repo bs =
 	stream_update_index repo $ map (\b -> merge_tree_index b h repo) bs
 
@@ -48,8 +50,6 @@ merge_index h repo bs =
 update_index :: Repo -> [String] -> IO ()
 update_index repo ls = stream_update_index repo [\s -> mapM_ s ls]
 
-type Streamer = (String -> IO ()) -> IO ()
-
 {- Streams content into update-index. -}
 stream_update_index :: Repo -> [Streamer] -> IO ()
 stream_update_index repo as = do
@@ -66,22 +66,22 @@ stream_update_index repo as = do
 
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}
-update_index_line :: String -> FilePath -> String
-update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
+update_index_line :: Sha -> FilePath -> String
+update_index_line sha file = "100644 blob " ++ show sha ++ "\t" ++ file
 
-{- Gets the contents of a tree. -}
-ls_tree :: String -> Repo -> Streamer
-ls_tree x repo streamer = mapM_ streamer =<< pipeNullSplit params repo
+{- Gets the current tree for a ref. -}
+ls_tree :: Ref -> Repo -> Streamer
+ls_tree (Ref x) repo streamer = mapM_ streamer =<< pipeNullSplit params repo
 	where
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: String -> String -> CatFileHandle -> Repo -> Streamer
-merge_trees x y h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
+merge_trees :: Ref -> Ref -> CatFileHandle -> Repo -> Streamer
+merge_trees (Ref x) (Ref y) h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: String -> CatFileHandle -> Repo -> Streamer
-merge_tree_index x h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
+merge_tree_index :: Ref -> CatFileHandle -> Repo -> Streamer
+merge_tree_index (Ref x) h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
 diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
@@ -101,7 +101,7 @@ calc_merge ch differ repo streamer = gendiff >>= go
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
 mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
-mergeFile info file h repo = case filter (/= nullsha) [asha, bsha] of
+mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
@@ -110,11 +110,11 @@ mergeFile info file h repo = case filter (/= nullsha) [asha, bsha] of
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
-		nullsha = replicate shaSize '0'
+		nullsha = Ref $ replicate shaSize '0'
 		unionmerge = L.unlines . nub . L.lines
 
-{- Injects some content into git, returning its hash. -}
-hashObject :: L.ByteString -> Repo -> IO String
+{- Injects some content into git, returning its Sha. -}
+hashObject :: L.ByteString -> Repo -> IO Sha
 hashObject content repo = getSha subcmd $ do
 	(h, s) <- pipeWriteRead (map Param params) content repo
 	L.length s `seq` do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 6a46ad8a16..e76d99b3ec 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -86,7 +86,7 @@ logFiles dir = return . filter (".log" `isSuffixOf`)
 
 push :: Annex ()
 push = do
-	origin_master <- Annex.Branch.refExists "origin/master"
+	origin_master <- Annex.Branch.refExists $ Git.Ref "origin/master"
 	origin_gitannex <- Annex.Branch.hasOrigin
 	case (origin_master, origin_gitannex) of
 		(_, True) -> do
@@ -103,7 +103,7 @@ push = do
 			Annex.Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
-			inRepo $ Git.run "push" [Param "origin", Param Annex.Branch.name]
+			inRepo $ Git.run "push" [Param "origin", Param $ show Annex.Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
 			-- know about the new branch
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 10ae842177..1cec4a0f85 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -37,7 +37,7 @@ parseArgs = do
 
 main :: IO ()
 main = do
-	[aref, bref, newref] <- parseArgs
+	[aref, bref, newref] <- map Git.Ref <$> parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g

From 6d9f525f647fb24f4955e4a51156741f216d3931 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 11:59:17 -0400
Subject: [PATCH 2535/2835] clarify

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 35ba25115d..e91e5a0e33 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -441,7 +441,7 @@ file contents are present at either of two repositories.
 * --copies=number
 
   Matches only files that git-annex believes to have the specified number
-  of copies.
+  of copies, or more.
 
 * --not
 

From 1ffd54ef781b0c240d7de6c80cfcb01328efc25f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 18:56:06 -0400
Subject: [PATCH 2536/2835] ensure branch exists before trying to update it

The branch may not exist, if .git/annex has been copied over from another
repo (or a corrupted repo). I suppose it could also have gotten deleted
somehow. Without this, there is a confusing failure.
---
 Annex/Branch.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a62a1384c9..ccc6145552 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -130,6 +130,8 @@ commit message = whenM journalDirty $ lockJournal $ do
  -}
 update :: Annex ()
 update = onceonly $ do
+	-- ensure branch exists
+	create
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches

From abd4e1192f675de8d8afeec7af467069443d6e40 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 17 Nov 2011 21:51:55 +0000
Subject: [PATCH 2537/2835]

---
 ...nterrupting_migration_causes_problems.mdwn | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 doc/bugs/interrupting_migration_causes_problems.mdwn

diff --git a/doc/bugs/interrupting_migration_causes_problems.mdwn b/doc/bugs/interrupting_migration_causes_problems.mdwn
new file mode 100644
index 0000000000..3298a1fa65
--- /dev/null
+++ b/doc/bugs/interrupting_migration_causes_problems.mdwn
@@ -0,0 +1,50 @@
+Killing a migration from WORM to SHA256 with ^C breaks things; future attempts to do the migration fail:
+
+    #!/bin/bash
+    
+    BASE=/tmp/migrate-bug
+    
+    set -x
+    
+    chmod -R +w $BASE
+    rm -rf $BASE
+    mkdir -p $BASE
+    cd $BASE
+    
+    # create annex
+    git init .
+    git annex init
+    
+    # make a big (sparse) file and add it
+    dd if=/dev/zero of=bigfile bs=1 count=0 seek=1G
+    git annex add --backend WORM bigfile
+    git commit -m 'added bigfile'
+    
+    # look at status
+    git annex status
+    
+    # now migrate it, but kill migration during checksum
+    # Simulate ^C by making a new process group and sending SIGINT
+    setsid git annex migrate --backend SHA256 bigfile &
+    PID=$!
+    sleep 1
+    kill -INT -$PID
+    wait
+    
+    # look at status
+    git annex status
+    
+    # this migration fails
+    git annex migrate --backend SHA256 bigfile
+    
+    # but fsck says everything's OK
+    git annex fsck
+
+The error:
+
+    migrate bigfile 
+    git-annex: /tmp/migrate-bug/.git/annex/objects/K9/V1/WORM-s1073741824-m1321566308--bigfile/WORM-s1073741824-m1321566308--bigfile: createLink: already exists (File exists)
+    failed
+    git-annex: migrate: 1 failed
+
+    

From b3bbc1cbb61d18d049ba334dc4fd85d0ca922afa Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 17 Nov 2011 21:56:55 +0000
Subject: [PATCH 2538/2835]

---
 ...ata_isn__39__t_unused_after_migration.mdwn | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn

diff --git a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
new file mode 100644
index 0000000000..322c354a21
--- /dev/null
+++ b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
@@ -0,0 +1,47 @@
+Old data isn't listed as unused after migrating backends:
+
+    #!/bin/bash
+    
+    BASE=/tmp/migrate-bug-2
+    set -x
+    chmod -R +w $BASE
+    rm -rf $BASE
+    mkdir -p $BASE
+    cd $BASE
+    
+    # create annex
+    git init .
+    git annex init
+    
+    # make a big (sparse) file and add it
+    dd if=/dev/zero of=bigfile bs=1 count=0 seek=1G
+    git annex add --backend WORM bigfile
+    git commit -m 'added bigfile'
+    
+    # migrate it
+    git annex migrate --backend SHA256 bigfile
+    
+    # status shows 2 keys taking up 2G
+    git annex status
+    
+    # but nothing is unused
+    git annex unused
+
+Output:
+
+    ++ git annex status
+    supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            ede95a82-1166-11e1-a475-475d55eb0f8f -- here
+    local annex keys: 2
+    local annex size: 2 gigabytes
+    visible annex keys: 1
+    visible annex size: 1 gigabyte
+    backend usage: 
+            WORM: 1
+            SHA256: 1
+    ++ git annex unused
+    unused . (checking for unused data...) (checking master...) ok
+
+The two files are hardlinked, so it's not taking up extra space, but it would be nice to be able to remove the old keys.

From d66fac1ec82ea350e5d668df0cc5d02a345ca75e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 17 Nov 2011 18:17:34 -0400
Subject: [PATCH 2539/2835] fix typo introduced with the Ref type

---
 Command/Unused.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index ccb9c48d81..7f9edfef25 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -153,7 +153,7 @@ excludeReferenced l = do
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
 		refs = map Git.Ref . 
-			last .
+			map last .
 			nubBy cmpheads .
 			filter ourbranches .
 			map words . lines . L.unpack

From 8b892901a98421b9b70c131bcc7af3a5e2ce62d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 17 Nov 2011 18:20:06 -0400
Subject: [PATCH 2540/2835] analysis; not a bug but a feature

---
 doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
index 322c354a21..9d1bbc4e07 100644
--- a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
+++ b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
@@ -45,3 +45,11 @@ Output:
     unused . (checking for unused data...) (checking master...) ok
 
 The two files are hardlinked, so it's not taking up extra space, but it would be nice to be able to remove the old keys.
+
+> `git annex unused` checks the content of all branches, and assumes that,
+> when a branch contains a file that points to a key, that key is still 
+> used. In this case, the migration has staged a change to the file,
+> but it is not yet committed, so when it checks the master branch, it
+> still finds a file referring to the old key. 
+> 
+> So, slightly surprising, but not a bug. --[[Joey]] [[done]]

From c70b78d40a1f27176cc01c8ae02355b50d5cb607 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 17 Nov 2011 18:29:28 -0400
Subject: [PATCH 2541/2835] migrate: Don't fall over a stale temp file.

---
 Command/Migrate.hs                                   | 5 +++--
 debian/changelog                                     | 1 +
 doc/bugs/interrupting_migration_causes_problems.mdwn | 4 +++-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 860c4bd470..045b8f9b1d 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -48,9 +48,10 @@ perform file oldkey newbackend = do
 	src <- fromRepo $ gitAnnexLocation oldkey
 	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp  takeFileName file
+	cleantmp tmpfile
 	liftIO $ createLink src tmpfile
 	k <- Backend.genKey tmpfile $ Just newbackend
-	liftIO $ cleantmp tmpfile
+	cleantmp tmpfile
 	case k of
 		Nothing -> stop
 		Just (newkey, _) -> do
@@ -70,7 +71,7 @@ perform file oldkey newbackend = do
 					next $ Command.Add.cleanup file newkey True
 				else stop
 	where
-		cleantmp t = whenM (doesFileExist t) $ removeFile t
+		cleantmp t = liftIO $ whenM (doesFileExist t) $ removeFile t
 		link src newkey = getViaTmpUnchecked newkey $ \t -> do
 			-- Make a hard link to the old backend's
 			-- cached key, to avoid wasting disk space.
diff --git a/debian/changelog b/debian/changelog
index fd989546a3..6976fd13a8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -16,6 +16,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * Fix support for insteadOf url remapping. Closes: #644278
   * When not run in a git repository, git-annex can still display a usage
     message, and "git annex version" even works.
+  * migrate: Don't fall over a stale temp file.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/bugs/interrupting_migration_causes_problems.mdwn b/doc/bugs/interrupting_migration_causes_problems.mdwn
index 3298a1fa65..68426e54af 100644
--- a/doc/bugs/interrupting_migration_causes_problems.mdwn
+++ b/doc/bugs/interrupting_migration_causes_problems.mdwn
@@ -47,4 +47,6 @@ The error:
     failed
     git-annex: migrate: 1 failed
 
-    
+> Fixed it to delete the stale temp file. [[done]]
+> 
+> Thanks for making such clear test cases, Jim! --[[Joey]] 

From 15164588851fe65415a814630e0d00fdb28d5b92 Mon Sep 17 00:00:00 2001
From: "http://ertai.myopenid.com/" 
Date: Fri, 18 Nov 2011 14:11:08 +0000
Subject: [PATCH 2542/2835]

---
 doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn

diff --git a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
new file mode 100644
index 0000000000..5e2fac99d4
--- /dev/null
+++ b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
@@ -0,0 +1,4 @@
+When using `git annex get foo` where foo is available in a rsync remote with encryption I got an error saying that rsync cannot
+find the required file but extra ' are here.
+
+I attached a patch for this.

From ed55a750d5255ac07bcd04adf4365a3255b3f605 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Nov 2011 11:58:55 -0400
Subject: [PATCH 2543/2835] response

---
 .../extraneous_shell_escaping_for_rsync_remotes.mdwn     | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
index 5e2fac99d4..0b292c5c92 100644
--- a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
+++ b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
@@ -2,3 +2,12 @@ When using `git annex get foo` where foo is available in a rsync remote with enc
 find the required file but extra ' are here.
 
 I attached a patch for this.
+
+> But you didn't, sadly. :(
+>
+> I don't seem to see the problem, set up a rsync over ssh with encryption
+> and sent over a file "foo", and then got it back from rsync, without
+> trouble.
+> 
+> Ah, you're not using rsync over ssh, but just to a local directory,
+> right? --[[Joey]]

From 1326bb863516b5e7efbfd9fba2754a3fe7289315 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Nov 2011 12:53:48 -0400
Subject: [PATCH 2544/2835] Avoid excessive escaping for rsync special remotes
 that are not accessed over ssh.

This is actually tricky, 45bbf210a1210172c7c7b87879ed74f7c8ccbdba added
the escaping because it's needed for rsync that does go over ssh.
So I had to detect whether the remote's rsync url will use ssh or not,
and vary the escaping.
---
 Remote/Rsync.hs                                  |  9 +++++++--
 Utility/RsyncFile.hs                             | 16 ++++++++++++++++
 debian/changelog                                 |  2 ++
 ...raneous_shell_escaping_for_rsync_remotes.mdwn |  2 ++
 4 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 86ff2ea5b1..5cd27a6099 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -81,13 +81,18 @@ rsyncSetup u c = do
 	gitConfigSpecialRemote u c' "rsyncurl" url
 	return c'
 
+rsyncEscape :: RsyncOpts -> String -> String
+rsyncEscape o s
+	| rsyncUrlIsShell (rsyncUrl o) = shellEscape s
+	| otherwise = s
+
 rsyncKey :: RsyncOpts -> Key -> String
-rsyncKey o k = rsyncUrl o  hashDirMixed k  shellEscape (f  f)
+rsyncKey o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (f  f)
         where
                 f = keyFile k
 
 rsyncKeyDir :: RsyncOpts -> Key -> String
-rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  shellEscape (keyFile k)
+rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (keyFile k)
 
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = rsyncSend o k =<< fromRepo (gitAnnexLocation k)
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index c5006a30fa..a691d0a0e6 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -8,6 +8,7 @@
 module Utility.RsyncFile where
 
 import Data.String.Utils
+import Data.List
 
 import Utility.SafeCommand
 
@@ -48,3 +49,18 @@ rsync = boolSystem "rsync"
 
 rsyncExec :: [CommandParam] -> IO ()
 rsyncExec params = executeFile "rsync" True (toCommand params) Nothing
+
+{- Checks if an rsync url involves the remote shell (ssh or rsh).
+ - Use of such urls with rsync or rsyncExec requires additional shell
+ - escaping. -}
+rsyncUrlIsShell :: String -> Bool
+rsyncUrlIsShell s
+	| "rsync://" `isPrefixOf` s = False
+	| otherwise = go s
+	where
+		-- host:dir is rsync protocol, while host:dir is ssh/rsh
+		go [] = False
+		go (c:cs)
+			| c == '/' = False -- got to directory with no colon
+			| c == ':' = not $ ":" `isPrefixOf` cs
+			| otherwise = go cs
diff --git a/debian/changelog b/debian/changelog
index 6976fd13a8..52d08f3037 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,8 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * When not run in a git repository, git-annex can still display a usage
     message, and "git annex version" even works.
   * migrate: Don't fall over a stale temp file.
+  * Avoid excessive escaping for rsync special remotes that are not accessed
+    over ssh.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
index 0b292c5c92..c4ee8d5bda 100644
--- a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
+++ b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
@@ -11,3 +11,5 @@ I attached a patch for this.
 > 
 > Ah, you're not using rsync over ssh, but just to a local directory,
 > right? --[[Joey]]
+
+>> [[fixed|done]] --[[Joey]] 

From c50a5fbeb4f90041f16cc84bf6b39577655d42b6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Nov 2011 13:22:48 -0400
Subject: [PATCH 2545/2835] status: Include all special remotes in the list of
 repositories.

Special remotes do not always have a description listed in uuid.log,
and such ones were not listed before.
---
 Command/Status.hs |  2 +-
 Logs/Trust.hs     | 28 ++++++++++++++++------------
 Logs/UUID.hs      |  6 +-----
 Remote.hs         |  6 +++++-
 debian/changelog  |  1 +
 5 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 47b500b90a..7448615cdd 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -102,7 +102,7 @@ supported_remote_types = stat "supported remote types" $
 
 remote_list :: TrustLevel -> String -> Stat
 remote_list level desc = stat n $ lift $ do
-	us <- uuidList
+	us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap)
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
 	return $ if null s then "0" else show (length rs) ++ "\n" ++ init s
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 072ea41d6d..e447fbebc8 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -20,20 +20,30 @@ import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
 import Logs.UUIDBased
-import Logs.UUID
 
 {- Filename of trust.log. -}
 trustLog :: FilePath
 trustLog = "trust.log"
 
-{- Returns a list of UUIDs at the specified trust level. -}
+{- Returns a list of UUIDs that the trustLog indicates have the
+ - specified trust level.
+ - Note that the list can be incomplete for SemiTrusted, since that's
+ - the default. -}
 trustGet :: TrustLevel -> Annex [UUID]
-trustGet SemiTrusted = do -- special case; trustMap does not contain all these
-	others <- M.keys . M.filter (/= SemiTrusted) <$> trustMap
-	alluuids <- uuidList
-	return $ alluuids \\ others
 trustGet level = M.keys . M.filter (== level) <$> trustMap
 
+{- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
+trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
+trustPartition level ls
+	| level == SemiTrusted = do
+		t <- trustGet Trusted
+		u <- trustGet UnTrusted
+		let uncandidates = t ++ u
+		return $ partition (`notElem` uncandidates) ls
+	| otherwise = do
+		candidates <- trustGet level
+		return $ partition (`elem` candidates) ls
+
 {- Read the trustLog into a map, overriding with any
  - values from forcetrust. The map is cached for speed. -}
 trustMap :: Annex TrustMap
@@ -72,9 +82,3 @@ trustSet uuid@(UUID _) level = do
 		showLog showTrust . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
 trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
-
-{- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
-trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
-trustPartition level ls = do
-	candidates <- trustGet level
-	return $ partition (`elem` candidates) ls
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 17b0330c11..20f43d15ca 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -16,8 +16,7 @@
 module Logs.UUID (
 	describeUUID,
 	recordUUID,
-	uuidMap,
-	uuidList
+	uuidMap
 ) where
 
 import qualified Data.Map as M
@@ -88,6 +87,3 @@ uuidMap = do
 	return $ M.insertWith' preferold u "" m
 	where
 		preferold = flip const
-
-uuidList :: Annex [UUID]
-uuidList = M.keys <$> uuidMap
diff --git a/Remote.hs b/Remote.hs
index 7c0362d2a7..b1be60ec4a 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -16,6 +16,7 @@ module Remote (
 	hasKeyCheap,
 
 	remoteTypes,
+	remoteMap,
 	byName,
 	prettyPrintUUIDs,
 	remotesWithUUID,
@@ -83,6 +84,10 @@ genList = do
 			u <- getRepoUUID r
 			generate t r u (M.lookup u m)
 
+{- Map of UUIDs of Remotes and their names. -}
+remoteMap :: Annex (M.Map UUID String)
+remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
+
 {- Looks up a remote by name. (Or by UUID.) Only finds currently configured
  - git remotes. -}
 byName :: String -> Annex (Remote Annex)
@@ -139,7 +144,6 @@ prettyPrintUUIDs desc uuids = do
 			| d == n = d
 			| null d = n
 			| otherwise = n ++ " (" ++ d ++ ")"
-		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
 		prettify m hereu u
 			| not (null d) = fromUUID u ++ " -- " ++ d
diff --git a/debian/changelog b/debian/changelog
index 52d08f3037..12bbba11d4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     in constant space.
   * status: Now displays trusted, untrusted, and semitrusted repositories
     separately.
+  * status: Include all special remotes in the list of repositories.
   * status: Fix --json mode (only the repository lists are currently
     displayed)
   * status: --fast is back

From 1de00df63602bc67a19a72ec2a07821edea6619d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Fri, 18 Nov 2011 21:12:30 +0000
Subject: [PATCH 2546/2835]

---
 doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
index 9d1bbc4e07..46febe72e0 100644
--- a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
+++ b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
@@ -53,3 +53,9 @@ The two files are hardlinked, so it's not taking up extra space, but it would be
 > still finds a file referring to the old key. 
 > 
 > So, slightly surprising, but not a bug. --[[Joey]] [[done]]
+
+>> Thanks for the explanation.  In my real repository, it was a bit trickier:
+>> the migration was commited to `master`, but other *remote* branches still
+>> referenced those keys.  I was just doing a `git pull` from a central repo, but
+>> needed a `git remote update` to remove those references from `remotes/foo/master` too.
+>> --Jim

From 1b90918cec5a649d490a56d6fc8417ca623cbb6a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Nov 2011 17:09:10 -0400
Subject: [PATCH 2547/2835] avoid error message when doing get --from on file
 not present on remote

---
 Command/Get.hs  | 12 +++++++++---
 Command/Move.hs | 14 +++++++++-----
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/Command/Get.hs b/Command/Get.hs
index f7d953bb65..093cd2cc59 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -27,14 +27,20 @@ start numcopies file (key, _) = do
 	if inannex
 		then stop
 		else autoCopies key (<) numcopies $ do
-			showStart "get" file
 			from <- Annex.getState Annex.fromremote
 			case from of
-				Nothing -> next $ perform key
+				Nothing -> go $ perform key
 				Just name -> do
 					-- get --from = copy --from
 					src <- Remote.byName name
-					next $ Command.Move.fromPerform src False key
+					ok <- Command.Move.fromOk src key
+					if ok
+						then go $ Command.Move.fromPerform src False key
+						else stop
+	where
+		go a = do
+			showStart "get" file
+			next a	
 
 perform :: Key -> CommandPerform
 perform key = do
diff --git a/Command/Move.hs b/Command/Move.hs
index 155f4d605f..4c4534c49d 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -117,13 +117,17 @@ fromStart src move file key
 		if ishere then stop else go
 	where
 		go = do
-			u <- getUUID
-			remotes <- Remote.keyPossibilities key
-			if u == Remote.uuid src || not (any (== src) remotes)
-				then stop
-				else do
+			ok <- fromOk src key
+			if ok
+				then do
 					showMoveAction move file
 					next $ fromPerform src move key
+				else stop
+fromOk :: Remote.Remote Annex -> Key -> Annex Bool
+fromOk src key = do
+	u <- getUUID
+	remotes <- Remote.keyPossibilities key
+	return $ u /= Remote.uuid src && any (== src) remotes
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = moveLock move key $ do
 	ishere <- inAnnex key

From 3905053a18011eb92f00754cb1f7e5331370e2ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Nov 2011 15:16:38 -0400
Subject: [PATCH 2548/2835] update comment to explain non-obvious temp file

---
 Command/Migrate.hs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 045b8f9b1d..7a329080f7 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -39,12 +39,16 @@ start b file (key, oldbackend) = do
 upgradableKey :: Key -> Bool
 upgradableKey key = isNothing $ Types.Key.keySize key
 
+{- Store the old backend's key in the new backend
+ - The old backend's key is not dropped from it, because there may
+ - be other files still pointing at that key.
+ -
+ - Use the same filename as the file for the temp file name, to support
+ - backends that allow the filename to influence the keys they
+ - generate.
+ -}
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
-	-- Store the old backend's cached key in the new backend
-	-- (the file can't be stored as usual, because it's already a symlink).
-	-- The old backend's key is not dropped from it, because there may
-	-- be other files still pointing at that key.
 	src <- fromRepo $ gitAnnexLocation oldkey
 	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp  takeFileName file

From 32d9813b1dc50a3300f24f3ddbf534c6cc529d6e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Nov 2011 15:27:10 -0400
Subject: [PATCH 2549/2835] tweak

---
 Command.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index b662171926..4d5bbeb363 100644
--- a/Command.hs
+++ b/Command.hs
@@ -92,11 +92,10 @@ isBareRepo = fromRepo Git.repoIsLocalBare
  - copies of the key is > or < than the numcopies setting, before running
  - the action. -}
 autoCopies :: Key -> (Int -> Int -> Bool) -> Maybe Int -> CommandStart -> CommandStart
-autoCopies key vs numcopiesattr a = do
-	auto <- Annex.getState Annex.auto
-	if auto
-		then do
+autoCopies key vs numcopiesattr a = Annex.getState Annex.auto >>= auto
+	where
+		auto False = a
+		auto True = do
 			needed <- getNumCopies numcopiesattr
 			(_, have) <- trustPartition UnTrusted =<< keyLocations key
 			if length have `vs` needed then a else stop
-		else a

From 0fa1d136dc74828eab820d7c74c811d707cc746a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Nov 2011 15:40:40 -0400
Subject: [PATCH 2550/2835] tweak

---
 Annex/UUID.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 6fc04c0f09..3b64bf3d82 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -39,10 +39,11 @@ genUUID = pOpen ReadFromPipe command params $ liftM toUUID . hGetLine
 			-- uuidgen generates random uuid by default
 			else []
 
+{- Get current repository's UUID. -}
 getUUID :: Annex UUID
 getUUID = getRepoUUID =<< gitRepo
 
-{- Looks up a repo's UUID. May return "" if none is known. -}
+{- Looks up a repo's UUID, caching it in .git/config if it's not already. -}
 getRepoUUID :: Git.Repo -> Annex UUID
 getRepoUUID r = do
 	c <- fromRepo cached
@@ -54,7 +55,7 @@ getRepoUUID r = do
 			return u
 		else return c
 	where
-		cached g = toUUID $ Git.configGet cachekey "" g
+		cached = toUUID . Git.configGet cachekey ""
 		updatecache u = do
 			g <- gitRepo
 			when (g /= r) $ storeUUID cachekey u

From 128b4bd01509bcdcdd6120a29d24527cff82d3ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Nov 2011 15:57:08 -0400
Subject: [PATCH 2551/2835] tweaks

---
 Annex/UUID.hs |  2 +-
 Config.hs     | 10 ++++------
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 3b64bf3d82..e510a7ccd3 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -59,7 +59,7 @@ getRepoUUID r = do
 		updatecache u = do
 			g <- gitRepo
 			when (g /= r) $ storeUUID cachekey u
-		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
+		cachekey = remoteConfig r "uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
 getUncachedUUID = toUUID . Git.configGet configkey ""
diff --git a/Config.hs b/Config.hs
index dbd13ad3fd..cc0c929531 100644
--- a/Config.hs
+++ b/Config.hs
@@ -28,13 +28,13 @@ getConfig r key def = do
 	def' <- fromRepo $ Git.configGet ("annex." ++ key) def
 	fromRepo $ Git.configGet (remoteConfig r key) def'
 
+{- Looks up a per-remote config setting in git config. -}
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 
 {- Calculates cost for a remote. Either the default, or as configured 
  - by remote..annex-cost, or if remote..annex-cost-command
- - is set and prints a number, that is used.
- -}
+ - is set and prints a number, that is used. -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
 	cmd <- getConfig r "cost-command" ""
@@ -55,7 +55,7 @@ semiCheapRemoteCost = 110
 expensiveRemoteCost :: Int
 expensiveRemoteCost = 200
 
-{- Adjust's a remote's cost to reflect it being encrypted. -}
+{- Adjusts a remote's cost to reflect it being encrypted. -}
 encryptedRemoteCostAdj :: Int
 encryptedRemoteCostAdj = 50
 
@@ -74,9 +74,7 @@ prop_cost_sane = False `notElem`
  - setting, or on command-line options. Allows command-line to override
  - annex-ignore. -}
 repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = do
-	ignored <- getConfig r "ignore" "false"
-	return $ not $ Git.configTrue ignored
+repoNotIgnored r = not . Git.configTrue <$> getConfig r "ignore" "false"
 
 {- If a value is specified, it is used; otherwise the default is looked up
  - in git config. forcenumcopies overrides everything. -}

From d675f1c82e7a3c7aa3f0b3f67284433cce111781 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Nov 2011 14:12:48 -0400
Subject: [PATCH 2552/2835] status --json now shows most things

Left out the backend usage graph for now, and bad/temp directory sizes
are only displayed when present. Also, disk usage is returned as a string
with units, which I can see changing later.
---
 Command/Status.hs | 45 ++++++++++++++++++++++++++++-----------------
 Messages.hs       | 24 +++++++++++-------------
 debian/changelog  |  3 +--
 3 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 7448615cdd..a47f21b91a 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -11,6 +11,7 @@ import Control.Monad.State
 import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
+import Text.JSON
 
 import Common.Annex
 import qualified Types.Backend as B
@@ -78,12 +79,21 @@ start = do
 		return True
 	stop
 
-stat :: String -> StatState String -> Stat
-stat desc a = return $ Just (desc, a)
+stat :: String -> (String -> StatState String) -> Stat
+stat desc a = return $ Just (desc, a desc)
 
 nostat :: Stat
 nostat = return Nothing
 
+json :: JSON j => (j -> String) -> StatState j -> String -> StatState String
+json serialize a desc = do
+	j <- a
+	lift $ maybeShowJSON [(desc, j)]
+	return $ serialize j
+
+nojson :: StatState String -> String -> StatState String
+nojson a _ = a
+
 showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
@@ -93,15 +103,15 @@ showStat s = calc =<< s
 		calc Nothing = return ()
 
 supported_backends :: Stat
-supported_backends = stat "supported backends" $ 
-	return $ unwords $ map B.name Backend.list
+supported_backends = stat "supported backends" $ json unwords $
+	return $ map B.name Backend.list
 
 supported_remote_types :: Stat
-supported_remote_types = stat "supported remote types" $
-	return $ unwords $ map R.typename Remote.remoteTypes
+supported_remote_types = stat "supported remote types" $ json unwords $
+	return $ map R.typename Remote.remoteTypes
 
 remote_list :: TrustLevel -> String -> Stat
-remote_list level desc = stat n $ lift $ do
+remote_list level desc = stat n $ nojson $ lift $ do
 	us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap)
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
@@ -110,20 +120,20 @@ remote_list level desc = stat n $ lift $ do
 		n = desc ++ " repositories"
 
 local_annex_size :: Stat
-local_annex_size = stat "local annex size" $
+local_annex_size = stat "local annex size" $ json id $
 	keySizeSum <$> cachedKeysPresent
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" $
-	show . S.size <$> cachedKeysPresent
+local_annex_keys = stat "local annex keys" $ json show $
+	S.size <$> cachedKeysPresent
 
 visible_annex_size :: Stat
-visible_annex_size = stat "visible annex size" $
+visible_annex_size = stat "visible annex size" $ json id $
 	keySizeSum <$> cachedKeysReferenced
 
 visible_annex_keys :: Stat
-visible_annex_keys = stat "visible annex keys" $
-	show . S.size <$> cachedKeysReferenced
+visible_annex_keys = stat "visible annex keys" $ json show $
+	S.size <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -132,7 +142,8 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced <*> cachedKeysPresent
+backend_usage = stat "backend usage" $ nojson $
+	usage <$> cachedKeysReferenced <*> cachedKeysPresent
 	where
 		usage a b = pp "" $ reverse . sort $ map swap $ splits $ S.toList $ S.union a b
 		splits :: [Key] -> [(String, Integer)]
@@ -179,9 +190,9 @@ staleSize label dirspec = do
 	keys <- lift (Command.Unused.staleKeys dirspec)
 	if null keys
 		then nostat
-		else stat label $ do
-			let s = keySizeSum $ S.fromList keys
-			return $ s ++ aside "clean up with git-annex unused"
+		else do
+			stat label $ json (++ aside "clean up with git-annex unused") $
+				return $ keySizeSum $ S.fromList keys
 
 aside :: String -> String
 aside s = " (" ++ s ++ ")"
diff --git a/Messages.hs b/Messages.hs
index 57b7068041..6ea347ca47 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -68,17 +68,17 @@ showEndFail :: Annex ()
 showEndFail = showEndResult False
 
 showEndResult :: Bool -> Annex ()
-showEndResult b = handle (JSON.end b) $ putStrLn msg
+showEndResult ok = handle (JSON.end ok) $ putStrLn msg
 	where
 		msg
-			| b = "ok"
+			| ok = "ok"
 			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
 showErr e = warning' $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
-warning w = warning' (indent w)
+warning = warning' . indent
 
 warning' :: String -> Annex ()
 warning' w = do
@@ -88,7 +88,7 @@ warning' w = do
 		hPutStrLn stderr w
 
 indent :: String -> String
-indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
+indent = join "\n" . map (\l -> "  " ++ l) . lines
 
 {- Shows a JSON value only when in json mode. -}
 maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
@@ -105,9 +105,8 @@ showCustom command a = do
 	handle (JSON.end r) q
 
 showHeader :: String -> Annex ()
-showHeader h = handle q $ do
-	putStr $ h ++ ": "
-	hFlush stdout
+showHeader h = handle q $
+	flushed $ putStr $ h ++ ": "
 
 showRaw :: String -> Annex ()
 showRaw s = handle q $ putStrLn s
@@ -126,12 +125,11 @@ setupConsole = do
 	hSetBinaryMode stderr True
 
 handle :: IO () -> IO () -> Annex ()
-handle json normal = do
-	output <- Annex.getState Annex.output
-	case output of
-		Annex.NormalOutput -> liftIO normal
-		Annex.QuietOutput -> q
-		Annex.JSONOutput -> liftIO json
+handle json normal = Annex.getState Annex.output >>= go
+	where
+		go Annex.NormalOutput = liftIO normal
+		go Annex.QuietOutput = q
+		go Annex.JSONOutput = liftIO json
 
 q :: Monad m => m ()
 q = return ()
diff --git a/debian/changelog b/debian/changelog
index 12bbba11d4..9f96a4fffe 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -11,8 +11,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * status: Now displays trusted, untrusted, and semitrusted repositories
     separately.
   * status: Include all special remotes in the list of repositories.
-  * status: Fix --json mode (only the repository lists are currently
-    displayed)
+  * status: Fix --json mode.
   * status: --fast is back
   * Fix support for insteadOf url remapping. Closes: #644278
   * When not run in a git repository, git-annex can still display a usage

From 2768be71362c97f9c8fed15e2cd9528cceb7a4dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Nov 2011 14:19:03 -0400
Subject: [PATCH 2553/2835] response

---
 doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
index 46febe72e0..9d468bdc7d 100644
--- a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
+++ b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
@@ -59,3 +59,8 @@ The two files are hardlinked, so it's not taking up extra space, but it would be
 >> referenced those keys.  I was just doing a `git pull` from a central repo, but
 >> needed a `git remote update` to remove those references from `remotes/foo/master` too.
 >> --Jim
+
+>>> I have considered making unused ignore remote tracking branches. 
+>>> On the one hand, it can be a little bit confusing, and those branches
+>>> can be out of date. On the other hand, it can be useful to know you're
+>>> not dropping anything that some remote might still refer to. --[[Joey]] 

From 0f0169fa990167aa21d236c5b247ec49998a7e43 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Nov 2011 22:49:53 -0400
Subject: [PATCH 2554/2835] comment update

---
 Command/Move.hs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 4c4534c49d..fd1ed90198 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -23,10 +23,6 @@ def = [dontCheck toOpt $ dontCheck fromOpt $
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed $ start True]
 
-{- Move (or copy) a file either --to or --from a repository.
- -
- - This only operates on the cached file content; it does not involve
- - moving data in the key-value backend. -}
 start :: Bool -> FilePath -> (Key, Backend Annex) -> CommandStart
 start move file (key, _) = do
 	noAuto

From 6c0448d94c2be5e3c0658334ff882452bf4e8000 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Mon, 21 Nov 2011 22:24:04 +0000
Subject: [PATCH 2555/2835]

---
 doc/forum/--print0_option_as_in___34__find__34__.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/--print0_option_as_in___34__find__34__.mdwn

diff --git a/doc/forum/--print0_option_as_in___34__find__34__.mdwn b/doc/forum/--print0_option_as_in___34__find__34__.mdwn
new file mode 100644
index 0000000000..cd537f8adb
--- /dev/null
+++ b/doc/forum/--print0_option_as_in___34__find__34__.mdwn
@@ -0,0 +1,3 @@
+It would be nice if git annex find supported a --print0 option as GNU
+find does.  That way, file names that are printed could be piped to
+xargs even if they have spaces. 

From 35b47069582ff45f7e5bfdd1b43cb22134b0d646 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Tue, 22 Nov 2011 07:09:24 +0000
Subject: [PATCH 2556/2835]

---
 ...tory_remote_and_case_sensitivity_on_FAT.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
new file mode 100644
index 0000000000..d1b21a9e7b
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
@@ -0,0 +1,17 @@
+I was copying files to a directory remote with `git annex copy`.  Out of 114 files, 9 of them failed with no message, just:
+
+    copy data/foo.dat (to usbdrive...) failed
+    copy data/bar.dat (to usbdrive...) failed
+
+According to strace:
+
+    31338 mkdir("/media/annex/Zp/9v/SHA256-s1362999320--d650297c8cf8c2dc0575110a52d0c5cc0ff266f294a0599f85796a6b44b23492", 0777) = -1 ENOENT (No such file or directory)
+    31338 mkdir("/media/annex/Zp/9v", 0777) = -1 ENOENT (No such file or directory)
+    31338 mkdir("/media/annex/Zp", 0777)    = -1 EEXIST (File exists)
+    31338 stat("/media/annex/Zp", 0x7f8449f170d0) = -1 ENOENT (No such file or directory)
+
+The filesystem is FAT32 and has weird case semantics.  This was mounted by udisks with its default options:
+
+    /dev/sdb1 on /media/annex type vfat (rw,nosuid,nodev,uhelper=udisks,uid=1000,gid=1000,shortname=mixed,dmask=0077,utf8=1,showexec)
+
+I wonder if the directory remote should use hashDirLower instead of hashDirMixed?

From fc2f0e8b1a4bd016ac29606381dfb7034c88e9f5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 12:37:51 -0400
Subject: [PATCH 2557/2835] response; cannot reproduce

---
 .../directory_remote_and_case_sensitivity_on_FAT.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
index d1b21a9e7b..b686304e59 100644
--- a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
@@ -15,3 +15,14 @@ The filesystem is FAT32 and has weird case semantics.  This was mounted by udisk
     /dev/sdb1 on /media/annex type vfat (rw,nosuid,nodev,uhelper=udisks,uid=1000,gid=1000,shortname=mixed,dmask=0077,utf8=1,showexec)
 
 I wonder if the directory remote should use hashDirLower instead of hashDirMixed?
+
+> git-annex intentionally uses the same layout for directory and rsync
+> special remotes as it does for the .git/annex directory. As far
+> as I know it works ok on case-insensative filesystems.
+> 
+> Based on your strace, if you `ls /media/annex/Zp`, you will see
+> "No such file or directory", but if you `mkdir /media/annex/Zp` it will
+> fail with "File exists". Doesn't make much sense to me.
+> 
+> I cannot reproduce this problem using a vfat filesystem 
+> mounted using the same options you show (linux 3.0.0). --[[Joey]]

From 7f7ae7a3b1cdfbc61879189dfe04a637690015aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 14:06:31 -0400
Subject: [PATCH 2558/2835] find: Support --print0

It would be nice if command-specific options were supported. The first
difficulty is that which command is being called is not known until after
getopt; but that could be worked around by finding the first non-dashed
parameter. Storing the settings without putting them in the annex monad is
the next difficulty; it could perhaps be handled by making the seek stage
pass applicable settings into the start stage (and from there on to perform
as needed). But that still leaves a problem, what data type to use to
represent the options between getopt and seek?
---
 Annex.hs                                              | 2 ++
 Command/Find.hs                                       | 8 ++++++--
 GitAnnex.hs                                           | 3 +++
 debian/changelog                                      | 1 +
 doc/forum/--print0_option_as_in___34__find__34__.mdwn | 2 ++
 doc/git-annex.mdwn                                    | 3 +++
 6 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 24cc78a647..6d245a92d1 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -60,6 +60,7 @@ data AnnexState = AnnexState
 	, force :: Bool
 	, fast :: Bool
 	, auto :: Bool
+	, print0 :: Bool
 	, branchstate :: BranchState
 	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
@@ -82,6 +83,7 @@ newState gitrepo = AnnexState
 	, force = False
 	, fast = False
 	, auto = False
+	, print0 = False
 	, branchstate = startBranchState
 	, catfilehandle = Nothing
 	, forcebackend = Nothing
diff --git a/Command/Find.hs b/Command/Find.hs
index c816ff0712..47058fa257 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Command
 import Annex.Content
 import Limit
+import qualified Annex
 
 def :: [Command]
 def = [command "find" paramPaths seek "lists available files"]
@@ -22,6 +23,9 @@ start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
-	whenM (liftM2 (||) (inAnnex key) limited) $
-		liftIO $ putStrLn file
+	whenM (liftM2 (||) (inAnnex key) limited) $ do
+		print0 <- Annex.getState Annex.print0
+		if print0
+			then liftIO $ putStr (file ++ "\0")
+			else liftIO $ putStrLn file
 	stop
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 7b51602be6..f563c08cbf 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -103,6 +103,8 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
+	, Option [] ["print0"] (NoArg (setprint0 True))
+		"terminate filename with null"
 	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
 	, Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote)
@@ -114,6 +116,7 @@ options = commonOptions ++
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
+		setprint0 v = Annex.changeState $ \s -> s { Annex.print0 = v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
 			newg <- inRepo $ Git.configStore v
diff --git a/debian/changelog b/debian/changelog
index 9f96a4fffe..3c4fa05aac 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,6 +19,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * migrate: Don't fall over a stale temp file.
   * Avoid excessive escaping for rsync special remotes that are not accessed
     over ssh.
+  * find: Support --print0
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/forum/--print0_option_as_in___34__find__34__.mdwn b/doc/forum/--print0_option_as_in___34__find__34__.mdwn
index cd537f8adb..7d9a2284dd 100644
--- a/doc/forum/--print0_option_as_in___34__find__34__.mdwn
+++ b/doc/forum/--print0_option_as_in___34__find__34__.mdwn
@@ -1,3 +1,5 @@
 It would be nice if git annex find supported a --print0 option as GNU
 find does.  That way, file names that are printed could be piped to
 xargs even if they have spaces. 
+
+> Done. --[[Joey]] 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e91e5a0e33..c225529de3 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -232,6 +232,9 @@ subdirectories).
   With no parameters, defaults to finding all files in the current directory
   and its subdirectories.
 
+  To output filenames terminated with nulls, for use with xargs -0,
+  specify --print0.
+
 * whereis [path ...]
 
   Displays a list of repositories known to contain the content of the

From 322d9b1cc044c960339779b9d88aa39e21f41d9e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 14:40:11 -0400
Subject: [PATCH 2559/2835] releasing version 3.20111122

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 3c4fa05aac..185ffecebf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111112) UNRELEASED; urgency=low
+git-annex (3.20111122) unstable; urgency=low
 
   * merge: Improve commit messages to mention what was merged.
   * Avoid doing auto-merging in commands that don't need fully current
@@ -21,7 +21,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     over ssh.
   * find: Support --print0
 
- -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
+ -- Joey Hess   Tue, 22 Nov 2011 14:31:45 -0400
 
 git-annex (3.20111111) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index b2638c3ac7..f31c7a86e8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111111
+Version: 3.20111122
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 2d9099531b7c8d237e4fee1327f1a6955412e70b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 14:41:52 -0400
Subject: [PATCH 2560/2835] add news item for git-annex 3.20111122

---
 doc/news/version_3.20111011.mdwn | 30 ------------------------------
 doc/news/version_3.20111122.mdwn | 22 ++++++++++++++++++++++
 2 files changed, 22 insertions(+), 30 deletions(-)
 delete mode 100644 doc/news/version_3.20111011.mdwn
 create mode 100644 doc/news/version_3.20111122.mdwn

diff --git a/doc/news/version_3.20111011.mdwn b/doc/news/version_3.20111011.mdwn
deleted file mode 100644
index 0346cbca82..0000000000
--- a/doc/news/version_3.20111011.mdwn
+++ /dev/null
@@ -1,30 +0,0 @@
-git-annex 3.20111011 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * This version of git-annex only works with git 1.7.7 and newer.
-     The breakage with old versions is subtle, and affects the
-     annex.numcopies settings in .gitattributes, so be sure to upgrade git
-     to 1.7.7. (Debian package now depends on that version.)
-   * Don't pass absolute paths to git show-attr, as it started following
-     symlinks when that's done in 1.7.7. Instead, use relative paths,
-     which show-attr only handles 100% correctly in 1.7.7. Closes: #[645046](http://bugs.debian.org/645046)
-   * Fix referring to remotes by uuid.
-   * New or changed repository descriptions in uuid.log now have a timestamp,
-     which is used to ensure the newest description is used when the uuid.log
-     has been merged.
-   * Note that older versions of git-annex will display the timestamp as part
-     of the repository description, which is ugly but otherwise harmless.
-   * Add timestamps to trust.log and remote.log too.
-   * git-annex-shell: Added the --uuid option.
-   * git-annex now asks git-annex-shell to verify that it's operating in
-     the expected repository.
-   * Note that this git-annex will not interoperate with remotes using
-     older versions of git-annex-shell.
-   * Now supports git's insteadOf configuration, to modify the url
-     used to access a remote. Note that pushInsteadOf is not used;
-     that and pushurl are reserved for actual git pushes. Closes: #[644278](http://bugs.debian.org/644278)
-   * status: List all known repositories.
-   * When displaying a list of repositories, show git remote names
-     in addition to their descriptions.
-   * Add locking to avoid races when changing the git-annex branch.
-   * Various speed improvements gained by using ByteStrings.
-   * Contain the zombie hordes."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111122.mdwn b/doc/news/version_3.20111122.mdwn
new file mode 100644
index 0000000000..193394de98
--- /dev/null
+++ b/doc/news/version_3.20111122.mdwn
@@ -0,0 +1,22 @@
+git-annex 3.20111122 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * merge: Improve commit messages to mention what was merged.
+   * Avoid doing auto-merging in commands that don't need fully current
+     information from the git-annex branch. In particular, git annex add
+     no longer needs to auto-merge.
+   * init: When run in an already initalized repository, and without
+     a description specified, don't delete the old description.
+   * Optimised union merging; now only runs git cat-file once, and runs
+     in constant space.
+   * status: Now displays trusted, untrusted, and semitrusted repositories
+     separately.
+   * status: Include all special remotes in the list of repositories.
+   * status: Fix --json mode.
+   * status: --fast is back
+   * Fix support for insteadOf url remapping. Closes: #[644278](http://bugs.debian.org/644278)
+   * When not run in a git repository, git-annex can still display a usage
+     message, and "git annex version" even works.
+   * migrate: Don't fall over a stale temp file.
+   * Avoid excessive escaping for rsync special remotes that are not accessed
+     over ssh.
+   * find: Support --print0"""]]
\ No newline at end of file

From b1c601ac8cfd2d30ee8299a508d36f4ea5c1cc1a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Tue, 22 Nov 2011 18:51:04 +0000
Subject: [PATCH 2561/2835] Added a comment: Case sensitivity

---
 ..._bcac9fd7b3f4a2ac28bee59bae674fa0._comment | 79 +++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment
new file mode 100644
index 0000000000..be8b8b0a72
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment
@@ -0,0 +1,79 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="Case sensitivity"
+ date="2011-11-22T18:51:03Z"
+ content="""
+I agree, it's weird, but that's what I'm seeing:
+
+    #!/bin/sh
+    
+    if [ $UID != 0 ] ; then echo \"need root\" ; exit 1 ; fi
+    
+    set -x
+    
+    # make image
+    cd /tmp
+    dd if=/dev/zero of=diskimage bs=1M count=40
+    DEV=$(losetup --find --show diskimage)
+    
+    # make FAT32 fs
+    mkfs.vfat -F 32 $DEV
+    
+    # mount it
+    mkdir annex
+    mount -o shortname=mixed,utf8=1 $DEV annex
+    
+    # show bug
+    ( 
+        cd annex
+        mkdir zP
+        mkdir Zp
+        ls Zp
+        ls
+        touch zP
+        touch Zp
+    )
+    
+    # cleanup
+    umount annex
+    rm -r annex
+    losetup -d $DEV
+    rm diskimage
+    
+    # info
+    uname -a
+
+Output:
+
+    + cd /tmp
+    + dd if=/dev/zero of=diskimage bs=1M count=40
+    40+0 records in
+    40+0 records out
+    41943040 bytes (42 MB) copied, 0.0847729 s, 495 MB/s
+    ++ losetup --find --show diskimage
+    + DEV=/dev/loop0
+    + mkfs.vfat -F 32 /dev/loop0
+    mkfs.vfat 3.0.9 (31 Jan 2010)
+    Loop device does not match a floppy size, using default hd params
+    + mkdir annex
+    + mount -o shortname=mixed,utf8=1 /dev/loop0 annex
+    + cd annex
+    + mkdir zP
+    + mkdir Zp
+    mkdir: cannot create directory `Zp': File exists
+    + ls Zp
+    ls: cannot access Zp: No such file or directory
+    + ls
+    zP
+    + touch zP
+    + touch Zp
+    touch: cannot touch `Zp': File exists
+    + umount annex
+    + rm -r annex
+    + losetup -d /dev/loop0
+    + rm diskimage
+    + uname -a
+    Linux pilot 3.0.3+ #1 SMP Mon Aug 29 15:21:18 EDT 2011 x86_64 GNU/Linux
+
+"""]]

From eeda534e9a0ff81e7498a69dd79577865deb76e9 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 22 Nov 2011 19:56:55 +0000
Subject: [PATCH 2562/2835] Added a comment

---
 ...2_c9088060fb9133b66951f1a3075981e8._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment
new file mode 100644
index 0000000000..5040b3120f
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-11-22T19:56:55Z"
+ content="""
+All right, I see the same thing with linux 3.1.0. It seems this behavior has changed since linux 3.0.0. Mounting with shortname=lower avoids the problem. 
+
+I feel a good case could be made that this new behavior is a linux bug. Your example with touch particularly shows how weird it is.
+
+
+$ touch Foo
+$ echo hi > foo
+sh: cannot create foo: File exists
+$ rm foo
+rm: cannot remove `foo': No such file or directory
+

+"""]]

From 6d7e990dcaf4f327fd957196ff71d1ee966ae490 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Tue, 22 Nov 2011 20:35:03 +0000
Subject: [PATCH 2563/2835] Added a comment

---
 ..._5bf34466187cfc9b34bd3ca8c89a07c6._comment | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment
new file mode 100644
index 0000000000..54d6ff50ab
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 3"
+ date="2011-11-22T20:35:01Z"
+ content="""
+I see the same results (\"`touch: cannot touch 'Zp': File exists`\") on these Debian systems:
+
+    Linux pilot 3.0.3+ #1 SMP Mon Aug 29 15:21:18 EDT 2011 x86_64 GNU/Linux
+    Linux neurosis 3.0.0-1-amd64 #1 SMP Sun Jul 24 02:24:44 UTC 2011 x86_64 GNU/Linux
+    Linux bucket 2.6.39-2-amd64 #1 SMP Tue Jul 5 02:51:22 UTC 2011 x86_64 GNU/Linux
+    Linux psychosis 2.6.37-trunk-amd64 #1 SMP Thu Jan 6 14:13:28 UTC 2011 x86_64 GNU/Linux
+    Linux bacon 2.6.32-5-amd64 #1 SMP Thu Aug 12 13:01:50 UTC 2010 x86_64 GNU/Linux
+
+It does NOT happen on this Ubuntu system:
+
+    Linux esensor 3.0.0-12-generic #20-Ubuntu SMP Fri Oct 7 14:56:25 UTC 2011 x86_64 x86_64 x86_64 GNU/Linux
+
+So really it seems like only the Ubuntu kernel is the outlier here?  Maybe it has something to do with charsets or something; I think FAT is a mess in that regard and even long versus short filenames can behave differently.
+"""]]

From 112399bf1a924e91473aa4e6d0cf84061a3bd482 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 22 Nov 2011 20:59:55 +0000
Subject: [PATCH 2564/2835] Added a comment

---
 .../comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment
new file mode 100644
index 0000000000..406a6b18ee
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-11-22T20:59:55Z"
+ content="""
+Your ubuntu system has 3.0.0 which as noted does not have the problem.
+"""]]

From fd81b5047b314ecc96b34e0c9f017f8f5a62f072 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 22 Nov 2011 21:01:14 +0000
Subject: [PATCH 2565/2835] Added a comment

---
 .../comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment
new file mode 100644
index 0000000000..1656ff2075
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-11-22T21:01:14Z"
+ content="""
+I am surprised if it happens on 2.6.x though. Debian 3.0.0 seemed to not have the problem but perhaps my test was bad.
+"""]]

From 75a590bdd893e579c5e375e5ad797022f5847496 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 18:20:55 -0400
Subject: [PATCH 2566/2835] Put a workaround in the directory special remote
 for strange behavior with VFAT filesystems on Linux (mounted with
 shortname=mixed)

---
 Remote/Directory.hs                           | 73 +++++++++++++------
 debian/changelog                              |  7 ++
 ...ry_remote_and_case_sensitivity_on_FAT.mdwn | 23 +++++-
 3 files changed, 76 insertions(+), 27 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index b592f41ff0..cadd5e7597 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -62,56 +62,81 @@ directorySetup u c = do
 	gitConfigSpecialRemote u c' "directory" dir
 	return $ M.delete "directory" c'
 
-dirKey :: FilePath -> Key -> FilePath
-dirKey d k = d  hashDirMixed k  f  f
+{- Where to store a given Key in the Directory.
+ -
+ - There are two possible locations to try; this had to be done because
+ - on Linux, vfat filesystem mounted with shortname=mixed have a
+ - variant of case insensativity that causes miserable failure when
+ - hashDirMixed produces eg, "xx" and "XX". The first directory to be
+ - created wins the namespace, and the second one cannot then be created.
+ - But unlike behavior with shortname=lower, "XX/foo" won't look in
+ - "xx/foo".
+ -}
+locations :: FilePath -> Key -> [FilePath]
+locations d k = [using hashDirMixed, using hashDirLower]
 	where
+		using h = d  h k  f  f
 		f = keyFile k
 
+withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
+withCheckedFile _ [] _ _ = return False
+withCheckedFile check d k a = go $ locations d k
+	where
+		go [] = return False
+		go (f:fs) = do
+			use <- check f
+			if use
+				then a f
+				else go fs
+
+withStoredFile :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
+withStoredFile = withCheckedFile doesFileExist
+
 store :: FilePath -> Key -> Annex Bool
 store d k = do
 	src <- fromRepo $ gitAnnexLocation k
-	let dest = dirKey d k
-	liftIO $ catchBoolIO $ storeHelper dest $ copyFileExternal src dest
+	liftIO $ catchBoolIO $ storeHelper d k $ copyFileExternal src
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
 	src <- fromRepo $ gitAnnexLocation k
-	let dest = dirKey d enck
-	liftIO $ catchBoolIO $ storeHelper dest $ encrypt src dest
+	liftIO $ catchBoolIO $ storeHelper d enck $ encrypt src
 	where
 		encrypt src dest = do
 			withEncryptedContent cipher (L.readFile src) $ L.writeFile dest
 			return True
 
-storeHelper :: FilePath -> IO Bool -> IO Bool
-storeHelper dest a = do
-	let dir = parentDir dest
-	createDirectoryIfMissing True dir
-	allowWrite dir	
-	ok <- a
-	when ok $ do
-		preventWrite dest
-		preventWrite dir
-	return ok
+storeHelper :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
+storeHelper d key a = withCheckedFile check d key go
+	where
+		check dest = isJust <$> mkdir (parentDir dest)
+		mkdir = catchMaybeIO . createDirectoryIfMissing True
+		go dest = do
+			let dir = parentDir dest
+			allowWrite dir
+			ok <- a dest
+			when ok $ do
+				preventWrite dest
+				preventWrite dir
+			return ok
 
 retrieve :: FilePath -> Key -> FilePath -> Annex Bool
-retrieve d k f = liftIO $ copyFileExternal (dirKey d k) f
+retrieve d k f = liftIO $ withStoredFile d k $ \file -> copyFileExternal file f
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =
-	liftIO $ catchBoolIO $ do
-		withDecryptedContent cipher (L.readFile (dirKey d enck)) $ L.writeFile f
+	liftIO $ withStoredFile d enck $ \file -> catchBoolIO $ do
+		withDecryptedContent cipher (L.readFile file) $ L.writeFile f
 		return True
 
 remove :: FilePath -> Key -> Annex Bool
-remove d k = liftIO $ catchBoolIO $ do
+remove d k = liftIO $ withStoredFile d k $ \file -> catchBoolIO $ do
+	let dir = parentDir file
 	allowWrite dir
 	removeFile file
 	removeDirectory dir
 	return True
-	where
-		file = dirKey d k
-		dir = parentDir file
 
 checkPresent :: FilePath -> Key -> Annex (Either String Bool)
-checkPresent d k = liftIO $ catchMsgIO $ doesFileExist (dirKey d k)
+checkPresent d k = liftIO $ catchMsgIO $ withStoredFile d k $
+	const $ return True -- withStoredFile checked that it exists
diff --git a/debian/changelog b/debian/changelog
index 185ffecebf..e043f6c9fa 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20111123) UNRELEASED; urgency=low
+
+  * Put a workaround in the directory special remote for strange behavior
+    with VFAT filesystems on Linux (mounted with shortname=mixed)
+
+ -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
+
 git-annex (3.20111122) unstable; urgency=low
 
   * merge: Improve commit messages to mention what was merged.
diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
index b686304e59..ae653d6197 100644
--- a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
@@ -18,11 +18,28 @@ I wonder if the directory remote should use hashDirLower instead of hashDirMixed
 
 > git-annex intentionally uses the same layout for directory and rsync
 > special remotes as it does for the .git/annex directory. As far
-> as I know it works ok on case-insensative filesystems.
+> as I know it works ok on (truely) case-insensative filesystems.
 > 
 > Based on your strace, if you `ls /media/annex/Zp`, you will see
 > "No such file or directory", but if you `mkdir /media/annex/Zp` it will
 > fail with "File exists". Doesn't make much sense to me.
 > 
-> I cannot reproduce this problem using a vfat filesystem 
-> mounted using the same options you show (linux 3.0.0). --[[Joey]]
+> The (default) VFAT mount option shortname=mixed causes this behavior.
+> With shortname=lower it works ok. --[[Joey]]
+> 
+>> So, the options for fixing this bug seem to be to fix Linux (which would
+>> be a good idea IMHO but I don't really want to go there), or generally
+>> convert git-annex to using lowercase for its hashing (which would be a
+>> large amount of pain to rewrite all the symlinks in every git repo),
+>> or some special hack around this problem. 
+>> 
+>> I've put in a workaround for the problem in the directory special
+>> remote; it will use mixed case but fall-back to lowercase as necessary.
+>> 
+>> That does leave the case of a bare git repository with annexed content
+>> stored on VFAT. More special casing could fix it, but that is, I
+>> think, an unusual configuration. Leaving the bug open for that case,
+>> and for the even more unlikely configuration of a rsync special remote
+>> stored on VFAT. --[[Joey]] 
+
+[[!meta title="bare git repository not supported on VFAT"]]

From 3dd66fd2f03fa1bffa2368496ef6b8a9793f881e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 22:13:11 -0400
Subject: [PATCH 2567/2835] update url

---
 doc/install/FreeBSD.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/FreeBSD.mdwn b/doc/install/FreeBSD.mdwn
index 23152ac204..72b402c380 100644
--- a/doc/install/FreeBSD.mdwn
+++ b/doc/install/FreeBSD.mdwn
@@ -1,2 +1,2 @@
 git-annex is in FreeBSD ports in
-[devel/git-annex](http://www.freshports.org/devel/git-annex/)
+[devel/git-annex](http://www.freshports.org/devel/hs-git-annex/)

From 709acf3f240171fa1e3782c8e0362ff4f3239bfb Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Thu, 24 Nov 2011 01:59:45 +0000
Subject: [PATCH 2568/2835]

---
 doc/forum/Podcast_syncing_use-case.mdwn | 34 +++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/forum/Podcast_syncing_use-case.mdwn

diff --git a/doc/forum/Podcast_syncing_use-case.mdwn b/doc/forum/Podcast_syncing_use-case.mdwn
new file mode 100644
index 0000000000..6b3c81cabc
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case.mdwn
@@ -0,0 +1,34 @@
+I've been trying to use git-annex with the following strategy.
+
+* Download podcasts into the annex `gpodder-downloads`
+* Check the podcasts into the annex using `git annex add`.
+* Copy the podcasts over to my mp3 player in the annex `usb-ariaz`.
+  This is a FAT-formatted mp3 player, so I have been using a bare
+  repository.  
+* Move the podcasts to a different annex called `gpodder-on-usbdisk`
+  to indicate that they have been successfully put on the mp3 player.
+* `chmod` the files on the mp3 player to `0600` so that I can delete
+  them from the player when I am done listening to them.
+  
+Then I go for a run or something and listen to a bunch of podcasts,
+deleting them after I have listened to them.  When I get back, I would
+like to find the files that I have listened to and remove them from
+the annexes that are not on the mp3 player.  What I have been hoping
+is that something like
+
+    ~/gpodder-on-usbdisk $ git annex find --not --in usb-ariaz --print0 | xargs -0 git rm
+    ~/gpodder-on-usbdisk $ git annex unused
+    ~/gpodder-on-usbdisk $ git annex dropunused `seq X`
+	
+would work.  However, it appears that `git-annex find` does not
+actually check to see that the file contents are present, but only
+looks at the `git-annex` branch of the `usb-ariaz` repository.  Since
+I have not changed that with my sneaky deletions, it has no way of
+knowing that the files have been deleted.
+
+Is there any way to do this properly?  (And by properly, I don't mean
+"don't delete the files".  That is really the only way I have of
+marking that I have listened to podcasts on this particular mp3 player.)
+
+I tried setting the `usb-ariaz` repository to be untrusted, but that
+did not change the behavior of `git annex find`.

From ba1ec2c60f26bfb39ea69e4e62a09bb85e19cb4a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Fri, 25 Nov 2011 05:56:59 +0000
Subject: [PATCH 2569/2835]

---
 ...9__t_handle_double_spaces_in_filename.mdwn | 83 +++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn

diff --git a/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
new file mode 100644
index 0000000000..930e23ea9d
--- /dev/null
+++ b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
@@ -0,0 +1,83 @@
+Unused files with double spaces in their name are not removed by `dropunused`:
+
+Script:
+
+    #!/bin/bash
+    
+    BASE=/tmp/unused-bug
+    
+    # setup
+    set -x
+    chmod -R +w $BASE
+    rm -rf $BASE
+    mkdir -p $BASE
+    cd $BASE
+    
+    # create annex
+    git init .
+    git annex init
+    
+    # make a file with two spaces
+    echo hello > 'foo  bar'
+    
+    # add it
+    git annex add --backend WORM 'foo  bar'
+    git commit -m 'add'
+    
+    # remove it
+    git rm 'foo  bar'
+    git commit -m 'remove'
+    
+    # unused
+    git annex unused
+    git annex dropunused 1
+    git annex unused
+
+Output:
+
+    + chmod -R +w /tmp/unused-bug
+    + rm -rf /tmp/unused-bug
+    + mkdir -p /tmp/unused-bug
+    + cd /tmp/unused-bug
+    + git init .
+    Initialized empty Git repository in /tmp/unused-bug/.git/
+    + git annex init
+    init  ok
+    + echo hello
+    + git annex add --backend WORM 'foo  bar'
+    add foo  bar ok
+    (Recording state in git...)
+    + git commit -m add
+    [master (root-commit) 926f7f5] add
+     1 files changed, 1 insertions(+), 0 deletions(-)
+     create mode 120000 foo  bar
+    + git rm 'foo  bar'
+    rm 'foo  bar'
+    + git commit -m remove
+    [master d025e3f] remove
+     1 files changed, 0 insertions(+), 1 deletions(-)
+     delete mode 120000 foo  bar
+    + git annex unused
+    unused . (checking for unused data...) (checking master...) 
+      Some annexed data is no longer used by any files:
+        NUMBER  KEY
+        1       WORM-s6-m1322200438--foo  bar
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      
+      To remove unwanted data: git-annex dropunused NUMBER
+      
+    ok
+    + git annex dropunused 1
+    dropunused 1 ok
+    + git annex unused
+    unused . (checking for unused data...) (checking master...) 
+      Some annexed data is no longer used by any files:
+        NUMBER  KEY
+        1       WORM-s6-m1322200438--foo  bar
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      
+      To remove unwanted data: git-annex dropunused NUMBER
+      
+    ok
+
+Strange that `dropunused` still said "ok" when it didn't succeed at removing the file.

From 12243d2279cae65ed163bdec1edb268817514265 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 25 Nov 2011 11:46:07 -0400
Subject: [PATCH 2570/2835] Flush json output, avoiding a buffering problem
 that could result in doubled output.

The bug was that with --json, output lines were sometimes doubled. For
example, git annex init --json would output two lines, despite only running
one thing. Adding to the weirdness, this only occurred when the output
was redirected to a pipe or a file.

Strace showed two processes outputting the same buffered output.
The second process was this writer process (only needed to work around
bug #624389):

                _ <- forkProcess $ do
                        hPutStr toh $ unlines paths
                        hClose toh
                        exitSuccess

The doubled output occurs when this process exits, and ghc flushes the
inherited stdout buffer. Why only when piping? I don't know, but ghc may
be behaving differently when stdout is not a terminal.

While this is quite possibly a ghc bug, there is a nice fix in git-annex.
Explicitly flushing after each chunk of json is output works around the
problem, and as a side effect, json is streamed rather than being output
all at the end when performing an expensive operaition.

However, note that this means all uses of putStr in git-annex must be
explicitly flushed. The others were, already.
---
 Messages.hs      | 2 +-
 debian/changelog | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Messages.hs b/Messages.hs
index 6ea347ca47..a7f14f4853 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -129,7 +129,7 @@ handle json normal = Annex.getState Annex.output >>= go
 	where
 		go Annex.NormalOutput = liftIO normal
 		go Annex.QuietOutput = q
-		go Annex.JSONOutput = liftIO json
+		go Annex.JSONOutput = liftIO $ flushed $ json
 
 q :: Monad m => m ()
 q = return ()
diff --git a/debian/changelog b/debian/changelog
index e043f6c9fa..cbf69b7410 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (3.20111123) UNRELEASED; urgency=low
 
   * Put a workaround in the directory special remote for strange behavior
     with VFAT filesystems on Linux (mounted with shortname=mixed)
+  * Flush json output, avoiding a buffering problem that could result in
+    doubled output.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 

From afe9e78401f7842a41bc353b27036baa2e3046c2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Nov 2011 12:03:01 -0400
Subject: [PATCH 2571/2835] error handling cleanup

Use Control.Exception bracket_; want to catch all errors.

System.Posix.Env.getEnv doesn't fail, no need to try it.
---
 Git.hs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/Git.hs b/Git.hs
index ba5d831fea..5bdd4afd4e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -72,7 +72,7 @@ module Git (
 
 import System.Posix.Directory
 import System.Posix.User
-import IO (bracket_, try)
+import Control.Exception (bracket_)
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
 import Data.Char
@@ -438,12 +438,12 @@ reap = do
  - index file. -}
 useIndex :: FilePath -> IO (IO ())
 useIndex index = do
-	res <- try $ getEnv var
+	res <- getEnv var
 	setEnv var index True
 	return $ reset res
 	where
 		var = "GIT_INDEX_FILE"
-		reset (Right (Just v)) = setEnv var v True
+		reset (Just v) = setEnv var v True
 		reset _ = unsetEnv var
 
 {- Runs an action that causes a git subcommand to emit a sha, and strips
@@ -484,10 +484,8 @@ configRead repo@(Repo { location = Dir d }) = do
 	{- Cannot use pipeRead because it relies on the config having
 	   been already read. Instead, chdir to the repo. -}
 	cwd <- getCurrentDirectory
-	bracket_ (changeWorkingDirectory d)
-		(\_ -> changeWorkingDirectory cwd) $
-			pOpen ReadFromPipe "git" ["config", "--list"] $
-				hConfigRead repo
+	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
+		pOpen ReadFromPipe "git" ["config", "--list"] $ hConfigRead repo
 configRead r = assertLocal r $ error "internal"
 
 {- Reads git config from a handle and populates a repo with it. -}

From 041d32412535ecaba103806f95a7d74234034416 Mon Sep 17 00:00:00 2001
From: Mark Wright 
Date: Sat, 26 Nov 2011 23:39:47 +1100
Subject: [PATCH 2572/2835] Remove haskell98 to build with ghc 7.2.2, also
 built with ghc 7.0.4

Signed-off-by: Joey Hess 
---
 Locations.hs        | 4 ++--
 Logs/Location.hs    | 2 ++
 Utility/TempFile.hs | 2 +-
 git-annex.cabal     | 2 +-
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 83897f488f..3cb632aae1 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -26,8 +26,8 @@ module Locations (
 	prop_idempotent_fileKey
 ) where
 
-import Bits
-import Word
+import Data.Bits
+import Data.Word
 import Data.Hash.MD5
 
 import Common
diff --git a/Logs/Location.hs b/Logs/Location.hs
index ab29ffcadd..cb21a2d1c0 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -1,3 +1,5 @@
+{-# LANGUAGE BangPatterns #-}
+
 {- git-annex location log
  -
  - git-annex keeps track of which repositories have the contents of annexed
diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs
index 8d50dd8b2c..3887b422b6 100644
--- a/Utility/TempFile.hs
+++ b/Utility/TempFile.hs
@@ -7,7 +7,7 @@
 
 module Utility.TempFile where
 
-import IO (bracket)
+import Control.Exception (bracket)
 import System.IO
 import System.Posix.Process hiding (executeFile)
 import System.Directory
diff --git a/git-annex.cabal b/git-annex.cabal
index f31c7a86e8..c257eaff63 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -28,7 +28,7 @@ Description:
 
 Executable git-annex
   Main-Is: git-annex.hs
-  Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
+  Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
    base < 5, monad-control, json

From a72f0ecc273a28bb4b12d8454e2e0f99278012ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Nov 2011 12:06:03 -0400
Subject: [PATCH 2573/2835] changelog

---
 debian/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/changelog b/debian/changelog
index cbf69b7410..35f3a7c8d7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20111123) UNRELEASED; urgency=low
     with VFAT filesystems on Linux (mounted with shortname=mixed)
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
+  * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 

From 9a67f9cb8d0ce6da314b080228ac2cea55ee612a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Nov 2011 12:08:54 -0400
Subject: [PATCH 2574/2835] use Control.Exception's brackets

---
 test.hs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/test.hs b/test.hs
index d4c1366d09..e625fbd756 100644
--- a/test.hs
+++ b/test.hs
@@ -11,7 +11,7 @@ import Test.QuickCheck
 
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
-import IO (bracket_, bracket)
+import Control.Exception (bracket_, bracket)
 import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
@@ -523,8 +523,7 @@ indir dir a = do
 	-- Assertion failures throw non-IO errors; catch
 	-- any type of error and change back to cwd before
 	-- rethrowing.
-	r <- bracket_ (changeToTmpDir dir)
-		(\_ -> changeWorkingDirectory cwd)
+	r <- bracket_ (changeToTmpDir dir) (changeWorkingDirectory cwd)
 		(E.try (a)::IO (Either E.SomeException ()))
 	case r of
 		Right () -> return ()

From 2bf3addf4997023584e32812a9d8cbc46833d672 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Nov 2011 13:50:05 -0400
Subject: [PATCH 2575/2835] Bugfix: dropunused did not drop keys with two
 spaces in their name.

---
 Command/DropUnused.hs                                         | 4 ++--
 debian/changelog                                              | 1 +
 ...punused_doesn__39__t_handle_double_spaces_in_filename.mdwn | 4 ++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 2c3bb296ad..3df9ab6c2b 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -73,6 +73,6 @@ readUnusedLog prefix = do
 		then M.fromList . map parse . lines <$> liftIO (readFile f)
 		else return M.empty
 	where
-		parse line = (head ws, fromJust $ readKey $ unwords $ tail ws)
+		parse line = (num, fromJust $ readKey $ tail rest)
 			where
-				ws = words line
+				(num, rest) = break (== ' ') line
diff --git a/debian/changelog b/debian/changelog
index 35f3a7c8d7..943d1e01cd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20111123) UNRELEASED; urgency=low
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
+  * Bugfix: dropunused did not drop keys with two spaces in their name.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 
diff --git a/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
index 930e23ea9d..a6b44cd2a3 100644
--- a/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
+++ b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
@@ -81,3 +81,7 @@ Output:
     ok
 
 Strange that `dropunused` still said "ok" when it didn't succeed at removing the file.
+
+> It was misparsing the unused file, so it thought you'd asked it to drop a
+> key that didn't exist (which means already dropped) so no error. I've
+> fixed the bug. [[done]] --[[Joey]] 

From a3f4ea1a6916ccaab8be26591bc9fc424649f692 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 27 Nov 2011 17:56:31 +0000
Subject: [PATCH 2576/2835] Added a comment

---
 ...comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment

diff --git a/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment b/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment
new file mode 100644
index 0000000000..fe396c39f3
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-11-27T17:56:31Z"
+ content="""
+Right, --in goes by git-annex's [[location_tracking]] information; actually checking if a remote still has the files would make --in too expensive in many cases.
+
+So you need to give `gpodder-on-usbdisk` current information. You can do that by going to `usb-ariaz` and doing a `git annex fsck`. That will find the deleted files and update the location information. Then, back on `gpodder-on-usbdisk`, `git pull usb-ariaz`, and then you can proceed with the commands you showed.
+"""]]

From faf55ac2b702de1e4f2f308f87bb22284a2bc128 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Nov 2011 13:57:32 -0400
Subject: [PATCH 2577/2835] update

---
 doc/location_tracking.mdwn | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn
index 85bb3d1b55..d40a7206fd 100644
--- a/doc/location_tracking.mdwn
+++ b/doc/location_tracking.mdwn
@@ -1,8 +1,7 @@
 git-annex keeps track of in which repositories it last saw a file's content.
 This location tracking information is stored in the git-annex branch.
 Repositories record their UUID and the date when they get or drop 
-a file's content. (Git is configured to use a union merge for this file,
-so the lines may be in arbitrary order, but it will never conflict.)
+a file's content.
 
 This location tracking information is useful if you have multiple
 repositories, and not all are always accessible. For example, perhaps one
@@ -10,7 +9,7 @@ is on a home file server, and you are away from home. Then git-annex can
 tell you what git remote it needs access to in order to get a file:
 
 	# git annex get myfile 
-	get myfile(not available)
+	get myfile (not available)
 	  I was unable to access these remotes: home
 
 Another way the location tracking comes in handy is if you put repositories

From 178a49af75166999c690fd92a02d74d2fbe47270 Mon Sep 17 00:00:00 2001
From: "http://hcs.furuvik.net/" 
Date: Sun, 27 Nov 2011 20:52:15 +0000
Subject: [PATCH 2578/2835]

---
 ...le_to_have_annex_on_a_separate_filesystem.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn

diff --git a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
new file mode 100644
index 0000000000..3e05b60931
--- /dev/null
+++ b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
@@ -0,0 +1,15 @@
+I belive I have found a regression.
+
+Inspired by 
+
+I tried to only have .git/annex/objects (also tested moving .git/annex) on NFS while having the rest on local SSD disk.
+
+But when trying to add files i get:
+
+     > git annex add testfile
+    add testfile (checksum...) 
+    git-annex: testfile: rename: unsupported operation (Invalid cross-device link)
+    failed
+    git-annex: add: 1 failed
+
+I have tried both using bind-mount and with a sym-link.

From ff2d9c828379ce29e5feb6ac770996be04ac072f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Nov 2011 17:09:29 -0400
Subject: [PATCH 2579/2835] response

---
 ...ossible_to_have_annex_on_a_separate_filesystem.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
index 3e05b60931..e9a3ee95a4 100644
--- a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
+++ b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
@@ -13,3 +13,13 @@ But when trying to add files i get:
     git-annex: add: 1 failed
 
 I have tried both using bind-mount and with a sym-link.
+
+> I don't think this was a reversion; the forum post doesn't really
+> indicate it ever worked.
+> 
+> Grepping for `renameFile` and `createLink` will find all the places
+> in git-annex that assume one filesystem. These would have to be changed
+> to catch errors and fall back to expensive copying.
+> 
+> Putting a separate repository on the file server could work better
+> depending on what you're trying to do. --[[Joey]] 

From d9e770a836ffefa7749ddfba06e54301f34cec2e Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Sun, 27 Nov 2011 22:10:45 +0000
Subject: [PATCH 2580/2835] Added a comment

---
 .../comment_2_930a6620b4d516e69ed952f9da5371bb._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment

diff --git a/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment b/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment
new file mode 100644
index 0000000000..97eb3c681c
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 2"
+ date="2011-11-27T22:10:44Z"
+ content="""
+Thanks, that works perfectly!
+"""]]

From 6869e6023e21698038da7e4a858cbaf6f7b7bbed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 15:26:27 -0400
Subject: [PATCH 2581/2835] support .git/annex on a different disk than the
 rest of the repo

The only fully supported thing is to have the main repository on one disk,
and .git/annex on another. Only commands that move data in/out of the annex
will need to copy it across devices.

There is only partial support for putting arbitrary subdirectories of
.git/annex on different devices. For one thing, but this can require more
copies to be done. For example, when .git/annex/tmp is on one device, and
.git/annex/journal on another, every journal write involves a call to
mv(1). Also, there are a few places that make hard links between various
subdirectories of .git/annex with createLink, that are not handled.

In the common case without cross-device, the new moveFile is actually
faster than renameFile, avoiding an unncessary stat to check that a file
(not a directory) is being moved. Of course if a cross-device move is
needed, it is as slow as mv(1) of the data.
---
 Annex/Branch.hs                               |  2 +-
 Annex/Content.hs                              |  8 +--
 Command/Add.hs                                |  2 +-
 Command/Unlock.hs                             |  2 +-
 Common.hs                                     |  1 +
 Utility/Directory.hs                          | 51 +++++++++++++++++++
 debian/changelog                              |  2 +
 ...o_have_annex_on_a_separate_filesystem.mdwn | 13 +++--
 ...it_on_ssd__44___annex_on_spindle_disk.mdwn | 11 +++-
 9 files changed, 81 insertions(+), 11 deletions(-)
 create mode 100644 Utility/Directory.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index ccc6145552..a92f05b2cc 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -312,7 +312,7 @@ setJournalFile file content = do
 			let jfile = journalFile g file
 			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
 			writeBinaryFile tmpfile content
-			renameFile tmpfile jfile
+			moveFile tmpfile jfile
 
 {- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 83839ea135..f5571b54af 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -113,7 +113,7 @@ logStatus key status = do
 	u <- getUUID
 	logChange key u status
 
-{- Runs an action, passing it a temporary filename to download,
+{- Runs an action, passing it a temporary filename to get,
  - and if the action succeeds, moves the temp file into 
  - the annex as a key's content. -}
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
@@ -221,7 +221,7 @@ moveAnnex key src = do
 		else liftIO $ do
 			createDirectoryIfMissing True dir
 			allowWrite dir -- in case the directory already exists
-			renameFile src dest
+			moveFile src dest
 			preventWrite dest
 			preventWrite dir
 
@@ -243,7 +243,7 @@ fromAnnex :: Key -> FilePath -> Annex ()
 fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
 	allowWrite dir
 	allowWrite file
-	renameFile file dest
+	moveFile file dest
 	removeDirectory dir
 
 {- Moves a key out of .git/annex/objects/ into .git/annex/bad, and
@@ -256,7 +256,7 @@ moveBad key = do
 	liftIO $ do
 		createDirectoryIfMissing True (parentDir dest)
 		allowWrite (parentDir src)
-		renameFile src dest
+		moveFile src dest
 		removeDirectory (parentDir src)
 	logStatus key InfoMissing
 	return dest
diff --git a/Command/Add.hs b/Command/Add.hs
index ab104b53cc..130f5e3110 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -61,7 +61,7 @@ undo file key e = do
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do
 			src <- fromRepo $ gitAnnexLocation key
-			liftIO $ renameFile src file
+			liftIO $ moveFile src file
 
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
 cleanup file key hascontent = do
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 22f9ce7108..b6f39488da 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -46,7 +46,7 @@ perform dest key = do
                 then do
 			liftIO $ do
 				removeFile dest
-				renameFile tmpdest dest
+				moveFile tmpdest dest
 				allowWrite dest
 			next $ return True
                 else error "copy failed!"
diff --git a/Common.hs b/Common.hs
index e0132d9e96..a3802da5f2 100644
--- a/Common.hs
+++ b/Common.hs
@@ -23,3 +23,4 @@ import Utility.Misc as X
 import Utility.Conditional as X
 import Utility.SafeCommand as X
 import Utility.Path as X
+import Utility.Directory as X
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
new file mode 100644
index 0000000000..7f8822fca5
--- /dev/null
+++ b/Utility/Directory.hs
@@ -0,0 +1,51 @@
+{- directory manipulation
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Directory where
+
+import System.IO.Error
+import System.Posix.Files
+import System.Directory
+import Control.Exception (throw)
+
+import Utility.SafeCommand
+import Utility.Conditional
+import Utility.TempFile
+
+{- Moves one filename to another.
+ - First tries a rename, but falls back to moving across devices if needed. -}
+moveFile :: FilePath -> FilePath -> IO ()
+moveFile src dest = try (rename src dest) >>= onrename
+	where
+		onrename (Right _) = return ()
+		onrename (Left e)
+			| isPermissionError e = rethrow
+			| isDoesNotExistError e = rethrow
+			| otherwise = do
+				-- copyFile is likely not as optimised as
+				-- the mv command, so we'll use the latter.
+				-- But, mv will move into a directory if
+				-- dest is one, which is not desired.
+				whenM (isdir dest) rethrow
+				viaTmp mv dest undefined
+			where
+				rethrow = throw e
+				mv tmp _ = do
+					ok <- boolSystem "mv" [Param "-f",
+						Param src, Param tmp]
+					if ok
+						then return ()
+						else do
+							-- delete any partial
+							_ <- try $
+								removeFile tmp
+							rethrow
+		isdir f = do
+			r <- try (getFileStatus f)
+			case r of
+				(Left _) -> return False
+				(Right s) -> return $ isDirectory s
diff --git a/debian/changelog b/debian/changelog
index 943d1e01cd..265ba71846 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (3.20111123) UNRELEASED; urgency=low
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
   * Bugfix: dropunused did not drop keys with two spaces in their name.
+  * Support for storing .git/annex on a different device than the rest of the
+    git repository.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 
diff --git a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
index e9a3ee95a4..7daf03284b 100644
--- a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
+++ b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
@@ -14,12 +14,19 @@ But when trying to add files i get:
 
 I have tried both using bind-mount and with a sym-link.
 
-> I don't think this was a reversion; the forum post doesn't really
-> indicate it ever worked.
-> 
 > Grepping for `renameFile` and `createLink` will find all the places
 > in git-annex that assume one filesystem. These would have to be changed
 > to catch errors and fall back to expensive copying.
 > 
 > Putting a separate repository on the file server could work better
 > depending on what you're trying to do. --[[Joey]] 
+
+>> I've added support for putting `.git/annex` on a separate filesystem
+>> from the rest of the git repository.
+>> 
+>> Putting individual subdirectories like `.git/annex/objects` on separate
+>> filesystems from other subdirectories is not fully supported; it may
+>> work but it may be slow and a few things (like `git annex migrate`) are
+>> known to fail due to using hard links. I don't think this is worth
+>> supporting. [[done]]
+>> --[[Joey]]
diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn
index a04c8b040b..f70c127025 100644
--- a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn
+++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn
@@ -1,3 +1,12 @@
 This works with bind-mount, I might try with softlinks as well.
 
-Going through git's data on push/pull can take ages on a spindle disk even if the repo is rather small in size. This is especially true if you are used to ssd speeds, but ssd storage is expensive. Storing the annex objects on a cheap spindle disk and everything else on a ssd makes things a _lot_ faster.
+Going through git's data on push/pull can take ages on a spindle disk even
+if the repo is rather small in size. This is especially true if you are
+used to ssd speeds, but ssd storage is expensive. Storing the annex objects
+on a cheap spindle disk and everything else on a ssd makes things a _lot_
+faster.
+
+> Update: git-annex supports `.git/annex/` being moved to a different disk
+> than the rest of the repisitory, but does *not* support individual 
+> subdirectories, like `.git/annex/objects/` being on a different disk
+> than the main `.git/annex/` directory. --[[Joey]]

From e32ab766b08b9a64bf12fe5ab6bf33f2d82b0fb7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 17:37:15 -0400
Subject: [PATCH 2582/2835] --inbackend can be used to make git-annex only
 operate on files whose content is stored using a specified key-value backend.

---
 GitAnnex.hs        | 2 ++
 Limit.hs           | 7 +++++++
 debian/changelog   | 2 ++
 doc/git-annex.mdwn | 5 +++++
 4 files changed, 16 insertions(+)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index f563c08cbf..42a6b7fd78 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -111,6 +111,8 @@ options = commonOptions ++
 		"skip files not present in a remote"
 	, Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber)
 		"skip files with fewer copies"
+	, Option ['B'] ["inbackend"] (ReqArg Limit.addInBackend paramName)
+		"skip files not using a key-value backend"
 	] ++ matcherOptions
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
diff --git a/Limit.hs b/Limit.hs
index 3ae949bfb9..c68c3bdd8b 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -88,3 +88,10 @@ addCopies num =
 		handle n (Just (key, _)) = do
 			us <- keyLocations key
 			return $ length us >= n
+
+{- Adds a limit to skip files not using a specified key-value backend. -}
+addInBackend :: String -> Annex ()
+addInBackend name = addLimit $ Backend.lookupFile >=> check
+	where
+		wanted = Backend.lookupBackendName name
+		check = return . maybe False ((==) wanted . snd)
diff --git a/debian/changelog b/debian/changelog
index 265ba71846..9cd915885c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,8 @@ git-annex (3.20111123) UNRELEASED; urgency=low
   * Bugfix: dropunused did not drop keys with two spaces in their name.
   * Support for storing .git/annex on a different device than the rest of the
     git repository.
+  * --inbackend can be used to make git-annex only operate on files
+    whose content is stored using a specified key-value backend.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index c225529de3..cedb2e782c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -446,6 +446,11 @@ file contents are present at either of two repositories.
   Matches only files that git-annex believes to have the specified number
   of copies, or more.
 
+* --inbackend=name
+
+  Matches only files whose content is stored using the specified key-value
+  backend.
+
 * --not
 
   Inverts the next file matching option. For example, to only act on

From 2b3c120506f1f25b4c3d0e19342b9826bde0b3b5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 17:49:03 -0400
Subject: [PATCH 2583/2835] clarify extent of limit checks

---
 doc/git-annex.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index cedb2e782c..e98c89fe39 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -435,7 +435,8 @@ file contents are present at either of two repositories.
 * --in=repository
 
   Matches only files that git-annex believes have their contents present
-  in a repository.
+  in a repository. Note that it does not check the repository to verify
+  that it still has the content.
 
   The repository should be specified using the name of a configured remote,
   or the UUID or description of a repository. For the current repository,
@@ -444,7 +445,8 @@ file contents are present at either of two repositories.
 * --copies=number
 
   Matches only files that git-annex believes to have the specified number
-  of copies, or more.
+  of copies, or more. Note that it does not check remotes to verify that
+  the copies still exist.
 
 * --inbackend=name
 

From da9cd315beb03570b96f83063a39e799fe01b166 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 22:43:51 -0400
Subject: [PATCH 2584/2835] add support for using hashDirLower in addition to
 hashDirMixed

Supporting multiple directory hash types will allow converting to a
different one, without a flag day.

gitAnnexLocation now checks which of the possible locations have a file.
This means more statting of files. Several places currently use
gitAnnexLocation and immediately check if the returned file exists;
those need to be optimised.
---
 Annex/Content.hs    | 17 ++++++++---------
 Backend/SHA.hs      |  2 +-
 Command/Add.hs      |  2 +-
 Command/Fsck.hs     |  4 ++--
 Command/Migrate.hs  |  2 +-
 Command/SendKey.hs  |  2 +-
 Command/Unannex.hs  |  2 +-
 Command/Unlock.hs   |  2 +-
 Locations.hs        | 35 +++++++++++++++++++++++++----------
 Remote/Bup.hs       |  4 ++--
 Remote/Directory.hs |  4 ++--
 Remote/Git.hs       | 29 ++++++++++++++++++++++-------
 Remote/Hook.hs      |  4 ++--
 Remote/Rsync.hs     |  4 ++--
 Remote/S3real.hs    |  4 ++--
 15 files changed, 73 insertions(+), 44 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index f5571b54af..90bde2975a 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -43,12 +43,12 @@ import Annex.Exception
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
-inAnnex = inAnnex' doesFileExist
+inAnnex = inAnnex' $ doesFileExist
 inAnnex' :: (FilePath -> IO a) -> Key -> Annex a
 inAnnex' a key = do
 	whenM (fromRepo Git.repoIsUrl) $
 		error "inAnnex cannot check remote repo"
-	inRepo $ a . gitAnnexLocation key
+	inRepo $ \g -> gitAnnexLocation key g >>= a
 
 {- A safer check; the key's content must not only be present, but
  - is not in the process of being removed. -}
@@ -70,7 +70,7 @@ inAnnexSafe = inAnnex' $ \f -> openForLock f False >>= check
  - it. (If the content is not present, no locking is done.) -}
 lockContent :: Key -> Annex a -> Annex a
 lockContent key a = do
-	file <- fromRepo $ gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	bracketIO (openForLock file True >>= lock) unlock a
 	where
 		lock Nothing = return Nothing
@@ -100,9 +100,8 @@ calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
 	cwd <- liftIO getCurrentDirectory
 	let absfile = fromMaybe whoops $ absNormPath cwd file
-	top <- fromRepo Git.workTree
-	return $ relPathDirToFile (parentDir absfile) 
-			top  ".git"  annexLocation key
+	loc <- inRepo $ gitAnnexLocation key
+	return $ relPathDirToFile (parentDir absfile) loc
 	where
 		whoops = error $ "unable to normalize " ++ file
 
@@ -213,7 +212,7 @@ checkDiskSpace' adjustment key = do
  -}
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
-	dest <- fromRepo $ gitAnnexLocation key
+	dest <- inRepo $ gitAnnexLocation key
 	let dir = parentDir dest
 	e <- liftIO $ doesFileExist dest
 	if e
@@ -227,7 +226,7 @@ moveAnnex key src = do
 
 withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
 withObjectLoc key a = do
-	file <- fromRepo $gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	let dir = parentDir file
 	a (dir, file)
 
@@ -250,7 +249,7 @@ fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
  - returns the file it was moved to. -}
 moveBad :: Key -> Annex FilePath
 moveBad key = do
-	src <- fromRepo $ gitAnnexLocation key
+	src <- inRepo $ gitAnnexLocation key
 	bad <- fromRepo gitAnnexBadDir
 	let dest = bad  takeFileName src
 	liftIO $ do
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index a3846a410b..2ae0cfcf42 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -99,7 +99,7 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
 	fast <- Annex.getState Annex.fast
-	file <- fromRepo $ gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	if not present || fast
 		then return True
diff --git a/Command/Add.hs b/Command/Add.hs
index 130f5e3110..9fdbdcaa69 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -60,7 +60,7 @@ undo file key e = do
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do
-			src <- fromRepo $ gitAnnexLocation key
+			src <- inRepo $ gitAnnexLocation key
 			liftIO $ moveFile src file
 
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 99dda99e5f..a803207e20 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -87,7 +87,7 @@ verifyLocationLog key desc = do
 	-- Since we're checking that a key's file is present, throw
 	-- in a permission fixup here too.
 	when present $ do
-		f <- fromRepo $ gitAnnexLocation key
+		f <- inRepo $ gitAnnexLocation key
 		liftIO $ do
 			preventWrite f
 			preventWrite (parentDir f)
@@ -118,7 +118,7 @@ verifyLocationLog key desc = do
  - the key's metadata, if available. -}
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
-	file <- fromRepo $ gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	case (present, Types.Key.keySize key) of
 		(_, Nothing) -> return True
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 7a329080f7..c85d7c2ac3 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -49,7 +49,7 @@ upgradableKey key = isNothing $ Types.Key.keySize key
  -}
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
-	src <- fromRepo $ gitAnnexLocation oldkey
+	src <- inRepo $ gitAnnexLocation oldkey
 	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp  takeFileName file
 	cleantmp tmpfile
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 5737478671..7b1cd3ecae 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -21,7 +21,7 @@ seek = [withKeys start]
 
 start :: Key -> CommandStart
 start key = do
-	file <- fromRepo $ gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	whenM (inAnnex key) $
 		liftIO $ rsyncServerSend file -- does not return
 	warning "requested key is not present"
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index b9190ce044..e97b6d05d8 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -55,7 +55,7 @@ cleanup file key = do
 	if fast
 		then do
 			-- fast mode: hard link to content in annex
-			src <- fromRepo $ gitAnnexLocation key
+			src <- inRepo $ gitAnnexLocation key
 			liftIO $ do
 				createLink src file
 				allowWrite file
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index b6f39488da..673a7038a0 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -37,7 +37,7 @@ perform dest key = do
 	
 	checkDiskSpace key
 
-	src <- fromRepo $ gitAnnexLocation key
+	src <- inRepo $ gitAnnexLocation key
 	tmpdest <- fromRepo $ gitAnnexTmpLocation key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showAction "copying"
diff --git a/Locations.hs b/Locations.hs
index 3cb632aae1..425e4fdcf9 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -9,7 +9,7 @@ module Locations (
 	keyFile,
 	fileKey,
 	gitAnnexLocation,
-	annexLocation,
+	annexLocations,
 	gitAnnexDir,
 	gitAnnexObjectDir,
 	gitAnnexTmpDir,
@@ -58,17 +58,33 @@ annexDir = addTrailingPathSeparator "annex"
 objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
-{- Annexed file's location relative to the .git directory. -}
-annexLocation :: Key -> FilePath
-annexLocation key = objectDir  hashDirMixed key  f  f
+{- Annexed file's possible locations relative to the .git directory.
+ - There are two different possibilities, using different hashes;
+ - the first is the default for new content. -}
+annexLocations :: Key -> [FilePath]
+annexLocations key = [using hashDirMixed, using hashDirLower]
 	where
+		using h = objectDir  h key  f  f
 		f = keyFile key
 
-{- Annexed file's absolute location in a repository. -}
-gitAnnexLocation :: Key -> Git.Repo -> FilePath
+{- Annexed file's absolute location in a repository.
+ - Out of the possible annexLocations, returns the one where the file
+ - is actually present. When the file is not present, returns the
+ - one where the file should be put.
+ -}
+gitAnnexLocation :: Key -> Git.Repo -> IO FilePath
 gitAnnexLocation key r
-	| Git.repoIsLocalBare r = Git.workTree r  annexLocation key
-	| otherwise = Git.workTree r  ".git"  annexLocation key
+	| Git.repoIsLocalBare r =
+		go (Git.workTree r) $ annexLocations key
+	| otherwise =
+		go (Git.workTree r  ".git") $ annexLocations key
+	where
+		go dir locs = fromMaybe (dir  head locs) <$> check dir locs
+		check _ [] = return Nothing
+		check dir (l:ls) = do
+			let f = dir  l
+			e <- doesFileExist f
+			if e then return (Just f) else check dir ls
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
@@ -76,8 +92,7 @@ gitAnnexDir r
 	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  annexDir
 	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  annexDir
 
-{- The part of the annex directory where file contents are stored.
- -}
+{- The part of the annex directory where file contents are stored. -}
 gitAnnexObjectDir :: Git.Repo -> FilePath
 gitAnnexObjectDir r
 	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  objectDir
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 4c826498da..589dea91d9 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -102,13 +102,13 @@ bupSplitParams r buprepo k src = do
 
 store :: Git.Repo -> BupRepo -> Key -> Annex Bool
 store r buprepo k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo k (File src)
 	liftIO $ boolSystem "bup" params
 
 storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r buprepo (cipher, enck) k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo enck (Param "-")
 	liftIO $ catchBoolIO $
 		withEncryptedHandle cipher (L.readFile src) $ \h ->
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index cadd5e7597..891a19ef6b 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -94,12 +94,12 @@ withStoredFile = withCheckedFile doesFileExist
 
 store :: FilePath -> Key -> Annex Bool
 store d k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	liftIO $ catchBoolIO $ storeHelper d k $ copyFileExternal src
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	liftIO $ catchBoolIO $ storeHelper d enck $ encrypt src
 	where
 		encrypt src dest = do
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 541d8e5f65..07afc02742 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -134,7 +134,14 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = checklocal
 	where
-		checkhttp = liftIO $ catchMsgIO $ Url.exists $ keyUrl r key
+		checkhttp = liftIO $ go undefined $ keyUrls r key
+			where
+				go e [] = return $ Left e
+				go _ (u:us) = do
+					res <- catchMsgIO $ Url.exists u
+					case res of
+						Left e -> go e us
+						v -> return v
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			onRemote r (check, unknown) "inannex" [Param (show key)]
@@ -169,8 +176,10 @@ onLocal r a = do
 		liftIO Git.reap
 		return ret
 
-keyUrl :: Git.Repo -> Key -> String
-keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
+keyUrls :: Git.Repo -> Key -> [String]
+keyUrls r key = map tourl (annexLocations key)
+	where
+		tourl l = Git.repoLocation r ++ "/" ++ l
 
 dropKey :: Git.Repo -> Key -> Annex Bool
 dropKey r key
@@ -185,16 +194,22 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = do
 		params <- rsyncParams r
-		rsyncOrCopyFile params (gitAnnexLocation key r) file
+		loc <- liftIO $ gitAnnexLocation key r
+		rsyncOrCopyFile params loc file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| Git.repoIsHttp r = liftIO $ Url.download (keyUrl r key) file
+	| Git.repoIsHttp r = liftIO $ downloadurls $ keyUrls r key
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
+	where
+		downloadurls [] = return False
+		downloadurls (u:us) = do
+			ok <- Url.download u file
+			if ok then return ok else downloadurls us
 
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
 	| not $ Git.repoIsUrl r = do
-		keysrc <- fromRepo $ gitAnnexLocation key
+		keysrc <- inRepo $ gitAnnexLocation key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
@@ -203,7 +218,7 @@ copyToRemote r key
 			Annex.Content.saveState
 			return ok
 	| Git.repoIsSsh r = do
-		keysrc <- fromRepo $ gitAnnexLocation key
+		keysrc <- inRepo $ gitAnnexLocation key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 03976fc709..ab84533b28 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -97,12 +97,12 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h
 
 store :: String -> Key -> Annex Bool
 store h k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	runHook h "store" k (Just src) $ return True
 
 storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	runHook h "store" enck (Just tmp) $ return True
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 5cd27a6099..836b93b314 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -95,11 +95,11 @@ rsyncKeyDir :: RsyncOpts -> Key -> String
 rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (keyFile k)
 
 store :: RsyncOpts -> Key -> Annex Bool
-store o k = rsyncSend o k =<< fromRepo (gitAnnexLocation k)
+store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
 
 storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	rsyncSend o enck tmp
 
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 97ac648218..b79939b902 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -112,7 +112,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 store :: Remote Annex -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
-	dest <- fromRepo $ gitAnnexLocation k
+	dest <- inRepo $ gitAnnexLocation k
 	res <- liftIO $ storeHelper (conn, bucket) r k dest
 	s3Bool res
 
@@ -121,7 +121,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) ->
 	-- To get file size of the encrypted content, have to use a temp file.
 	-- (An alternative would be chunking to to a constant size.)
 	withTmp enck $ \tmp -> do
-		f <- fromRepo $ gitAnnexLocation k
+		f <- inRepo $ gitAnnexLocation k
 		liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 		res <- liftIO $ storeHelper (conn, bucket) r enck tmp
 		s3Bool res

From f4bf444ae0933f80b4ec1849ea1c44da43008499 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 22:47:11 -0400
Subject: [PATCH 2585/2835] store content in hashDirLower directories in bare
 repositories

When storing content in bare repositories, use the hashDirLower
directories. Bare repositories can be on USB drives, which might
use the FAT filesystem, and fall afoul of recent bugs in linux's handling
of mixed case on FAT. Using hashDirLower avoids that.
---
 Locations.hs                                              | 5 ++++-
 debian/changelog                                          | 8 ++++++--
 ...nsitivity_on_FAT.mdwn => case_sensitivity_on_FAT.mdwn} | 3 ++-
 3 files changed, 12 insertions(+), 4 deletions(-)
 rename doc/bugs/{directory_remote_and_case_sensitivity_on_FAT.mdwn => case_sensitivity_on_FAT.mdwn} (95%)

diff --git a/Locations.hs b/Locations.hs
index 425e4fdcf9..fb5ee517ee 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -75,7 +75,10 @@ annexLocations key = [using hashDirMixed, using hashDirLower]
 gitAnnexLocation :: Key -> Git.Repo -> IO FilePath
 gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
-		go (Git.workTree r) $ annexLocations key
+		-- bare repositories default to hashDirLower for new
+		-- content, as it's more portable, so check locations
+		-- in reverse order
+		go (Git.workTree r) $ reverse $ annexLocations key
 	| otherwise =
 		go (Git.workTree r  ".git") $ annexLocations key
 	where
diff --git a/debian/changelog b/debian/changelog
index 9cd915885c..8da74af2f7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,11 @@
 git-annex (3.20111123) UNRELEASED; urgency=low
 
-  * Put a workaround in the directory special remote for strange behavior
-    with VFAT filesystems on Linux (mounted with shortname=mixed)
+  * The VFAT filesystem on recent versions of Linux, when mounted with
+    shortname=mixed, does not get along well with git-annex's mixed case
+    .git/annex/objects hash directories. To better support it, bare
+    repositories (and the directory special remote) now store content
+    in all-lowercase hash directories. Mixed case hash directories are
+    still used for non-bare directories, which cannot be put in VFAT.
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn b/doc/bugs/case_sensitivity_on_FAT.mdwn
similarity index 95%
rename from doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
rename to doc/bugs/case_sensitivity_on_FAT.mdwn
index ae653d6197..cb3424e34d 100644
--- a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/case_sensitivity_on_FAT.mdwn
@@ -42,4 +42,5 @@ I wonder if the directory remote should use hashDirLower instead of hashDirMixed
 >> and for the even more unlikely configuration of a rsync special remote
 >> stored on VFAT. --[[Joey]] 
 
-[[!meta title="bare git repository not supported on VFAT"]]
+>>> Bare repositories now use lowercase. rsync is the only remaining
+>>> unsupported possibility. --[[Joey]]

From e6ef66cea39b8c97a1e5115251e5ed0163c66fb3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 23:08:11 -0400
Subject: [PATCH 2586/2835] optimize gitAnnexLocation

For non-bare it's back to doing no work.
---
 Locations.hs | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index fb5ee517ee..a7a1f0cd1f 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -59,28 +59,35 @@ objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
 {- Annexed file's possible locations relative to the .git directory.
- - There are two different possibilities, using different hashes;
- - the first is the default for new content. -}
+ - There are two different possibilities, using different hashes. -}
 annexLocations :: Key -> [FilePath]
-annexLocations key = [using hashDirMixed, using hashDirLower]
+annexLocations key = map (annexLocation key) [hashDirMixed, hashDirLower]
+annexLocation :: Key -> (Key -> FilePath) -> FilePath
+annexLocation key hasher = objectDir  hasher key  f  f
 	where
-		using h = objectDir  h key  f  f
 		f = keyFile key
 
 {- Annexed file's absolute location in a repository.
- - Out of the possible annexLocations, returns the one where the file
- - is actually present. When the file is not present, returns the
- - one where the file should be put.
+ -
+ - When there are multiple possible locations, returns the one where the
+ - file is actually present.
+ -
+ - When the file is not present, returns the location where the file should
+ - be stored.
  -}
 gitAnnexLocation :: Key -> Git.Repo -> IO FilePath
 gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
-		-- bare repositories default to hashDirLower for new
-		-- content, as it's more portable, so check locations
-		-- in reverse order
-		go (Git.workTree r) $ reverse $ annexLocations key
+		{- Bare repositories default to hashDirLower for new
+		 - content, as it's more portable. -}
+		go (Git.workTree r) $
+			map (annexLocation key) [hashDirLower, hashDirMixed]
 	| otherwise =
-		go (Git.workTree r  ".git") $ annexLocations key
+		{- Non-bare repositories only use hashDirMixed, so
+		 - don't need to do any work to check if the file is
+		 - present. -}
+		return $ Git.workTree r  ".git" 
+				annexLocation key hashDirMixed
 	where
 		go dir locs = fromMaybe (dir  head locs) <$> check dir locs
 		check _ [] = return Nothing

From bff6ca2634e5f796d83c1632a66ac00adab0be43 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 23:20:31 -0400
Subject: [PATCH 2587/2835] refactor

---
 Locations.hs        | 11 ++++++++++-
 Remote/Directory.hs | 14 ++------------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index a7a1f0cd1f..53a80043ac 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,6 +20,7 @@ module Locations (
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
 	isLinkToAnnex,
+	annexHashes,
 	hashDirMixed,
 	hashDirLower,
 
@@ -58,10 +59,18 @@ annexDir = addTrailingPathSeparator "annex"
 objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
+{- Two different directory hashes may be used. The mixed case hash
+ - came first, and is fine, except for the problem of case-strict
+ - filesystems such as Linux VFAT (mounted with shortname=mixed),
+ - which do not allow using a directory "XX" when "xx" already exists.
+ - To support that, some repositories will use a lower case hash. -}
+annexHashes :: [Key -> FilePath]
+annexHashes = [hashDirMixed, hashDirLower]
+
 {- Annexed file's possible locations relative to the .git directory.
  - There are two different possibilities, using different hashes. -}
 annexLocations :: Key -> [FilePath]
-annexLocations key = map (annexLocation key) [hashDirMixed, hashDirLower]
+annexLocations key = map (annexLocation key) annexHashes
 annexLocation :: Key -> (Key -> FilePath) -> FilePath
 annexLocation key hasher = objectDir  hasher key  f  f
 	where
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 891a19ef6b..83302b65a5 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -62,20 +62,10 @@ directorySetup u c = do
 	gitConfigSpecialRemote u c' "directory" dir
 	return $ M.delete "directory" c'
 
-{- Where to store a given Key in the Directory.
- -
- - There are two possible locations to try; this had to be done because
- - on Linux, vfat filesystem mounted with shortname=mixed have a
- - variant of case insensativity that causes miserable failure when
- - hashDirMixed produces eg, "xx" and "XX". The first directory to be
- - created wins the namespace, and the second one cannot then be created.
- - But unlike behavior with shortname=lower, "XX/foo" won't look in
- - "xx/foo".
- -}
+{- Locations to try to access a given Key in the Directory. -}
 locations :: FilePath -> Key -> [FilePath]
-locations d k = [using hashDirMixed, using hashDirLower]
+locations d k = map (\h -> d  h k  f  f) annexHashes
 	where
-		using h = d  h k  f  f
 		f = keyFile k
 
 withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool

From 998d8f796857132688c6d8a539aaf18135439205 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 23:23:14 -0400
Subject: [PATCH 2588/2835] clarify

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 8da74af2f7..816a571e38 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,7 +3,7 @@ git-annex (3.20111123) UNRELEASED; urgency=low
   * The VFAT filesystem on recent versions of Linux, when mounted with
     shortname=mixed, does not get along well with git-annex's mixed case
     .git/annex/objects hash directories. To better support it, bare
-    repositories (and the directory special remote) now store content
+    repositories (and the directory special remote) now store new content
     in all-lowercase hash directories. Mixed case hash directories are
     still used for non-bare directories, which cannot be put in VFAT.
   * Flush json output, avoiding a buffering problem that could result in

From 598eb2e2daac5d2338b9788814e45bb5b2c16e61 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Nov 2011 12:01:15 -0400
Subject: [PATCH 2589/2835] cleanup

---
 Annex/Branch.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a92f05b2cc..a890668810 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -186,7 +186,7 @@ tryFastForwardTo :: [Git.Ref] -> Annex Bool
 tryFastForwardTo [] = return True
 tryFastForwardTo (first:rest) = do
 	-- First, check that the git-annex branch does not contain any
-	-- new commits that are in the first other branch. If it does,
+	-- new commits that are not in the first other branch. If it does,
 	-- cannot fast-forward.
 	diverged <- changedBranch first fullname
 	if diverged
@@ -195,7 +195,8 @@ tryFastForwardTo (first:rest) = do
 	where
 		no_ff = return False
 		do_ff branch = do
-			inRepo $ Git.run "update-ref" [Param $ show fullname, Param $ show branch]
+			inRepo $ Git.run "update-ref"
+				[Param $ show fullname, Param $ show branch]
 			return True
 		findbest c [] = return $ Just c
 		findbest c (r:rs)

From 3953f7ab81f4edd540ab66fe25ebec56dc9c7ebc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Dec 2011 17:40:26 -0400
Subject: [PATCH 2590/2835] idea

---
 doc/todo/add_-all_option.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/todo/add_-all_option.mdwn

diff --git a/doc/todo/add_-all_option.mdwn b/doc/todo/add_-all_option.mdwn
new file mode 100644
index 0000000000..cb4722fa78
--- /dev/null
+++ b/doc/todo/add_-all_option.mdwn
@@ -0,0 +1,16 @@
+`--all` would make git-annex operate on either every key with content
+present (or in some cases like `get` and `copy --from` on 
+every keys with content not present).
+
+This would be useful when a repository has a history with deleted files
+whose content you want to keep (so you're not using `dropunused`).
+Or when you have a lot of branches and just want to be able to fsck
+every file referenced in any branch.
+
+A problem with the idea is that `.gitattributes` values for keys not
+currently in the tree would not be available (without horrific anounts of
+grubbing thru history to find where/when the key used to exist). So
+`numcopies` set via `.gitattributes` would not work. This would be a
+particular problem for `drop` and for `--auto`.
+
+--[[Joey]]

From 97f809c0069c7e7e107f10dab614e3f765255abe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 14:18:55 -0400
Subject: [PATCH 2591/2835] wording

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 816a571e38..45088db522 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,10 +2,10 @@ git-annex (3.20111123) UNRELEASED; urgency=low
 
   * The VFAT filesystem on recent versions of Linux, when mounted with
     shortname=mixed, does not get along well with git-annex's mixed case
-    .git/annex/objects hash directories. To better support it, bare
+    .git/annex/objects hash directories. To avoid this problem, bare
     repositories (and the directory special remote) now store new content
     in all-lowercase hash directories. Mixed case hash directories are
-    still used for non-bare directories, which cannot be put in VFAT.
+    still used for non-bare repositories, which cannot be put on FAT.
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.

From 0815cc2fc1ffccd89bb942a9129a2c29e291b038 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 14:39:47 -0400
Subject: [PATCH 2592/2835] refactor

---
 Locations.hs        | 52 ++++++++++++++++++++++++++++-----------------
 Remote/Directory.hs |  4 +---
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 53a80043ac..1b5f8108df 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -1,6 +1,6 @@
 {- git-annex file locations
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -8,6 +8,7 @@
 module Locations (
 	keyFile,
 	fileKey,
+	keyPaths,
 	gitAnnexLocation,
 	annexLocations,
 	gitAnnexDir,
@@ -59,22 +60,12 @@ annexDir = addTrailingPathSeparator "annex"
 objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
-{- Two different directory hashes may be used. The mixed case hash
- - came first, and is fine, except for the problem of case-strict
- - filesystems such as Linux VFAT (mounted with shortname=mixed),
- - which do not allow using a directory "XX" when "xx" already exists.
- - To support that, some repositories will use a lower case hash. -}
-annexHashes :: [Key -> FilePath]
-annexHashes = [hashDirMixed, hashDirLower]
-
 {- Annexed file's possible locations relative to the .git directory.
  - There are two different possibilities, using different hashes. -}
 annexLocations :: Key -> [FilePath]
 annexLocations key = map (annexLocation key) annexHashes
-annexLocation :: Key -> (Key -> FilePath) -> FilePath
-annexLocation key hasher = objectDir  hasher key  f  f
-	where
-		f = keyFile key
+annexLocation :: Key -> Hasher -> FilePath
+annexLocation key hasher = objectDir  keyPath key hasher
 
 {- Annexed file's absolute location in a repository.
  -
@@ -150,7 +141,7 @@ gitAnnexJournalLock r = gitAnnexDir r  "journal.lck"
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
-{- Converts a key into a filename fragment.
+{- Converts a key into a filename fragment without any directory.
  -
  - Escape "/" in the key name, to keep a flat tree of files and avoid
  - issues with keys containing "/../" or ending with "/" etc. 
@@ -166,6 +157,22 @@ keyFile :: Key -> FilePath
 keyFile key = replace "/" "%" $ replace ":" "&c" $
 	replace "%" "&s" $ replace "&" "&a"  $ show key
 
+{- A location to store a key on the filesystem. A directory hash is used,
+ - to protect against filesystems that dislike having many items in a
+ - single directory.
+ -
+ - The file is put in a directory with the same name, this allows
+ - write-protecting the directory to avoid accidental deletion of the file.
+ -}
+keyPath :: Key -> Hasher -> FilePath
+keyPath key hasher = hasher key  f  f
+	where
+		f = keyFile key
+
+{- All possibile locations to store a key using different directory hashes. -}
+keyPaths :: Key -> [FilePath]
+keyPaths key = map (keyPath key) annexHashes
+
 {- Reverses keyFile, converting a filename fragment (ie, the basename of
  - the symlink target) into a key. -}
 fileKey :: FilePath -> Maybe Key
@@ -178,17 +185,22 @@ prop_idempotent_fileKey :: String -> Bool
 prop_idempotent_fileKey s = Just k == fileKey (keyFile k)
 	where k = stubKey { keyName = s, keyBackendName = "test" }
 
-{- Given a key, generates a short directory name to put it in,
- - to do hashing to protect against filesystems that dislike having
- - many items in a single directory. -}
-hashDirMixed :: Key -> FilePath
+{- Two different directory hashes may be used. The mixed case hash
+ - came first, and is fine, except for the problem of case-strict
+ - filesystems such as Linux VFAT (mounted with shortname=mixed),
+ - which do not allow using a directory "XX" when "xx" already exists.
+ - To support that, some repositories will use a lower case hash. -}
+type Hasher = Key -> FilePath
+annexHashes :: [Hasher]
+annexHashes = [hashDirMixed, hashDirLower]
+
+hashDirMixed :: Hasher
 hashDirMixed k = addTrailingPathSeparator $ take 2 dir  drop 2 dir
 	where
 		dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d]
 		ABCD (a,b,c,d) = md5 $ Str $ show k
 
-{- Generates a hash directory that is all lower case. -}
-hashDirLower :: Key -> FilePath
+hashDirLower :: Hasher
 hashDirLower k = addTrailingPathSeparator $ take 3 dir  drop 3 dir
 	where
 		dir = take 6 $ md5s $ Str $ show k
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 83302b65a5..5f294f0bef 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -64,9 +64,7 @@ directorySetup u c = do
 
 {- Locations to try to access a given Key in the Directory. -}
 locations :: FilePath -> Key -> [FilePath]
-locations d k = map (\h -> d  h k  f  f) annexHashes
-	where
-		f = keyFile k
+locations d k = map (d ) (keyLocations k)
 
 withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
 withCheckedFile _ [] _ _ = return False

From db5b479f3f9c68c05bd172b90fe5cab0336f378d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 14:56:48 -0400
Subject: [PATCH 2593/2835] use lowercase hash by default; non-bare repos are a
 special case

Directory special remotes will now always store keys in the lowercase name,
which avoids the complication of catching failures to create the mixed case
name.

Git remotes using http will now try the lowercase name first.
---
 Locations.hs        |  8 +++-----
 Remote/Directory.hs | 24 +++++++++++-------------
 2 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 1b5f8108df..2f4a9200db 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -21,7 +21,6 @@ module Locations (
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
 	isLinkToAnnex,
-	annexHashes,
 	hashDirMixed,
 	hashDirLower,
 
@@ -80,8 +79,7 @@ gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
 		{- Bare repositories default to hashDirLower for new
 		 - content, as it's more portable. -}
-		go (Git.workTree r) $
-			map (annexLocation key) [hashDirLower, hashDirMixed]
+		go (Git.workTree r) (annexLocations key)
 	| otherwise =
 		{- Non-bare repositories only use hashDirMixed, so
 		 - don't need to do any work to check if the file is
@@ -189,10 +187,10 @@ prop_idempotent_fileKey s = Just k == fileKey (keyFile k)
  - came first, and is fine, except for the problem of case-strict
  - filesystems such as Linux VFAT (mounted with shortname=mixed),
  - which do not allow using a directory "XX" when "xx" already exists.
- - To support that, some repositories will use a lower case hash. -}
+ - To support that, most repositories use the lower case hash for new data. -}
 type Hasher = Key -> FilePath
 annexHashes :: [Hasher]
-annexHashes = [hashDirMixed, hashDirLower]
+annexHashes = [hashDirLower, hashDirMixed]
 
 hashDirMixed :: Hasher
 hashDirMixed k = addTrailingPathSeparator $ take 2 dir  drop 2 dir
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 5f294f0bef..a6077d813c 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -64,7 +64,7 @@ directorySetup u c = do
 
 {- Locations to try to access a given Key in the Directory. -}
 locations :: FilePath -> Key -> [FilePath]
-locations d k = map (d ) (keyLocations k)
+locations d k = map (d ) (keyPaths k)
 
 withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
 withCheckedFile _ [] _ _ = return False
@@ -95,18 +95,16 @@ storeEncrypted d (cipher, enck) k = do
 			return True
 
 storeHelper :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
-storeHelper d key a = withCheckedFile check d key go
-	where
-		check dest = isJust <$> mkdir (parentDir dest)
-		mkdir = catchMaybeIO . createDirectoryIfMissing True
-		go dest = do
-			let dir = parentDir dest
-			allowWrite dir
-			ok <- a dest
-			when ok $ do
-				preventWrite dest
-				preventWrite dir
-			return ok
+storeHelper d key a = do
+	let dest = head $ locations d key
+	let dir = parentDir dest
+	createDirectoryIfMissing True dir
+	allowWrite dir
+	ok <- a dest
+	when ok $ do
+		preventWrite dest
+		preventWrite dir
+	return ok
 
 retrieve :: FilePath -> Key -> FilePath -> Annex Bool
 retrieve d k f = liftIO $ withStoredFile d k $ \file -> copyFileExternal file f

From fb68a7881f725a7b097f8b0f1b347f24dfea5d59 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 15:50:27 -0400
Subject: [PATCH 2594/2835] convert rsync special backend to using both hash
 directory types

---
 Locations.hs     |  1 +
 Remote/Rsync.hs  | 64 +++++++++++++++++++++++++++++-------------------
 debian/changelog |  9 ++++---
 3 files changed, 45 insertions(+), 29 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 2f4a9200db..1179886ade 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -21,6 +21,7 @@ module Locations (
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
 	isLinkToAnnex,
+	annexHashes,
 	hashDirMixed,
 	hashDirLower,
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 836b93b314..651ed4de87 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -86,13 +86,26 @@ rsyncEscape o s
 	| rsyncUrlIsShell (rsyncUrl o) = shellEscape s
 	| otherwise = s
 
-rsyncKey :: RsyncOpts -> Key -> String
-rsyncKey o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (f  f)
-        where
+rsyncUrls :: RsyncOpts -> Key -> [String]
+rsyncUrls o k = map use annexHashes
+	where
+		use h = rsyncUrl o  h k  rsyncEscape o (f  f)
                 f = keyFile k
 
-rsyncKeyDir :: RsyncOpts -> Key -> String
-rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (keyFile k)
+rsyncUrlDirs :: RsyncOpts -> Key -> [String]
+rsyncUrlDirs o k = map use annexHashes
+	where
+		use h = rsyncUrl o  h k  rsyncEscape o (keyFile k)
+
+withRsyncUrl :: RsyncOpts -> Key -> (FilePath -> Annex Bool) -> Annex Bool
+withRsyncUrl o k a = go $ rsyncUrls o k
+	where
+		go [] = return False
+		go (u:us) = do
+			ok <- a u
+			if ok
+				then return ok
+				else go us
 
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
@@ -104,10 +117,10 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
 	rsyncSend o enck tmp
 
 retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
-retrieve o k f = rsyncRemote o
+retrieve o k f = withRsyncUrl o k $ \u -> rsyncRemote o
 	-- use inplace when retrieving to support resuming
 	[ Param "--inplace"
-	, Param $ rsyncKey o k
+	, Param u
 	, Param f
 	]
 
@@ -121,27 +134,30 @@ retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
 		else return res
 
 remove :: RsyncOpts -> Key -> Annex Bool
-remove o k = withRsyncScratchDir $ \tmp -> do
-	{- Send an empty directory to rysnc as the parent directory
-         - of the file to remove. -}
-	let dummy = tmp  keyFile k
-	liftIO $ createDirectoryIfMissing True dummy
-	liftIO $ rsync $ rsyncOptions o ++
-		[ Params "--delete --recursive"
-		, partialParams
-		, Param $ addTrailingPathSeparator dummy
-		, Param $ rsyncKeyDir o k
-		]
+remove o k = any (== True) <$> sequence (map go (rsyncUrlDirs o k))
+	where
+		go d = withRsyncScratchDir $ \tmp -> liftIO $ do
+			{- Send an empty directory to rysnc as the
+			 - parent directory of the file to remove. -}
+			let dummy = tmp  keyFile k
+			createDirectoryIfMissing True dummy
+			rsync $ rsyncOptions o ++
+				[ Params "--quiet --delete --recursive"
+				, partialParams
+				, Param $ addTrailingPathSeparator dummy
+				, Param d
+				]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool)
 checkPresent r o k = do
 	showAction $ "checking " ++ Git.repoDescribe r
-	-- note: Does not currently differnetiate between rsync failing
+	-- note: Does not currently differentiate between rsync failing
 	-- to connect, and the file not being present.
-	res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd]
-	return $ Right res
+	Right <$> check
 	where
-		cmd = "rsync --quiet " ++ shellEscape (rsyncKey o k) ++ " 2>/dev/null"
+ 		check = withRsyncUrl o k $ \u ->
+			liftIO $ boolSystem "sh" [Param "-c", Param (cmd u)]
+		cmd u = "rsync --quiet " ++ shellEscape u ++ " 2>/dev/null"
 
 {- Rsync params to enable resumes of sending files safely,
  - ensure that files are only moved into place once complete
@@ -182,7 +198,7 @@ rsyncRemote o params = do
    directories. -}
 rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool
 rsyncSend o k src = withRsyncScratchDir $ \tmp -> do
-	let dest = tmp  hashDirMixed k  f  f
+	let dest = tmp  head (keyPaths k)
 	liftIO $ createDirectoryIfMissing True $ parentDir dest
 	liftIO $ createLink src dest
 	rsyncRemote o
@@ -192,5 +208,3 @@ rsyncSend o k src = withRsyncScratchDir $ \tmp -> do
 		, Param $ addTrailingPathSeparator tmp
 		, Param $ rsyncUrl o
 		]
-	where
-		f = keyFile k
diff --git a/debian/changelog b/debian/changelog
index 45088db522..d038c7849c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,10 +2,11 @@ git-annex (3.20111123) UNRELEASED; urgency=low
 
   * The VFAT filesystem on recent versions of Linux, when mounted with
     shortname=mixed, does not get along well with git-annex's mixed case
-    .git/annex/objects hash directories. To avoid this problem, bare
-    repositories (and the directory special remote) now store new content
-    in all-lowercase hash directories. Mixed case hash directories are
-    still used for non-bare repositories, which cannot be put on FAT.
+    .git/annex/objects hash directories. To avoid this problem, new content
+    is now stored in all-lowercase hash directories. Except for non-bare
+    repositories which would be a pain to transition and cannot be put on FAT.
+    (Old mixed-case hash directories are still tried for backwards
+    compatibility.)
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.

From e19dc8554723a148e6b809da4989a747f3aa925e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:10:52 -0400
Subject: [PATCH 2595/2835] factor out untilTrue

---
 Remote/Git.hs          |  5 +----
 Remote/Rsync.hs        | 50 +++++++++++++++++-------------------------
 Remote/Web.hs          |  6 ++---
 Utility/Conditional.hs |  6 +++++
 4 files changed, 29 insertions(+), 38 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 07afc02742..99ca9fe8e2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -200,10 +200,7 @@ copyFromRemote r key file
 	| Git.repoIsHttp r = liftIO $ downloadurls $ keyUrls r key
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
 	where
-		downloadurls [] = return False
-		downloadurls (u:us) = do
-			ok <- Url.download u file
-			if ok then return ok else downloadurls us
+		downloadurls us = untilTrue us $ \u -> Url.download u file
 
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 651ed4de87..81107cb561 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -97,16 +97,6 @@ rsyncUrlDirs o k = map use annexHashes
 	where
 		use h = rsyncUrl o  h k  rsyncEscape o (keyFile k)
 
-withRsyncUrl :: RsyncOpts -> Key -> (FilePath -> Annex Bool) -> Annex Bool
-withRsyncUrl o k a = go $ rsyncUrls o k
-	where
-		go [] = return False
-		go (u:us) = do
-			ok <- a u
-			if ok
-				then return ok
-				else go us
-
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
 
@@ -117,12 +107,13 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
 	rsyncSend o enck tmp
 
 retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
-retrieve o k f = withRsyncUrl o k $ \u -> rsyncRemote o
-	-- use inplace when retrieving to support resuming
-	[ Param "--inplace"
-	, Param u
-	, Param f
-	]
+retrieve o k f = untilTrue (rsyncUrls o k) $ \u ->
+	rsyncRemote o
+		-- use inplace when retrieving to support resuming
+		[ Param "--inplace"
+		, Param u
+		, Param f
+		]
 
 retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
@@ -134,19 +125,18 @@ retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
 		else return res
 
 remove :: RsyncOpts -> Key -> Annex Bool
-remove o k = any (== True) <$> sequence (map go (rsyncUrlDirs o k))
-	where
-		go d = withRsyncScratchDir $ \tmp -> liftIO $ do
-			{- Send an empty directory to rysnc as the
-			 - parent directory of the file to remove. -}
-			let dummy = tmp  keyFile k
-			createDirectoryIfMissing True dummy
-			rsync $ rsyncOptions o ++
-				[ Params "--quiet --delete --recursive"
-				, partialParams
-				, Param $ addTrailingPathSeparator dummy
-				, Param d
-				]
+remove o k = untilTrue (rsyncUrlDirs o k) $ \d ->
+	withRsyncScratchDir $ \tmp -> liftIO $ do
+		{- Send an empty directory to rysnc as the
+		 - parent directory of the file to remove. -}
+		let dummy = tmp  keyFile k
+		createDirectoryIfMissing True dummy
+		rsync $ rsyncOptions o ++
+			[ Params "--quiet --delete --recursive"
+			, partialParams
+			, Param $ addTrailingPathSeparator dummy
+			, Param d
+			]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool)
 checkPresent r o k = do
@@ -155,7 +145,7 @@ checkPresent r o k = do
 	-- to connect, and the file not being present.
 	Right <$> check
 	where
- 		check = withRsyncUrl o k $ \u ->
+ 		check = untilTrue (rsyncUrls o k) $ \u ->
 			liftIO $ boolSystem "sh" [Param "-c", Param (cmd u)]
 		cmd u = "rsync --quiet " ++ shellEscape u ++ " 2>/dev/null"
 
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 64fcd51aaa..5871ae8dae 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -71,8 +71,6 @@ checkKey key = do
 		then return $ Right False
 		else return . Right =<< checkKey' us
 checkKey' :: [URLString] -> Annex Bool
-checkKey' [] = return False
-checkKey' (u:us) = do
+checkKey' us = untilTrue us $ \u -> do
 	showAction $ "checking " ++ u
-	e <- liftIO $ Url.exists u
-	if e then return e else checkKey' us
+	liftIO $ Url.exists u
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
index 85e39ec64c..7a0df4b482 100644
--- a/Utility/Conditional.hs
+++ b/Utility/Conditional.hs
@@ -9,6 +9,12 @@ module Utility.Conditional where
 
 import Control.Monad (when, unless)
 
+untilTrue :: Monad m => [v] -> (v -> m Bool) -> m Bool
+untilTrue [] _ = return False
+untilTrue (v:vs) a = do
+	ok <- a v
+	if ok then return ok else untilTrue vs a
+
 whenM :: Monad m => m Bool -> m () -> m ()
 whenM c a = c >>= flip when a
 

From 6f29f9db725131d37af86e2746ff83a6765b4239 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:15:18 -0400
Subject: [PATCH 2596/2835] rename

---
 ...tree_options.mdwn => on--git-dir_and_--work-tree_options.mdwn} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/bugs/{--git-dir_and_--work-tree_options.mdwn => on--git-dir_and_--work-tree_options.mdwn} (100%)

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/on--git-dir_and_--work-tree_options.mdwn
similarity index 100%
rename from doc/bugs/--git-dir_and_--work-tree_options.mdwn
rename to doc/bugs/on--git-dir_and_--work-tree_options.mdwn

From 6d8d25262c40d7f422743ddbfa13e362635721e8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:26:56 -0400
Subject: [PATCH 2597/2835] close

---
 doc/bugs/case_sensitivity_on_FAT.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/case_sensitivity_on_FAT.mdwn b/doc/bugs/case_sensitivity_on_FAT.mdwn
index cb3424e34d..682acc71d7 100644
--- a/doc/bugs/case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/case_sensitivity_on_FAT.mdwn
@@ -44,3 +44,6 @@ I wonder if the directory remote should use hashDirLower instead of hashDirMixed
 
 >>> Bare repositories now use lowercase. rsync is the only remaining
 >>> unsupported possibility. --[[Joey]]
+>>>> Everything now uses lowercase, with the exception of non-bare
+>>>> repos, which cannot be on FAT anyway due to using symlinks. [[done]]
+>>>> --[[Joey]] 

From 7b08584c5553c22b322a2d9c268fda855666f4ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:59:09 -0400
Subject: [PATCH 2598/2835] close

---
 doc/bugs/case_sensitivity_on_FAT.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/case_sensitivity_on_FAT.mdwn b/doc/bugs/case_sensitivity_on_FAT.mdwn
index cb3424e34d..682acc71d7 100644
--- a/doc/bugs/case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/case_sensitivity_on_FAT.mdwn
@@ -44,3 +44,6 @@ I wonder if the directory remote should use hashDirLower instead of hashDirMixed
 
 >>> Bare repositories now use lowercase. rsync is the only remaining
 >>> unsupported possibility. --[[Joey]]
+>>>> Everything now uses lowercase, with the exception of non-bare
+>>>> repos, which cannot be on FAT anyway due to using symlinks. [[done]]
+>>>> --[[Joey]] 

From 251c01d51e22dc295359ba1f85144afc4c178e7a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:59:55 -0400
Subject: [PATCH 2599/2835] dead: A command which says that a repository is
 gone for good and you don't want git-annex to mention it again.

---
 GitAnnex.hs                                    |  2 ++
 Logs/Location.hs                               | 10 ++++++++--
 Logs/Trust.hs                                  |  6 ++++--
 Types/TrustLevel.hs                            |  2 +-
 debian/changelog                               |  2 ++
 doc/git-annex.mdwn                             |  5 +++++
 .../what_to_do_when_you_lose_a_repository.mdwn | 18 +++++++++---------
 doc/trust.mdwn                                 | 10 +++++++++-
 8 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index 42a6b7fd78..d768499ddb 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -46,6 +46,7 @@ import qualified Command.Uninit
 import qualified Command.Trust
 import qualified Command.Untrust
 import qualified Command.Semitrust
+import qualified Command.Dead
 import qualified Command.AddUrl
 import qualified Command.Map
 import qualified Command.Upgrade
@@ -70,6 +71,7 @@ cmds = concat
 	, Command.Trust.def
 	, Command.Untrust.def
 	, Command.Semitrust.def
+	, Command.Dead.def
 	, Command.AddUrl.def
 	, Command.FromKey.def
 	, Command.DropKey.def
diff --git a/Logs/Location.hs b/Logs/Location.hs
index cb21a2d1c0..27b4d709e7 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -27,6 +27,7 @@ module Logs.Location (
 import Common.Annex
 import qualified Annex.Branch
 import Logs.Presence
+import Logs.Trust
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Key -> UUID -> LogStatus -> Annex ()
@@ -34,9 +35,14 @@ logChange key (UUID u) s = addLog (logFile key) =<< logNow s u
 logChange _ NoUUID _ = return ()
 
 {- Returns a list of repository UUIDs that, according to the log, have
- - the value of a key. -}
+ - the value of a key.
+ -
+ - Dead repositories are skipped.
+ -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations key = map toUUID <$> (currentLog . logFile) key
+keyLocations key = do
+	l <- map toUUID <$> (currentLog . logFile) key
+	snd <$> trustPartition DeadTrusted l
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index e447fbebc8..f18f425110 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -67,12 +67,14 @@ parseTrust s
 		w = words s
 		parse "1" = Trusted
 		parse "0" = UnTrusted
+		parse "X" = DeadTrusted
 		parse _ = SemiTrusted
 
 showTrust :: TrustLevel -> String
-showTrust SemiTrusted = "?"
-showTrust UnTrusted = "0"
 showTrust Trusted = "1"
+showTrust UnTrusted = "0"
+showTrust DeadTrusted = "X"
+showTrust SemiTrusted = "?"
 
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
diff --git a/Types/TrustLevel.hs b/Types/TrustLevel.hs
index ddb8e45e49..99d7497303 100644
--- a/Types/TrustLevel.hs
+++ b/Types/TrustLevel.hs
@@ -14,7 +14,7 @@ import qualified Data.Map as M
 
 import Types.UUID
 
-data TrustLevel = SemiTrusted | UnTrusted | Trusted
+data TrustLevel = Trusted | SemiTrusted | UnTrusted | DeadTrusted
 	deriving Eq
 
 type TrustMap = M.Map UUID TrustLevel
diff --git a/debian/changelog b/debian/changelog
index d038c7849c..4a2a3dd06b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -15,6 +15,8 @@ git-annex (3.20111123) UNRELEASED; urgency=low
     git repository.
   * --inbackend can be used to make git-annex only operate on files
     whose content is stored using a specified key-value backend.
+  * dead: A command which says that a repository is gone for good
+    and you don't want git-annex to mention it again.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e98c89fe39..9df5c3c6d1 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -173,6 +173,11 @@ subdirectories).
 
   Returns a repository to the default semi trusted state.
 
+* dead [repository ...]
+
+  Indicates that the repository has been irretrevably lost.
+  (To undo, use semitrust.)
+
 # REPOSITORY MAINTENANCE COMMANDS
 
 * fsck [path ...]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository.mdwn b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
index 16a55b37b3..3be13b8abd 100644
--- a/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
+++ b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
@@ -4,16 +4,16 @@ drive died or some other misfortune has befallen your data.
 Unless you configured backups, git-annex can't get your data back. But it
 can help you deal with the loss.
 
-First, go somewhere that knows about the lost repository, and mark it as
-untrusted.
+Go somewhere that knows about the lost repository, and mark it as
+dead:
 
-	git annex untrust usbdrive
+	git annex dead usbdrive
 
-To remind yourself later what happened, you can change its description, too:
+This retains the [[location_tracking]] information for the repository,
+but avoids trying to access it, or list it as a location where files
+are present.
 
-	git annex describe usbdrive "USB drive lost in Timbuktu. Probably gone forever."
+If you later found the drive, you could let git-annex know it's found
+like so:
 
-This retains the [[location_tracking]] information for the repository.
-Maybe you'll find the drive later. Maybe that's impossible. Either way,
-this lets git-annex tell you why a file is no longer accessible, and
-it avoids it relying on that drive to hold any content.
+	git annex semitrusted usbdrive
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index 7505a7af65..1fd47fd1d3 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -1,8 +1,9 @@
-Git-annex supports three levels of trust of a repository:
+Git-annex supports several levels of trust of a repository:
 
 * semitrusted (default)
 * untrusted
 * trusted
+* dead
 
 ## semitrusted
 
@@ -49,3 +50,10 @@ trust temporarily.
 
 To configure a repository as fully and permanently trusted,
 use the `git annex trust` command.
+
+## dead
+
+This is used to indicate that you have no trust that the repository
+exists at all. It's appropriate to use when a drive has been lost,
+or a directory irretrevably deleted. It will make git-annex avoid
+even showing the repository as a place where data might still reside.

From f0cc42685e42a493c83eb85de02e61c54f69e4f0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 19:21:56 -0400
Subject: [PATCH 2600/2835] fix display of dead repositories in status

---
 Command/Status.hs | 1 +
 Logs/Trust.hs     | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index a47f21b91a..0fefda1f6b 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -58,6 +58,7 @@ fast_stats =
 	, remote_list Trusted "trusted"
 	, remote_list SemiTrusted "semitrusted"
 	, remote_list UnTrusted "untrusted"
+	, remote_list DeadTrusted "dead"
 	]
 slow_stats :: [Stat]
 slow_stats = 
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index f18f425110..196666a84e 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -38,7 +38,8 @@ trustPartition level ls
 	| level == SemiTrusted = do
 		t <- trustGet Trusted
 		u <- trustGet UnTrusted
-		let uncandidates = t ++ u
+		d <- trustGet DeadTrusted
+		let uncandidates = t ++ u ++ d
 		return $ partition (`notElem` uncandidates) ls
 	| otherwise = do
 		candidates <- trustGet level

From b5930f6d076d266b337b415447f448fbb14d9ea3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 19:22:43 -0400
Subject: [PATCH 2601/2835] add

---
 Command/Dead.hs | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 Command/Dead.hs

diff --git a/Command/Dead.hs b/Command/Dead.hs
new file mode 100644
index 0000000000..192551e207
--- /dev/null
+++ b/Command/Dead.hs
@@ -0,0 +1,32 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Dead where
+
+import Common.Annex
+import Command
+import qualified Remote
+import Logs.Trust
+
+def :: [Command]
+def = [command "dead" (paramRepeating paramRemote) seek
+	"hide a lost repository"]
+
+seek :: [CommandSeek]
+seek = [withWords start]
+
+start :: [String] -> CommandStart
+start ws = do
+	let name = unwords ws
+	showStart "dead " name
+	u <- Remote.nameToUUID name
+	next $ perform u
+
+perform :: UUID -> CommandPerform
+perform uuid = do
+	trustSet uuid DeadTrusted
+	next $ return True

From 64672c6262048e9498be667f8dc7460c0a9189e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Dec 2011 09:10:23 -0400
Subject: [PATCH 2602/2835] refactor

---
 Common.hs              | 1 +
 Locations.hs           | 6 +-----
 Remote/Web.hs          | 1 -
 Utility/Conditional.hs | 6 ------
 Utility/Monad.hs       | 4 ++++
 5 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/Common.hs b/Common.hs
index a3802da5f2..7e8dd9a2a5 100644
--- a/Common.hs
+++ b/Common.hs
@@ -24,3 +24,4 @@ import Utility.Conditional as X
 import Utility.SafeCommand as X
 import Utility.Path as X
 import Utility.Directory as X
+import Utility.Monad as X
diff --git a/Locations.hs b/Locations.hs
index 1179886ade..3843495f9c 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -89,11 +89,7 @@ gitAnnexLocation key r
 				annexLocation key hashDirMixed
 	where
 		go dir locs = fromMaybe (dir  head locs) <$> check dir locs
-		check _ [] = return Nothing
-		check dir (l:ls) = do
-			let f = dir  l
-			e <- doesFileExist f
-			if e then return (Just f) else check dir ls
+		check dir = firstM $ \f -> doesFileExist $ dir  f
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 5871ae8dae..d5acd7d862 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -13,7 +13,6 @@ import qualified Git
 import Config
 import Logs.Web
 import qualified Utility.Url as Url
-import Utility.Monad
 
 remote :: RemoteType Annex
 remote = RemoteType {
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
index 7a0df4b482..85e39ec64c 100644
--- a/Utility/Conditional.hs
+++ b/Utility/Conditional.hs
@@ -9,12 +9,6 @@ module Utility.Conditional where
 
 import Control.Monad (when, unless)
 
-untilTrue :: Monad m => [v] -> (v -> m Bool) -> m Bool
-untilTrue [] _ = return False
-untilTrue (v:vs) a = do
-	ok <- a v
-	if ok then return ok else untilTrue vs a
-
 whenM :: Monad m => m Bool -> m () -> m ()
 whenM c a = c >>= flip when a
 
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 9523e17165..0d1675fa49 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -24,3 +24,7 @@ firstM p (x:xs) = do
  - stopping once one is found. -}
 anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
 anyM p = liftM isJust . firstM p
+
+{- Runs an action on values from a list until it succeeds. -}
+untilTrue :: (Monad m) => [a] -> (a -> m Bool) -> m Bool
+untilTrue = flip anyM

From 2f8d75638ac70e968b48a9675be32d2eead6dbc5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Dec 2011 21:01:22 -0400
Subject: [PATCH 2603/2835] update for dead repos

---
 doc/internals.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 5559d122b9..d84b3c4898 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -62,8 +62,8 @@ Records the [[trust]] information for repositories. Does not exist unless
 [[trust]] values are configured.
 
 The file format is one line per repository, with the uuid followed by a
-space, and then either 1 (trusted), 0 (untrusted), or ? (semi-trusted),
-and finally a timestamp.
+space, and then either `1` (trusted), `0` (untrusted), `?` (semi-trusted),
+`X` (dead) and finally a timestamp.
 
 Example:
 

From ff5df842ea85378288519b4e964faf28d67b6dbb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Dec 2011 21:13:21 -0400
Subject: [PATCH 2604/2835] releasing version 3.20111203

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4a2a3dd06b..41c251b849 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111123) UNRELEASED; urgency=low
+git-annex (3.20111203) unstable; urgency=low
 
   * The VFAT filesystem on recent versions of Linux, when mounted with
     shortname=mixed, does not get along well with git-annex's mixed case
@@ -18,7 +18,7 @@ git-annex (3.20111123) UNRELEASED; urgency=low
   * dead: A command which says that a repository is gone for good
     and you don't want git-annex to mention it again.
 
- -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
+ -- Joey Hess   Sat, 03 Dec 2011 21:01:45 -0400
 
 git-annex (3.20111122) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index c257eaff63..7be78053f8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111122
+Version: 3.20111203
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 58567045b737d1ffbcd143934a85e22e62da8044 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Dec 2011 21:13:29 -0400
Subject: [PATCH 2605/2835] add news item for git-annex 3.20111203

---
 doc/news/version_3.20111025.mdwn |  8 --------
 doc/news/version_3.20111203.mdwn | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_3.20111025.mdwn
 create mode 100644 doc/news/version_3.20111203.mdwn

diff --git a/doc/news/version_3.20111025.mdwn b/doc/news/version_3.20111025.mdwn
deleted file mode 100644
index 90e4227bae..0000000000
--- a/doc/news/version_3.20111025.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 3.20111025 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * A remote can have a annexUrl configured, that is used by git-annex
-     instead of its usual url. (Similar to pushUrl.)
-   * migrate: Copy url logs for keys when migrating.
-   * git-annex-shell: GIT\_ANNEX\_SHELL\_READONLY and GIT\_ANNEX\_SHELL\_LIMITED
-     environment variables can be set to limit what commands can be run.
-     This is used by gitolite's new git-annex support!"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111203.mdwn b/doc/news/version_3.20111203.mdwn
new file mode 100644
index 0000000000..5be6e21424
--- /dev/null
+++ b/doc/news/version_3.20111203.mdwn
@@ -0,0 +1,19 @@
+git-annex 3.20111203 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * The VFAT filesystem on recent versions of Linux, when mounted with
+     shortname=mixed, does not get along well with git-annex's mixed case
+     .git/annex/objects hash directories. To avoid this problem, new content
+     is now stored in all-lowercase hash directories. Except for non-bare
+     repositories which would be a pain to transition and cannot be put on FAT.
+     (Old mixed-case hash directories are still tried for backwards
+     compatibility.)
+   * Flush json output, avoiding a buffering problem that could result in
+     doubled output.
+   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
+   * Bugfix: dropunused did not drop keys with two spaces in their name.
+   * Support for storing .git/annex on a different device than the rest of the
+     git repository.
+   * --inbackend can be used to make git-annex only operate on files
+     whose content is stored using a specified key-value backend.
+   * dead: A command which says that a repository is gone for good
+     and you don't want git-annex to mention it again."""]]
\ No newline at end of file

From b6c8a0119ad68f466e24ec2bab3a2bacf6ea726e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Dec 2011 12:23:10 -0400
Subject: [PATCH 2606/2835] map: Fix a failure to detect a loop when both
 repositories are local and refer to each other with relative paths.

---
 Command/Map.hs   | 9 +++++----
 debian/changelog | 7 +++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 8755bc7c2d..6b1e8d5bbc 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -138,15 +138,16 @@ spider' (r:rs) known
 
 		-- The remotes will be relative to r', and need to be
 		-- made absolute for later use.
-		let remotes = map (absRepo r') (Git.remotes r')
+		remotes <- mapM (absRepo r') (Git.remotes r')
 		let r'' = Git.remotesAdd r' remotes
 
 		spider' (rs ++ remotes) (r'':known)
 
-absRepo :: Git.Repo -> Git.Repo -> Git.Repo
+{- Converts repos to a common absolute form. -}
+absRepo :: Git.Repo -> Git.Repo -> Annex Git.Repo
 absRepo reference r
-	| Git.repoIsUrl reference = Git.localToUrl reference r
-	| otherwise = r
+	| Git.repoIsUrl reference = return $ Git.localToUrl reference r
+	| otherwise = liftIO $ Git.repoFromAbsPath =<< absPath (Git.workTree r)
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
diff --git a/debian/changelog b/debian/changelog
index 41c251b849..7feb8f8ca9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20111204) UNRELEASED; urgency=low
+
+  * map: Fix a failure to detect a loop when both repositories are local
+    and refer to each other with relative paths.
+
+ -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
+
 git-annex (3.20111203) unstable; urgency=low
 
   * The VFAT filesystem on recent versions of Linux, when mounted with

From 04e5bf3644b5999d9b5251dc5fa0ee43bac6b435 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Mon, 5 Dec 2011 19:09:14 +0000
Subject: [PATCH 2607/2835]

---
 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn

diff --git a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
new file mode 100644
index 0000000000..b2124ae793
--- /dev/null
+++ b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
@@ -0,0 +1 @@
+Git-annex doesn't compile with the latest version of monad-control. Would it be hard to support that new version?

From 622c2b9b8b780a7df73fa5f02fb8b68116584d9d Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Mon, 5 Dec 2011 19:13:20 +0000
Subject: [PATCH 2608/2835]

---
 doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn

diff --git a/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
new file mode 100644
index 0000000000..474d9c9d8f
--- /dev/null
+++ b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
@@ -0,0 +1 @@
+A failure during "make test" should be signalled to the caller by means of a non-zero exit code. Without that signal, it's very hard to run the regression test suite in an automated fashion.

From fd5c9791212c049594f4dfc4daf5522c4655cb46 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Dec 2011 15:38:04 -0400
Subject: [PATCH 2609/2835] fixed a long time ago

---
 ...Please_abort_build_if___34__make_test__34___fails.mdwn | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
index 474d9c9d8f..592b5e0773 100644
--- a/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
+++ b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
@@ -1 +1,7 @@
-A failure during "make test" should be signalled to the caller by means of a non-zero exit code. Without that signal, it's very hard to run the regression test suite in an automated fashion.
+A failure during "make test" should be signalled to the caller by means of
+a non-zero exit code. Without that signal, it's very hard to run the
+regression test suite in an automated fashion.
+
+> git-annex used to have a Makefile that ignored make test exit status,
+> but that was fixed in commit dab5bddc64ab4ad479a1104748c15d194e138847,
+> in October 6th. [[done]] --[[Joey]] 

From 2a1e3bceb30a51f38bebc93e9321ed12d9567ec0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Dec 2011 15:42:02 -0400
Subject: [PATCH 2610/2835] respond

---
 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
index b2124ae793..8b88f103ec 100644
--- a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
+++ b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
@@ -1 +1,4 @@
 Git-annex doesn't compile with the latest version of monad-control. Would it be hard to support that new version?
+
+> I hope not. I have been waiting for it to land in Debian before trying to 
+> deal with its changes. --[[Joey]]

From 20d729514adaac04a33f0fbf99016a1276782f81 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Dec 2011 22:54:30 -0400
Subject: [PATCH 2611/2835] added a branch for the new monad-control 0.3

---
 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
index 8b88f103ec..ca68c2c913 100644
--- a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
+++ b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
@@ -1,4 +1,7 @@
 Git-annex doesn't compile with the latest version of monad-control. Would it be hard to support that new version?
 
-> I hope not. I have been waiting for it to land in Debian before trying to 
-> deal with its changes. --[[Joey]]
+> I have been waiting for it to land in Debian before trying to 
+> deal with its changes.
+> 
+> There is now a branch in git called `new-monad-control` that will build
+> with the new monad-control. --[[Joey]]

From 27ce7ba5b49cf2ec4c8a778f90320d714fd17155 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 12:43:56 +0000
Subject: [PATCH 2612/2835]

---
 ...nex_add_crash_and_subsequent_recovery.mdwn | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn b/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn
new file mode 100644
index 0000000000..3f3b943a0b
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn
@@ -0,0 +1,25 @@
+Perhaps stupidly I added some very large bare git repos into a git-annex.
+
+This took a very long time, used lot's of memory, and then crashed.  I didn't catch the error (which is annoying) - sorry about that.  IIRC it is the same error if one Ctrl-c's the addition.
+
+I ran `git annex add .` a second time and eventually killed it (I perhaps should have waited - I now think it was working).
+
+A `git annex unannex` fixed up some files but somehow I managed to end up with tonnes of files all sym-linked into the git annex object directory but not somehow recognised as annexed files.  I'm assuming that they somehow didn't make it into git annex's meta-data layer (or equivalent).
+
+Commands such as `git annex {fsck,whereis,unannex} weirdfile` immediately returned without error.
+
+I've now spent a lot of manual time copying the files back. Doing the following, not the cleverest but I was a little panicky about my data...
+
+    find . -type l -exec mv \{} \{}.link \; #Move link names out of the way
+    find . -type l -exec cp \{} \{}.cp \; #Copy follows links so we can copy target back to link location
+    find . -type f -name "*.link.cp" | xargs -n 1 rename 's/\.link\.cp//' #Change to original name
+    find . -type l -exec rm \{} \; #Ditch the links
+    git annex unused
+    git annex dropunused `seq 9228`
+
+9228 files were found to be unused, this gives an idea of the scale of the number of "lost" files for want of a better term.
+
+A pretty poor bug report as these things go.  Anyone any idea what might have happened (it didn't seem space or memory related)?  Or how I might have fixed it a little more cleverly?
+
+For reference I am using stable Debian, git annex version 3.20111011.
+

From d62b060f0f8f765b8b7f79c453b5f1c363403f19 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 12:50:28 +0000
Subject: [PATCH 2613/2835] Added a comment

---
 ...1_062d0153a379c1ba1df8585b90220d3d._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment
new file mode 100644
index 0000000000..e879441ff8
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 1"
+ date="2011-12-06T12:50:27Z"
+ content="""
+Ah HA!  Looks like I found the cause of this.
+
+    [matt@rss01:~/files/matt_ford]0> git annex add mhs
+    add mhs/Accessing_Web_Manager_V10.pdf ok
+    ....
+    add mhs/MAHSC Costing Request Form Dual
+    Organisations - FINAL v20 Oct 2010.xls git-annex: unknown response from git cat-file refs/heads/git-annex:8d5/ed4/WORM-s568832-m1323164214--MAHSC Costing Request Form Dual missing
+
+Spot the file name with a newline character in it!  This causes the error message above.  It seems that the files proceeding this badly named file are sym-linked but not registered.
+
+Perhaps a bug?
+"""]]

From 01f7c74d1fb581ed79c46aa16841a36f5e23e945 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 13:05:11 +0000
Subject: [PATCH 2614/2835]

---
 .../bad_behaviour_with_file_names_with_newline_in_them.mdwn    | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn

diff --git a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
new file mode 100644
index 0000000000..5673d8060c
--- /dev/null
+++ b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
@@ -0,0 +1,3 @@
+Found this out the hard way.  See the comment in the below post for what happens.
+
+[[/forum/git_annex_add_crash_and_subsequent_recovery/]]

From c3d3bf332952811b59a2404ee4c568799245472f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 13:36:35 +0000
Subject: [PATCH 2615/2835]

---
 doc/forum/git_pull_remote_git-annex.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex.mdwn

diff --git a/doc/forum/git_pull_remote_git-annex.mdwn b/doc/forum/git_pull_remote_git-annex.mdwn
new file mode 100644
index 0000000000..349610693b
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex.mdwn
@@ -0,0 +1,11 @@
+I thought I'd followed the walk through when initially setting up my repos.
+
+However I find that I have to do the following to sync my annex's.
+
+    git pull remote master
+    git checkout git-annex
+    git pull remote git-annex
+    git checkout master
+    git annex get .
+
+Has something gone wrong? I see no mention of syncing git-annex repos in the walk-through...

From 340d206efee926f0b39fedddb6afe0dfec43682f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 6 Dec 2011 16:43:29 +0000
Subject: [PATCH 2616/2835] Added a comment

---
 ..._9c245db3518d8b889ecdf5115ad9e053._comment | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment b/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment
new file mode 100644
index 0000000000..989ab9bcd8
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-06T16:43:29Z"
+ content="""
+You're taking a very long and strange way to a place that you can reach as follows:
+
+
+git pull remote
+git annex get .
+

+
+Which is just as shown in [[walkthrough/getting_file_content]]. 
+
+In particular, \"git pull remote\" first fetches all branches from the remote, including the git-annex branch.
+When you say \"git pull remote master\", you're preventing it from fetching the git-annex branch. 
+If for some reason you want the slightly longer way around, it is:
+
+
+git pull remote master
+git fetch remote git-annex
+git annex get .
+

+
+Or, eqivilantly but with less network connections:
+
+
+git fetch remote
+git merge remote/master
+git annex get .
+

+
+BTW, notice that this is all bog-standard git branch pulling stuff, not specific to git-annex in the least.
+Consult your extensive and friendly git documentation for details. :)
+"""]]

From cf5353acb4746cd0c2e736eecd066bd505555af3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 6 Dec 2011 16:49:32 +0000
Subject: [PATCH 2617/2835] Added a comment

---
 ...comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment

diff --git a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment
new file mode 100644
index 0000000000..7ff8f8e3d9
--- /dev/null
+++ b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-06T16:49:32Z"
+ content="""
+This only happens with the WORM backend (or possibly with SHA1E if the file's extension has a newline).
+
+The problem is not the newline in the file, but the newline in the key generated for the file. It's probably best to just disallow such keys being created.
+"""]]

From 480495beb4a3422f006ee529df807a20cc944727 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Dec 2011 13:02:50 -0400
Subject: [PATCH 2618/2835] Prevent key names from containing newlines.

There are several places where it's assumed a key can be written on one
line. One is in the format of the .git/annex/unused files. The difficult
one is that filenames derived from keys are fed into git cat-file --batch,
which has a line based input. (And no -z option.)

So, for now it's best to block such keys being created.
---
 Backend.hs                                             |  8 +++++++-
 Backend/SHA.hs                                         | 10 ++++++----
 debian/changelog                                       |  1 +
 ...behaviour_with_file_names_with_newline_in_them.mdwn |  2 ++
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index f7990c22c1..136c2eb7a6 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -64,7 +64,13 @@ genKey' (b:bs) file = do
 	r <- (B.getKey b) file
 	case r of
 		Nothing -> genKey' bs file
-		Just k -> return $ Just (k, b)
+		Just k -> return $ Just (makesane k, b)
+	where
+		-- keyNames should not contain newline characters.
+		makesane k = k { keyName = map fixbadchar (keyName k) }
+		fixbadchar c
+			| c == '\n' = '_'
+			| otherwise = c
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 2ae0cfcf42..7935b6d262 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -90,10 +90,12 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 			, keyBackendName = shaNameE size
 			}
 		naiveextension = takeExtension file
-		extension = 
-			if length naiveextension > 6
-				then "" -- probably not really an extension
-				else naiveextension
+		extension
+			-- long or newline containing extensions are 
+			-- probably not really an extension
+			| length naiveextension > 6 ||
+			  '\n' `elem` naiveextension = ""
+			| otherwise = naiveextension
 
 {- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
diff --git a/debian/changelog b/debian/changelog
index 7feb8f8ca9..a4d3a6492a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20111204) UNRELEASED; urgency=low
 
   * map: Fix a failure to detect a loop when both repositories are local
     and refer to each other with relative paths.
+  * Prevent key names from containing newlines.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
index 5673d8060c..530a8da5d5 100644
--- a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
+++ b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
@@ -1,3 +1,5 @@
 Found this out the hard way.  See the comment in the below post for what happens.
 
 [[/forum/git_annex_add_crash_and_subsequent_recovery/]]
+
+> [[fixed|done]] --[[Joey]]

From ba74e2069cdf3c69a5c729c53abd73ce11330828 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 6 Dec 2011 17:08:37 +0000
Subject: [PATCH 2619/2835] Added a comment

---
 ..._6fc6be43c488c468a4811cd0a1360225._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment
new file mode 100644
index 0000000000..38f2434f49
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-06T17:08:37Z"
+ content="""
+The bug with newlines is now fixed.
+
+Thought I'd mention how to clean up from interrupting `git annex add`.
+When you do that, it doesn't get a chance to `git add` the files it's
+added (this is normally done at the end, or sometimes at points in the middle when you're adding a *lot* of files).
+Which is also why fsck, whereis, and unannex wouldn't operate on them, since they only deal with files in git.
+
+So the first step is to manually use `git add` on any symlinks.
+
+Then, `git commit` as usual.
+
+At that point, `git annex unannex` would get you back to your starting state.
+"""]]

From adb1dc65bc9342d76d641fa06ec187f3e2c7b783 Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Tue, 6 Dec 2011 20:25:06 +0000
Subject: [PATCH 2620/2835]

---
 ...rsync_remotes_with_encryption_enabled.mdwn | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
new file mode 100644
index 0000000000..105e24aace
--- /dev/null
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -0,0 +1,40 @@
+Somehow git-annex has again lost a complete rsync remote with encryption enabled...
+
+Both *remoteserver* and *localserver* are rsync remotes with enabled encryption.
+All commands are executed on the git repository on my laptop.
+Target of origin is a gitolite repository without annex support (thus the two rsync remotes).
+
+Is there a way in git-annex to verify that all files fullfill the numcopies, in my case
+numcopies=2, and can be read from the remotes their are on?
+I thought that *copy* would verify that, but seems not.
+    
+    % g a copy --to remoteserver tools
+    copy tools/md5_sha1_utility.exe (gpg) (checking remoteserver...) ok
+    copy tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking remoteserver...) ok
+    
+    % g a copy --to localserver tools
+    copy tools/md5_sha1_utility.exe (gpg) (checking localserver...) ok
+    copy tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking localserver...) ok
+    
+    % g a drop tools
+    drop tools/md5_sha1_utility.exe (gpg) (checking localserver...) (checking remoteserver...) (unsafe)
+      Could only verify the existence of 1 out of 2 necessary copies
+      
+      Try making some of these repositories available:
+            718a9b5c-1b4a-11e1-8211-6f094f20e050 -- remoteserver (remote backupserver)
+      
+      (Use --force to override this check, or adjust annex.numcopies.)
+    failed
+    drop tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking localserver...) (checking remoteserver...) (unsafe)
+      Could only verify the existence of 1 out of 2 necessary copies
+      
+      Try making some of these repositories available:
+            718a9b5c-1b4a-11e1-8211-6f094f20e050 -- remoteserver (remote backupserver)
+      
+      (Use --force to override this check, or adjust annex.numcopies.)
+    failed
+    git-annex: drop: 2 failed
+    
+    % g a fsck tools
+    fsck tools/md5_sha1_utility.exe (checksum...) ok
+    fsck tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checksum...) ok

From 6f221f1fc3647de5b28179574125f6caa211fe0c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Dec 2011 17:06:08 -0400
Subject: [PATCH 2621/2835] response

---
 ...ing_rsync_remotes_with_encryption_enabled.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 105e24aace..914b45d184 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -1,5 +1,9 @@
 Somehow git-annex has again lost a complete rsync remote with encryption enabled...
 
+> "once again" ? When did it do it before?
+
+> "lost" ? How is the remote lost?
+
 Both *remoteserver* and *localserver* are rsync remotes with enabled encryption.
 All commands are executed on the git repository on my laptop.
 Target of origin is a gitolite repository without annex support (thus the two rsync remotes).
@@ -38,3 +42,14 @@ I thought that *copy* would verify that, but seems not.
     % g a fsck tools
     fsck tools/md5_sha1_utility.exe (checksum...) ok
     fsck tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checksum...) ok
+
+> Copy does do an explicit check that the content is present on remoteserver,
+> and based on the above, the content was found to be already there,
+> which is why it did not copy it again.
+> 
+> Drop does an indentical check that the content is present, and 
+> since it failed to find it, I am left thinking something must have
+> happened to the remove in between the copy and the drop to cause the
+> content to go away.
+> 
+> What happens if you copy the data to remoteserver again? --[[Joey]]

From 42c81e5dc00988964c1ea264c2a13ca2ce95c78a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 23:23:31 +0000
Subject: [PATCH 2622/2835] Added a comment

---
 ...ent_2_0f7f4a311b0ec1d89613e80847e69b42._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment b/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment
new file mode 100644
index 0000000000..198f95cee8
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 2"
+ date="2011-12-06T23:23:29Z"
+ content="""
+Doh! Total brain melt on my part. Thanks for the additional info.  Not taking my time and reading things properly - kept assuming that the full remote pull failed due to the warning:
+
+    You asked to pull from the remote 'rss', but did not specify
+    a branch. Because this is not the default configured remote
+    for your current branch, you must specify a branch on the command line.
+
+Rookie mistake indeed.
+"""]]

From ee26ebe6eb63bc58156bfc418d7692a912390a6a Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Wed, 7 Dec 2011 03:14:24 +0000
Subject: [PATCH 2623/2835]

---
 doc/users/gebi.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/users/gebi.mdwn

diff --git a/doc/users/gebi.mdwn b/doc/users/gebi.mdwn
new file mode 100644
index 0000000000..121bedbdd7
--- /dev/null
+++ b/doc/users/gebi.mdwn
@@ -0,0 +1 @@
+Michael Gebetsroither 

From 919d58667a65df2ae702ba438dd53c8f202dee8b Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Wed, 7 Dec 2011 03:20:43 +0000
Subject: [PATCH 2624/2835]

---
 ...nex_losing_rsync_remotes_with_encryption_enabled.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 914b45d184..968f296dd0 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -2,8 +2,15 @@ Somehow git-annex has again lost a complete rsync remote with encryption enabled
 
 > "once again" ? When did it do it before?
 
+It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]]
+
 > "lost" ? How is the remote lost?
 
+git-annex is not able to find any files on the encrypted rsync remote anymore.
+Copy does not copy the content again but drop doesn't find it, thus it's somehow "lost" and in an strange state.
+I've also had the state where the content was already on the remote side but git-annex copy would copy it again,
+ignoring all the data on the remote side. --[[gebi]]
+
 Both *remoteserver* and *localserver* are rsync remotes with enabled encryption.
 All commands are executed on the git repository on my laptop.
 Target of origin is a gitolite repository without annex support (thus the two rsync remotes).
@@ -53,3 +60,5 @@ I thought that *copy* would verify that, but seems not.
 > content to go away.
 > 
 > What happens if you copy the data to remoteserver again? --[[Joey]]
+
+The commands above are executed within a few seconds and completely repeatable. --[[gebi]]

From 51b7b828700438523d56f44d08e07679ef3e849f Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Wed, 7 Dec 2011 05:48:47 +0000
Subject: [PATCH 2625/2835]

---
 .../git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 968f296dd0..d727d69174 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -1,5 +1,7 @@
 Somehow git-annex has again lost a complete rsync remote with encryption enabled...
 
+git-annex version was 3.20111111
+
 > "once again" ? When did it do it before?
 
 It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]]

From c929f9df0f45db80445f9247f644bb87e8ca0a35 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Wed, 7 Dec 2011 07:39:16 +0000
Subject: [PATCH 2626/2835] Added a comment

---
 ...comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment
new file mode 100644
index 0000000000..b58f81c5b7
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 3"
+ date="2011-12-07T07:39:15Z"
+ content="""
+Ah - very good to know that recovery is easier than the method I used.  
+
+I wonder if it could be made a feature to automatically and safely recover/resume from an interrupted `git add`?
+"""]]

From 5926be6f307c03f5ed1a02dffff5dce0d987e689 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Dec 2011 12:36:15 -0400
Subject: [PATCH 2627/2835] response

---
 ...rsync_remotes_with_encryption_enabled.mdwn | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index d727d69174..93588b49c1 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -4,14 +4,14 @@ git-annex version was 3.20111111
 
 > "once again" ? When did it do it before?
 
-It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]]
+>> It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]]
 
 > "lost" ? How is the remote lost?
 
-git-annex is not able to find any files on the encrypted rsync remote anymore.
-Copy does not copy the content again but drop doesn't find it, thus it's somehow "lost" and in an strange state.
-I've also had the state where the content was already on the remote side but git-annex copy would copy it again,
-ignoring all the data on the remote side. --[[gebi]]
+>> git-annex is not able to find any files on the encrypted rsync remote anymore.
+>> Copy does not copy the content again but drop doesn't find it, thus it's somehow "lost" and in an strange state.
+>> I've also had the state where the content was already on the remote side but git-annex copy would copy it again,
+>> ignoring all the data on the remote side. --[[gebi]]
 
 Both *remoteserver* and *localserver* are rsync remotes with enabled encryption.
 All commands are executed on the git repository on my laptop.
@@ -64,3 +64,15 @@ I thought that *copy* would verify that, but seems not.
 > What happens if you copy the data to remoteserver again? --[[Joey]]
 
 The commands above are executed within a few seconds and completely repeatable. --[[gebi]]
+
+> In that case, why don't you run the commands with `-d` to see the actual
+> rsync command it's running to check if the content is present. 
+> Then you can try repeatedly running the command by hand and see why it
+> sometimes succeeds and sometimes fail.
+> 
+> The command will be something like this:
+> `rsync --quiet hostname:/dir/file 2>/dev/null`
+> 
+> The exit status is what's used to see if content is present -- and
+> currently any failure even a failure to connect is taken to mean it's not
+> present. --[[Joey]]

From d2ff311a3496fc498ad540b194767853ffdc1fc0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Dec 2011 13:17:00 -0400
Subject: [PATCH 2628/2835] change footer

---
 doc/git-annex-shell.mdwn | 2 +-
 doc/git-annex.mdwn       | 2 +-
 doc/git-union-merge.mdwn | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index fc5bc6c2d6..7a65f10775 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -78,4 +78,4 @@ Joey Hess 
 
 
 
-Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD).  Edit with care
+Warning: Automatically converted into a man page by mdwn2man. Edit with care
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9df5c3c6d1..08def0d62b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -623,4 +623,4 @@ Joey Hess 
 
 
 
-Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD).  Edit with care
+Warning: Automatically converted into a man page by mdwn2man. Edit with care
diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
index ed1778910b..8e3c34f8f1 100644
--- a/doc/git-union-merge.mdwn
+++ b/doc/git-union-merge.mdwn
@@ -35,4 +35,4 @@ Joey Hess 
 
 
 
-Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD).  Edit with care
+Warning: Automatically converted into a man page by mdwn2man. Edit with care

From 8047bba5b92a6f77ef305c1a74e59b5dacbcc9a2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Dec 2011 16:53:53 -0400
Subject: [PATCH 2629/2835] add: If interrupted, add can leave files converted
 to symlinks but not yet added to git. Running the add again will now clean up
 this situtation.

---
 Command.hs       |  8 ++++----
 Command/Add.hs   | 20 ++++++++++++++------
 debian/changelog |  2 ++
 3 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/Command.hs b/Command.hs
index 4d5bbeb363..0cd0bf4910 100644
--- a/Command.hs
+++ b/Command.hs
@@ -13,7 +13,7 @@ module Command (
 	prepCommand,
 	doCommand,
 	whenAnnexed,
-	notAnnexed,
+	ifAnnexed,
 	notBareRepo,
 	isBareRepo,
 	autoCopies,
@@ -71,10 +71,10 @@ doCommand = start
 {- Modifies an action to only act on files that are already annexed,
  - and passes the key and backend on to it. -}
 whenAnnexed :: (FilePath -> (Key, Backend Annex) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a)
-whenAnnexed a file = maybe (return Nothing) (a file) =<< Backend.lookupFile file
+whenAnnexed a file = ifAnnexed file (a file) (return Nothing)
 
-notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
-notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
+ifAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex a) -> (Annex a) -> Annex a
+ifAnnexed file yes no = maybe no yes =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
diff --git a/Command/Add.hs b/Command/Add.hs
index 9fdbdcaa69..9410601b8b 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -29,13 +29,21 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
 start :: BackendFile -> CommandStart
-start p@(_, file) = notBareRepo $ notAnnexed file $ do
-	s <- liftIO $ getSymbolicLinkStatus file
-	if isSymbolicLink s || not (isRegularFile s)
-		then stop
-		else do
+start p@(_, file) = notBareRepo $ ifAnnexed file fixup add
+	where
+		add = do
+			s <- liftIO $ getSymbolicLinkStatus file
+			if isSymbolicLink s || not (isRegularFile s)
+				then stop
+				else do
+					showStart "add" file
+					next $ perform p
+		fixup (key, _) = do
+			-- fixup from an interrupted add; the symlink
+			-- is present but not yet added to git
 			showStart "add" file
-			next $ perform p
+			liftIO $ removeFile file
+			next $ next $ cleanup file key =<< inAnnex key
 
 perform :: BackendFile -> CommandPerform
 perform (backend, file) = Backend.genKey file backend >>= go
diff --git a/debian/changelog b/debian/changelog
index a4d3a6492a..588ea6d5be 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
   * map: Fix a failure to detect a loop when both repositories are local
     and refer to each other with relative paths.
   * Prevent key names from containing newlines.
+  * add: If interrupted, add can leave files converted to symlinks but not
+    yet added to git. Running the add again will now clean up this situtation.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 

From 2515bb65209cd42c0291be949eaced4365babf8f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 7 Dec 2011 20:54:51 +0000
Subject: [PATCH 2630/2835] Added a comment

---
 .../comment_4_c560eae40867512b0af2cbef161fc8ac._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment
new file mode 100644
index 0000000000..8fca16cada
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-12-07T20:54:51Z"
+ content="""
+Good idea! I've made `git annex add` recover when ran a second time.
+"""]]

From 2568beee07928d1bd462d631372881f146b190ee Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Thu, 8 Dec 2011 14:24:08 +0000
Subject: [PATCH 2631/2835]

---
 ..._rsync_remotes_with_encryption_enabled.mdwn | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 93588b49c1..af1fed2eed 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -69,10 +69,26 @@ The commands above are executed within a few seconds and completely repeatable.
 > rsync command it's running to check if the content is present. 
 > Then you can try repeatedly running the command by hand and see why it
 > sometimes succeeds and sometimes fail.
-> 
+
+The commands fail and succeed consistently, not either or.
+git annex copy succeeds consistently with not copying the content to remote because it checks and it's already there.
+
+git annex drop fails consistently with error because content is missing on the exact same remote git annex copy checks
+and thinks the content is there. --[[gebi]]
+
 > The command will be something like this:
 > `rsync --quiet hostname:/dir/file 2>/dev/null`
 > 
 > The exit status is what's used to see if content is present -- and
 > currently any failure even a failure to connect is taken to mean it's not
 > present. --[[Joey]]
+
+hm... thats interesting, git annex drop and git annex copy check for different hashes on the same file at the same remote...
+
+git annex drop -d tools/md5_sha1_utility.exe
+> Running: sh ["-c","rsync --quiet 'REMOVED_HOST:annex/work/JF/z7/'\"'\"'GPGHMACSHA1--7ffb3840f0e37aee964352e98808403655e8473a/GPGHMACSHA1--7ffb3840f0e37aee964352e98808403655e8473a'\"'\"'' 2>/dev/null"]
+
+git annex copy --to remoteserver -d tools/md5_sha1_utility.exe
+> Running: sh ["-c","rsync --quiet 'REMOVED_HOST:annex/work/1F/PQ/'\"'\"'GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35/GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35'\"'\"'' 2>/dev/null"]
+
+And yes, only the hash *annex copy* is checking for exists on the remote side. --[[gebi]]

From e3f1568e0ff7dc872f3782115c74b9e7d8c291b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 8 Dec 2011 16:01:46 -0400
Subject: [PATCH 2632/2835] Fix caching of decrypted ciphers, which failed when
 drop had to check multiple different encrypted special remotes.

---
 Annex.hs                                      |  5 ++--
 Remote/Helper/Encryptable.hs                  | 23 +++++++++++--------
 Types/Crypto.hs                               |  2 ++
 debian/changelog                              |  2 ++
 ...rsync_remotes_with_encryption_enabled.mdwn |  7 ++++++
 5 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 6d245a92d1..d8d9c63b40 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -36,6 +36,7 @@ import Types.BranchState
 import Types.TrustLevel
 import Types.UUID
 import qualified Utility.Matcher
+import qualified Data.Map as M
 
 -- git-annex's monad
 newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
@@ -70,7 +71,7 @@ data AnnexState = AnnexState
 	, limit :: Either [Utility.Matcher.Token (FilePath -> Annex Bool)] (Utility.Matcher.Matcher (FilePath -> Annex Bool))
 	, forcetrust :: [(UUID, TrustLevel)]
 	, trustmap :: Maybe TrustMap
-	, cipher :: Maybe Cipher
+	, ciphers :: M.Map EncryptedCipher Cipher
 	}
 
 newState :: Git.Repo -> AnnexState
@@ -93,7 +94,7 @@ newState gitrepo = AnnexState
 	, limit = Left []
 	, forcetrust = []
 	, trustmap = Nothing
-	, cipher = Nothing
+	, ciphers = M.empty
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 85d269a213..99f48fe7b0 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -61,19 +61,22 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 		withkey a k = cip k >>= maybe (a k) (a . snd)
 		cip = cipherKey c
 
-{- Gets encryption Cipher. The decrypted Cipher is cached in the Annex
+{- Gets encryption Cipher. The decrypted Ciphers are cached in the Annex
  - state. -}
 remoteCipher :: RemoteConfig -> Annex (Maybe Cipher)
-remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher
+remoteCipher c = go $ extractCipher c
 	where
-		cached cipher = return $ Just cipher
-		expensive = case extractCipher c of
-			Nothing -> return Nothing
-			Just encipher -> do
-				showNote "gpg"
-				cipher <- liftIO $ decryptCipher c encipher
-				Annex.changeState (\s -> s { Annex.cipher = Just cipher })
-				return $ Just cipher
+		go Nothing = return Nothing
+		go (Just encipher) = do
+			cache <- Annex.getState Annex.ciphers
+			case M.lookup encipher cache of
+				Just cipher -> return $ Just cipher
+				Nothing -> decrypt encipher cache
+		decrypt encipher cache = do
+			showNote "gpg"
+			cipher <- liftIO $ decryptCipher c encipher
+			Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache })
+			return $ Just cipher
 
 {- Gets encryption Cipher, and encrypted version of Key. -}
 cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))
diff --git a/Types/Crypto.hs b/Types/Crypto.hs
index a9d3dddc59..29a4cd099c 100644
--- a/Types/Crypto.hs
+++ b/Types/Crypto.hs
@@ -11,5 +11,7 @@ module Types.Crypto where
 newtype Cipher = Cipher String
 
 data EncryptedCipher = EncryptedCipher String KeyIds
+	deriving (Ord, Eq)
 
 newtype KeyIds = KeyIds [String]
+	deriving (Ord, Eq)
diff --git a/debian/changelog b/debian/changelog
index 588ea6d5be..04e9116720 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
   * Prevent key names from containing newlines.
   * add: If interrupted, add can leave files converted to symlinks but not
     yet added to git. Running the add again will now clean up this situtation.
+  * Fix caching of decrypted ciphers, which failed when drop had to check
+    multiple different encrypted special remotes.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index af1fed2eed..0dad8856e8 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -92,3 +92,10 @@ git annex copy --to remoteserver -d tools/md5_sha1_utility.exe
 > Running: sh ["-c","rsync --quiet 'REMOVED_HOST:annex/work/1F/PQ/'\"'\"'GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35/GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35'\"'\"'' 2>/dev/null"]
 
 And yes, only the hash *annex copy* is checking for exists on the remote side. --[[gebi]]
+
+> Ok, this is due to too aggressive caching of the decrypted cipher
+> for a remote. When dopping, it decrypts localserver's cipher, 
+> caches it, and then when checking remoteserver it says hey, 
+> here's an already decrypted cipher -- it must be the right one!
+> 
+> Problem reproduced here, and fixed. [[done]] --[[Joey]]

From e0e40964ab2d0a305fe2503215f306e1228aac24 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Thu, 8 Dec 2011 22:42:29 +0000
Subject: [PATCH 2633/2835]

---
 ..._no_fixed_hostname_and_optimising_ssh.mdwn | 72 +++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn

diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
new file mode 100644
index 0000000000..db108f48b9
--- /dev/null
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
@@ -0,0 +1,72 @@
+## Intro
+
+This tip is based on my experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address.
+
+I'm not using a bare repository that allows pushing (an alternate way to sort the issue) nor did I fancy allowing `git push` to run against my desktop checked out repository.
+
+None of this stuff is really `git annex` specific but is useful to know...
+
+## Dealing with no fixed hostname
+
+Essentially set up two repos as per the [[walkthrough]].
+
+Desktop as follows:
+
+    cd ~/annex
+    git init
+    git annex init "desktop"
+
+And the laptop like this
+
+    git clone ssh://desktop/annex
+    git init
+    git annex init "laptop"
+
+Now we want to add the the repos as remotes of each other.  
+
+For the laptop it is easy:
+
+    git remote add desktop ssh://desktop/~/annex 
+
+However for the desktop to add an ever changing laptops hostname it's a little tricky.  We make use of remote SSH tunnels to do this.  Essentially we have the laptop which always knows it's own name and address and that of the desktop create a tunnel starting on an arbitrary port on the desktop and heading back to itself on it's own SSH server port (22).
+
+To do this make part of your laptops SSH config look like this:
+
+    Host desktop
+    User matt
+    HostName desktop.example.org
+    RemoteForward 2222 localhost:22
+
+Now on the desktop to connect over the tunnel to the laptops SSH port you need this:
+
+    Host laptop
+    User matt
+    HostName localhost
+    port 2222
+
+So to add the desktops remote:
+
+a) From the laptop ensure the tunnel is up
+
+    ssh desktop
+
+b) From the desktop add the remote
+
+    git remote add laptop ssh://laptop/~/annex
+
+So now you can work on the train, pop on the wifi at work upon arrival and sync up with a `git pull && git annex get`.
+
+An alternate solution maybe to use direct tunnels over Openvpn.
+
+## Optimising SSH
+
+Running a `git annex get .`, at least in the version I have, creates a new SSH connection for every file transfer (maybe this is a feature request?)
+
+Lot's of new small files in an _annex_ cause lot's of connections to be made quickly: this is an relatively expensive overhead and is enough for connection limiting to start in my case.  The process can be made much faster by using SSH's connection sharing capabilities.  An SSH config like this should do it:
+
+    # Global Settings
+    ControlMaster auto
+    ControlPersist 30
+    ControlPath ~/.ssh/master-%r@%h:%p
+
+This will create a master connection for sharing if one isn't present, maintain it for 30 seconds after closing down the connection (just-in-cases') and automatically use the master connection for subsequent connections.  Wins all round!

From b3ac4af6b0722db35b8295d1131b41c827cdd708 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Thu, 8 Dec 2011 22:50:45 +0000
Subject: [PATCH 2634/2835]

---
 ...h_no_fixed_hostname_and_optimising_ssh.mdwn | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
index db108f48b9..3f02ea47d3 100644
--- a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
@@ -2,9 +2,9 @@
 
 This tip is based on my experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address.
 
-I'm not using a bare repository that allows pushing (an alternate way to sort the issue) nor did I fancy allowing `git push` to run against my desktop checked out repository.
+I'm not using a bare repository that allows pushing (an alternative solution) nor do I fancy allowing `git push` to run against my desktop checked out repository (perhaps I worry over nothing?)
 
-None of this stuff is really `git annex` specific but is useful to know...
+None of this is really `git annex` specific but I think it is useful to know...
 
 ## Dealing with no fixed hostname
 
@@ -28,23 +28,23 @@ For the laptop it is easy:
 
     git remote add desktop ssh://desktop/~/annex 
 
-However for the desktop to add an ever changing laptops hostname it's a little tricky.  We make use of remote SSH tunnels to do this.  Essentially we have the laptop which always knows it's own name and address and that of the desktop create a tunnel starting on an arbitrary port on the desktop and heading back to itself on it's own SSH server port (22).
+However for the desktop to add an ever changing laptops hostname it's a little tricky.  We make use of remote SSH tunnels to do this.  Essentially we have the laptop (which always knows it's own name and address and knows the address of the desktop) create a tunnel starting on an arbitrary port at the desktop and heads back to the laptop on it's own SSH server port (22).
 
-To do this make part of your laptops SSH config look like this:
+To do this make part of your laptop's SSH config look like this:
 
     Host desktop
     User matt
     HostName desktop.example.org
     RemoteForward 2222 localhost:22
 
-Now on the desktop to connect over the tunnel to the laptops SSH port you need this:
+Now on the desktop to connect over the tunnel to the laptop's SSH port you need this:
 
     Host laptop
     User matt
     HostName localhost
     port 2222
 
-So to add the desktops remote:
+So to add the desktop's remote:
 
 a) From the laptop ensure the tunnel is up
 
@@ -54,13 +54,13 @@ b) From the desktop add the remote
 
     git remote add laptop ssh://laptop/~/annex
 
-So now you can work on the train, pop on the wifi at work upon arrival and sync up with a `git pull && git annex get`.
+So now you can work on the train, pop on the wifi at work upon arrival, and sync up with a `git pull && git annex get`.
 
-An alternate solution maybe to use direct tunnels over Openvpn.
+An alternative solution may be to use direct tunnels over Openvpn.
 
 ## Optimising SSH
 
-Running a `git annex get .`, at least in the version I have, creates a new SSH connection for every file transfer (maybe this is a feature request?)
+Running a `git annex get .`, at least in the version I have, creates a new SSH connection for every file transfer (maybe this should be a feature request?)
 
 Lot's of new small files in an _annex_ cause lot's of connections to be made quickly: this is an relatively expensive overhead and is enough for connection limiting to start in my case.  The process can be made much faster by using SSH's connection sharing capabilities.  An SSH config like this should do it:
 

From d64132a43ae176e8a1353d5463c5387a93da9ad7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 01:57:13 -0400
Subject: [PATCH 2635/2835] hslint

---
 Annex/Branch.hs      |  2 +-
 Annex/Content.hs     |  2 +-
 Annex/Ssh.hs         |  2 +-
 CmdLine.hs           |  2 +-
 Command.hs           |  2 +-
 Command/Map.hs       |  2 +-
 Command/Status.hs    |  5 ++---
 Command/Unused.hs    |  5 ++---
 Git/UnionMerge.hs    |  2 +-
 Logs/UUID.hs         |  6 +++---
 Remote/Git.hs        |  2 +-
 Upgrade/V2.hs        |  2 +-
 Utility/DataUnits.hs |  2 +-
 Utility/Directory.hs | 13 ++++++-------
 configure.hs         |  2 +-
 test.hs              |  3 +--
 16 files changed, 25 insertions(+), 29 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a890668810..8f0a09fd6b 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -141,7 +141,7 @@ update = onceonly $ do
 		let merge_desc = if null branches
 			then "update" 
 			else "merging " ++
-				(unwords $ map (show . Git.refDescribe) branches) ++ 
+				unwords (map (show . Git.refDescribe) branches) ++ 
 				" into " ++ show name
 		unless (null branches) $ do
 			showSideAction merge_desc
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 90bde2975a..3f1db37b53 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -43,7 +43,7 @@ import Annex.Exception
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
-inAnnex = inAnnex' $ doesFileExist
+inAnnex = inAnnex' doesFileExist
 inAnnex' :: (FilePath -> IO a) -> Key -> Annex a
 inAnnex' a key = do
 	whenM (fromRepo Git.repoIsUrl) $
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index f8cd5d9bc8..6893f94ef4 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -43,7 +43,7 @@ git_annex_shell r command params
 		shellcmd = "git-annex-shell"
 		shellopts = Param command : File dir : params
 		sshcmd uuid = unwords $
-			shellcmd : (map shellEscape $ toCommand shellopts) ++
+			shellcmd : map shellEscape (toCommand shellopts) ++
 			uuidcheck uuid
 		uuidcheck NoUUID = []
 		uuidcheck (UUID u) = ["--uuid", u]
diff --git a/CmdLine.hs b/CmdLine.hs
index 78f46a2e39..672969c30a 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -32,7 +32,7 @@ dispatch args cmds options header getgitrepo = do
 	setupConsole
 	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo)
 	case r of
-		Left e -> maybe (throw e) id (cmdnorepo cmd)
+		Left e -> fromMaybe (throw e) (cmdnorepo cmd)
 		Right g -> do
 			state <- Annex.new g
 			(actions, state') <- Annex.run state $ do
diff --git a/Command.hs b/Command.hs
index 0cd0bf4910..86a83e30c0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -73,7 +73,7 @@ doCommand = start
 whenAnnexed :: (FilePath -> (Key, Backend Annex) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a)
 whenAnnexed a file = ifAnnexed file (a file) (return Nothing)
 
-ifAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex a) -> (Annex a) -> Annex a
+ifAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex a) -> Annex a -> Annex a
 ifAnnexed file yes no = maybe no yes =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
diff --git a/Command/Map.hs b/Command/Map.hs
index 6b1e8d5bbc..57b48d5030 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -203,7 +203,7 @@ tryScan r
 					"git config --list"
 				dir = Git.workTree r
 				cddir
-					| take 2 dir == "/~" =
+					| "/~" `isPrefixOf` dir =
 						let (userhome, reldir) = span (/= '/') (drop 1 dir)
 						in "cd " ++ userhome ++ " && cd " ++ shellEscape (drop 1 reldir)
 					| otherwise = "cd " ++ shellEscape dir
diff --git a/Command/Status.hs b/Command/Status.hs
index 0fefda1f6b..09da41987b 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -191,9 +191,8 @@ staleSize label dirspec = do
 	keys <- lift (Command.Unused.staleKeys dirspec)
 	if null keys
 		then nostat
-		else do
-			stat label $ json (++ aside "clean up with git-annex unused") $
-				return $ keySizeSum $ S.fromList keys
+		else stat label $ json (++ aside "clean up with git-annex unused") $
+			return $ keySizeSum $ S.fromList keys
 
 aside :: String -> String
 aside s = " (" ++ s ++ ")"
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 7f9edfef25..be0107752e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -152,13 +152,12 @@ excludeReferenced l = do
 		(S.fromList l)
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
-		refs = map Git.Ref . 
-			map last .
+		refs = map (Git.Ref .  last) .
 			nubBy cmpheads .
 			filter ourbranches .
 			map words . lines . L.unpack
 		cmpheads a b = head a == head b
-		ourbranchend = '/' : show (Annex.Branch.name)
+		ourbranchend = '/' : show Annex.Branch.name
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
 		removewith [] s = return $ S.toList s
 		removewith (a:as) s
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index edc8cb20ba..ddbff6a82f 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -48,7 +48,7 @@ merge_index h repo bs =
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Repo -> [String] -> IO ()
-update_index repo ls = stream_update_index repo [\s -> mapM_ s ls]
+update_index repo ls = stream_update_index repo [(`mapM_` ls)]
 
 {- Streams content into update-index. -}
 stream_update_index :: Repo -> [Streamer] -> IO ()
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 20f43d15ca..b325c78b6f 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -55,15 +55,15 @@ fixBadUUID = M.fromList . map fixup . M.toList
 			| otherwise = (k, v)
 			where
 				kuuid = fromUUID k
-				isbad = (not $ isuuid kuuid) && isuuid lastword
+				isbad = not (isuuid kuuid) && isuuid lastword
 				ws = words $ value v
 				lastword = last ws
 				fixeduuid = toUUID lastword
-				fixedvalue = unwords $ kuuid:(take (length ws - 1) ws)
+				fixedvalue = unwords $ kuuid: init ws
 		-- For the fixed line to take precidence, it should be
 		-- slightly newer, but only slightly.
 		newertime (LogEntry (Date d) _) = d + minimumPOSIXTimeSlice
-		newertime (LogEntry (Unknown) _) = minimumPOSIXTimeSlice
+		newertime (LogEntry Unknown _) = minimumPOSIXTimeSlice
 		minimumPOSIXTimeSlice = 0.000001
 		isuuid s = length s == 36 && length (split "-" s) == 5
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 99ca9fe8e2..d172ec7c04 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -165,7 +165,7 @@ onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
 	-- Avoid re-reading the repository's configuration if it was
 	-- already read.
-	state <- if (M.null $ Git.configMap r)
+	state <- if M.null $ Git.configMap r
 		then Annex.new r
 		else return $ Annex.newState r
 	Annex.eval state $ do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index e76d99b3ec..08bf83e837 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -53,7 +53,7 @@ upgrade = do
 
 	when e $ do
 		inRepo $ Git.run "rm" [Param "-r", Param "-f", Param "-q", File old]
-		unless bare $ inRepo $ gitAttributesUnWrite
+		unless bare $ inRepo gitAttributesUnWrite
 	showProgress
 
 	unless bare push
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index e7552f52fb..5d80a04b9c 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -99,7 +99,7 @@ bandwidthUnits = error "stop trying to rip people off"
 
 {- Do you yearn for the days when men were men and megabytes were megabytes? -}
 oldSchoolUnits :: [Unit]
-oldSchoolUnits = map mingle $ zip storageUnits memoryUnits
+oldSchoolUnits = zipWith (curry mingle) storageUnits memoryUnits
 	where
 		mingle (Unit _ a n, Unit s' _ _) = Unit s' a n
 
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index 7f8822fca5..249ed69356 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -11,6 +11,7 @@ import System.IO.Error
 import System.Posix.Files
 import System.Directory
 import Control.Exception (throw)
+import Control.Monad
 
 import Utility.SafeCommand
 import Utility.Conditional
@@ -37,13 +38,11 @@ moveFile src dest = try (rename src dest) >>= onrename
 				mv tmp _ = do
 					ok <- boolSystem "mv" [Param "-f",
 						Param src, Param tmp]
-					if ok
-						then return ()
-						else do
-							-- delete any partial
-							_ <- try $
-								removeFile tmp
-							rethrow
+					unless ok $ do
+						-- delete any partial
+						_ <- try $
+							removeFile tmp
+						rethrow
 		isdir f = do
 			r <- try (getFileStatus f)
 			case r of
diff --git a/configure.hs b/configure.hs
index cb73af2a99..0d96b39554 100644
--- a/configure.hs
+++ b/configure.hs
@@ -71,7 +71,7 @@ checkGitVersion = do
 		dotted = sum . mult 1 . reverse . extend 10 . map readi . split "." 
 		extend n l = l ++ replicate (n - length l) 0
 		mult _ [] = []
-		mult n (x:xs) = (n*x) : (mult (n*100) xs)
+		mult n (x:xs) = (n*x) : mult (n*100) xs
 		readi :: String -> Integer
 		readi s = case reads s of
 			((x,_):_) -> x
diff --git a/test.hs b/test.hs
index e625fbd756..91c11873da 100644
--- a/test.hs
+++ b/test.hs
@@ -11,11 +11,10 @@ import Test.QuickCheck
 
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
-import Control.Exception (bracket_, bracket)
+import Control.Exception (bracket_, bracket, throw)
 import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
-import Control.Exception (throw)
 import qualified Data.Map as M
 import System.IO.HVFS (SystemFS(..))
 

From e664af5d8cf310ac3dcb7e27a5a6846360917fbf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Fri, 9 Dec 2011 10:03:20 +0000
Subject: [PATCH 2636/2835]

---
 ...ing_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
index 3f02ea47d3..8fb2bf9db1 100644
--- a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
@@ -1,6 +1,6 @@
 ## Intro
 
-This tip is based on my experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address.
+This tip is based on my (Matt Ford) experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address.
 
 I'm not using a bare repository that allows pushing (an alternative solution) nor do I fancy allowing `git push` to run against my desktop checked out repository (perhaps I worry over nothing?)
 

From 3f5f28b48754bc91620a6354ca70afe4c61c9894 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 12:23:45 -0400
Subject: [PATCH 2637/2835] factor out a stopUnless

code melt for lunch
---
 Command.hs         |  7 +++++++
 Command/AddUrl.hs  | 21 +++++++++------------
 Command/Drop.hs    | 28 +++++++++-------------------
 Command/DropKey.hs | 17 +++++------------
 Command/Fix.hs     |  9 +++------
 Command/Get.hs     | 32 ++++++++++++--------------------
 Command/Migrate.hs | 24 ++++++++++--------------
 Command/Move.hs    | 14 ++++----------
 Command/Unannex.hs | 24 ++++++++++--------------
 9 files changed, 69 insertions(+), 107 deletions(-)

diff --git a/Command.hs b/Command.hs
index 86a83e30c0..813a239cb0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -10,6 +10,7 @@ module Command (
 	noRepo,
 	next,
 	stop,
+	stopUnless,
 	prepCommand,
 	doCommand,
 	whenAnnexed,
@@ -49,6 +50,12 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
+{- Stops unless a condition is met. -}
+stopUnless :: Annex Bool -> Annex (Maybe a) -> Annex (Maybe a)
+stopUnless c a = do
+	ok <- c
+	if ok then a else stop
+
 {- Prepares to run a command via the check and seek stages, returning a
  - list of actions to perform to run the command. -}
 prepCommand :: Command -> [String] -> Annex [CommandCleanup]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 945848e9f7..75ca740314 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -45,18 +45,15 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	tmp <- fromRepo $ gitAnnexTmpLocation dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- liftIO $ Url.download url tmp
-	if ok
-		then do
-			[(backend, _)] <- Backend.chooseBackends [file]
-			k <- Backend.genKey tmp backend
-			case k of
-				Nothing -> stop
-				Just (key, _) -> do
-					moveAnnex key tmp
-					setUrlPresent key url
-					next $ Command.Add.cleanup file key True
-		else stop
+	stopUnless (liftIO $ Url.download url tmp) $ do
+		[(backend, _)] <- Backend.chooseBackends [file]
+		k <- Backend.genKey tmp backend
+		case k of
+			Nothing -> stop
+			Just (key, _) -> do
+				moveAnnex key tmp
+				setUrlPresent key url
+				next $ Command.Add.cleanup file key True
 
 nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
diff --git a/Command/Drop.hs b/Command/Drop.hs
index ee3583869b..0a4c9dfd6f 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -37,13 +37,9 @@ start numcopies file (key, _) = autoCopies key (>) numcopies $ do
 				else startRemote file numcopies key remote
 
 startLocal :: FilePath -> Maybe Int -> Key -> CommandStart
-startLocal file numcopies key = do
-	present <- inAnnex key
-	if present
-		then do
-			showStart "drop" file
-			next $ performLocal key numcopies
-		else stop
+startLocal file numcopies key = stopUnless (inAnnex key) $ do
+	showStart "drop" file
+	next $ performLocal key numcopies
 
 startRemote :: FilePath -> Maybe Int -> Key -> Remote.Remote Annex -> CommandStart
 startRemote file numcopies key remote = do
@@ -55,12 +51,9 @@ performLocal key numcopies = lockContent key $ do
 	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
 	untrusteduuids <- trustGet UnTrusted
 	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
-	success <- canDropKey key numcopies trusteduuids tocheck []
-	if success
-		then do
-			whenM (inAnnex key) $ removeAnnex key
-			next $ cleanupLocal key
-		else stop
+	stopUnless (canDropKey key numcopies trusteduuids tocheck []) $ do
+		whenM (inAnnex key) $ removeAnnex key
+		next $ cleanupLocal key
 
 performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
 performRemote key numcopies remote = lockContent key $ do
@@ -75,12 +68,9 @@ performRemote key numcopies remote = lockContent key $ do
 	untrusteduuids <- trustGet UnTrusted
 	let tocheck = filter (/= remote) $
 		Remote.remotesWithoutUUID remotes (have++untrusteduuids)
-	success <- canDropKey key numcopies have tocheck [uuid]
-	if success
-		then do
-			ok <- Remote.removeKey remote key
-			next $ cleanupRemote key remote ok
-		else stop
+	stopUnless (canDropKey key numcopies have tocheck [uuid]) $ do
+		ok <- Remote.removeKey remote key
+		next $ cleanupRemote key remote ok
 	where
 		uuid = Remote.uuid remote
 
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index b63d481bf3..ae2ad8b6a8 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -21,18 +21,11 @@ seek :: [CommandSeek]
 seek = [withKeys start]
 
 start :: Key -> CommandStart
-start key = do
-	present <- inAnnex key
-	if not present
-		then stop
-		else do
-			checkforced
-			showStart "dropkey" (show key)
-			next $ perform key
-	where
-		checkforced = 
-			unlessM (Annex.getState Annex.force) $
-				error "dropkey can cause data loss; use --force if you're sure you want to do this"
+start key = stopUnless (not <$> inAnnex key) $ do
+	unlessM (Annex.getState Annex.force) $
+		error "dropkey can cause data loss; use --force if you're sure you want to do this"
+	showStart "dropkey" (show key)
+	next $ perform key
 
 perform :: Key -> CommandPerform
 perform key = lockContent key $ do
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 27c4b167e5..f264106c3f 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -23,12 +23,9 @@ seek = [withFilesInGit $ whenAnnexed start]
 start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = do
 	link <- calcGitLink file key
-	l <- liftIO $ readSymbolicLink file
-	if link == l
-		then stop
-		else do
-			showStart "fix" file
-			next $ perform file link
+	stopUnless ((/=) link <$> liftIO (readSymbolicLink file)) $ do
+		showStart "fix" file
+		next $ perform file link
 
 perform :: FilePath -> FilePath -> CommandPerform
 perform file link = do
diff --git a/Command/Get.hs b/Command/Get.hs
index 093cd2cc59..b7023e2de8 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -22,32 +22,24 @@ seek :: [CommandSeek]
 seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
 start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
-start numcopies file (key, _) = do
-	inannex <- inAnnex key
-	if inannex
-		then stop
-		else autoCopies key (<) numcopies $ do
-			from <- Annex.getState Annex.fromremote
-			case from of
-				Nothing -> go $ perform key
-				Just name -> do
-					-- get --from = copy --from
-					src <- Remote.byName name
-					ok <- Command.Move.fromOk src key
-					if ok
-						then go $ Command.Move.fromPerform src False key
-						else stop
+start numcopies file (key, _) = stopUnless (not <$> inAnnex key) $
+	autoCopies key (<) numcopies $ do
+		from <- Annex.getState Annex.fromremote
+		case from of
+			Nothing -> go $ perform key
+			Just name -> do
+				-- get --from = copy --from
+				src <- Remote.byName name
+				stopUnless (Command.Move.fromOk src key) $
+					go $ Command.Move.fromPerform src False key
 	where
 		go a = do
 			showStart "get" file
 			next a	
 
 perform :: Key -> CommandPerform
-perform key = do
-	ok <- getViaTmp key (getKeyFile key)
-	if ok
-		then next $ return True -- no cleanup needed
-		else stop
+perform key = stopUnless (getViaTmp key $ getKeyFile key) $ do
+	next $ return True -- no cleanup needed
 
 {- Try to find a copy of the file in one of the remotes,
  - and copy it to here. -}
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index c85d7c2ac3..30288fc162 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -58,22 +58,18 @@ perform file oldkey newbackend = do
 	cleantmp tmpfile
 	case k of
 		Nothing -> stop
-		Just (newkey, _) -> do
-			ok <- link src newkey
-			if ok
-				then do
-					-- Update symlink to use the new key.
-					liftIO $ removeFile file
+		Just (newkey, _) -> stopUnless (link src newkey) $ do
+			-- Update symlink to use the new key.
+			liftIO $ removeFile file
 
-					-- If the old key had some
-					-- associated urls, record them for
-					-- the new key as well.
-					urls <- getUrls oldkey
-					unless (null urls) $
-						mapM_ (setUrlPresent newkey) urls
+			-- If the old key had some
+			-- associated urls, record them for
+			-- the new key as well.
+			urls <- getUrls oldkey
+			unless (null urls) $
+				mapM_ (setUrlPresent newkey) urls
 
-					next $ Command.Add.cleanup file newkey True
-				else stop
+			next $ Command.Add.cleanup file newkey True
 	where
 		cleantmp t = liftIO $ whenM (doesFileExist t) $ removeFile t
 		link src newkey = getViaTmpUnchecked newkey $ \t -> do
diff --git a/Command/Move.hs b/Command/Move.hs
index fd1ed90198..cc26eecda1 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -108,17 +108,11 @@ toPerform dest move key = moveLock move key $ do
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
 fromStart src move file key
 	| move = go
-	| otherwise = do
-		ishere <- inAnnex key
-		if ishere then stop else go
+	| otherwise = stopUnless (inAnnex key) go
 	where
-		go = do
-			ok <- fromOk src key
-			if ok
-				then do
-					showMoveAction move file
-					next $ fromPerform src move key
-				else stop
+		go = stopUnless (fromOk src key) $ do
+			showMoveAction move file
+			next $ fromPerform src move key
 fromOk :: Remote.Remote Annex -> Key -> Annex Bool
 fromOk src key = do
 	u <- getUUID
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index e97b6d05d8..263ff88b40 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -25,21 +25,17 @@ seek = [withFilesInGit $ whenAnnexed start]
 
 {- The unannex subcommand undoes an add. -}
 start :: FilePath -> (Key, Backend Annex) -> CommandStart
-start file (key, _) = do
-	ishere <- inAnnex key
-	if ishere
-		then do
-			force <- Annex.getState Annex.force
-			unless force $ do
-				top <- fromRepo Git.workTree
-				staged <- inRepo $ LsFiles.staged [top]
-				unless (null staged) $
-					error "This command cannot be run when there are already files staged for commit."
-				Annex.changeState $ \s -> s { Annex.force = True }
+start file (key, _) = stopUnless (inAnnex key) $ do
+	force <- Annex.getState Annex.force
+	unless force $ do
+		top <- fromRepo Git.workTree
+		staged <- inRepo $ LsFiles.staged [top]
+		unless (null staged) $
+			error "This command cannot be run when there are already files staged for commit."
+		Annex.changeState $ \s -> s { Annex.force = True }
 
-			showStart "unannex" file
-			next $ perform file key
-		else stop
+	showStart "unannex" file
+	next $ perform file key
 
 perform :: FilePath -> Key -> CommandPerform
 perform file key = next $ cleanup file key

From 14e9b87d44831fe0b6510422ffb4266f253496d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 13:07:31 -0400
Subject: [PATCH 2638/2835] unannex improvements

Added files don't have to be committed before they can be unannexed.

unannex no longer commits existing staged changes

unannex of the last file in a directory now works, before it failed because
git rm deleted the directory out from under it,
---
 Command/Unannex.hs | 30 ++++++++++++------------------
 debian/changelog   |  2 ++
 2 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 263ff88b40..aafc189d00 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -23,17 +23,8 @@ def = [command "unannex" paramPaths seek "undo accidential add command"]
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed start]
 
-{- The unannex subcommand undoes an add. -}
 start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = stopUnless (inAnnex key) $ do
-	force <- Annex.getState Annex.force
-	unless force $ do
-		top <- fromRepo Git.workTree
-		staged <- inRepo $ LsFiles.staged [top]
-		unless (null staged) $
-			error "This command cannot be run when there are already files staged for commit."
-		Annex.changeState $ \s -> s { Annex.force = True }
-
 	showStart "unannex" file
 	next $ perform file key
 
@@ -43,9 +34,17 @@ perform file key = next $ cleanup file key
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
 	liftIO $ removeFile file
-	inRepo $ Git.run "rm" [Params "--quiet --", File file]
-	-- git rm deletes empty directories; put them back
-	liftIO $ createDirectoryIfMissing True (parentDir file)
+	-- git rm deletes empty directory without --cached
+	inRepo $ Git.run "rm" [Params "--cached --quiet --", File file]
+	
+	-- If the file was already committed, it is now staged for removal.
+	-- Commit that removal now, to avoid later confusing the
+	-- pre-commit hook if this file is later added back to
+	-- git as a normal, non-annexed file.
+	whenM (not . null <$> inRepo (LsFiles.staged [file])) $ do
+		inRepo $ Git.run "commit" [
+			Param "-m", Param "content removed from git annex",
+			Param "--", File file]
 
 	fast <- Annex.getState Annex.fast
 	if fast
@@ -58,10 +57,5 @@ cleanup file key = do
 		else do
 			fromAnnex key file
 			logStatus key InfoMissing
-	
-	-- Commit staged changes at end to avoid confusing the
-	-- pre-commit hook if this file is later added back to
-	-- git as a normal, non-annexed file.
-	Annex.Queue.add "commit" [Param "-m", Param "content removed from git annex"] []
-	
+
 	return True
diff --git a/debian/changelog b/debian/changelog
index 04e9116720..1c27ad5666 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
     yet added to git. Running the add again will now clean up this situtation.
   * Fix caching of decrypted ciphers, which failed when drop had to check
     multiple different encrypted special remotes.
+  * unannex: Can be run on files that have been added to the annex, but not
+    yet committed.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 

From 252b2e92b0ddd5c97565d37c84771b471b6b5082 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 13:31:51 -0400
Subject: [PATCH 2639/2835] cleanup

---
 Command/Unannex.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index aafc189d00..bed857b060 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -10,7 +10,6 @@ module Command.Unannex where
 import Common.Annex
 import Command
 import qualified Annex
-import qualified Annex.Queue
 import Utility.FileMode
 import Logs.Location
 import Annex.Content

From 95e748cbd4bb858a3b87621e60f5b43d53b50480 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 13:32:09 -0400
Subject: [PATCH 2640/2835] inverted logic

---
 Command/DropKey.hs | 2 +-
 Command/Move.hs    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index ae2ad8b6a8..aaaa224661 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -21,7 +21,7 @@ seek :: [CommandSeek]
 seek = [withKeys start]
 
 start :: Key -> CommandStart
-start key = stopUnless (not <$> inAnnex key) $ do
+start key = stopUnless (inAnnex key) $ do
 	unlessM (Annex.getState Annex.force) $
 		error "dropkey can cause data loss; use --force if you're sure you want to do this"
 	showStart "dropkey" (show key)
diff --git a/Command/Move.hs b/Command/Move.hs
index cc26eecda1..85fdff7398 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -108,7 +108,7 @@ toPerform dest move key = moveLock move key $ do
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
 fromStart src move file key
 	| move = go
-	| otherwise = stopUnless (inAnnex key) go
+	| otherwise = stopUnless (not <$> inAnnex key) go
 	where
 		go = stopUnless (fromOk src key) $ do
 			showMoveAction move file

From d69cf79e202b7d60465ce2068f736243aab14b3b Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Fri, 9 Dec 2011 20:47:00 +0000
Subject: [PATCH 2641/2835]

---
 doc/forum/pure_git-annex_only_workflow.mdwn | 46 +++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow.mdwn

diff --git a/doc/forum/pure_git-annex_only_workflow.mdwn b/doc/forum/pure_git-annex_only_workflow.mdwn
new file mode 100644
index 0000000000..e0cbcd5384
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow.mdwn
@@ -0,0 +1,46 @@
+I’m using git annex to manage my movie collection on various devices – my laptop, a NSLU tucked away somewhere with lots of space, some external hard drives. For this use case, I do not need the full power of git as a version control system, so having to run "git commit" and coming up with commit messages is annoying. Also, this makes sense for a version control system, but not for my media collection:
+
+	$ git annex add Hot\ Fuzz\ -\ English.mkv 
+	add Hot Fuzz - English.mkv (checksum...) ok
+	(Recording state in git...)
+	$ git commit -m 'another movie added'
+	[master 851dc8a] another movie added
+	 1 files changed, 1 insertions(+), 0 deletions(-)
+	 create mode 120000 00 Noch nicht gesehen/Hot Fuzz - English.mkv
+	$ git push jeff
+	Counting objects: 38, done.
+	Delta compression using up to 2 threads.
+	Compressing objects: 100% (20/20), done.
+	Writing objects: 100% (26/26), 2.00 KiB, done.
+	Total 26 (delta 11), reused 0 (delta 0)
+	remote: error: refusing to update checked out branch: refs/heads/master
+	remote: error: By default, updating the current branch in a non-bare repository
+	remote: error: is denied, because it will make the index and work tree inconsistent
+	remote: error: with what you pushed, and will require 'git reset --hard' to match
+	remote: error: the work tree to HEAD.
+	remote: error: 
+	remote: error: You can set 'receive.denyCurrentBranch' configuration variable to
+	remote: error: 'ignore' or 'warn' in the remote repository to allow pushing into
+	remote: error: its current branch; however, this is not recommended unless you
+	remote: error: arranged to update its work tree to match what you pushed in some
+	remote: error: other way.
+	remote: error: 
+	remote: error: To squelch this message and still keep the default behaviour, set
+	remote: error: 'receive.denyCurrentBranch' configuration variable to 'refuse'.
+	To jeff:/mnt/media/Movies
+	 ! [rejected]        git-annex -> git-annex (non-fast-forward)
+	 ! [remote rejected] master -> master (branch is currently checked out)
+	error: failed to push some refs to 'jeff:/mnt/media/Movies'
+	To prevent you from losing history, non-fast-forward updates were rejected
+	Merge the remote changes (e.g. 'git pull') before pushing again.  See the
+	'Note about fast-forwards' section of 'git push --help' for details.
+
+It seems that to successfully make the new files known to the other side, I have to log into jeff and pull _from_ my currenct machine.	
+
+What I would like to have is that
+
+* git annex add does not require a commit afterwards.
+* Changes to the files are automatically picked up with the next git-annex call (similar to how etckeeper works).
+* Commands "git annex push" and "git annex pull" that will syn the metadata (i.e. the list of files) in both directions without further manual intervention, at least not until the two repositories have diverged in a way that is not possible to merge sensible.
+
+Summay: git-annex is great. git is not always. Please make it possible to use git annex without having to use git.

From 07c32dd49ce389cd0ba1888b42429ae5fe54a972 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Fri, 9 Dec 2011 20:57:00 +0000
Subject: [PATCH 2642/2835] typo

---
 doc/forum/pure_git-annex_only_workflow.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/pure_git-annex_only_workflow.mdwn b/doc/forum/pure_git-annex_only_workflow.mdwn
index e0cbcd5384..48226671c1 100644
--- a/doc/forum/pure_git-annex_only_workflow.mdwn
+++ b/doc/forum/pure_git-annex_only_workflow.mdwn
@@ -35,7 +35,7 @@ I’m using git annex to manage my movie collection on various devices – my la
 	Merge the remote changes (e.g. 'git pull') before pushing again.  See the
 	'Note about fast-forwards' section of 'git push --help' for details.
 
-It seems that to successfully make the new files known to the other side, I have to log into jeff and pull _from_ my currenct machine.	
+It seems that to successfully make the new files known to the other side, I have to log into jeff and pull _from_ my current machine.	
 
 What I would like to have is that
 

From 2cd22c1a13d6b4e1c6d2407d249f2f8d92643fbd Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Fri, 9 Dec 2011 21:25:37 +0000
Subject: [PATCH 2643/2835] brr, so many typos today.

---
 doc/forum/pure_git-annex_only_workflow.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/pure_git-annex_only_workflow.mdwn b/doc/forum/pure_git-annex_only_workflow.mdwn
index 48226671c1..36648a9058 100644
--- a/doc/forum/pure_git-annex_only_workflow.mdwn
+++ b/doc/forum/pure_git-annex_only_workflow.mdwn
@@ -41,6 +41,6 @@ What I would like to have is that
 
 * git annex add does not require a commit afterwards.
 * Changes to the files are automatically picked up with the next git-annex call (similar to how etckeeper works).
-* Commands "git annex push" and "git annex pull" that will syn the metadata (i.e. the list of files) in both directions without further manual intervention, at least not until the two repositories have diverged in a way that is not possible to merge sensible.
+* Commands "git annex push" and "git annex pull" that will sync the metadata (i.e. the list of files) in both directions without further manual intervention, at least not until the two repositories have diverged in a way that is not possible to merge sensible.
 
 Summay: git-annex is great. git is not always. Please make it possible to use git annex without having to use git.

From 28699c95a7de284f07a5c0e34fb1755868301f3c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 18:10:41 -0400
Subject: [PATCH 2644/2835] some work on avoiding partial functions

There are still hundreds of places that use partial functions head, tail,
init, and last.
---
 Command/DropUnused.hs |  4 ++--
 Git.hs                | 18 ++++++------------
 Utility/BadPrelude.hs | 24 ++++++++++++++++++++++++
 Utility/Misc.hs       | 13 +++++++++++++
 4 files changed, 45 insertions(+), 14 deletions(-)
 create mode 100644 Utility/BadPrelude.hs

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 3df9ab6c2b..244f378d97 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -73,6 +73,6 @@ readUnusedLog prefix = do
 		then M.fromList . map parse . lines <$> liftIO (readFile f)
 		else return M.empty
 	where
-		parse line = (num, fromJust $ readKey $ tail rest)
+		parse line = (num, fromJust $ readKey rest)
 			where
-				(num, rest) = break (== ' ') line
+				(num, rest) = separate (== ' ') line
diff --git a/Git.hs b/Git.hs
index 5bdd4afd4e..84153be5d8 100644
--- a/Git.hs
+++ b/Git.hs
@@ -507,11 +507,7 @@ configStore s repo = do
 configParse :: String -> M.Map String String
 configParse s = M.fromList $ map pair $ lines s
 	where
-		pair l = (key l, val l)
-		key l = head $ keyval l
-		val l = join sep $ drop 1 $ keyval l
-		keyval l = split sep l :: [String]
-		sep = "="
+		pair = separate (== '=')
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
 configRemotes :: Repo -> IO [Repo]
@@ -550,13 +546,11 @@ genRemote s repo = gen $ calcloc s
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
 			where
-				bits = split ":" v
-				host = head bits
-				dir = join ":" $ drop 1 bits
-				slash d	| d == "" = "/~/" ++ dir
-					| head d == '/' = dir
-					| head d == '~' = '/':dir
-					| otherwise = "/~/" ++ dir
+				(host, dir) = separate (== ':') v
+				slash d	| d == "" = "/~/" ++ d
+					| "/" `isPrefixOf` d = d
+					| "~" `isPrefixOf` d = '/':d
+					| otherwise = "/~/" ++ d
 
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
new file mode 100644
index 0000000000..1bb70adfb3
--- /dev/null
+++ b/Utility/BadPrelude.hs
@@ -0,0 +1,24 @@
+{- Some stuff from Prelude should not be used, as it tends to be a source
+ - of bugs.
+ -
+ - This exports functions that conflict with the prelude, which avoids
+ - them being accidentially used.
+ -}
+
+module Utility.BadPrelude where
+
+{- head is a partial function; head [] is an error -}
+head :: [a] -> a
+head = Prelude.head
+
+{- tail is also partial -}
+tail :: [a] -> a
+tail = Prelude.tail
+
+{- init too -}
+init :: [a] -> a
+init = Prelude.init
+
+{- last too -}
+last :: [a] -> a
+last = Prelude.last
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index 7285987231..541e150b77 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -27,6 +27,19 @@ readMaybe s = case reads s of
 	((x,_):_) -> Just x
 	_ -> Nothing
 
+{- Like break, but the character matching the condition is not included
+ - in the second result list.
+ -
+ - separate (== ':') "foo:bar" = ("foo", "bar")
+ - separate (== ':') "foobar" = ("foo, "")
+ -}
+separate :: (a -> Bool) -> [a] -> ([a], [a])
+separate c l = unbreak $ break c l
+	where
+		unbreak r@(a, b)
+			| null b = r
+			| otherwise = (a, tail b)
+
 {- Catches IO errors and returns a Bool -}
 catchBoolIO :: IO Bool -> IO Bool
 catchBoolIO a = catchDefaultIO a False

From be67294a1ebf1a49eddaf9698c662888d80d6a7a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 9 Dec 2011 22:56:11 +0000
Subject: [PATCH 2645/2835] Added a comment

---
 ..._a32f7efd18d174845099a4ed59e6feae._comment | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment b/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment
new file mode 100644
index 0000000000..def1794a3e
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-09T22:56:11Z"
+ content="""
+First, you need a bare git repository that you can push to, and pull from. This simplifies most git workflow.
+
+Secondly, I use [mr](http://kitenet.net/~joey/code/mr/), with this in `.mrconfig`:
+
+
+[DEFAULT]
+lib =
+        annexupdate() {
+                git commit -a -m update || true
+                git pull \"$@\"
+                git annex merge
+                git push || true
+        }
+
+[lib/sound]
+update = annexupdate
+[lib/big]
+update = annexupdate
+

+
+Which makes \"mr update\" in repositories where I rarely care about git details take care of syncing my changes.
+
+I also make \"mr update\" do a \"git annex get\" of some files in some repositories that I want to always populate. git-annex and mr go well together. :)
+
+Perhaps my annexupdate above should be available as \"git annex sync\"?
+"""]]

From 4b3c4c0c2b88aa17a9b7822470e3e5dd2a3e7c2b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 18:57:09 -0400
Subject: [PATCH 2646/2835] avoid some read

---
 Config.hs             | 7 ++++---
 Git.hs                | 2 +-
 Utility/BadPrelude.hs | 4 ++++
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Config.hs b/Config.hs
index cc0c929531..c6107fc8eb 100644
--- a/Config.hs
+++ b/Config.hs
@@ -79,9 +79,10 @@ repoNotIgnored r = not . Git.configTrue <$> getConfig r "ignore" "false"
 {- If a value is specified, it is used; otherwise the default is looked up
  - in git config. forcenumcopies overrides everything. -}
 getNumCopies :: Maybe Int -> Annex Int
-getNumCopies v = 
-	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
+getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 	where
 		use (Just n) = return n
-		use Nothing = read <$> fromRepo (Git.configGet config "1")
+		use Nothing = perhaps (return 1) =<< 
+			readMaybe <$> fromRepo (Git.configGet config "1")
+		perhaps fallback = maybe fallback (return . id)
 		config = "annex.numcopies"
diff --git a/Git.hs b/Git.hs
index 84153be5d8..8bc32b7cc7 100644
--- a/Git.hs
+++ b/Git.hs
@@ -345,7 +345,7 @@ urlPort :: Repo -> Maybe Integer
 urlPort r = 
 	case urlAuthPart uriPort r of
 		":" -> Nothing
-		(':':p) -> Just (read p)
+		(':':p) -> readMaybe p
 		_ -> Nothing
 
 {- Hostname of an URL repo, including any username (ie, "user@host") -}
diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 1bb70adfb3..7921a7e9b7 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -22,3 +22,7 @@ init = Prelude.init
 {- last too -}
 last :: [a] -> a
 last = Prelude.last
+
+{- read should be avoided, as it throws an error -}
+read :: Read a => String -> a
+read = Prelude.read

From fb8231f3a1eb67e61a7df0381dc82be2f0ca0417 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 20:27:22 -0400
Subject: [PATCH 2647/2835] sync: New command that synchronises the local
 repository and default remote, by running git commit, pull, and push for you.

---
 Command/Sync.hs    | 64 ++++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |  4 ++-
 debian/changelog   |  2 ++
 doc/git-annex.mdwn | 11 ++++++++
 4 files changed, 80 insertions(+), 1 deletion(-)
 create mode 100644 Command/Sync.hs

diff --git a/Command/Sync.hs b/Command/Sync.hs
new file mode 100644
index 0000000000..c0d7f37ad0
--- /dev/null
+++ b/Command/Sync.hs
@@ -0,0 +1,64 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Sync where
+
+import Common.Annex
+import Command
+import qualified Annex.Branch
+import qualified Git
+
+import qualified Data.ByteString.Lazy.Char8 as L
+
+def :: [Command]
+def = [command "sync" paramPaths seek "synchronize local repository with remote"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStart
+start = do
+        showStart "sync" "."
+	showOutput
+        next perform
+
+perform :: CommandPerform
+perform = do
+	remote <- defaultRemote =<< currentBranch
+	checkRemote remote
+	commit
+	inRepo $ Git.run "pull" [Param remote]
+	Annex.Branch.update
+	inRepo $ Git.run "push" [Param remote, matchingbranches]
+	next $ return True
+	where
+		-- git push may be configured to not push matching
+		-- branches; this should ensure it always does.
+		matchingbranches = Param ":"
+
+commit :: Annex ()
+commit = do
+	-- Commit will fail when the tree is clean (or when in a confliced
+	-- merge, etc). Ignore failure.
+	_ <- inRepo $ Git.runBool "commit" [Param "-a", Param "-m", Param "sync"]
+	return ()
+
+-- the remote defaults to origin when not configured
+defaultRemote :: String -> Annex String
+defaultRemote branch =
+	fromRepo $ Git.configGet ("branch." ++ branch ++ ".remote") "origin"
+
+currentBranch :: Annex String
+currentBranch = last . split "/" . L.unpack . head . L.lines <$>
+	inRepo (Git.pipeRead [Param "symbolic-ref", Param "HEAD"])
+
+checkRemote :: String -> Annex ()
+checkRemote remote = do
+	remoteurl <- fromRepo $
+		Git.configGet ("remote." ++ remote ++ ".url") ""
+	when (null remoteurl) $ do
+		error $ "No url is configured for the remote: " ++ remote
diff --git a/GitAnnex.hs b/GitAnnex.hs
index d768499ddb..7871638e44 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -47,6 +47,7 @@ import qualified Command.Trust
 import qualified Command.Untrust
 import qualified Command.Semitrust
 import qualified Command.Dead
+import qualified Command.Sync
 import qualified Command.AddUrl
 import qualified Command.Map
 import qualified Command.Upgrade
@@ -61,6 +62,8 @@ cmds = concat
 	, Command.Copy.def
 	, Command.Unlock.def
 	, Command.Lock.def
+	, Command.Sync.def
+	, Command.AddUrl.def
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def
@@ -72,7 +75,6 @@ cmds = concat
 	, Command.Untrust.def
 	, Command.Semitrust.def
 	, Command.Dead.def
-	, Command.AddUrl.def
 	, Command.FromKey.def
 	, Command.DropKey.def
 	, Command.Fix.def
diff --git a/debian/changelog b/debian/changelog
index 1c27ad5666..b481d99997 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
     multiple different encrypted special remotes.
   * unannex: Can be run on files that have been added to the annex, but not
     yet committed.
+  * sync: New command that synchronises the local repository and default
+    remote, by running git commit, pull, and push for you.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 08def0d62b..d7a51663fd 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -120,6 +120,17 @@ subdirectories).
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
+* sync
+
+  Use this command when you want to synchronize the local repository
+  with its default remote (typically "origin"). The sync process involves
+  first committing all local changes, then pulling and merging any changes
+  from the remote, and finally pushing the repository's state to the remote.
+  You can use standard git commands to do each of those steps by hand,
+  or if you don't want to worry about the details, you can use sync.
+
+  Note that sync does not transfer any file contents from or to the remote.
+
 * addurl [url ...]
 
   Downloads each url to a file, which is added to the annex.

From 4200b8038a79aa788ce0b62579bd5820b7d75e3b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 12:21:22 -0400
Subject: [PATCH 2648/2835] separate operations

---
 Command/Sync.hs | 54 +++++++++++++++++++++++++++----------------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index c0d7f37ad0..7dc5f4d24c 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -17,39 +17,45 @@ import qualified Data.ByteString.Lazy.Char8 as L
 def :: [Command]
 def = [command "sync" paramPaths seek "synchronize local repository with remote"]
 
+-- syncing involves several operations, any of which can independantly fail
 seek :: [CommandSeek]
-seek = [withNothing start]
+seek = map withNothing [commit, pull, push]
 
-start :: CommandStart
-start = do
-        showStart "sync" "."
-	showOutput
-        next perform
+commit :: CommandStart
+commit = do
+	showStart "commit" ""
+	next $ next $ do
+		showOutput
+		-- Commit will fail when the tree is clean, so ignore failure.
+		_ <- inRepo $ Git.runBool "commit" [Param "-a", Param "-m", Param "sync"]
+		return True
 
-perform :: CommandPerform
-perform = do
-	remote <- defaultRemote =<< currentBranch
-	checkRemote remote
-	commit
-	inRepo $ Git.run "pull" [Param remote]
-	Annex.Branch.update
-	inRepo $ Git.run "push" [Param remote, matchingbranches]
-	next $ return True
+pull :: CommandStart
+pull = do
+	remote <- defaultRemote
+	showStart "pull" remote
+	next $ next $ do
+		showOutput
+		checkRemote remote
+		inRepo $ Git.runBool "pull" [Param remote]
+
+push :: CommandStart
+push = do
+	remote <- defaultRemote
+	showStart "push" remote
+	next $ next $ do
+		Annex.Branch.update
+		showOutput
+		inRepo $ Git.runBool "push" [Param remote, matchingbranches]
 	where
 		-- git push may be configured to not push matching
 		-- branches; this should ensure it always does.
 		matchingbranches = Param ":"
 
-commit :: Annex ()
-commit = do
-	-- Commit will fail when the tree is clean (or when in a confliced
-	-- merge, etc). Ignore failure.
-	_ <- inRepo $ Git.runBool "commit" [Param "-a", Param "-m", Param "sync"]
-	return ()
-
 -- the remote defaults to origin when not configured
-defaultRemote :: String -> Annex String
-defaultRemote branch =
+defaultRemote :: Annex String
+defaultRemote = do
+	branch <- currentBranch
 	fromRepo $ Git.configGet ("branch." ++ branch ++ ".remote") "origin"
 
 currentBranch :: Annex String

From e0e9d1cabd0aca52aa410be37de7251533b5e237 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sat, 10 Dec 2011 16:28:29 +0000
Subject: [PATCH 2649/2835] Added a comment

---
 ...nt_2_66dc9b65523a9912411db03c039ba848._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment b/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment
new file mode 100644
index 0000000000..473a0287d0
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 2"
+ date="2011-12-10T16:28:29Z"
+ content="""
+Thanks for the tips so far. I guess a bare-only repo helps, but as well is something that I don’t _need_ (for my use case), any only have to do because git works like this.
+
+Also, if I have a mobile device that I want to push to, then I’d have to have two repositories on the device, as I might not be able to reach my main bare repository when traveling, but I cannot push to the „real“ repo on the mobile device from my computer. I guess I am spoiled by darcs, which will happily push to a checked out 
+remote repository, updating the checkout if possible without conflict.
+
+If I introduce a central bare repository to push to and from; I’d still have to have the other non-bare repos as remotes, so that git-annex will know about them and their files, right?
+
+I’d appreciate a \"git annex sync\" that does what you described (commit all, pull, merge, push). Especially if it comes in a \"git annex sync --all\" variant that syncs all reachable repositories.
+"""]]

From eecaf424856e42290d653e0b4b4043665da18b00 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 12:30:31 -0400
Subject: [PATCH 2650/2835] no need to show, it's a string

---
 Annex/Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 8f0a09fd6b..37237ec667 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -141,7 +141,7 @@ update = onceonly $ do
 		let merge_desc = if null branches
 			then "update" 
 			else "merging " ++
-				unwords (map (show . Git.refDescribe) branches) ++ 
+				unwords (map Git.refDescribe branches) ++ 
 				" into " ++ show name
 		unless (null branches) $ do
 			showSideAction merge_desc

From c5267802f38eb020edee8143d5a097296232bb35 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 12:49:44 -0400
Subject: [PATCH 2651/2835] version dependency on old monad-control

This should let cabal build it with the right version.
---
 debian/changelog | 1 +
 git-annex.cabal  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index b481d99997..8dbcb07dc6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -11,6 +11,7 @@ git-annex (3.20111204) UNRELEASED; urgency=low
     yet committed.
   * sync: New command that synchronises the local repository and default
     remote, by running git commit, pull, and push for you.
+  * Version monad-control dependency in cabal file.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 7be78053f8..8f5d6ebb86 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5, monad-control, json
+   base < 5, monad-control < 0.3, json
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From 6cf28585b66264e174eee45496e01d23667e0d20 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 10 Dec 2011 19:43:04 +0000
Subject: [PATCH 2652/2835] Added a comment

---
 ...mment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment b/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment
new file mode 100644
index 0000000000..9b6e6d7c4d
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-12-10T19:43:04Z"
+ content="""
+Git can actually push into a non-bare repository, so long as the branch you change there is not a checked out one. Pushing into `remotes/$foo/master` and `remotes/$foo/git-annex` would work, however determining the value that the repository expects for `$foo` is something git cannot do on its own. And of course you'd still have to `git merge remotes/$foo/master` to get the changes. 
+
+Yes, you still keep the non-bare repos as remotes when adding a bare repository, so git-annex knows how to get to them.
+
+I've made `git annex sync` run the simple script above. Perhaps it can later be improved to sync all repositories.
+"""]]

From bfdc9f28fc6ef19f7e3ec2ea724a540af104a386 Mon Sep 17 00:00:00 2001
From: "http://schnouki.net/" 
Date: Sat, 10 Dec 2011 21:40:01 +0000
Subject: [PATCH 2653/2835]

---
 ...__git-annex_get__34___with_3.20111203.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn

diff --git a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
new file mode 100644
index 0000000000..2e77713862
--- /dev/null
+++ b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
@@ -0,0 +1,21 @@
+Hi there,
+
+After updating to 3.20111203 (on Arch Linux) I noticed I was not able to use `git annex get` from a SSH remote (server running Arch Linux, same version of git-annex): "requested key is not present". Same behavior with current master (commit 6cf28585). I had no issue with the previous version (3.20111122).
+
+On this server, I was able to track down the issue using `git-annex-shell inannex` and `strace`:
+
+    $ strace -f -o log git-annex-shell inannex ~/photos-annex.git WORM-s369360-m1321602916--2011-11-17.jpg   
+    $ echo $?
+    1
+    $ tail -n20 log
+    [...]
+    25623 chdir("/home/schnouki/git-annex") = 0
+    25623 stat("/home/schnouki/photos-annex.git/annex/objects/082/676/WORM-s369360-m1321602916--2011-11-17.jpg/WORM-s369360-m1321602916--2011-11-17.jpg", {st_mode=S_IFREG|0400, st_size=369360, ...}) = 0
+    25623 open("annex/objects/082/676/WORM-s369360-m1321602916--2011-11-17.jpg/WORM-s369360-m1321602916--2011-11-17.jpg", O_RDONLY) = -1 ENOENT (No such file or directory)
+    [...]
+
+Note there is a call to `stat()` with the full path to the requested file, and *then* a call to `open()` with a relative path -- which calls this call to fail, and git-annex-shell to return 1. With 3.20111122, there was no call to `stat()`, just a successful call to `open()` with a full absolute path.
+
+Using `git bisect` I was able to determine that this bug appeared in commit 64672c62 ("refactor"). Reverting it makes `git-annex-shell` work as expected, but I'm sure there are better ways to fix this. However I don't know enough Haskell to do it myself.
+
+Could you please try to fix this in a future version?

From 10e8028a42fe65e8586b7424557b792cf36e215c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 18:45:55 -0400
Subject: [PATCH 2654/2835] Fix bug in last version in getting contents from
 bare repositories.

---
 Locations.hs                                             | 9 ++++-----
 debian/changelog                                         | 1 +
 ..._39__t___34__git-annex_get__34___with_3.20111203.mdwn | 5 +++++
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 3843495f9c..080766841e 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -80,16 +80,15 @@ gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
 		{- Bare repositories default to hashDirLower for new
 		 - content, as it's more portable. -}
-		go (Git.workTree r) (annexLocations key)
+		check (map inrepo $ annexLocations key)
 	| otherwise =
 		{- Non-bare repositories only use hashDirMixed, so
 		 - don't need to do any work to check if the file is
 		 - present. -}
-		return $ Git.workTree r  ".git" 
-				annexLocation key hashDirMixed
+		return $ inrepo ".git"  annexLocation key hashDirMixed
 	where
-		go dir locs = fromMaybe (dir  head locs) <$> check dir locs
-		check dir = firstM $ \f -> doesFileExist $ dir  f
+		inrepo = () (Git.workTree r)
+		check locs = fromMaybe (head locs) <$> firstM doesFileExist locs
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
diff --git a/debian/changelog b/debian/changelog
index 8dbcb07dc6..5c68fde2db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (3.20111204) UNRELEASED; urgency=low
   * sync: New command that synchronises the local repository and default
     remote, by running git commit, pull, and push for you.
   * Version monad-control dependency in cabal file.
+  * Fix bug in last version in getting contents from bare repositories.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
index 2e77713862..00ab9f3f07 100644
--- a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
+++ b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
@@ -19,3 +19,8 @@ Note there is a call to `stat()` with the full path to the requested file, and *
 Using `git bisect` I was able to determine that this bug appeared in commit 64672c62 ("refactor"). Reverting it makes `git-annex-shell` work as expected, but I'm sure there are better ways to fix this. However I don't know enough Haskell to do it myself.
 
 Could you please try to fix this in a future version?
+
+> Thanks for a very good bug report. 
+> 
+> I've fixed this stupid mistake introduced in the code refactoring.
+> --[[Joey]]

From 9ba99a544b79f9304839a5f4e756d8025e9f18f0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 18:51:01 -0400
Subject: [PATCH 2655/2835] update

---
 Remote/Bup.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 589dea91d9..e705bbb344 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -182,7 +182,7 @@ onBupRemote r a command params = do
  - local bup repositories to see if they are available, and getting their
  - uuid (which may be different from the stored uuid for the bup remote).
  -
- - If a bup repository is not available, returns a dummy uuid of "".
+ - If a bup repository is not available, returns NoUUID.
  - This will cause checkPresent to indicate nothing from the bup remote
  - is known to be present.
  -

From f44f715f51aad6cf164501b0d53b7163f3a5e758 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 19:38:27 -0400
Subject: [PATCH 2656/2835] ensure local remote is initialized when copying to
 it

Needed due to this scenario: Bare repo origin is made, foo is cloned from it;
foo is initalized; a file is added to foo's annex; git annex move --to origin

Since the git-annex branch has not yet been pushed to origin, it doesn't
auto-initialize. When the content is sent to it, it's stored, but
the remote has NoUUID, and so nothing is logged in the location log.
Then the content is removed from the local repo, and git-annex has lost
track of it.

git annex fsck in origin will find the lost content, but let's not let this
happen. Content should only be sent to initalized remotes.

This cannot happen for non-local remotes, since git-annex-shell always
checks that the repo is initialized.
---
 Remote/Git.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index d172ec7c04..05743a28d5 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -210,6 +210,7 @@ copyToRemote r key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
+			ensureInitialized
 			ok <- Annex.Content.getViaTmp key $
 				rsyncOrCopyFile params keysrc
 			Annex.Content.saveState

From 583ba809923948eb3ce9b3f91892c7492e5db14f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 20:53:42 -0400
Subject: [PATCH 2657/2835] better syntax

---
 Locations.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Locations.hs b/Locations.hs
index 080766841e..f7784a56d5 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -87,7 +87,7 @@ gitAnnexLocation key r
 		 - present. -}
 		return $ inrepo ".git"  annexLocation key hashDirMixed
 	where
-		inrepo = () (Git.workTree r)
+		inrepo d = Git.workTree r  d
 		check locs = fromMaybe (head locs) <$> firstM doesFileExist locs
 
 {- The annex directory of a repository. -}

From bf6a3b757adebe92b330c4216fb32fbf1a6d64d2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 01:15:27 -0400
Subject: [PATCH 2658/2835] close

---
 .../Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
index 00ab9f3f07..ea56c37320 100644
--- a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
+++ b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
@@ -23,4 +23,5 @@ Could you please try to fix this in a future version?
 > Thanks for a very good bug report. 
 > 
 > I've fixed this stupid mistake introduced in the code refactoring.
+> [[done]]
 > --[[Joey]]

From 0ba4b1de18acf9e2318da6c8e7d80ce4bb216850 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 14:14:28 -0400
Subject: [PATCH 2659/2835] move a file location to Locations.hs

---
 Annex/Branch.hs | 6 +-----
 Locations.hs    | 5 +++++
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 37237ec667..44ee69e627 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -43,10 +43,6 @@ fullname = Git.Ref $ "refs/heads/" ++ show name
 originname :: Git.Ref
 originname = Git.Ref $ "origin/" ++ show name
 
-{- A separate index file for the branch. -}
-index :: Git.Repo -> FilePath
-index g = gitAnnexDir g  "index"
-
 {- Populates the branch's index file with the current branch contents.
  - 
  - Usually, this is only done when the index doesn't yet exist, and
@@ -62,7 +58,7 @@ withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
-	f <- fromRepo index
+	f <- fromRepo gitAnnexIndex
 	bracketIO (Git.useIndex f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
diff --git a/Locations.hs b/Locations.hs
index f7784a56d5..fdb8232844 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,6 +20,7 @@ module Locations (
 	gitAnnexUnusedLog,
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
+	gitAnnexIndex,
 	isLinkToAnnex,
 	annexHashes,
 	hashDirMixed,
@@ -131,6 +132,10 @@ gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r  "journal"
 gitAnnexJournalLock :: Git.Repo -> FilePath
 gitAnnexJournalLock r = gitAnnexDir r  "journal.lck"
 
+{- .git/annex/index is used to stage changes to the git-annex branch -}
+gitAnnexIndex :: Git.Repo -> FilePath
+gitAnnexIndex r = gitAnnexDir r  "index"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s

From 59971c923029a6f10c47a526e7878130637c139e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 14:47:44 -0400
Subject: [PATCH 2660/2835] new bug

---
 doc/bugs/git-annex_branch_corruption.mdwn | 89 +++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 doc/bugs/git-annex_branch_corruption.mdwn

diff --git a/doc/bugs/git-annex_branch_corruption.mdwn b/doc/bugs/git-annex_branch_corruption.mdwn
new file mode 100644
index 0000000000..63633fb080
--- /dev/null
+++ b/doc/bugs/git-annex_branch_corruption.mdwn
@@ -0,0 +1,89 @@
+Below is a test case which shows a way that the git-annex branch
+can become corrupted and lose data, including location log records and
+uuid.log lines.
+
+At the end, a commit on the git-annex branch removes one of the 2 lines
+from the uuid.log; which should never happen.
+
+The actual problem occurs earlier, at the "push point". Here a repo is
+cloned from the main one, initialized (adding the last uuid.log line),
+and then pushed back to the main one. That push is a fast-forward, so is
+allowed to directly update the git-annex branch in the main repo:
+
+	b884fe5..c497739  git-annex -> git-annex
+
+Now the git-annex branch has a change that is not reflected in
+`.git/annex/index`, so the next time a change is made, it's committed
+using the out of date index, which causes a reversion of the changes
+that were pushed to the branch.
+
+---
+
+## Thoughts
+
+This is essentially the same reason why git blocks pushes to the checked-out
+branch of a non-bare repository. 
+
+This problem only affects workflows that involve pushing. Pulling workflows
+do not directly update the local git-annex branch, so avoid the problem.
+
+And while bare repos are pushed to, they rarely have changes made directly
+to their git-annex branches, so while I think the same problem could
+happen with pushing to a bare repo, it's unlikely.
+
+None of which is to say this is not a bad bug that needs to be comprehensively
+fixed. 
+
+Probably git-annex needs to record which ref of the git-annex branch
+corresponds to its index, and if the branch is at a different ref,
+merge it into the index. I am still considering how to do that atomically;
+what if a push comes in while git-annex is updating its index?
+
+---
+
+## Workaround
+
+Users who want to prevent this bug from occuring when pushing to their
+non-bare repositories can install this script as `.git/hooks/update`
+
+
+#!/bin/sh
+if [ "$1" = refs/heads/git-annex ]; then
+	exit 1
+fi
+

+
+--[[Joey]] 
+
+---
+
+## Test Case
+
+#!/bin/sh
+mkdir annextest
+cd annextest
+
+git init dir1
+cd dir1
+git annex init
+touch foo 
+echo hi > bar
+git annex add
+git commit -m add
+
+cd ..
+git clone dir1 dir2
+cd dir2
+git annex init otherdir
+git annex get
+# push point
+git push
+
+cd ..
+cd dir1
+echo "before"
+git show git-annex:uuid.log
+git annex drop foo --force
+echo "after"
+git show git-annex:uuid.log
+


From 8680c415dedcc1fdbbcc0b9cdd7e37e860b22130 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 14:51:20 -0400
Subject: [PATCH 2661/2835] slow, stupid, and safe index updating

Always merge the git-annex branch into .git/annex/index before making a
commit from the index.

This ensures that, when the branch has been changed in any way
(by a push being received, or changes pulled directly into it, or
even by the user checking it out, and committing a change), the index
reflects those changes.

This is much too slow; it needs to be optimised to only update the
index when the branch has really changed, not every time.

Also, there is an unhandled race, when a change is made to the branch
right after the index gets updated. I left it in for now because it's
unlikely and I didn't want to complicate things with additional locking
yet.
---
 Annex/Branch.hs                           | 37 ++++++++++++++---------
 debian/changelog                          |  5 +++
 doc/bugs/git-annex_branch_corruption.mdwn |  3 ++
 doc/internals.mdwn                        | 14 ++-------
 4 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 44ee69e627..be19a9be0c 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -45,14 +45,26 @@ originname = Git.Ref $ "origin/" ++ show name
 
 {- Populates the branch's index file with the current branch contents.
  - 
- - Usually, this is only done when the index doesn't yet exist, and
- - the index is used to build up changes to be commited to the branch,
- - and merge in changes from other branches.
+ - This is only done when the index doesn't yet exist, and the index 
+ - is used to build up changes to be commited to the branch, and merge
+ - in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
 genIndex g = Git.UnionMerge.stream_update_index g
 	[Git.UnionMerge.ls_tree fullname g]
 
+{- Merges the specified branches into the index.
+ - Any changes staged in the index will be preserved. -}
+mergeIndex :: [Git.Ref] -> Annex ()
+mergeIndex branches = do
+	h <- catFileHandle
+	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+
+{- Updates the branch's index to reflect the current contents of the branch.
+ - Any changes staged in the index will be preserved. -}
+updateIndex :: Annex ()
+updateIndex = withIndex $ mergeIndex [fullname]
+
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
@@ -66,6 +78,8 @@ withIndex' bootstrapping a = do
 			unless bootstrapping $ inRepo genIndex
 		a
 
+{- Runs an action using the branch's index file, first making sure that
+ - the branch and index are up-to-date. -}
 withIndexUpdate :: Annex a -> Annex a
 withIndexUpdate a = update >> withIndex a
 
@@ -106,11 +120,12 @@ create = unlessM hasBranch $ do
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
+	updateIndex
 	stageJournalFiles
 	withIndex $ inRepo $ Git.commit message fullname [fullname]
 
-{- Ensures that the branch is up-to-date; should be called before data is
- - read from it. Runs only once per git-annex run.
+{- Ensures that the branch and index are is up-to-date; should be
+ - called before data is read from it. Runs only once per git-annex run.
  -
  - Before refs are merged into the index, it's important to first stage the
  - journal into the index. Otherwise, any changes in the journal would
@@ -126,8 +141,9 @@ commit message = whenM journalDirty $ lockJournal $ do
  -}
 update :: Annex ()
 update = onceonly $ do
-	-- ensure branch exists
+	-- ensure branch exists, and index is up-to-date
 	create
+	updateIndex
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches
@@ -141,14 +157,7 @@ update = onceonly $ do
 				" into " ++ show name
 		unless (null branches) $ do
 			showSideAction merge_desc
-			{- Note: This merges the branches into the index.
-			 - Any unstaged changes in the git-annex branch
-			 - (if it's checked out) will be removed. So,
-			 - documentation advises users not to directly
-			 - modify the branch.
-			 -}
-			h <- catFileHandle
-			inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+			mergeIndex branches
 		ff <- if dirty then return False else tryFastForwardTo refs
 		unless ff $ inRepo $
 			Git.commit merge_desc fullname (nub $ fullname:refs)
diff --git a/debian/changelog b/debian/changelog
index 5c68fde2db..405e98b749 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,11 @@ git-annex (3.20111204) UNRELEASED; urgency=low
     remote, by running git commit, pull, and push for you.
   * Version monad-control dependency in cabal file.
   * Fix bug in last version in getting contents from bare repositories.
+  * Ensure that git-annex branch changes are merged into git-annex's index,
+    which fixes a bug that could cause changes that were pushed to the
+    git-annex branch to get reverted. As a side effect, it's now safe
+    for users to check out and commit changes directly to the git-annex
+    branch.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/bugs/git-annex_branch_corruption.mdwn b/doc/bugs/git-annex_branch_corruption.mdwn
index 63633fb080..5249e63d82 100644
--- a/doc/bugs/git-annex_branch_corruption.mdwn
+++ b/doc/bugs/git-annex_branch_corruption.mdwn
@@ -39,6 +39,9 @@ corresponds to its index, and if the branch is at a different ref,
 merge it into the index. I am still considering how to do that atomically;
 what if a push comes in while git-annex is updating its index?
 
+> Now git-annex always updates the index with the git-annex branch, which
+> is a slow, but safe way to avoid this problem. [[done]] --[[Joey]]
+
 ---
 
 ## Workaround
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index d84b3c4898..68cc7c3cd9 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -22,17 +22,9 @@ deleting or changing the file contents.
 This branch is managed by git-annex, with the contents listed below.
 
 The file `.git/annex/index` is a separate git index file it uses
-to accumulate changes for the git-annex. Also, `.git/annex/journal/` is used
-to record changes before they are added to git.
-
-Note that for speed reasons, git-annex assumes only it will modify this
-branch. If you go in and make changes directly, it will probably revert
-your changes in its next commit to the branch.
-
-The best way to make changes to the git-annex branch is instead
-to create a branch of it, with a name like "my/git-annex", and then
-use "git annex merge" to automerge your branch into the main git-annex
-branch.
+to accumulate changes for the git-annex branch.
+Also, `.git/annex/journal/` is used to record changes before they
+are added to git.
 
 ### `uuid.log`
 

From cfbbda99f4dd3e510e52dbb499d132300ad203e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 16:11:13 -0400
Subject: [PATCH 2662/2835] optimize index updating

The last branch ref that the index was updated to is stored in
.git/annex/index.lck, and the index only updated when the current
branch ref differs.

(The .lck file should later be used for locking too.)

Some more optimization is still needed, since there is some redundancy in
calls to git show-ref.
---
 Annex/Branch.hs    | 54 ++++++++++++++++++++++++++++++++++++----------
 Git.hs             |  5 +++--
 Locations.hs       |  5 +++++
 git-union-merge.hs |  2 +-
 4 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index be19a9be0c..52e82e25cc 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -61,9 +61,26 @@ mergeIndex branches = do
 	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
 
 {- Updates the branch's index to reflect the current contents of the branch.
- - Any changes staged in the index will be preserved. -}
+ - Any changes staged in the index will be preserved.
+ -
+ - Compares the ref stored in the lock file with the current
+ - ref of the branch to see if an update is needed.
+ -}
 updateIndex :: Annex ()
-updateIndex = withIndex $ mergeIndex [fullname]
+updateIndex = do
+        lock <- fromRepo gitAnnexIndexLock
+	lockref <- firstRef <$> liftIO (catchDefaultIO (readFileStrict lock) "")
+	branchref <- getRef fullname
+	when (lockref /= branchref) $ do
+		withIndex $ mergeIndex [fullname]
+		setIndexRef branchref
+
+{- Record that the branch's index has been updated to correspond to a
+ - given ref of the branch. -}
+setIndexRef :: Git.Ref -> Annex ()
+setIndexRef ref = do
+        lock <- fromRepo gitAnnexIndexLock
+	liftIO $ writeFile lock $ show ref ++ "\n"
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
@@ -109,12 +126,13 @@ getCache file = getState >>= go
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM hasBranch $ do
-	e <- hasOrigin
-	if e
-		then inRepo $ Git.run "branch"
-			[Param $ show name, Param $ show originname]
-		else withIndex' True $
+create = unlessM hasBranch $ hasOrigin >>= go >>= setIndexRef
+	where
+		go True = do
+			inRepo $ Git.run "branch"
+				[Param $ show name, Param $ show originname]
+			getRef fullname
+		go False = withIndex' True $
 			inRepo $ Git.commit "branch created" fullname []
 
 {- Stages the journal, and commits staged changes to the branch. -}
@@ -122,7 +140,8 @@ commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
 	updateIndex
 	stageJournalFiles
-	withIndex $ inRepo $ Git.commit message fullname [fullname]
+	withIndex $ 
+		setIndexRef =<< inRepo (Git.commit message fullname [fullname])
 
 {- Ensures that the branch and index are is up-to-date; should be
  - called before data is read from it. Runs only once per git-annex run.
@@ -159,8 +178,9 @@ update = onceonly $ do
 			showSideAction merge_desc
 			mergeIndex branches
 		ff <- if dirty then return False else tryFastForwardTo refs
-		unless ff $ inRepo $
-			Git.commit merge_desc fullname (nub $ fullname:refs)
+		unless ff $ 
+			setIndexRef =<< 
+				inRepo (Git.commit merge_desc fullname (nub $ fullname:refs))
 		invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
@@ -253,6 +273,18 @@ siblingBranches = do
 		gen l = (Git.Ref $ head l, Git.Ref $ last l)
 		uref (a, _) (b, _) = a == b
 
+{- Get the ref of a branch. -}
+getRef :: Git.Ref -> Annex Git.Ref
+getRef branch = firstRef . L.unpack <$> showref
+	where
+		showref = inRepo $ Git.pipeRead [Param "show-ref",
+			Param "--hash", -- get the hash
+			Param "--verify", -- only exact match
+			Param $ show branch]
+
+firstRef :: String-> Git.Ref
+firstRef = Git.Ref . takeWhile (/= '\n')
+
 {- Applies a function to modifiy the content of a file.
  -
  - Note that this does not cause the branch to be merged, it only
diff --git a/Git.hs b/Git.hs
index 8bc32b7cc7..1da5997c18 100644
--- a/Git.hs
+++ b/Git.hs
@@ -463,8 +463,8 @@ shaSize :: Int
 shaSize = 40
 
 {- Commits the index into the specified branch (or other ref), 
- - with the specified parent refs. -}
-commit :: String -> Ref -> [Ref] -> Repo -> IO ()
+ - with the specified parent refs, and returns the new ref -}
+commit :: String -> Ref -> [Ref] -> Repo -> IO Ref
 commit message newref parentrefs repo = do
 	tree <- getSha "write-tree" $ asString $
 		pipeRead [Param "write-tree"] repo
@@ -473,6 +473,7 @@ commit message newref parentrefs repo = do
 			(map Param $ ["commit-tree", show tree] ++ ps)
 			(L.pack message) repo
 	run "update-ref" [Param $ show newref, Param $ show sha] repo
+	return sha
 	where
 		ignorehandle a = snd <$> a
 		asString a = L.unpack <$> a
diff --git a/Locations.hs b/Locations.hs
index fdb8232844..85fcb98887 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -21,6 +21,7 @@ module Locations (
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
 	gitAnnexIndex,
+	gitAnnexIndexLock,
 	isLinkToAnnex,
 	annexHashes,
 	hashDirMixed,
@@ -136,6 +137,10 @@ gitAnnexJournalLock r = gitAnnexDir r  "journal.lck"
 gitAnnexIndex :: Git.Repo -> FilePath
 gitAnnexIndex r = gitAnnexDir r  "index"
 
+{- Lock file for .git/annex/index. -}
+gitAnnexIndexLock :: Git.Repo -> FilePath
+gitAnnexIndexLock r = gitAnnexDir r  "index.lck"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 1cec4a0f85..edd9330c80 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -42,5 +42,5 @@ main = do
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
 	Git.UnionMerge.merge aref bref g
-	Git.commit "union merge" newref [aref, bref] g
+	_ <- Git.commit "union merge" newref [aref, bref] g
 	cleanup g

From 0236bb020b7b055676b06d1cdf582746864f3d78 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 16:19:54 -0400
Subject: [PATCH 2663/2835] update

---
 doc/bugs/git-annex_branch_corruption.mdwn | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/doc/bugs/git-annex_branch_corruption.mdwn b/doc/bugs/git-annex_branch_corruption.mdwn
index 5249e63d82..9c864d85f0 100644
--- a/doc/bugs/git-annex_branch_corruption.mdwn
+++ b/doc/bugs/git-annex_branch_corruption.mdwn
@@ -36,11 +36,14 @@ fixed.
 
 Probably git-annex needs to record which ref of the git-annex branch
 corresponds to its index, and if the branch is at a different ref,
-merge it into the index. I am still considering how to do that atomically;
-what if a push comes in while git-annex is updating its index?
+merge it into the index.
 
-> Now git-annex always updates the index with the git-annex branch, which
-> is a slow, but safe way to avoid this problem. [[done]] --[[Joey]]
+> And now that's [[done]]. I managed to do it with very little slowdown.
+> 
+> A side benefit is that users can now safely check out the git-annex
+> branch and commit changes to it, and git-annex will notice them.
+> Before, it was documented to ignore such changes.
+> --[[Joey]] 
 
 ---
 

From 81f311103d99ec5bfd31ae5a76d6add05ff40121 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 16:41:56 -0400
Subject: [PATCH 2664/2835] a new bug report to track a race

---
 doc/bugs/git-annex_branch_push_race.mdwn | 43 ++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 doc/bugs/git-annex_branch_push_race.mdwn

diff --git a/doc/bugs/git-annex_branch_push_race.mdwn b/doc/bugs/git-annex_branch_push_race.mdwn
new file mode 100644
index 0000000000..257c477bff
--- /dev/null
+++ b/doc/bugs/git-annex_branch_push_race.mdwn
@@ -0,0 +1,43 @@
+The fix for the [[git-annex_branch_corruption]] bug is subject to a race.
+With that fix, git-annex does this when committing a change to the branch:
+
+1. lock the journal file (this avoids git-annex racing itself, FWIW)
+2. check what the head of the branch points to, to see if a newer branch
+   has appeared
+3. if so, updates the index file from the branch
+4. stages changes in the index
+5. commits to the branch using the index file
+
+If a push to the branch comes in during 2-5, then
+[[git-annex_branch_corruption]] could still occur.
+
+---
+
+## approach 1, using locking
+
+Add an update hook and a post-update hook. The update hook
+will use locking to ensure that no git-annex is currently running
+a commit, and block any git-annex's from starting one. It
+will background itself, and remain running during the push.
+The post-update hook will signal it to exit.
+
+I don't like this approach much, since it involves a daemon, two hooks,
+and lots of things to go wrong. And it blocks using git-annex during a
+push. This approach should be a last resort.
+
+## approach 2, lockless method
+
+After a commit is made to the branch, check to see if the parent of
+the commit is the same ref that the index file was last updated to. If it's
+not, then the race occurred.
+
+How to recover from the race? Well, just union merging the parent of the
+commit into the index file and re-committing should work, I think. When
+the race occurs, the commit reverts its parent's changes, and this will
+redo them. 
+
+(Of course, this re-commit will also be subject to the race, and
+will need the same check for the race as the other commits. It won't loop
+forever, I hope.)
+
+--[[Joey]]

From c4c965d602687a6277d43ae003c12dc4c132ba28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 18:39:53 -0400
Subject: [PATCH 2665/2835] detect and recover from branch push/commit race

Dealing with a race without using locking is exceedingly difficult and tricky.
Fully tested, I hope.

There are three places left where the branch can be updated, that are not
covered by the race recovery code. Let's prove they're all immune to the
race:

1. tryFastForwardTo checks to see if a fast-forward can be done,
   and then does git-update-ref on the branch to fast-forward it.

   If a push comes in before the check, then either no fast-forward
   will be done (ok), or the push set the branch to a ref that can
   still be fast-forwarded (also ok)

   If a push comes in after the check, the git-update-ref will
   undo the ref change made by the push. It's as if the push did not come
   in, and the next git-push will see this, and try to re-do it.
   (acceptable)

2. When creating the branch for the very first time, an empty index
   is created, and a commit of it made to the branch. The commit's ref
   is recorded as the current state of the index. If a push came in
   during that, it will be noticed the next time a commit is made to the
   branch, since the branch will have changed. (ok)

3. Creating the branch from an existing remote branch involves making
   the branch, and then getting its ref, and recording that the index
   reflects that ref.

   If a push creates the branch first, git-branch will fail (ok).

   If the branch is created and a racing push is then able to change it
   (highly unlikely!) we're still ok, because it first records the ref into
   the index.lck, and then updating the index. The race can cause the
   index.lck to have the old branch ref, while the index has the newly pushed
   branch merged into it, but that only results in an unnecessary update of
   the index file later on.
---
 Annex/Branch.hs                          | 122 ++++++++++++++++-------
 Annex/CatFile.hs                         |   6 ++
 doc/bugs/git-annex_branch_push_race.mdwn |   2 +
 3 files changed, 94 insertions(+), 36 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 52e82e25cc..0ac4199944 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -60,28 +60,6 @@ mergeIndex branches = do
 	h <- catFileHandle
 	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
 
-{- Updates the branch's index to reflect the current contents of the branch.
- - Any changes staged in the index will be preserved.
- -
- - Compares the ref stored in the lock file with the current
- - ref of the branch to see if an update is needed.
- -}
-updateIndex :: Annex ()
-updateIndex = do
-        lock <- fromRepo gitAnnexIndexLock
-	lockref <- firstRef <$> liftIO (catchDefaultIO (readFileStrict lock) "")
-	branchref <- getRef fullname
-	when (lockref /= branchref) $ do
-		withIndex $ mergeIndex [fullname]
-		setIndexRef branchref
-
-{- Record that the branch's index has been updated to correspond to a
- - given ref of the branch. -}
-setIndexRef :: Git.Ref -> Annex ()
-setIndexRef ref = do
-        lock <- fromRepo gitAnnexIndexLock
-	liftIO $ writeFile lock $ show ref ++ "\n"
-
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
@@ -95,6 +73,80 @@ withIndex' bootstrapping a = do
 			unless bootstrapping $ inRepo genIndex
 		a
 
+{- Updates the branch's index to reflect the current contents of the branch.
+ - Any changes staged in the index will be preserved.
+ -
+ - Compares the ref stored in the lock file with the current
+ - ref of the branch to see if an update is needed.
+ -}
+updateIndex :: Annex (Maybe Git.Ref)
+updateIndex = do
+	branchref <- getRef fullname
+	go branchref
+	return branchref
+	where
+		go Nothing = return ()
+		go (Just branchref) = do
+			lock <- fromRepo gitAnnexIndexLock
+			lockref <- firstRef <$> liftIO (catchDefaultIO (readFileStrict lock) "")
+			when (lockref /= branchref) $ do
+				withIndex $ mergeIndex [fullname]
+				setIndexRef branchref
+
+{- Record that the branch's index has been updated to correspond to a
+ - given ref of the branch. -}
+setIndexRef :: Git.Ref -> Annex ()
+setIndexRef ref = do
+        lock <- fromRepo gitAnnexIndexLock
+	liftIO $ writeFile lock $ show ref ++ "\n"
+
+{- Commits the staged changes in the index to the branch.
+ - 
+ - Ensures that the branch's index file is first updated to include the
+ - current state of the branch, before running the commit action. This
+ - is needed because the branch may have had changes pushed to it, that
+ - are not yet reflected in the index.
+ -
+ - Also safely handles a race that can occur if a change is being pushed
+ - into the branch at the same time. When the race happens, the commit will
+ - be made on top of the newly pushed change, but without the index file
+ - being updated to include it. The result is that the newly pushed
+ - change is reverted. This race is detected and another commit made
+ - to fix it.
+ -}
+commitBranch :: String -> [Git.Ref] -> Annex ()
+commitBranch message parents = do
+	expected <- updateIndex
+	committedref <- inRepo $ Git.commit message fullname parents
+	setIndexRef committedref
+	parentrefs <- commitparents <$> catObject committedref
+	when (racedetected expected parentrefs) $
+		fixrace committedref parentrefs
+	where
+		-- look for "parent ref" lines and return the refs
+		commitparents = map (Git.Ref . snd) . filter isparent .
+			map (toassoc . L.unpack) . L.lines
+		toassoc = separate (== ' ')
+		isparent (k,_) = k == "parent"
+		
+ 		{- The race can be detected by checking the commit's
+		 - parent, which will be the newly pushed branch,
+		 - instead of the expected ref that the index was updated to. -}
+		racedetected Nothing parentrefs
+			| null parentrefs = False -- first commit, no parents
+			| otherwise = True -- race on first commit
+		racedetected (Just expectedref) parentrefs
+			| expectedref `elem` parentrefs = False -- good parent
+			| otherwise = True -- race!
+		
+		{- To recover from the race, union merge the lost refs
+		 - into the index, and recommit on top of the bad commit. -}
+		fixrace committedref lostrefs = do
+			mergeIndex lostrefs
+			commitBranch racemessage [committedref]
+		
+		racemessage = message ++ " (recovery from race)"
+
 {- Runs an action using the branch's index file, first making sure that
  - the branch and index are up-to-date. -}
 withIndexUpdate :: Annex a -> Annex a
@@ -126,22 +178,20 @@ getCache file = getState >>= go
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM hasBranch $ hasOrigin >>= go >>= setIndexRef
+create = unlessM hasBranch $ hasOrigin >>= go
 	where
 		go True = do
 			inRepo $ Git.run "branch"
 				[Param $ show name, Param $ show originname]
-			getRef fullname
-		go False = withIndex' True $
-			inRepo $ Git.commit "branch created" fullname []
+			maybe (return ()) setIndexRef =<< getRef fullname
+		go False = withIndex' True $ 
+			setIndexRef =<< (inRepo $ Git.commit "branch created" fullname [])
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
-	updateIndex
 	stageJournalFiles
-	withIndex $ 
-		setIndexRef =<< inRepo (Git.commit message fullname [fullname])
+	withIndex $ commitBranch message [fullname]
 
 {- Ensures that the branch and index are is up-to-date; should be
  - called before data is read from it. Runs only once per git-annex run.
@@ -162,7 +212,7 @@ update :: Annex ()
 update = onceonly $ do
 	-- ensure branch exists, and index is up-to-date
 	create
-	updateIndex
+	_ <- updateIndex
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches
@@ -178,9 +228,7 @@ update = onceonly $ do
 			showSideAction merge_desc
 			mergeIndex branches
 		ff <- if dirty then return False else tryFastForwardTo refs
-		unless ff $ 
-			setIndexRef =<< 
-				inRepo (Git.commit merge_desc fullname (nub $ fullname:refs))
+		unless ff $ commitBranch merge_desc (nub $ fullname:refs)
 		invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
@@ -274,13 +322,15 @@ siblingBranches = do
 		uref (a, _) (b, _) = a == b
 
 {- Get the ref of a branch. -}
-getRef :: Git.Ref -> Annex Git.Ref
-getRef branch = firstRef . L.unpack <$> showref
+getRef :: Git.Ref -> Annex (Maybe Git.Ref)
+getRef branch = process . L.unpack <$> showref
 	where
 		showref = inRepo $ Git.pipeRead [Param "show-ref",
 			Param "--hash", -- get the hash
-			Param "--verify", -- only exact match
+			Params "--verify", -- only exact match
 			Param $ show branch]
+		process [] = Nothing
+		process s = Just $ firstRef s
 
 firstRef :: String-> Git.Ref
 firstRef = Git.Ref . takeWhile (/= '\n')
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 1d996edfd5..bcf44551e2 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -7,6 +7,7 @@
 
 module Annex.CatFile (
 	catFile,
+	catObject,
 	catFileHandle
 ) where
 
@@ -22,6 +23,11 @@ catFile branch file = do
 	h <- catFileHandle
 	liftIO $ Git.CatFile.catFile h branch file
 
+catObject :: Git.Ref -> Annex L.ByteString
+catObject ref = do
+	h <- catFileHandle
+	liftIO $ Git.CatFile.catObject h ref
+
 catFileHandle :: Annex Git.CatFile.CatFileHandle
 catFileHandle = maybe startup return =<< Annex.getState Annex.catfilehandle
 	where
diff --git a/doc/bugs/git-annex_branch_push_race.mdwn b/doc/bugs/git-annex_branch_push_race.mdwn
index 257c477bff..013ff70dd5 100644
--- a/doc/bugs/git-annex_branch_push_race.mdwn
+++ b/doc/bugs/git-annex_branch_push_race.mdwn
@@ -40,4 +40,6 @@ redo them.
 will need the same check for the race as the other commits. It won't loop
 forever, I hope.)
 
+> [[done]] and tested.
+
 --[[Joey]]

From 29b88ad657b4d84e84bdc366617d765726e29bb0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 21:40:36 -0400
Subject: [PATCH 2666/2835] avoid redundant call to updateIndex

commitBranch calls updateIndex
---
 Annex/Branch.hs | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 0ac4199944..3da8bb1986 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -203,38 +203,42 @@ commit message = whenM journalDirty $ lockJournal $ do
  - (It would be cleaner to handle the merge by updating the journal, not the
  - index, with changes from the branches.)
  -
- - The index is always updated using a union merge, as that's the most
- - efficient way to update it. However, if the branch can be
- - fast-forwarded, that is then done, rather than adding an unnecessary
- - commit to it.
+ - The branch is fast-forwarded if possible, otherwise a merge commit is
+ - made.
  -}
 update :: Annex ()
 update = onceonly $ do
-	-- ensure branch exists, and index is up-to-date
+	-- ensure branch exists
 	create
-	_ <- updateIndex
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches
 	let (refs, branches) = unzip c
-	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
-		when dirty stageJournalFiles
-		let merge_desc = if null branches
-			then "update" 
-			else "merging " ++
-				unwords (map Git.refDescribe branches) ++ 
-				" into " ++ show name
-		unless (null branches) $ do
-			showSideAction merge_desc
-			mergeIndex branches
-		ff <- if dirty then return False else tryFastForwardTo refs
-		unless ff $ commitBranch merge_desc (nub $ fullname:refs)
-		invalidateCache
+	if (not dirty && null refs)
+		then simpleupdate
+		else withIndex $ lockJournal $ do
+			when dirty stageJournalFiles
+			let merge_desc = if null branches
+				then "update" 
+				else "merging " ++
+					unwords (map Git.refDescribe branches) ++ 
+					" into " ++ show name
+			unless (null branches) $ do
+				showSideAction merge_desc
+				mergeIndex branches
+			ff <- if dirty then return False else tryFastForwardTo refs
+			if ff
+				then simpleupdate
+				else commitBranch merge_desc (nub $ fullname:refs)
+			invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
 			r <- a
 			disableUpdate
 			return r
+		simpleupdate = do
+			_ <- updateIndex
+			return ()
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}

From acb2d5a5a6878b43ba5fa52b6f4790c765f9f54d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 21:55:51 -0400
Subject: [PATCH 2667/2835] releasing version 3.20111211

---
 debian/changelog | 16 ++++++++--------
 git-annex.cabal  |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 405e98b749..b676c3c3d3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,11 @@
-git-annex (3.20111204) UNRELEASED; urgency=low
+git-annex (3.20111211) unstable; urgency=medium
 
+  * Fix bug in last version in getting contents from bare repositories.
+  * Ensure that git-annex branch changes are merged into git-annex's index,
+    which fixes a bug that could cause changes that were pushed to the
+    git-annex branch to get reverted. As a side effect, it's now safe
+    for users to check out and commit changes directly to the git-annex
+    branch.
   * map: Fix a failure to detect a loop when both repositories are local
     and refer to each other with relative paths.
   * Prevent key names from containing newlines.
@@ -12,14 +18,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
   * sync: New command that synchronises the local repository and default
     remote, by running git commit, pull, and push for you.
   * Version monad-control dependency in cabal file.
-  * Fix bug in last version in getting contents from bare repositories.
-  * Ensure that git-annex branch changes are merged into git-annex's index,
-    which fixes a bug that could cause changes that were pushed to the
-    git-annex branch to get reverted. As a side effect, it's now safe
-    for users to check out and commit changes directly to the git-annex
-    branch.
 
- -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
+ -- Joey Hess   Sun, 11 Dec 2011 21:24:39 -0400
 
 git-annex (3.20111203) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 8f5d6ebb86..35b3e690f9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111203
+Version: 3.20111211
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From d3d9c8a9a68fe74208d08ca4e59670934891d489 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 21:56:17 -0400
Subject: [PATCH 2668/2835] add news item for git-annex 3.20111211

---
 doc/news/version_3.20111105.mdwn | 27 ---------------------------
 doc/news/version_3.20111211.mdwn | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 27 deletions(-)
 delete mode 100644 doc/news/version_3.20111105.mdwn
 create mode 100644 doc/news/version_3.20111211.mdwn

diff --git a/doc/news/version_3.20111105.mdwn b/doc/news/version_3.20111105.mdwn
deleted file mode 100644
index 663c2ef1fc..0000000000
--- a/doc/news/version_3.20111105.mdwn
+++ /dev/null
@@ -1,27 +0,0 @@
-git-annex 3.20111105 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * The default backend used when adding files to the annex is changed
-     from WORM to SHA256.
-     To get old behavior, add a .gitattributes containing: * annex.backend=WORM
-   * Sped up some operations on remotes that are on the same host.
-   * copy --to: Fixed leak when copying many files to a remote on the same
-     host.
-   * uninit: Add guard against being run with the git-annex branch checked out.
-   * Fail if --from or --to is passed to commands that do not support them.
-   * drop --from is now supported to remove file content from a remote.
-   * status: Now always shows the current repository, even when it does not
-     appear in uuid.log.
-   * fsck: Now works in bare repositories. Checks location log information,
-     and file contents. Does not check that numcopies is satisfied, as
-     .gitattributes information about numcopies is not available in a bare
-     repository.
-   * unused, dropunused: Now work in bare repositories.
-   * Removed the setkey command, and added a reinject command with a more
-     useful interface.
-   * The fromkey command now takes the key as its first parameter. The --key
-     option is no longer used.
-   * Built without any filename containing .git being excluded. Closes: #[647215](http://bugs.debian.org/647215)
-   * Record uuid when auto-initializing a remote so it shows in status.
-   * Bugfix: Fixed git-annex init crash in a bare repository when there was
-     already an existing git-annex branch.
-   * Pass -t to rsync to preserve timestamps."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111211.mdwn b/doc/news/version_3.20111211.mdwn
new file mode 100644
index 0000000000..5d2c57e455
--- /dev/null
+++ b/doc/news/version_3.20111211.mdwn
@@ -0,0 +1,20 @@
+git-annex 3.20111211 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix bug in last version in getting contents from bare repositories.
+   * Ensure that git-annex branch changes are merged into git-annex's index,
+     which fixes a bug that could cause changes that were pushed to the
+     git-annex branch to get reverted. As a side effect, it's now safe
+     for users to check out and commit changes directly to the git-annex
+     branch.
+   * map: Fix a failure to detect a loop when both repositories are local
+     and refer to each other with relative paths.
+   * Prevent key names from containing newlines.
+   * add: If interrupted, add can leave files converted to symlinks but not
+     yet added to git. Running the add again will now clean up this situtation.
+   * Fix caching of decrypted ciphers, which failed when drop had to check
+     multiple different encrypted special remotes.
+   * unannex: Can be run on files that have been added to the annex, but not
+     yet committed.
+   * sync: New command that synchronises the local repository and default
+     remote, by running git commit, pull, and push for you.
+   * Version monad-control dependency in cabal file."""]]
\ No newline at end of file

From b9ac5854549636493449fea6830364a01159fbf6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 23:02:25 -0400
Subject: [PATCH 2669/2835] more efficient union merges

Tries to avoid generating a new object when the merged content has the same
lines that were in the old object.

I've noticed some merge commits that only move lines around, like this:

- 1323478057.181191s 1 be23c3ac-0ee5-11e0-b185-3b0f9b5b00c5
  1323204972.062151s 1 87e06c7a-7388-11e0-ba07-03cdf300bd87
++1323478057.181191s 1 be23c3ac-0ee5-11e0-b185-3b0f9b5b00c5

Unsure if this will really save anything in practice, since it only looks
at one of the two old objects, and maybe I didn't pick the best one.
---
 Git/UnionMerge.hs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index ddbff6a82f..27113c85ac 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -104,14 +104,17 @@ mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
 mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
-	shas -> do
-		content <- L.concat <$> mapM (catObject h) shas
-		sha <- hashObject (unionmerge content) repo
-		return $ Just $ update_index_line sha file
+	(sha:shas) -> do
+		origcontent <- L.lines <$> catObject h sha
+		content <- map L.lines <$> mapM (catObject h) shas
+		let newcontent = nub $ concat $ origcontent:content
+		newsha <- if (newcontent == origcontent)
+			then return sha
+			else hashObject (L.unlines $ newcontent) repo
+		return $ Just $ update_index_line newsha file
 	where
-		[_colonamode, _bmode, asha, bsha, _status] = words info
+		[_colonmode, _bmode, asha, bsha, _status] = words info
 		nullsha = Ref $ replicate shaSize '0'
-		unionmerge = L.unlines . nub . L.lines
 
 {- Injects some content into git, returning its Sha. -}
 hashObject :: L.ByteString -> Repo -> IO Sha

From 0cbab5de657e025057dd10b087a874d6b3a7b13e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 00:48:25 -0400
Subject: [PATCH 2670/2835] refactor

---
 Git/UnionMerge.hs | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 27113c85ac..89fcf83e02 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -105,20 +105,24 @@ mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	(sha:shas) -> do
-		origcontent <- L.lines <$> catObject h sha
-		content <- map L.lines <$> mapM (catObject h) shas
-		let newcontent = nub $ concat $ origcontent:content
-		newsha <- if (newcontent == origcontent)
-			then return sha
-			else hashObject (L.unlines $ newcontent) repo
+		newsha <- maybe (return sha) (hashObject repo . L.unlines) =<<
+			unionmerge
+				<$> (L.lines <$> catObject h sha)
+				<*> (map L.lines <$> mapM (catObject h) shas)
 		return $ Just $ update_index_line newsha file
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
 		nullsha = Ref $ replicate shaSize '0'
 
+		unionmerge origcontent content
+			| newcontent == origcontent = Nothing
+			| otherwise = Just newcontent
+			where
+				newcontent = nub $ concat $ origcontent:content
+
 {- Injects some content into git, returning its Sha. -}
-hashObject :: L.ByteString -> Repo -> IO Sha
-hashObject content repo = getSha subcmd $ do
+hashObject :: Repo -> L.ByteString -> IO Sha
+hashObject repo content = getSha subcmd $ do
 	(h, s) <- pipeWriteRead (map Param params) content repo
 	L.length s `seq` do
 		forceSuccess h

From acd7a52dfd2cad24fd946ffcf8c4b1d07eb474ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 01:33:02 -0400
Subject: [PATCH 2671/2835] always find optimal merge

Testing b9ac5854549636493449fea6830364a01159fbf6, it didn't find the
optimal union merge, the second sha was the one to use, at least in
the case I tried. Let's just try all shas to see if any can be reused.

I stopped using the expensive nub, so despite the use of sets to
sort/uniq file contents, this is probably as fast or faster than it
was before.
---
 Git/UnionMerge.hs | 34 ++++++++++++++++++++--------------
 debian/changelog  |  6 ++++++
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 89fcf83e02..0345af3994 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -15,8 +15,8 @@ module Git.UnionMerge (
 ) where
 
 import System.Cmd.Utils
-import Data.List
 import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Set as S
 
 import Common
 import Git
@@ -103,22 +103,14 @@ calc_merge ch differ repo streamer = gendiff >>= go
 mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
 mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
-	(sha:[]) -> return $ Just $ update_index_line sha file
-	(sha:shas) -> do
-		newsha <- maybe (return sha) (hashObject repo . L.unlines) =<<
-			unionmerge
-				<$> (L.lines <$> catObject h sha)
-				<*> (map L.lines <$> mapM (catObject h) shas)
-		return $ Just $ update_index_line newsha file
+	(sha:[]) -> use sha
+	shas -> use =<< either return (hashObject repo . L.unlines) =<<
+		calcMerge . zip shas <$> mapM getcontents shas
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
 		nullsha = Ref $ replicate shaSize '0'
-
-		unionmerge origcontent content
-			| newcontent == origcontent = Nothing
-			| otherwise = Just newcontent
-			where
-				newcontent = nub $ concat $ origcontent:content
+		getcontents s = L.lines <$> catObject h s
+		use sha = return $ Just $ update_index_line sha file
 
 {- Injects some content into git, returning its Sha. -}
 hashObject :: Repo -> L.ByteString -> IO Sha
@@ -131,3 +123,17 @@ hashObject repo content = getSha subcmd $ do
 	where
 		subcmd = "hash-object"
 		params = [subcmd, "-w", "--stdin"]
+
+{- Calculates a union merge between a list of refs, with contents.
+ -
+ - When possible, reuses the content of an existing ref, rather than
+ - generating new content.
+ -}
+calcMerge :: [(Ref, [L.ByteString])] -> Either Ref [L.ByteString]
+calcMerge shacontents
+	| null reuseable = Right $ new
+	| otherwise = Left $ fst $ head reuseable
+	where
+		reuseable = filter (\c -> sorteduniq (snd c) == new) shacontents
+		new = sorteduniq $ concat $ map snd shacontents
+		sorteduniq = S.toList . S.fromList
diff --git a/debian/changelog b/debian/changelog
index b676c3c3d3..db23decbb9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20111212) UNRELEASED; urgency=low
+
+  * Union merge now finds the least expensive way to represent the merge.
+
+ -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
+
 git-annex (3.20111211) unstable; urgency=medium
 
   * Fix bug in last version in getting contents from bare repositories.

From 2332afb4bc207ab04b4a1d7adfb3a2df7f5d7874 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 02:04:48 -0400
Subject: [PATCH 2672/2835] cleanup

---
 Annex/Branch.hs | 28 +++++++++++++---------------
 Utility/Misc.hs |  4 ++++
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 3da8bb1986..b108281ce7 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -88,7 +88,8 @@ updateIndex = do
 		go Nothing = return ()
 		go (Just branchref) = do
 			lock <- fromRepo gitAnnexIndexLock
-			lockref <- firstRef <$> liftIO (catchDefaultIO (readFileStrict lock) "")
+			lockref <- Git.Ref . firstLine <$>
+				liftIO (catchDefaultIO (readFileStrict lock) "")
 			when (lockref /= branchref) $ do
 				withIndex $ mergeIndex [fullname]
 				setIndexRef branchref
@@ -303,6 +304,17 @@ refExists :: Git.Ref -> Annex Bool
 refExists ref = inRepo $ Git.runBool "show-ref"
 	[Param "--verify", Param "-q", Param $ show ref]
 
+{- Get the ref of a branch. -}
+getRef :: Git.Branch -> Annex (Maybe Git.Ref)
+getRef branch = process . L.unpack <$> showref
+	where
+		showref = inRepo $ Git.pipeRead [Param "show-ref",
+			Param "--hash", -- get the hash
+			Params "--verify", -- only exact match
+			Param $ show branch]
+		process [] = Nothing
+		process s = Just $ Git.Ref $ firstLine s
+
 {- Does the main git-annex branch exist? -}
 hasBranch :: Annex Bool
 hasBranch = refExists fullname
@@ -325,20 +337,6 @@ siblingBranches = do
 		gen l = (Git.Ref $ head l, Git.Ref $ last l)
 		uref (a, _) (b, _) = a == b
 
-{- Get the ref of a branch. -}
-getRef :: Git.Ref -> Annex (Maybe Git.Ref)
-getRef branch = process . L.unpack <$> showref
-	where
-		showref = inRepo $ Git.pipeRead [Param "show-ref",
-			Param "--hash", -- get the hash
-			Params "--verify", -- only exact match
-			Param $ show branch]
-		process [] = Nothing
-		process s = Just $ firstRef s
-
-firstRef :: String-> Git.Ref
-firstRef = Git.Ref . takeWhile (/= '\n')
-
 {- Applies a function to modifiy the content of a file.
  -
  - Note that this does not cause the branch to be merged, it only
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index 541e150b77..e95ac4319d 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -40,6 +40,10 @@ separate c l = unbreak $ break c l
 			| null b = r
 			| otherwise = (a, tail b)
 
+{- Breaks out the first line. -}
+firstLine :: String-> String
+firstLine = takeWhile (/= '\n')
+
 {- Catches IO errors and returns a Bool -}
 catchBoolIO :: IO Bool -> IO Bool
 catchBoolIO a = catchDefaultIO a False

From c7e65bbb12db3db314d809e0186db0905c677d2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 02:24:37 -0400
Subject: [PATCH 2673/2835] optimiation

avoids reading the config of a local remote twice in a row
---
 Remote/Git.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 05743a28d5..0251da5583 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -86,9 +86,9 @@ tryGitConfigRead r
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsHttp r = store $ safely geturlconfig
 	| Git.repoIsUrl r = return r
-	| otherwise = store $ safely $ do
-		onLocal r ensureInitialized
-		Git.configRead r
+	| otherwise = store $ safely $ onLocal r $ do 
+		ensureInitialized
+		Annex.getState Annex.repo
 	where
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.

From f9cd3f6ad18ca04a1da7082c6c10215dc5435154 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 02:31:07 -0400
Subject: [PATCH 2674/2835] optimisation

avoids a useless diff from git-annex..refs/heads/git-annex
---
 Annex/Branch.hs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index b108281ce7..e6c92fbe54 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -213,7 +213,7 @@ update = onceonly $ do
 	create
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
-	c <- filterM (changedBranch name . snd) =<< siblingBranches
+	c <- filterM (changedBranch fullname . snd) =<< siblingBranches
 	let (refs, branches) = unzip c
 	if (not dirty && null refs)
 		then simpleupdate
@@ -244,7 +244,9 @@ update = onceonly $ do
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
 changedBranch :: Git.Branch -> Git.Branch -> Annex Bool
-changedBranch origbranch newbranch = not . L.null <$> diffs
+changedBranch origbranch newbranch
+	| origbranch == newbranch = return False
+	| otherwise = not . L.null <$> diffs
 	where
 		diffs = inRepo $ Git.pipeRead
 			[ Param "log"

From 79345ad5fc10fc96dcd3599f2e092b3967291549 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 03:30:47 -0400
Subject: [PATCH 2675/2835] optimisation

avoids a redundant call to git show-ref
---
 Annex/Branch.hs | 79 +++++++++++++++++++++++--------------------------
 1 file changed, 37 insertions(+), 42 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index e6c92fbe54..c657525b1d 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -79,20 +79,14 @@ withIndex' bootstrapping a = do
  - Compares the ref stored in the lock file with the current
  - ref of the branch to see if an update is needed.
  -}
-updateIndex :: Annex (Maybe Git.Ref)
-updateIndex = do
-	branchref <- getRef fullname
-	go branchref
-	return branchref
-	where
-		go Nothing = return ()
-		go (Just branchref) = do
-			lock <- fromRepo gitAnnexIndexLock
-			lockref <- Git.Ref . firstLine <$>
-				liftIO (catchDefaultIO (readFileStrict lock) "")
-			when (lockref /= branchref) $ do
-				withIndex $ mergeIndex [fullname]
-				setIndexRef branchref
+updateIndex :: Git.Ref -> Annex ()
+updateIndex branchref = do
+	lock <- fromRepo gitAnnexIndexLock
+	lockref <- Git.Ref . firstLine <$>
+		liftIO (catchDefaultIO (readFileStrict lock) "")
+	when (lockref /= branchref) $ do
+		withIndex $ mergeIndex [fullname]
+		setIndexRef branchref
 
 {- Record that the branch's index has been updated to correspond to a
  - given ref of the branch. -}
@@ -115,13 +109,13 @@ setIndexRef ref = do
  - change is reverted. This race is detected and another commit made
  - to fix it.
  -}
-commitBranch :: String -> [Git.Ref] -> Annex ()
-commitBranch message parents = do
-	expected <- updateIndex
+commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex ()
+commitBranch branchref message parents = do
+	updateIndex branchref
 	committedref <- inRepo $ Git.commit message fullname parents
 	setIndexRef committedref
 	parentrefs <- commitparents <$> catObject committedref
-	when (racedetected expected parentrefs) $
+	when (racedetected branchref parentrefs) $
 		fixrace committedref parentrefs
 	where
 		-- look for "parent ref" lines and return the refs
@@ -133,10 +127,7 @@ commitBranch message parents = do
  		{- The race can be detected by checking the commit's
 		 - parent, which will be the newly pushed branch,
 		 - instead of the expected ref that the index was updated to. -}
-		racedetected Nothing parentrefs
-			| null parentrefs = False -- first commit, no parents
-			| otherwise = True -- race on first commit
-		racedetected (Just expectedref) parentrefs
+		racedetected expectedref parentrefs
 			| expectedref `elem` parentrefs = False -- good parent
 			| otherwise = True -- race!
 		
@@ -144,7 +135,7 @@ commitBranch message parents = do
 		 - into the index, and recommit on top of the bad commit. -}
 		fixrace committedref lostrefs = do
 			mergeIndex lostrefs
-			commitBranch racemessage [committedref]
+			commitBranch committedref racemessage [committedref]
 		
 		racemessage = message ++ " (recovery from race)"
 
@@ -179,20 +170,31 @@ getCache file = getState >>= go
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM hasBranch $ hasOrigin >>= go
+create = do
+	_ <- getBranch
+	return ()
+
+{- Returns the ref of the branch, creating it first if necessary. -}
+getBranch :: Annex (Git.Ref)
+getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< getRef fullname
 	where
 		go True = do
 			inRepo $ Git.run "branch"
 				[Param $ show name, Param $ show originname]
-			maybe (return ()) setIndexRef =<< getRef fullname
-		go False = withIndex' True $ 
-			setIndexRef =<< (inRepo $ Git.commit "branch created" fullname [])
+			fromMaybe (error $ "failed to create " ++ show name)
+				<$> getRef fullname
+		go False = withIndex' True $ do
+			inRepo $ Git.commit "branch created" fullname []
+		use ref = do
+			setIndexRef ref
+			return ref
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
 	stageJournalFiles
-	withIndex $ commitBranch message [fullname]
+	ref <- getBranch
+	withIndex $ commitBranch ref message [fullname]
 
 {- Ensures that the branch and index are is up-to-date; should be
  - called before data is read from it. Runs only once per git-annex run.
@@ -209,14 +211,14 @@ commit message = whenM journalDirty $ lockJournal $ do
  -}
 update :: Annex ()
 update = onceonly $ do
-	-- ensure branch exists
-	create
+	-- ensure branch exists, and get its current ref
+	branchref <- getBranch
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch fullname . snd) =<< siblingBranches
 	let (refs, branches) = unzip c
 	if (not dirty && null refs)
-		then simpleupdate
+		then updateIndex branchref
 		else withIndex $ lockJournal $ do
 			when dirty stageJournalFiles
 			let merge_desc = if null branches
@@ -229,17 +231,15 @@ update = onceonly $ do
 				mergeIndex branches
 			ff <- if dirty then return False else tryFastForwardTo refs
 			if ff
-				then simpleupdate
-				else commitBranch merge_desc (nub $ fullname:refs)
+				then updateIndex branchref
+				else commitBranch branchref merge_desc
+					(nub $ fullname:refs)
 			invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
 			r <- a
 			disableUpdate
 			return r
-		simpleupdate = do
-			_ <- updateIndex
-			return ()
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
@@ -306,21 +306,16 @@ refExists :: Git.Ref -> Annex Bool
 refExists ref = inRepo $ Git.runBool "show-ref"
 	[Param "--verify", Param "-q", Param $ show ref]
 
-{- Get the ref of a branch. -}
+{- Get the ref of a branch. (Must be a fully qualified branch name) -}
 getRef :: Git.Branch -> Annex (Maybe Git.Ref)
 getRef branch = process . L.unpack <$> showref
 	where
 		showref = inRepo $ Git.pipeRead [Param "show-ref",
 			Param "--hash", -- get the hash
-			Params "--verify", -- only exact match
 			Param $ show branch]
 		process [] = Nothing
 		process s = Just $ Git.Ref $ firstLine s
 
-{- Does the main git-annex branch exist? -}
-hasBranch :: Annex Bool
-hasBranch = refExists fullname
-
 {- Does origin/git-annex exist? -}
 hasOrigin :: Annex Bool
 hasOrigin = refExists originname

From 6edaabd0407287becda24904ed19413a5371979c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 13:43:52 -0400
Subject: [PATCH 2676/2835] reinject: Add a sanity check for using an annexed
 file as the source file.

---
 Command/Reinject.hs | 5 +++--
 debian/changelog    | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index cfa0655ef1..906f7c5176 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -24,8 +24,9 @@ start :: [FilePath] -> CommandStart
 start (src:dest:[])
 	| src == dest = stop
 	| otherwise = do
-		showStart "reinject" dest
-		next $ whenAnnexed (perform src) dest
+		ifAnnexed src
+			(error $ "cannot used annexed file as src: " ++ src)
+			(next $ whenAnnexed (perform src) dest)
 start _ = error "specify a src file and a dest file"
 
 perform :: FilePath -> FilePath -> (Key, Backend Annex) -> CommandPerform
diff --git a/debian/changelog b/debian/changelog
index db23decbb9..5fc01f9ead 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20111212) UNRELEASED; urgency=low
 
   * Union merge now finds the least expensive way to represent the merge.
+  * reinject: Add a sanity check for using an annexed file as the source file.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From b2f934e07ad32cfe44ea1fab2ca154379f30efe8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 17:24:12 -0400
Subject: [PATCH 2677/2835] update comment

---
 Annex/Branch.hs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index c657525b1d..b79a8975d0 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -97,8 +97,8 @@ setIndexRef ref = do
 
 {- Commits the staged changes in the index to the branch.
  - 
- - Ensures that the branch's index file is first updated to include the
- - current state of the branch, before running the commit action. This
+ - Ensures that the branch's index file is first updated to the state
+ - of the brannch at branchref, before running the commit action. This
  - is needed because the branch may have had changes pushed to it, that
  - are not yet reflected in the index.
  -
@@ -108,6 +108,10 @@ setIndexRef ref = do
  - being updated to include it. The result is that the newly pushed
  - change is reverted. This race is detected and another commit made
  - to fix it.
+ - 
+ - The branchref value can have been obtained using getBranch at any
+ - previous point, though getting it a long time ago makes the race
+ - more likely to occur.
  -}
 commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex ()
 commitBranch branchref message parents = do

From 98dfc0c9b0024c156d0fea99bf8d2355e06244a7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 17:38:46 -0400
Subject: [PATCH 2678/2835] split out Annex/BranchState.hs

---
 Annex/Branch.hs      | 45 ++---------------------------------
 Annex/BranchState.hs | 56 ++++++++++++++++++++++++++++++++++++++++++++
 Remote/Git.hs        |  4 ++--
 3 files changed, 60 insertions(+), 45 deletions(-)
 create mode 100644 Annex/BranchState.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index b79a8975d0..699bc0323c 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -8,7 +8,6 @@
 module Annex.Branch (
 	create,
 	update,
-	disableUpdate,
 	get,
 	change,
 	commit,
@@ -25,7 +24,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common.Annex
 import Annex.Exception
-import Types.BranchState
+import Annex.BranchState
 import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
@@ -148,30 +147,6 @@ commitBranch branchref message parents = do
 withIndexUpdate :: Annex a -> Annex a
 withIndexUpdate a = update >> withIndex a
 
-getState :: Annex BranchState
-getState = Annex.getState Annex.branchstate
-
-setState :: BranchState -> Annex ()
-setState state = Annex.changeState $ \s -> s { Annex.branchstate = state }
-
-setCache :: FilePath -> String -> Annex ()
-setCache file content = do
-	state <- getState
-	setState state { cachedFile = Just file, cachedContent = content }
-
-invalidateCache :: Annex ()
-invalidateCache = do
-	state <- getState
-	setState state { cachedFile = Nothing, cachedContent = "" }
-
-getCache :: FilePath -> Annex (Maybe String)
-getCache file = getState >>= go
-	where
-		go state
-			| cachedFile state == Just file =
-				return $ Just $ cachedContent state
-			| otherwise = return Nothing
-
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
 create = do
@@ -214,7 +189,7 @@ commit message = whenM journalDirty $ lockJournal $ do
  - made.
  -}
 update :: Annex ()
-update = onceonly $ do
+update = runUpdateOnce $ do
 	-- ensure branch exists, and get its current ref
 	branchref <- getBranch
 	-- check what needs updating before taking the lock
@@ -239,11 +214,6 @@ update = onceonly $ do
 				else commitBranch branchref merge_desc
 					(nub $ fullname:refs)
 			invalidateCache
-	where
-		onceonly a = unlessM (branchUpdated <$> getState) $ do
-			r <- a
-			disableUpdate
-			return r
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
@@ -294,17 +264,6 @@ tryFastForwardTo (first:rest) = do
 				(False, True) -> findbest c rs -- worse
 				(False, False) -> findbest c rs -- same
 			
-{- Avoids updating the branch. A useful optimisation when the branch
- - is known to have not changed, or git-annex won't be relying on info
- - from it. -}
-disableUpdate :: Annex ()
-disableUpdate = Annex.changeState setupdated
-	where
-		setupdated s = s { Annex.branchstate = new }
-			where
-				new = old { branchUpdated = True }
-				old = Annex.branchstate s
-
 {- Checks if a git ref exists. -}
 refExists :: Git.Ref -> Annex Bool
 refExists ref = inRepo $ Git.runBool "show-ref"
diff --git a/Annex/BranchState.hs b/Annex/BranchState.hs
new file mode 100644
index 0000000000..0950e9a967
--- /dev/null
+++ b/Annex/BranchState.hs
@@ -0,0 +1,56 @@
+{- git-annex branch state management
+ -
+ - Runtime state about the git-annex branch, including a small read cache.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.BranchState where
+
+import Common.Annex
+import Types.BranchState
+import qualified Annex
+
+getState :: Annex BranchState
+getState = Annex.getState Annex.branchstate
+
+setState :: BranchState -> Annex ()
+setState state = Annex.changeState $ \s -> s { Annex.branchstate = state }
+
+setCache :: FilePath -> String -> Annex ()
+setCache file content = do
+	state <- getState
+	setState state { cachedFile = Just file, cachedContent = content }
+
+getCache :: FilePath -> Annex (Maybe String)
+getCache file = getState >>= go
+	where
+		go state
+			| cachedFile state == Just file =
+				return $ Just $ cachedContent state
+			| otherwise = return Nothing
+
+invalidateCache :: Annex ()
+invalidateCache = do
+	state <- getState
+	setState state { cachedFile = Nothing, cachedContent = "" }
+
+{- Runs an action to update the branch, if it's not been updated before
+ - in this run of git-annex. -}
+runUpdateOnce :: Annex () -> Annex ()
+runUpdateOnce a = unlessM (branchUpdated <$> getState) $ do
+	a
+	disableUpdate
+
+{- Avoids updating the branch. A useful optimisation when the branch
+ - is known to have not changed, or git-annex won't be relying on info
+ - from it. -}
+disableUpdate :: Annex ()
+disableUpdate = Annex.changeState setupdated
+	where
+		setupdated s = s { Annex.branchstate = new }
+			where
+				new = old { branchUpdated = True }
+				old = Annex.branchstate s
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 0251da5583..2f9288e1b4 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -19,7 +19,7 @@ import qualified Git
 import qualified Annex
 import Annex.UUID
 import qualified Annex.Content
-import qualified Annex.Branch
+import qualified Annex.BranchState
 import qualified Utility.Url as Url
 import Utility.TempFile
 import Config
@@ -171,7 +171,7 @@ onLocal r a = do
 	Annex.eval state $ do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
-		Annex.Branch.disableUpdate
+		Annex.BranchState.disableUpdate
 		ret <- a
 		liftIO Git.reap
 		return ret

From da95cbadca5b7ef3058b91a384d5f3a48cc39039 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 18:03:28 -0400
Subject: [PATCH 2679/2835] split out Annex/Journal.hs

---
 Annex/Branch.hs  | 92 ++++-------------------------------------------
 Annex/Journal.hs | 94 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+), 85 deletions(-)
 create mode 100644 Annex/Journal.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 699bc0323c..1dac8ef793 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -18,16 +18,15 @@ module Annex.Branch (
 	name	
 ) where
 
-import System.IO.Binary
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common.Annex
 import Annex.Exception
 import Annex.BranchState
+import Annex.Journal
 import qualified Git
 import qualified Git.UnionMerge
-import qualified Annex
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -171,7 +170,7 @@ getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< getRef fullname
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
-	stageJournalFiles
+	stageJournal
 	ref <- getBranch
 	withIndex $ commitBranch ref message [fullname]
 
@@ -199,7 +198,7 @@ update = runUpdateOnce $ do
 	if (not dirty && null refs)
 		then updateIndex branchref
 		else withIndex $ lockJournal $ do
-			when dirty stageJournalFiles
+			when dirty stageJournal
 			let merge_desc = if null branches
 				then "update" 
 				else "merging " ++
@@ -305,7 +304,7 @@ siblingBranches = do
 change :: FilePath -> (String -> String) -> Annex ()
 change file a = lockJournal $ getStale file >>= return . a >>= set file
 
-{- Records new content of a file into the journal. -}
+{- Records new content of a file into the journal and cache. -}
 set :: FilePath -> String -> Annex ()
 set file content = do
 	setJournalFile file content
@@ -346,44 +345,9 @@ files = withIndexUpdate $ do
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
 
-{- Records content for a file in the branch to the journal.
- -
- - Using the journal, rather than immediatly staging content to the index
- - avoids git needing to rewrite the index after every change. -}
-setJournalFile :: FilePath -> String -> Annex ()
-setJournalFile file content = do
-	g <- gitRepo
-	liftIO $ doRedo (write g) $ do
-		createDirectoryIfMissing True $ gitAnnexJournalDir g
-		createDirectoryIfMissing True $ gitAnnexTmpDir g
-	where
-		-- journal file is written atomically
-		write g = do
-			let jfile = journalFile g file
-			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
-			writeBinaryFile tmpfile content
-			moveFile tmpfile jfile
-
-{- Gets any journalled content for a file in the branch. -}
-getJournalFile :: FilePath -> Annex (Maybe String)
-getJournalFile file = inRepo $ \g -> catchMaybeIO $
-	readFileStrict $ journalFile g file
-
-{- List of files that have updated content in the journal. -}
-getJournalledFiles :: Annex [FilePath]
-getJournalledFiles = map fileJournal <$> getJournalFiles
-
-{- List of existing journal files. -}
-getJournalFiles :: Annex [FilePath]
-getJournalFiles = do
-	g <- gitRepo
-	fs <- liftIO $
-		catchDefaultIO (getDirectoryContents $ gitAnnexJournalDir g) []
-	return $ filter (`notElem` [".", ".."]) fs
-
-{- Stages the specified journalfiles. -}
-stageJournalFiles :: Annex ()
-stageJournalFiles = do
+{- Stages the journal into the index. -}
+stageJournal :: Annex ()
+stageJournal = do
 	fs <- getJournalFiles
 	g <- gitRepo
 	withIndex $ liftIO $ do
@@ -409,45 +373,3 @@ stageJournalFiles = do
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
 		git_hash_object = Git.gitCommandLine
 			[Param "hash-object", Param "-w", Param "--stdin-paths"]
-
-
-{- Checks if there are changes in the journal. -}
-journalDirty :: Annex Bool
-journalDirty = not . null <$> getJournalFiles
-
-{- Produces a filename to use in the journal for a file on the branch.
- -
- - The journal typically won't have a lot of files in it, so the hashing
- - used in the branch is not necessary, and all the files are put directly
- - in the journal directory.
- -}
-journalFile :: Git.Repo -> FilePath -> FilePath
-journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
-	where
-		mangle '/' = "_"
-		mangle '_' = "__"
-		mangle c = [c]
-
-{- Converts a journal file (relative to the journal dir) back to the
- - filename on the branch. -}
-fileJournal :: FilePath -> FilePath
-fileJournal = replace "//" "_" . replace "_" "/"
-
-{- Runs an action that modifies the journal, using locking to avoid
- - contention with other git-annex processes. -}
-lockJournal :: Annex a -> Annex a
-lockJournal a = do
-	file <- fromRepo gitAnnexJournalLock
-	bracketIO (lock file) unlock a
-	where
-		lock file = do
-			l <- doRedo (createFile file stdFileMode) $
-				createDirectoryIfMissing True $ takeDirectory file
-			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
-			return l
-		unlock = closeFd
-
-{- Runs an action, catching failure and running something to fix it up, and
- - retrying if necessary. -}
-doRedo :: IO a -> IO b -> IO a
-doRedo a b = catch a $ const $ b >> a
diff --git a/Annex/Journal.hs b/Annex/Journal.hs
new file mode 100644
index 0000000000..9c5be89b19
--- /dev/null
+++ b/Annex/Journal.hs
@@ -0,0 +1,94 @@
+{- management of the git-annex journal and cache
+ -
+ - The journal is used to queue up changes before they are committed to the
+ - git-annex branch. Amoung other things, it ensures that if git-annex is
+ - interrupted, its recorded data is not lost.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.Journal where
+
+import System.IO.Binary
+
+import Common.Annex
+import Annex.Exception
+import qualified Git
+
+{- Records content for a file in the branch to the journal.
+ -
+ - Using the journal, rather than immediatly staging content to the index
+ - avoids git needing to rewrite the index after every change. -}
+setJournalFile :: FilePath -> String -> Annex ()
+setJournalFile file content = do
+	g <- gitRepo
+	liftIO $ doRedo (write g) $ do
+		createDirectoryIfMissing True $ gitAnnexJournalDir g
+		createDirectoryIfMissing True $ gitAnnexTmpDir g
+	where
+		-- journal file is written atomically
+		write g = do
+			let jfile = journalFile g file
+			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
+			writeBinaryFile tmpfile content
+			moveFile tmpfile jfile
+
+{- Gets any journalled content for a file in the branch. -}
+getJournalFile :: FilePath -> Annex (Maybe String)
+getJournalFile file = inRepo $ \g -> catchMaybeIO $
+	readFileStrict $ journalFile g file
+
+{- List of files that have updated content in the journal. -}
+getJournalledFiles :: Annex [FilePath]
+getJournalledFiles = map fileJournal <$> getJournalFiles
+
+{- List of existing journal files. -}
+getJournalFiles :: Annex [FilePath]
+getJournalFiles = do
+	g <- gitRepo
+	fs <- liftIO $
+		catchDefaultIO (getDirectoryContents $ gitAnnexJournalDir g) []
+	return $ filter (`notElem` [".", ".."]) fs
+
+{- Checks if there are changes in the journal. -}
+journalDirty :: Annex Bool
+journalDirty = not . null <$> getJournalFiles
+
+{- Produces a filename to use in the journal for a file on the branch.
+ -
+ - The journal typically won't have a lot of files in it, so the hashing
+ - used in the branch is not necessary, and all the files are put directly
+ - in the journal directory.
+ -}
+journalFile :: Git.Repo -> FilePath -> FilePath
+journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
+	where
+		mangle '/' = "_"
+		mangle '_' = "__"
+		mangle c = [c]
+
+{- Converts a journal file (relative to the journal dir) back to the
+ - filename on the branch. -}
+fileJournal :: FilePath -> FilePath
+fileJournal = replace "//" "_" . replace "_" "/"
+
+{- Runs an action that modifies the journal, using locking to avoid
+ - contention with other git-annex processes. -}
+lockJournal :: Annex a -> Annex a
+lockJournal a = do
+	file <- fromRepo gitAnnexJournalLock
+	bracketIO (lock file) unlock a
+	where
+		lock file = do
+			l <- doRedo (createFile file stdFileMode) $
+				createDirectoryIfMissing True $ takeDirectory file
+			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
+			return l
+		unlock = closeFd
+
+{- Runs an action, catching failure and running something to fix it up, and
+ - retrying if necessary. -}
+doRedo :: IO a -> IO b -> IO a
+doRedo a b = catch a $ const $ b >> a

From 543d0d250104c1f5908e1b7b258d36d95488a029 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 18:23:24 -0400
Subject: [PATCH 2680/2835] split out Git/Ref.hs

---
 Annex/Branch.hs   | 57 ++++++++++++++++-------------------------------
 Command/Unused.hs |  3 ++-
 Git.hs            | 17 ++++----------
 Git/Ref.hs        | 47 ++++++++++++++++++++++++++++++++++++++
 Init.hs           |  2 +-
 Upgrade/V2.hs     |  3 ++-
 6 files changed, 75 insertions(+), 54 deletions(-)
 create mode 100644 Git/Ref.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 1dac8ef793..c8a538acd0 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -12,10 +12,9 @@ module Annex.Branch (
 	change,
 	commit,
 	files,
-	refExists,
+	name,
 	hasOrigin,
-	hasSomeBranch,
-	name	
+	hasSibling,
 ) where
 
 import System.Exit
@@ -27,6 +26,7 @@ import Annex.BranchState
 import Annex.Journal
 import qualified Git
 import qualified Git.UnionMerge
+import qualified Git.Ref
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -84,12 +84,12 @@ updateIndex branchref = do
 		liftIO (catchDefaultIO (readFileStrict lock) "")
 	when (lockref /= branchref) $ do
 		withIndex $ mergeIndex [fullname]
-		setIndexRef branchref
+		setIndexSha branchref
 
 {- Record that the branch's index has been updated to correspond to a
  - given ref of the branch. -}
-setIndexRef :: Git.Ref -> Annex ()
-setIndexRef ref = do
+setIndexSha :: Git.Ref -> Annex ()
+setIndexSha ref = do
         lock <- fromRepo gitAnnexIndexLock
 	liftIO $ writeFile lock $ show ref ++ "\n"
 
@@ -115,7 +115,7 @@ commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex ()
 commitBranch branchref message parents = do
 	updateIndex branchref
 	committedref <- inRepo $ Git.commit message fullname parents
-	setIndexRef committedref
+	setIndexSha committedref
 	parentrefs <- commitparents <$> catObject committedref
 	when (racedetected branchref parentrefs) $
 		fixrace committedref parentrefs
@@ -154,18 +154,19 @@ create = do
 
 {- Returns the ref of the branch, creating it first if necessary. -}
 getBranch :: Annex (Git.Ref)
-getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< getRef fullname
+getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
 	where
 		go True = do
 			inRepo $ Git.run "branch"
 				[Param $ show name, Param $ show originname]
 			fromMaybe (error $ "failed to create " ++ show name)
-				<$> getRef fullname
+				<$> branchsha
 		go False = withIndex' True $ do
 			inRepo $ Git.commit "branch created" fullname []
-		use ref = do
-			setIndexRef ref
-			return ref
+		use sha = do
+			setIndexSha sha
+			return sha
+		branchsha = inRepo $ Git.Ref.sha fullname
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
@@ -202,7 +203,7 @@ update = runUpdateOnce $ do
 			let merge_desc = if null branches
 				then "update" 
 				else "merging " ++
-					unwords (map Git.refDescribe branches) ++ 
+					unwords (map Git.Ref.describe branches) ++ 
 					" into " ++ show name
 			unless (null branches) $ do
 				showSideAction merge_desc
@@ -263,38 +264,18 @@ tryFastForwardTo (first:rest) = do
 				(False, True) -> findbest c rs -- worse
 				(False, False) -> findbest c rs -- same
 			
-{- Checks if a git ref exists. -}
-refExists :: Git.Ref -> Annex Bool
-refExists ref = inRepo $ Git.runBool "show-ref"
-	[Param "--verify", Param "-q", Param $ show ref]
-
-{- Get the ref of a branch. (Must be a fully qualified branch name) -}
-getRef :: Git.Branch -> Annex (Maybe Git.Ref)
-getRef branch = process . L.unpack <$> showref
-	where
-		showref = inRepo $ Git.pipeRead [Param "show-ref",
-			Param "--hash", -- get the hash
-			Param $ show branch]
-		process [] = Nothing
-		process s = Just $ Git.Ref $ firstLine s
-
 {- Does origin/git-annex exist? -}
 hasOrigin :: Annex Bool
-hasOrigin = refExists originname
+hasOrigin = inRepo $ Git.Ref.exists originname
 
-{- Does the git-annex branch or a foo/git-annex branch exist? -}
-hasSomeBranch :: Annex Bool
-hasSomeBranch = not . null <$> siblingBranches
+{- Does the git-annex branch or a sibling foo/git-annex branch exist? -}
+hasSibling :: Annex Bool
+hasSibling = not . null <$> siblingBranches
 
 {- List of git-annex (refs, branches), including the main one and any
  - from remotes. Duplicate refs are filtered out. -}
 siblingBranches :: Annex [(Git.Ref, Git.Branch)]
-siblingBranches = do
-	r <- inRepo $ Git.pipeRead [Param "show-ref", Param $ show name]
-	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
-	where
-		gen l = (Git.Ref $ head l, Git.Ref $ last l)
-		uref (a, _) (b, _) = a == b
+siblingBranches = inRepo $ Git.Ref.matching name
 
 {- Applies a function to modifiy the content of a file.
  -
diff --git a/Command/Unused.hs b/Command/Unused.hs
index be0107752e..cd1cd16024 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -20,6 +20,7 @@ import Utility.TempFile
 import Logs.Location
 import qualified Annex
 import qualified Git
+import qualified Git.Ref
 import qualified Git.LsFiles as LsFiles
 import qualified Git.LsTree as LsTree
 import qualified Backend
@@ -190,7 +191,7 @@ getKeysReferenced = do
 {- List of keys referenced by symlinks in a git ref. -}
 getKeysReferencedInGit :: Git.Ref -> Annex [Key]
 getKeysReferencedInGit ref = do
-	showAction $ "checking " ++ Git.refDescribe ref
+	showAction $ "checking " ++ Git.Ref.describe ref
 	findkeys [] =<< inRepo (LsTree.lsTree ref)
 	where
 		findkeys c [] = return c
diff --git a/Git.hs b/Git.hs
index 1da5997c18..9af68a1942 100644
--- a/Git.hs
+++ b/Git.hs
@@ -24,7 +24,6 @@ module Git (
 	repoIsHttp,
 	repoIsLocalBare,
 	repoDescribe,
-	refDescribe,
 	repoLocation,
 	workTree,
 	workTreeFile,
@@ -178,14 +177,6 @@ repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
 repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
-{- Converts a fully qualified git ref into a user-visible version. -}
-refDescribe :: Ref -> String
-refDescribe = remove "refs/heads/" . remove "refs/remotes/" . show
-	where
-		remove prefix s
-			| prefix `isPrefixOf` s = drop (length prefix) s
-			| otherwise = s
-
 {- Location of the repo, either as a path or url. -}
 repoLocation :: Repo -> String
 repoLocation Repo { location = Url url } = show url
@@ -463,16 +454,16 @@ shaSize :: Int
 shaSize = 40
 
 {- Commits the index into the specified branch (or other ref), 
- - with the specified parent refs, and returns the new ref -}
-commit :: String -> Ref -> [Ref] -> Repo -> IO Ref
-commit message newref parentrefs repo = do
+ - with the specified parent refs, and returns the committed sha -}
+commit :: String -> Branch -> [Ref] -> Repo -> IO Sha
+commit message branch parentrefs repo = do
 	tree <- getSha "write-tree" $ asString $
 		pipeRead [Param "write-tree"] repo
 	sha <- getSha "commit-tree" $ asString $
 		ignorehandle $ pipeWriteRead
 			(map Param $ ["commit-tree", show tree] ++ ps)
 			(L.pack message) repo
-	run "update-ref" [Param $ show newref, Param $ show sha] repo
+	run "update-ref" [Param $ show branch, Param $ show sha] repo
 	return sha
 	where
 		ignorehandle a = snd <$> a
diff --git a/Git/Ref.hs b/Git/Ref.hs
new file mode 100644
index 0000000000..723bea6817
--- /dev/null
+++ b/Git/Ref.hs
@@ -0,0 +1,47 @@
+{- git ref stuff
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Ref where
+
+import qualified Data.ByteString.Lazy.Char8 as L
+
+import Common
+import Git
+
+{- Converts a fully qualified git ref into a user-visible version. -}
+describe :: Ref -> String
+describe = remove "refs/heads/" . remove "refs/remotes/" . show
+	where
+		remove prefix s
+			| prefix `isPrefixOf` s = drop (length prefix) s
+			| otherwise = s
+
+{- Checks if a ref exists. -}
+exists :: Ref -> Repo -> IO Bool
+exists ref = runBool "show-ref" 
+	[Param "--verify", Param "-q", Param $ show ref]
+
+{- Get the sha of a fully qualified git ref, if it exists. -}
+sha :: Branch -> Repo -> IO (Maybe Sha)
+sha branch repo = process . L.unpack <$> showref repo
+	where
+		showref = pipeRead [Param "show-ref",
+			Param "--hash", -- get the hash
+			Param $ show branch]
+		process [] = Nothing
+		process s = Just $ Ref $ firstLine s
+
+{- List of (refs, branches) matching a given ref spec.
+ - Duplicate refs are filtered out. -}
+matching :: Ref -> Repo -> IO [(Git.Ref, Git.Branch)]
+matching ref repo = do
+	r <- Git.pipeRead [Param "show-ref", Param $ show ref] repo
+	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
+	where
+		gen l = (Git.Ref $ head l, Git.Ref $ last l)
+		uref (a, _) (b, _) = a == b
+
diff --git a/Init.hs b/Init.hs
index d03e10031f..c8deadf3b4 100644
--- a/Init.hs
+++ b/Init.hs
@@ -39,7 +39,7 @@ ensureInitialized :: Annex ()
 ensureInitialized = getVersion >>= maybe needsinit checkVersion
 	where
 		needsinit = do
-			annexed <- Annex.Branch.hasSomeBranch
+			annexed <- Annex.Branch.hasSibling
 			if annexed
 				then initialize Nothing
 				else error "First run: git-annex init"
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 08bf83e837..3440d504ba 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -9,6 +9,7 @@ module Upgrade.V2 where
 
 import Common.Annex
 import qualified Git
+import qualified Git.Ref
 import qualified Annex.Branch
 import Logs.Location
 import Annex.Content
@@ -86,7 +87,7 @@ logFiles dir = return . filter (".log" `isSuffixOf`)
 
 push :: Annex ()
 push = do
-	origin_master <- Annex.Branch.refExists $ Git.Ref "origin/master"
+	origin_master <- inRepo $ Git.Ref.exists $ Git.Ref "origin/master"
 	origin_gitannex <- Annex.Branch.hasOrigin
 	case (origin_master, origin_gitannex) of
 		(_, True) -> do

From 31a0c07ee91af9e3bf434f416a4d711d841aa223 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 21:12:51 -0400
Subject: [PATCH 2681/2835] broke out Git/Branch.hs and reorganized

---
 Annex/Branch.hs | 407 ++++++++++++++++++++++--------------------------
 Git/Branch.hs   |  60 +++++++
 2 files changed, 242 insertions(+), 225 deletions(-)
 create mode 100644 Git/Branch.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index c8a538acd0..42940f4ff5 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -6,15 +6,15 @@
  -}
 
 module Annex.Branch (
+	name,
+	hasOrigin,
+	hasSibling,
 	create,
 	update,
 	get,
 	change,
 	commit,
 	files,
-	name,
-	hasOrigin,
-	hasSibling,
 ) where
 
 import System.Exit
@@ -27,6 +27,7 @@ import Annex.Journal
 import qualified Git
 import qualified Git.UnionMerge
 import qualified Git.Ref
+import qualified Git.Branch
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -41,57 +42,132 @@ fullname = Git.Ref $ "refs/heads/" ++ show name
 originname :: Git.Ref
 originname = Git.Ref $ "origin/" ++ show name
 
-{- Populates the branch's index file with the current branch contents.
- - 
- - This is only done when the index doesn't yet exist, and the index 
- - is used to build up changes to be commited to the branch, and merge
- - in changes from other branches.
- -}
-genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.stream_update_index g
-	[Git.UnionMerge.ls_tree fullname g]
+{- Does origin/git-annex exist? -}
+hasOrigin :: Annex Bool
+hasOrigin = inRepo $ Git.Ref.exists originname
 
-{- Merges the specified branches into the index.
- - Any changes staged in the index will be preserved. -}
-mergeIndex :: [Git.Ref] -> Annex ()
-mergeIndex branches = do
-	h <- catFileHandle
-	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+{- Does the git-annex branch or a sibling foo/git-annex branch exist? -}
+hasSibling :: Annex Bool
+hasSibling = not . null <$> siblingBranches
 
-{- Runs an action using the branch's index file. -}
-withIndex :: Annex a -> Annex a
-withIndex = withIndex' False
-withIndex' :: Bool -> Annex a -> Annex a
-withIndex' bootstrapping a = do
-	f <- fromRepo gitAnnexIndex
-	bracketIO (Git.useIndex f) id $ do
-		unlessM (liftIO $ doesFileExist f) $ do
-			unless bootstrapping create
-			liftIO $ createDirectoryIfMissing True $ takeDirectory f
-			unless bootstrapping $ inRepo genIndex
-		a
+{- List of git-annex (refs, branches), including the main one and any
+ - from remotes. Duplicate refs are filtered out. -}
+siblingBranches :: Annex [(Git.Ref, Git.Branch)]
+siblingBranches = inRepo $ Git.Ref.matching name
 
-{- Updates the branch's index to reflect the current contents of the branch.
- - Any changes staged in the index will be preserved.
+{- Creates the branch, if it does not already exist. -}
+create :: Annex ()
+create = do
+	_ <- getBranch
+	return ()
+
+{- Returns the ref of the branch, creating it first if necessary. -}
+getBranch :: Annex (Git.Ref)
+getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
+	where
+		go True = do
+			inRepo $ Git.run "branch"
+				[Param $ show name, Param $ show originname]
+			fromMaybe (error $ "failed to create " ++ show name)
+				<$> branchsha
+		go False = withIndex' True $ do
+			inRepo $ Git.commit "branch created" fullname []
+		use sha = do
+			setIndexSha sha
+			return sha
+		branchsha = inRepo $ Git.Ref.sha fullname
+
+{- Ensures that the branch and index are is up-to-date; should be
+ - called before data is read from it. Runs only once per git-annex run.
  -
- - Compares the ref stored in the lock file with the current
- - ref of the branch to see if an update is needed.
+ - Before refs are merged into the index, it's important to first stage the
+ - journal into the index. Otherwise, any changes in the journal would
+ - later get staged, and might overwrite changes made during the merge.
+ -
+ - (It would be cleaner to handle the merge by updating the journal, not the
+ - index, with changes from the branches.)
+ -
+ - The branch is fast-forwarded if possible, otherwise a merge commit is
+ - made.
  -}
-updateIndex :: Git.Ref -> Annex ()
-updateIndex branchref = do
-	lock <- fromRepo gitAnnexIndexLock
-	lockref <- Git.Ref . firstLine <$>
-		liftIO (catchDefaultIO (readFileStrict lock) "")
-	when (lockref /= branchref) $ do
-		withIndex $ mergeIndex [fullname]
-		setIndexSha branchref
+update :: Annex ()
+update = runUpdateOnce $ do
+	-- ensure branch exists, and get its current ref
+	branchref <- getBranch
+	-- check what needs updating before taking the lock
+	dirty <- journalDirty
+	(refs, branches) <- unzip <$> newerSiblings
+	if (not dirty && null refs)
+		then updateIndex branchref
+		else withIndex $ lockJournal $ do
+			when dirty stageJournal
+			let merge_desc = if null branches
+				then "update" 
+				else "merging " ++
+					unwords (map Git.Ref.describe branches) ++ 
+					" into " ++ show name
+			unless (null branches) $ do
+				showSideAction merge_desc
+				mergeIndex branches
+			ff <- if dirty
+				then return False
+				else inRepo $ Git.Branch.fastForward fullname refs
+			if ff
+				then updateIndex branchref
+				else commitBranch branchref merge_desc
+					(nub $ fullname:refs)
+			invalidateCache
+	where
+		newerSiblings = filterM isnewer =<< siblingBranches
+		isnewer (_, b) = inRepo $ Git.Branch.changed fullname b
 
-{- Record that the branch's index has been updated to correspond to a
- - given ref of the branch. -}
-setIndexSha :: Git.Ref -> Annex ()
-setIndexSha ref = do
-        lock <- fromRepo gitAnnexIndexLock
-	liftIO $ writeFile lock $ show ref ++ "\n"
+{- Gets the content of a file on the branch, or content from the journal, or
+ - staged in the index.
+ -
+ - Returns an empty string if the file doesn't exist yet. -}
+get :: FilePath -> Annex String
+get = get' False
+
+{- Like get, but does not merge the branch, so the info returned may not
+ - reflect changes in remotes. (Changing the value this returns, and then
+ - merging is always the same as using get, and then changing its value.) -}
+getStale :: FilePath -> Annex String
+getStale = get' True
+
+get' :: Bool -> FilePath -> Annex String
+get' staleok file = fromcache =<< getCache file
+	where
+		fromcache (Just content) = return content
+		fromcache Nothing = fromjournal =<< getJournalFile file
+		fromjournal (Just content) = cache content
+		fromjournal Nothing
+			| staleok = withIndex frombranch
+			| otherwise = withIndexUpdate $ frombranch >>= cache
+		frombranch = L.unpack <$> catFile fullname file
+		cache content = do
+			setCache file content
+			return content
+
+{- Applies a function to modifiy the content of a file.
+ -
+ - Note that this does not cause the branch to be merged, it only
+ - modifes the current content of the file on the branch.
+ -}
+change :: FilePath -> (String -> String) -> Annex ()
+change file a = lockJournal $ getStale file >>= return . a >>= set file
+
+{- Records new content of a file into the journal and cache. -}
+set :: FilePath -> String -> Annex ()
+set file content = do
+	setJournalFile file content
+	setCache file content
+
+{- Stages the journal, and commits staged changes to the branch. -}
+commit :: String -> Annex ()
+commit message = whenM journalDirty $ lockJournal $ do
+	stageJournal
+	ref <- getBranch
+	withIndex $ commitBranch ref message [fullname]
 
 {- Commits the staged changes in the index to the branch.
  - 
@@ -141,183 +217,6 @@ commitBranch branchref message parents = do
 		
 		racemessage = message ++ " (recovery from race)"
 
-{- Runs an action using the branch's index file, first making sure that
- - the branch and index are up-to-date. -}
-withIndexUpdate :: Annex a -> Annex a
-withIndexUpdate a = update >> withIndex a
-
-{- Creates the branch, if it does not already exist. -}
-create :: Annex ()
-create = do
-	_ <- getBranch
-	return ()
-
-{- Returns the ref of the branch, creating it first if necessary. -}
-getBranch :: Annex (Git.Ref)
-getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
-	where
-		go True = do
-			inRepo $ Git.run "branch"
-				[Param $ show name, Param $ show originname]
-			fromMaybe (error $ "failed to create " ++ show name)
-				<$> branchsha
-		go False = withIndex' True $ do
-			inRepo $ Git.commit "branch created" fullname []
-		use sha = do
-			setIndexSha sha
-			return sha
-		branchsha = inRepo $ Git.Ref.sha fullname
-
-{- Stages the journal, and commits staged changes to the branch. -}
-commit :: String -> Annex ()
-commit message = whenM journalDirty $ lockJournal $ do
-	stageJournal
-	ref <- getBranch
-	withIndex $ commitBranch ref message [fullname]
-
-{- Ensures that the branch and index are is up-to-date; should be
- - called before data is read from it. Runs only once per git-annex run.
- -
- - Before refs are merged into the index, it's important to first stage the
- - journal into the index. Otherwise, any changes in the journal would
- - later get staged, and might overwrite changes made during the merge.
- -
- - (It would be cleaner to handle the merge by updating the journal, not the
- - index, with changes from the branches.)
- -
- - The branch is fast-forwarded if possible, otherwise a merge commit is
- - made.
- -}
-update :: Annex ()
-update = runUpdateOnce $ do
-	-- ensure branch exists, and get its current ref
-	branchref <- getBranch
-	-- check what needs updating before taking the lock
-	dirty <- journalDirty
-	c <- filterM (changedBranch fullname . snd) =<< siblingBranches
-	let (refs, branches) = unzip c
-	if (not dirty && null refs)
-		then updateIndex branchref
-		else withIndex $ lockJournal $ do
-			when dirty stageJournal
-			let merge_desc = if null branches
-				then "update" 
-				else "merging " ++
-					unwords (map Git.Ref.describe branches) ++ 
-					" into " ++ show name
-			unless (null branches) $ do
-				showSideAction merge_desc
-				mergeIndex branches
-			ff <- if dirty then return False else tryFastForwardTo refs
-			if ff
-				then updateIndex branchref
-				else commitBranch branchref merge_desc
-					(nub $ fullname:refs)
-			invalidateCache
-
-{- Checks if the second branch has any commits not present on the first
- - branch. -}
-changedBranch :: Git.Branch -> Git.Branch -> Annex Bool
-changedBranch origbranch newbranch
-	| origbranch == newbranch = return False
-	| otherwise = not . L.null <$> diffs
-	where
-		diffs = inRepo $ Git.pipeRead
-			[ Param "log"
-			, Param (show origbranch ++ ".." ++ show newbranch)
-			, Params "--oneline -n1"
-			]
-
-{- Given a set of refs that are all known to have commits not
- - on the git-annex branch, tries to update the branch by a
- - fast-forward.
- -
- - In order for that to be possible, one of the refs must contain
- - every commit present in all the other refs, as well as in the
- - git-annex branch.
- -}
-tryFastForwardTo :: [Git.Ref] -> Annex Bool
-tryFastForwardTo [] = return True
-tryFastForwardTo (first:rest) = do
-	-- First, check that the git-annex branch does not contain any
-	-- new commits that are not in the first other branch. If it does,
-	-- cannot fast-forward.
-	diverged <- changedBranch first fullname
-	if diverged
-		then no_ff
-		else maybe no_ff do_ff =<< findbest first rest
-	where
-		no_ff = return False
-		do_ff branch = do
-			inRepo $ Git.run "update-ref"
-				[Param $ show fullname, Param $ show branch]
-			return True
-		findbest c [] = return $ Just c
-		findbest c (r:rs)
-			| c == r = findbest c rs
-			| otherwise = do
-			better <- changedBranch c r
-			worse <- changedBranch r c
-			case (better, worse) of
-				(True, True) -> return Nothing -- divergent fail
-				(True, False) -> findbest r rs -- better
-				(False, True) -> findbest c rs -- worse
-				(False, False) -> findbest c rs -- same
-			
-{- Does origin/git-annex exist? -}
-hasOrigin :: Annex Bool
-hasOrigin = inRepo $ Git.Ref.exists originname
-
-{- Does the git-annex branch or a sibling foo/git-annex branch exist? -}
-hasSibling :: Annex Bool
-hasSibling = not . null <$> siblingBranches
-
-{- List of git-annex (refs, branches), including the main one and any
- - from remotes. Duplicate refs are filtered out. -}
-siblingBranches :: Annex [(Git.Ref, Git.Branch)]
-siblingBranches = inRepo $ Git.Ref.matching name
-
-{- Applies a function to modifiy the content of a file.
- -
- - Note that this does not cause the branch to be merged, it only
- - modifes the current content of the file on the branch.
- -}
-change :: FilePath -> (String -> String) -> Annex ()
-change file a = lockJournal $ getStale file >>= return . a >>= set file
-
-{- Records new content of a file into the journal and cache. -}
-set :: FilePath -> String -> Annex ()
-set file content = do
-	setJournalFile file content
-	setCache file content
-
-{- Gets the content of a file on the branch, or content from the journal, or
- - staged in the index.
- -
- - Returns an empty string if the file doesn't exist yet. -}
-get :: FilePath -> Annex String
-get = get' False
-
-{- Like get, but does not merge the branch, so the info returned may not
- - reflect changes in remotes. (Changing the value this returns, and then
- - merging is always the same as using get, and then changing its value.) -}
-getStale :: FilePath -> Annex String
-getStale = get' True
-
-get' :: Bool -> FilePath -> Annex String
-get' staleok file = fromcache =<< getCache file
-	where
-		fromcache (Just content) = return content
-		fromcache Nothing = fromjournal =<< getJournalFile file
-		fromjournal (Just content) = cache content
-		fromjournal Nothing
-			| staleok = withIndex frombranch
-			| otherwise = withIndexUpdate $ frombranch >>= cache
-		frombranch = L.unpack <$> catFile fullname file
-		cache content = do
-			setCache file content
-			return content
-
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
@@ -326,6 +225,64 @@ files = withIndexUpdate $ do
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
 
+
+{- Populates the branch's index file with the current branch contents.
+ - 
+ - This is only done when the index doesn't yet exist, and the index 
+ - is used to build up changes to be commited to the branch, and merge
+ - in changes from other branches.
+ -}
+genIndex :: Git.Repo -> IO ()
+genIndex g = Git.UnionMerge.stream_update_index g
+	[Git.UnionMerge.ls_tree fullname g]
+
+{- Merges the specified branches into the index.
+ - Any changes staged in the index will be preserved. -}
+mergeIndex :: [Git.Ref] -> Annex ()
+mergeIndex branches = do
+	h <- catFileHandle
+	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+
+{- Runs an action using the branch's index file. -}
+withIndex :: Annex a -> Annex a
+withIndex = withIndex' False
+withIndex' :: Bool -> Annex a -> Annex a
+withIndex' bootstrapping a = do
+	f <- fromRepo gitAnnexIndex
+	bracketIO (Git.useIndex f) id $ do
+		unlessM (liftIO $ doesFileExist f) $ do
+			unless bootstrapping create
+			liftIO $ createDirectoryIfMissing True $ takeDirectory f
+			unless bootstrapping $ inRepo genIndex
+		a
+
+{- Runs an action using the branch's index file, first making sure that
+ - the branch and index are up-to-date. -}
+withIndexUpdate :: Annex a -> Annex a
+withIndexUpdate a = update >> withIndex a
+
+{- Updates the branch's index to reflect the current contents of the branch.
+ - Any changes staged in the index will be preserved.
+ -
+ - Compares the ref stored in the lock file with the current
+ - ref of the branch to see if an update is needed.
+ -}
+updateIndex :: Git.Ref -> Annex ()
+updateIndex branchref = do
+	lock <- fromRepo gitAnnexIndexLock
+	lockref <- Git.Ref . firstLine <$>
+		liftIO (catchDefaultIO (readFileStrict lock) "")
+	when (lockref /= branchref) $ do
+		withIndex $ mergeIndex [fullname]
+		setIndexSha branchref
+
+{- Record that the branch's index has been updated to correspond to a
+ - given ref of the branch. -}
+setIndexSha :: Git.Ref -> Annex ()
+setIndexSha ref = do
+        lock <- fromRepo gitAnnexIndexLock
+	liftIO $ writeFile lock $ show ref ++ "\n"
+
 {- Stages the journal into the index. -}
 stageJournal :: Annex ()
 stageJournal = do
diff --git a/Git/Branch.hs b/Git/Branch.hs
new file mode 100644
index 0000000000..e69e96f288
--- /dev/null
+++ b/Git/Branch.hs
@@ -0,0 +1,60 @@
+{- git branch stuff
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Branch where
+
+import qualified Data.ByteString.Lazy.Char8 as L
+
+import Common
+import Git
+
+{- Checks if the second branch has any commits not present on the first
+ - branch. -}
+changed :: Branch -> Branch -> Repo -> IO Bool
+changed origbranch newbranch repo
+	| origbranch == newbranch = return False
+	| otherwise = not . L.null <$> diffs
+	where
+		diffs = Git.pipeRead
+			[ Param "log"
+			, Param (show origbranch ++ ".." ++ show newbranch)
+			, Params "--oneline -n1"
+			] repo
+
+{- Given a set of refs that are all known to have commits not
+ - on the branch, tries to update the branch by a fast-forward.
+ -
+ - In order for that to be possible, one of the refs must contain
+ - every commit present in all the other refs.
+ -}
+fastForward :: Branch -> [Ref] -> Repo -> IO Bool
+fastForward _ [] _ = return True
+fastForward branch (first:rest) repo = do
+	-- First, check that the branch does not contain any
+	-- new commits that are not in the first ref. If it does,
+	-- cannot fast-forward.
+	diverged <- changed first branch repo
+	if diverged
+		then no_ff
+		else maybe no_ff do_ff =<< findbest first rest
+	where
+		no_ff = return False
+		do_ff to = do
+			Git.run "update-ref"
+				[Param $ show branch, Param $ show to] repo
+			return True
+		findbest c [] = return $ Just c
+		findbest c (r:rs)
+			| c == r = findbest c rs
+			| otherwise = do
+			better <- changed c r repo
+			worse <- changed r c repo
+			case (better, worse) of
+				(True, True) -> return Nothing -- divergent fail
+				(True, False) -> findbest r rs -- better
+				(False, True) -> findbest c rs -- worse
+				(False, False) -> findbest c rs -- same

From 0e45b762a07d12dbc099936a8481bda9c02d0318 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 21:24:55 -0400
Subject: [PATCH 2682/2835] broke out Git/HashObject.hs

---
 Annex/Branch.hs   | 19 +++----------------
 Git/HashObject.hs | 29 +++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 16 deletions(-)
 create mode 100644 Git/HashObject.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 42940f4ff5..556df976fe 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -17,7 +17,6 @@ module Annex.Branch (
 	files,
 ) where
 
-import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common.Annex
@@ -25,9 +24,10 @@ import Annex.Exception
 import Annex.BranchState
 import Annex.Journal
 import qualified Git
-import qualified Git.UnionMerge
 import qualified Git.Ref
 import qualified Git.Branch
+import qualified Git.UnionMerge
+import qualified Git.HashObject
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -291,23 +291,10 @@ stageJournal = do
 	withIndex $ liftIO $ do
 		let dir = gitAnnexJournalDir g
 		let paths = map (dir ) fs
-		-- inject all the journal files directly into git
-		-- in one quick command
-		(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object g
-		_ <- forkProcess $ do
-			hPutStr toh $ unlines paths
-			hClose toh
-			exitSuccess
-		hClose toh
-		shas <- map Git.Ref . lines <$> hGetContents fromh
-		-- update the index, also in just one command
+		shas <- Git.HashObject.hashFiles paths g
 		Git.UnionMerge.update_index g $
 			index_lines shas (map fileJournal fs)
-		hClose fromh
-		forceSuccess pid
 		mapM_ removeFile paths
 	where
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
-		git_hash_object = Git.gitCommandLine
-			[Param "hash-object", Param "-w", Param "--stdin-paths"]
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
new file mode 100644
index 0000000000..f28d865b13
--- /dev/null
+++ b/Git/HashObject.hs
@@ -0,0 +1,29 @@
+{- git hash-object interface
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.HashObject where
+
+import Common
+import Git
+
+{- Injects a set of files into git, returning the shas of the objects. -}
+hashFiles :: [FilePath] -> Repo -> IO [Sha]
+hashFiles paths repo = do
+	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo
+	_ <- forkProcess (feeder toh)
+	hClose toh
+	shas <- map Git.Ref . lines <$> hGetContents fromh
+	hClose fromh
+	forceSuccess pid
+	return shas
+	where
+		git_hash_object = Git.gitCommandLine
+			[Param "hash-object", Param "-w", Param "--stdin-paths"]
+		feeder toh = do
+			hPutStr toh $ unlines paths
+			hClose toh
+			exitSuccess

From 46588674b081cd4ea5820680d8fc15c81ed175ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 21:41:37 -0400
Subject: [PATCH 2683/2835] avoid closing pipe before all the shas are read
 from it

Could have just used hGetContentsStrict here, but that would require
storing all the shas in memory. Since this is called at the end of a
git-annex run, it may have created a *lot* of shas, so I avoid that memory
use and stream them out like before.
---
 Annex/Branch.hs   |  3 ++-
 Git/HashObject.hs | 14 ++++++++------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 556df976fe..f9fa6cbb31 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -291,9 +291,10 @@ stageJournal = do
 	withIndex $ liftIO $ do
 		let dir = gitAnnexJournalDir g
 		let paths = map (dir ) fs
-		shas <- Git.HashObject.hashFiles paths g
+		(shas, cleanup) <- Git.HashObject.hashFiles paths g
 		Git.UnionMerge.update_index g $
 			index_lines shas (map fileJournal fs)
+		cleanup
 		mapM_ removeFile paths
 	where
 		index_lines shas = map genline . zip shas
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index f28d865b13..99b96afcb9 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -10,16 +10,15 @@ module Git.HashObject where
 import Common
 import Git
 
-{- Injects a set of files into git, returning the shas of the objects. -}
-hashFiles :: [FilePath] -> Repo -> IO [Sha]
+{- Injects a set of files into git, returning the shas of the objects
+ - and an IO action to call ones the the shas have been used. -}
+hashFiles :: [FilePath] -> Repo -> IO ([Sha], IO ())
 hashFiles paths repo = do
 	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo
 	_ <- forkProcess (feeder toh)
 	hClose toh
-	shas <- map Git.Ref . lines <$> hGetContents fromh
-	hClose fromh
-	forceSuccess pid
-	return shas
+	shas <- map Git.Ref . lines <$> hGetContentsStrict fromh
+	return (shas, ender fromh pid)
 	where
 		git_hash_object = Git.gitCommandLine
 			[Param "hash-object", Param "-w", Param "--stdin-paths"]
@@ -27,3 +26,6 @@ hashFiles paths repo = do
 			hPutStr toh $ unlines paths
 			hClose toh
 			exitSuccess
+		ender fromh pid = do
+			hClose fromh
+			forceSuccess pid

From 51b95fbc0792b6fc9ba546692896274987b48822 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Tue, 13 Dec 2011 18:16:09 +0000
Subject: [PATCH 2684/2835] Added a comment

---
 ..._dc8a3f75533906ad3756fcc47f7e96bb._comment | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment b/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment
new file mode 100644
index 0000000000..1ac9e798a8
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 4"
+ date="2011-12-13T18:16:08Z"
+ content="""
+I thought about this some more, and I think I have a pretty decent solution that avoids a central bare repository. Instead of pushing to master (which git does not like) or trying to guess the remote branch name on the other side, there is a well-known branch name, say git-annex-master. Then a sync command would do something like this (untested):
+
+    git commit -a -m 'git annex sync' # ideally with a description derived from the diff
+    git merge git-annex-master
+    git pull someremote git-annex-master # for all reachable remotes. Or better to use fetch and then merge everything in one command?
+    git branch -f git-annex-master # (or checkout git-annex-master, merge master, checkout master, but since we merged before this should have the same effect
+    git annex merge
+    git push someremote git-annex-master # for all reachable remotes
+
+The nice things are: One can push to any remote repository, and thus avoid the issue of pushing to a portable device; the merging happens on the master branch, so if it fails to merge automatically, regular git foo can resolve it, and all changes eventually reach every repository.
+
+What do you think?
+    
+"""]]

From 1a06455f5cedb52324c6da6b13b7eedd72abe430 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Tue, 13 Dec 2011 18:47:18 +0000
Subject: [PATCH 2685/2835] Added a comment

---
 ..._afe5035a6b35ed2c7e193fb69cc182e2._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment b/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment
new file mode 100644
index 0000000000..0847daae9d
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 5"
+ date="2011-12-13T18:47:18Z"
+ content="""
+After some experimentation, this seems to work better:
+
+    	git commit -a -m 'git annex sync'
+	git merge git-annex-master
+	for remote in $(git remote)
+	do
+		git fetch $remote
+		git merge $remote git-annex-master
+	done
+	git branch -f git-annex-master
+	git annex merge
+	for remote in $(git remote)
+	do
+		git push $remote git-annex git-annex-master
+	done
+
+Maybe this approach can be enhance to skip stuff gracefully if there is no git-annex-master branch and then be added to what \"git annex sync\" does, this way those who want to use the feature can do so by running \"git branch git-annex-master\" once. Or, if you like this and want to make it default, just make git-annex-init create the git-annex-master branch :-)
+"""]]

From 13fff71f2019ae098c3f8532ac2734cb1ab11498 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 13 Dec 2011 15:05:07 -0400
Subject: [PATCH 2686/2835] split out three modules from Git

Constructors and configuration make sense in separate modules.
A separate Git.Types is needed to avoid cycles.
---
 Annex.hs                 |   3 +-
 Annex/UUID.hs            |   5 +-
 Annex/Version.hs         |   4 +-
 Backend.hs               |   3 +-
 Command/Map.hs           |  10 +-
 Command/Sync.hs          |   5 +-
 Config.hs                |   9 +-
 Git.hs                   | 250 +--------------------------------------
 Git/Config.hs            |  58 +++++++++
 Git/Construct.hs         | 198 +++++++++++++++++++++++++++++++
 Git/Types.hs             |  36 ++++++
 GitAnnex.hs              |   7 +-
 Remote/Bup.hs            |  16 +--
 Remote/Git.hs            |   8 +-
 Remote/Helper/Special.hs |   3 +-
 Remote/Web.hs            |   3 +-
 git-annex-shell.hs       |   4 +-
 git-annex.cabal          |   2 +-
 git-union-merge.hs       |   4 +-
 test.hs                  |   6 +-
 20 files changed, 349 insertions(+), 285 deletions(-)
 create mode 100644 Git/Config.hs
 create mode 100644 Git/Construct.hs
 create mode 100644 Git/Types.hs

diff --git a/Annex.hs b/Annex.hs
index d8d9c63b40..acc38a5db9 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -27,6 +27,7 @@ import Control.Monad.State
 
 import Common
 import qualified Git
+import qualified Git.Config
 import Git.CatFile
 import Git.Queue
 import Types.Backend
@@ -99,7 +100,7 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState <$> Git.configRead gitrepo
+new gitrepo = newState <$> Git.Config.read gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index e510a7ccd3..48bf71f104 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -21,6 +21,7 @@ module Annex.UUID (
 
 import Common.Annex
 import qualified Git
+import qualified Git.Config
 import qualified Build.SysConfig as SysConfig
 import Config
 
@@ -55,14 +56,14 @@ getRepoUUID r = do
 			return u
 		else return c
 	where
-		cached = toUUID . Git.configGet cachekey ""
+		cached = toUUID . Git.Config.get cachekey ""
 		updatecache u = do
 			g <- gitRepo
 			when (g /= r) $ storeUUID cachekey u
 		cachekey = remoteConfig r "uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID = toUUID . Git.configGet configkey ""
+getUncachedUUID = toUUID . Git.Config.get configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
diff --git a/Annex/Version.hs b/Annex/Version.hs
index 9e694faf14..917859eae4 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -8,7 +8,7 @@
 module Annex.Version where
 
 import Common.Annex
-import qualified Git
+import qualified Git.Config
 import Config
 
 type Version = String
@@ -26,7 +26,7 @@ versionField :: String
 versionField = "annex.version"
 
 getVersion :: Annex (Maybe Version)
-getVersion = handle <$> fromRepo (Git.configGet versionField "")
+getVersion = handle <$> fromRepo (Git.Config.get versionField "")
 	where
 		handle [] = Nothing
 		handle v = Just v
diff --git a/Backend.hs b/Backend.hs
index 136c2eb7a6..c7cb57440b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -21,6 +21,7 @@ import System.Posix.Files
 
 import Common.Annex
 import qualified Git
+import qualified Git.Config
 import qualified Annex
 import Types.Key
 import qualified Types.Backend as B
@@ -47,7 +48,7 @@ orderedList = do
 			l' <- (lookupBackendName name :) <$> standard
 			Annex.changeState $ \s -> s { Annex.backends = l' }
 			return l'
-		standard = fromRepo $ parseBackendList . Git.configGet "annex.backends" ""
+		standard = fromRepo $ parseBackendList . Git.Config.get "annex.backends" ""
 		parseBackendList [] = list
 		parseBackendList s = map lookupBackendName $ words s
 
diff --git a/Command/Map.hs b/Command/Map.hs
index 57b48d5030..815b142e72 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -13,6 +13,8 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import Annex.UUID
 import Logs.UUID
 import Logs.Trust
@@ -146,8 +148,8 @@ spider' (r:rs) known
 {- Converts repos to a common absolute form. -}
 absRepo :: Git.Repo -> Git.Repo -> Annex Git.Repo
 absRepo reference r
-	| Git.repoIsUrl reference = return $ Git.localToUrl reference r
-	| otherwise = liftIO $ Git.repoFromAbsPath =<< absPath (Git.workTree r)
+	| Git.repoIsUrl reference = return $ Git.Construct.localToUrl reference r
+	| otherwise = liftIO $ Git.Construct.fromAbsPath =<< absPath (Git.workTree r)
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
@@ -182,7 +184,7 @@ tryScan :: Git.Repo -> Annex (Maybe Git.Repo)
 tryScan r
 	| Git.repoIsSsh r = sshscan
 	| Git.repoIsUrl r = return Nothing
-	| otherwise = safely $ Git.configRead r
+	| otherwise = safely $ Git.Config.read r
 	where
 		safely a = do
 			result <- liftIO (try a :: IO (Either SomeException Git.Repo))
@@ -191,7 +193,7 @@ tryScan r
 				Right r' -> return $ Just r'
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
-				Git.hConfigRead r
+				Git.Config.hRead r
 
 		configlist =
 			onRemote r (pipedconfig, Nothing) "configlist" []
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 7dc5f4d24c..987eb6138f 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Command
 import qualified Annex.Branch
 import qualified Git
+import qualified Git.Config
 
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -56,7 +57,7 @@ push = do
 defaultRemote :: Annex String
 defaultRemote = do
 	branch <- currentBranch
-	fromRepo $ Git.configGet ("branch." ++ branch ++ ".remote") "origin"
+	fromRepo $ Git.Config.get ("branch." ++ branch ++ ".remote") "origin"
 
 currentBranch :: Annex String
 currentBranch = last . split "/" . L.unpack . head . L.lines <$>
@@ -65,6 +66,6 @@ currentBranch = last . split "/" . L.unpack . head . L.lines <$>
 checkRemote :: String -> Annex ()
 checkRemote remote = do
 	remoteurl <- fromRepo $
-		Git.configGet ("remote." ++ remote ++ ".url") ""
+		Git.Config.get ("remote." ++ remote ++ ".url") ""
 	when (null remoteurl) $ do
 		error $ "No url is configured for the remote: " ++ remote
diff --git a/Config.hs b/Config.hs
index c6107fc8eb..322dc8af82 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,6 +9,7 @@ module Config where
 
 import Common.Annex
 import qualified Git
+import qualified Git.Config
 import qualified Annex
 
 type ConfigKey = String
@@ -18,15 +19,15 @@ setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
 	inRepo $ Git.run "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
-	newg <- inRepo Git.configRead
+	newg <- inRepo Git.Config.read
 	Annex.changeState $ \s -> s { Annex.repo = newg }
 
 {- Looks up a per-remote config setting in git config.
  - Failing that, tries looking for a global config option. -}
 getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
 getConfig r key def = do
-	def' <- fromRepo $ Git.configGet ("annex." ++ key) def
-	fromRepo $ Git.configGet (remoteConfig r key) def'
+	def' <- fromRepo $ Git.Config.get ("annex." ++ key) def
+	fromRepo $ Git.Config.get (remoteConfig r key) def'
 
 {- Looks up a per-remote config setting in git config. -}
 remoteConfig :: Git.Repo -> ConfigKey -> String
@@ -83,6 +84,6 @@ getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 	where
 		use (Just n) = return n
 		use Nothing = perhaps (return 1) =<< 
-			readMaybe <$> fromRepo (Git.configGet config "1")
+			readMaybe <$> fromRepo (Git.Config.get config "1")
 		perhaps fallback = maybe fallback (return . id)
 		config = "annex.numcopies"
diff --git a/Git.hs b/Git.hs
index 9af68a1942..cb7cc19c25 100644
--- a/Git.hs
+++ b/Git.hs
@@ -3,7 +3,7 @@
  - This is written to be completely independant of git-annex and should be
  - suitable for other uses.
  -
- - Copyright 2010,2011 Joey Hess 
+ - Copyright 2010, 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -14,11 +14,6 @@ module Git (
 	Branch,
 	Sha,
 	Tag,
-	repoFromCwd,
-	repoFromAbsPath,
-	repoFromUnknown,
-	repoFromUrl,
-	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
 	repoIsHttp,
@@ -34,11 +29,7 @@ module Git (
 	urlHostUser,
 	urlAuthority,
 	urlScheme,
-	configGet,
 	configMap,
-	configRead,
-	hConfigRead,
-	configStore,
 	configTrue,
 	gitCommandLine,
 	run,
@@ -51,14 +42,12 @@ module Git (
 	attributes,
 	remotes,
 	remotesAdd,
-	genRemote,
 	repoRemoteName,
 	repoRemoteNameSet,
 	repoRemoteNameFromKey,
 	checkAttr,
 	decodeGitFile,
 	encodeGitFile,
-	repoAbsPath,
 	reap,
 	useIndex,
 	getSha,
@@ -69,9 +58,6 @@ module Git (
 	prop_idempotent_deencode
 ) where
 
-import System.Posix.Directory
-import System.Posix.User
-import Control.Exception (bracket_)
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
 import Data.Char
@@ -83,92 +69,7 @@ import System.Posix.Env (setEnv, unsetEnv, getEnv)
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
-
-{- There are two types of repositories; those on local disk and those
- - accessed via an URL. -}
-data RepoLocation = Dir FilePath | Url URI | Unknown
-	deriving (Show, Eq)
-
-data Repo = Repo {
-	location :: RepoLocation,
-	config :: M.Map String String,
-	remotes :: [Repo],
-	-- remoteName holds the name used for this repo in remotes
-	remoteName :: Maybe String 
-} deriving (Show, Eq)
-
-{- A git ref. Can be a sha1, or a branch or tag name. -}
-newtype Ref = Ref String
-	deriving (Eq)
-
-instance Show Ref where
-	show (Ref v) = v
-
-{- Aliases for Ref. -}
-type Branch = Ref
-type Sha = Ref
-type Tag = Ref
-
-newFrom :: RepoLocation -> Repo
-newFrom l = 
-	Repo {
-		location = l,
-		config = M.empty,
-		remotes = [],
-		remoteName = Nothing
-	}
-
-{- Local Repo constructor, requires an absolute path to the repo be
- - specified. -}
-repoFromAbsPath :: FilePath -> IO Repo
-repoFromAbsPath dir
-	| "/" `isPrefixOf` dir = do
- 		-- Git always looks for "dir.git" in preference to
-		-- to "dir", even if dir ends in a "/".
-		let canondir = dropTrailingPathSeparator dir
-		let dir' = canondir ++ ".git"
-		e <- doesDirectoryExist dir'
-		if e
-			then ret dir'
-			else if "/.git" `isSuffixOf` canondir
-				then do
-					-- When dir == "foo/.git", git looks
-					-- for "foo/.git/.git", and failing
-					-- that, uses "foo" as the repository.
-					e' <- doesDirectoryExist $ dir  ".git"
-					if e'
-						then ret dir
-						else ret $ takeDirectory canondir
-				else ret dir
-	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
-	where
-		ret = return . newFrom . Dir
-
-{- Remote Repo constructor. Throws exception on invalid url. -}
-repoFromUrl :: String -> IO Repo
-repoFromUrl url
-	| startswith "file://" url = repoFromAbsPath $ uriPath u
-	| otherwise = return $ newFrom $ Url u
-		where
-			u = fromMaybe bad $ parseURI url
-			bad = error $ "bad url " ++ url
-
-{- Creates a repo that has an unknown location. -}
-repoFromUnknown :: Repo
-repoFromUnknown = newFrom Unknown
-
-{- Converts a Local Repo into a remote repo, using the reference repo
- - which is assumed to be on the same host. -}
-localToUrl :: Repo -> Repo -> Repo
-localToUrl reference r
-	| not $ repoIsUrl reference = error "internal error; reference repo not url"
-	| repoIsUrl r = r
-	| otherwise = r { location = Url $ fromJust $ parseURI absurl }
-	where
-		absurl =
-			urlScheme reference ++ "//" ++
-			urlAuthority reference ++
-			workTree r
+import Git.Types
 
 {- User-visible description of a git repo. -}
 repoDescribe :: Repo -> String
@@ -470,89 +371,10 @@ commit message branch parentrefs repo = do
 		asString a = L.unpack <$> a
 		ps = concatMap (\r -> ["-p", show r]) parentrefs
 
-{- Runs git config and populates a repo with its config. -}
-configRead :: Repo -> IO Repo
-configRead repo@(Repo { location = Dir d }) = do
-	{- Cannot use pipeRead because it relies on the config having
-	   been already read. Instead, chdir to the repo. -}
-	cwd <- getCurrentDirectory
-	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
-		pOpen ReadFromPipe "git" ["config", "--list"] $ hConfigRead repo
-configRead r = assertLocal r $ error "internal"
-
-{- Reads git config from a handle and populates a repo with it. -}
-hConfigRead :: Repo -> Handle -> IO Repo
-hConfigRead repo h = do
-	val <- hGetContentsStrict h
-	configStore val repo
-
-{- Stores a git config into a repo, returning the new version of the repo.
- - The git config may be multiple lines, or a single line. Config settings
- - can be updated inrementally. -}
-configStore :: String -> Repo -> IO Repo
-configStore s repo = do
-	let repo' = repo { config = configParse s `M.union` config repo }
-	rs <- configRemotes repo'
-	return $ repo' { remotes = rs }
-
-{- Parses git config --list output into a config map. -}
-configParse :: String -> M.Map String String
-configParse s = M.fromList $ map pair $ lines s
-	where
-		pair = separate (== '=')
-
-{- Calculates a list of a repo's configured remotes, by parsing its config. -}
-configRemotes :: Repo -> IO [Repo]
-configRemotes repo = mapM construct remotepairs
-	where
-		filterconfig f = filter f $ M.toList $ config repo
-		filterkeys f = filterconfig (\(k,_) -> f k)
-		remotepairs = filterkeys isremote
-		isremote k = startswith "remote." k && endswith ".url" k
-		construct (k,v) = repoRemoteNameFromKey k <$> genRemote v repo
-
-{- Generates one of a repo's remotes using a given location (ie, an url). -}
-genRemote :: String -> Repo -> IO Repo
-genRemote s repo = gen $ calcloc s
-	where
-		filterconfig f = filter f $ M.toList $ config repo
-		gen v	
-			| scpstyle v = repoFromUrl $ scptourl v
-			| isURI v = repoFromUrl v
-			| otherwise = repoFromRemotePath v repo
-		-- insteadof config can rewrite remote location
-		calcloc l
-			| null insteadofs = l
-			| otherwise = replacement ++ drop (length bestvalue) l
-			where
-				replacement = drop (length prefix) $
-					take (length bestkey - length suffix) bestkey
-				(bestkey, bestvalue) = maximumBy longestvalue insteadofs
-				longestvalue (_, a) (_, b) = compare b a
-				insteadofs = filterconfig $ \(k, v) -> 
-					startswith prefix k &&
-					endswith suffix k &&
-					startswith v l
-				(prefix, suffix) = ("url." , ".insteadof")
-		-- git remotes can be written scp style -- [user@]host:dir
-		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
-		scptourl v = "ssh://" ++ host ++ slash dir
-			where
-				(host, dir) = separate (== ':') v
-				slash d	| d == "" = "/~/" ++ d
-					| "/" `isPrefixOf` d = d
-					| "~" `isPrefixOf` d = '/':d
-					| otherwise = "/~/" ++ d
-
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 
-{- Returns a single git config setting, or a default value if not set. -}
-configGet :: String -> String -> Repo -> String
-configGet key defaultValue repo = 
-	M.findWithDefault defaultValue key (config repo)
-
 {- Access to raw config Map -}
 configMap :: Repo -> M.Map String String
 configMap = config
@@ -658,71 +480,3 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
 prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)
-
-{- Constructs a Repo from the path specified in the git remotes of
- - another Repo. -}
-repoFromRemotePath :: FilePath -> Repo -> IO Repo
-repoFromRemotePath dir repo = do
-	dir' <- expandTilde dir
-	repoFromAbsPath $ workTree repo  dir'
-
-{- Git remotes can have a directory that is specified relative
- - to the user's home directory, or that contains tilde expansions.
- - This converts such a directory to an absolute path.
- - Note that it has to run on the system where the remote is.
- -}
-repoAbsPath :: FilePath -> IO FilePath
-repoAbsPath d = do
-	d' <- expandTilde d
-	h <- myHomeDir
-	return $ h  d'
-
-expandTilde :: FilePath -> IO FilePath
-expandTilde = expandt True
-	where
-		expandt _ [] = return ""
-		expandt _ ('/':cs) = do
-			v <- expandt True cs
-			return ('/':v)
-		expandt True ('~':'/':cs) = do
-			h <- myHomeDir
-			return $ h  cs
-		expandt True ('~':cs) = do
-			let (name, rest) = findname "" cs
-			u <- getUserEntryForName name
-			return $ homeDirectory u  rest
-		expandt _ (c:cs) = do
-			v <- expandt False cs
-			return (c:v)
-		findname n [] = (n, "")
-		findname n (c:cs)
-			| c == '/' = (n, cs)
-			| otherwise = findname (n++[c]) cs
-
-{- Finds the current git repository, which may be in a parent directory. -}
-repoFromCwd :: IO Repo
-repoFromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
-	where
-		makerepo = return . newFrom . Dir
-		norepo = error "Not in a git repository."
-
-seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
-seekUp want dir = do
-	ok <- want dir
-	if ok
-		then return $ Just dir
-		else case parentDir dir of
-			"" -> return Nothing
-			d -> seekUp want d
-
-isRepoTop :: FilePath -> IO Bool
-isRepoTop dir = do
-	r <- isRepo
-	b <- isBareRepo
-	return (r || b)
-	where
-		isRepo = gitSignature ".git" ".git/config"
-		isBareRepo = gitSignature "objects" "config"
-		gitSignature subdir file = liftM2 (&&)
-			(doesDirectoryExist (dir ++ "/" ++ subdir))
-			(doesFileExist (dir ++ "/" ++ file))
diff --git a/Git/Config.hs b/Git/Config.hs
new file mode 100644
index 0000000000..5f0e3fdc21
--- /dev/null
+++ b/Git/Config.hs
@@ -0,0 +1,58 @@
+{- git repository configuration handling
+ -
+ - Copyright 2010,2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Config (
+	get,
+	read,
+	hRead,
+	store
+) where
+
+import Prelude hiding (read)
+import System.Posix.Directory
+import Control.Exception (bracket_)
+import qualified Data.Map as M
+
+import Common
+import Git
+import Git.Types
+import qualified Git.Construct
+
+{- Returns a single git config setting, or a default value if not set. -}
+get :: String -> String -> Repo -> String
+get key defaultValue repo = M.findWithDefault defaultValue key (config repo)
+
+{- Runs git config and populates a repo with its config. -}
+read :: Repo -> IO Repo
+read repo@(Repo { location = Dir d }) = do
+	{- Cannot use pipeRead because it relies on the config having
+	   been already read. Instead, chdir to the repo. -}
+	cwd <- getCurrentDirectory
+	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
+		pOpen ReadFromPipe "git" ["config", "--list"] $ hRead repo
+read r = assertLocal r $ error "internal"
+
+{- Reads git config from a handle and populates a repo with it. -}
+hRead :: Repo -> Handle -> IO Repo
+hRead repo h = do
+	val <- hGetContentsStrict h
+	store val repo
+
+{- Stores a git config into a repo, returning the new version of the repo.
+ - The git config may be multiple lines, or a single line. Config settings
+ - can be updated inrementally. -}
+store :: String -> Repo -> IO Repo
+store s repo = do
+	let repo' = repo { config = parse s `M.union` config repo }
+	rs <- Git.Construct.fromRemotes repo'
+	return $ repo' { remotes = rs }
+
+{- Parses git config --list output into a config map. -}
+parse :: String -> M.Map String String
+parse s = M.fromList $ map pair $ lines s
+	where
+		pair = separate (== '=')
diff --git a/Git/Construct.hs b/Git/Construct.hs
new file mode 100644
index 0000000000..9149ab9ec0
--- /dev/null
+++ b/Git/Construct.hs
@@ -0,0 +1,198 @@
+{- Construction of Git Repo objects
+ -
+ - Copyright 2010,2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Construct (
+	fromCwd,
+	fromAbsPath,
+	fromUrl,
+	fromUnknown,
+	localToUrl,
+	fromRemotes,
+	fromRemoteLocation,
+	repoAbsPath,
+) where
+
+import System.Posix.User
+import qualified Data.Map as M hiding (map, split)
+import Network.URI
+
+import Common
+import Git.Types
+import Git
+
+{- Finds the current git repository, which may be in a parent directory. -}
+fromCwd :: IO Repo
+fromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
+	where
+		makerepo = return . newFrom . Dir
+		norepo = error "Not in a git repository."
+
+{- Local Repo constructor, requires an absolute path to the repo be
+ - specified. -}
+fromAbsPath :: FilePath -> IO Repo
+fromAbsPath dir
+	| "/" `isPrefixOf` dir = do
+ 		-- Git always looks for "dir.git" in preference to
+		-- to "dir", even if dir ends in a "/".
+		let canondir = dropTrailingPathSeparator dir
+		let dir' = canondir ++ ".git"
+		e <- doesDirectoryExist dir'
+		if e
+			then ret dir'
+			else if "/.git" `isSuffixOf` canondir
+				then do
+					-- When dir == "foo/.git", git looks
+					-- for "foo/.git/.git", and failing
+					-- that, uses "foo" as the repository.
+					e' <- doesDirectoryExist $ dir  ".git"
+					if e'
+						then ret dir
+						else ret $ takeDirectory canondir
+				else ret dir
+	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
+	where
+		ret = return . newFrom . Dir
+
+{- Remote Repo constructor. Throws exception on invalid url. -}
+fromUrl :: String -> IO Repo
+fromUrl url
+	| startswith "file://" url = fromAbsPath $ uriPath u
+	| otherwise = return $ newFrom $ Url u
+		where
+			u = fromMaybe bad $ parseURI url
+			bad = error $ "bad url " ++ url
+
+{- Creates a repo that has an unknown location. -}
+fromUnknown :: Repo
+fromUnknown = newFrom Unknown
+
+{- Converts a local Repo into a remote repo, using the reference repo
+ - which is assumed to be on the same host. -}
+localToUrl :: Repo -> Repo -> Repo
+localToUrl reference r
+	| not $ repoIsUrl reference = error "internal error; reference repo not url"
+	| repoIsUrl r = r
+	| otherwise = r { location = Url $ fromJust $ parseURI absurl }
+	where
+		absurl =
+			urlScheme reference ++ "//" ++
+			urlAuthority reference ++
+			workTree r
+
+{- Calculates a list of a repo's configured remotes, by parsing its config. -}
+fromRemotes :: Repo -> IO [Repo]
+fromRemotes repo = mapM construct remotepairs
+	where
+		filterconfig f = filter f $ M.toList $ config repo
+		filterkeys f = filterconfig (\(k,_) -> f k)
+		remotepairs = filterkeys isremote
+		isremote k = startswith "remote." k && endswith ".url" k
+		construct (k,v) = repoRemoteNameFromKey k <$> fromRemoteLocation v repo
+
+{- Constructs a new Repo for one of a Repo's remotes using a given
+ - location (ie, an url). -}
+fromRemoteLocation :: String -> Repo -> IO Repo
+fromRemoteLocation s repo = gen $ calcloc s
+	where
+		filterconfig f = filter f $ M.toList $ config repo
+		gen v	
+			| scpstyle v = fromUrl $ scptourl v
+			| isURI v = fromUrl v
+			| otherwise = fromRemotePath v repo
+		-- insteadof config can rewrite remote location
+		calcloc l
+			| null insteadofs = l
+			| otherwise = replacement ++ drop (length bestvalue) l
+			where
+				replacement = drop (length prefix) $
+					take (length bestkey - length suffix) bestkey
+				(bestkey, bestvalue) = maximumBy longestvalue insteadofs
+				longestvalue (_, a) (_, b) = compare b a
+				insteadofs = filterconfig $ \(k, v) -> 
+					startswith prefix k &&
+					endswith suffix k &&
+					startswith v l
+				(prefix, suffix) = ("url." , ".insteadof")
+		-- git remotes can be written scp style -- [user@]host:dir
+		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
+		scptourl v = "ssh://" ++ host ++ slash dir
+			where
+				(host, dir) = separate (== ':') v
+				slash d	| d == "" = "/~/" ++ d
+					| "/" `isPrefixOf` d = d
+					| "~" `isPrefixOf` d = '/':d
+					| otherwise = "/~/" ++ d
+
+{- Constructs a Repo from the path specified in the git remotes of
+ - another Repo. -}
+fromRemotePath :: FilePath -> Repo -> IO Repo
+fromRemotePath dir repo = do
+	dir' <- expandTilde dir
+	fromAbsPath $ workTree repo  dir'
+
+{- Git remotes can have a directory that is specified relative
+ - to the user's home directory, or that contains tilde expansions.
+ - This converts such a directory to an absolute path.
+ - Note that it has to run on the system where the remote is.
+ -}
+repoAbsPath :: FilePath -> IO FilePath
+repoAbsPath d = do
+	d' <- expandTilde d
+	h <- myHomeDir
+	return $ h  d'
+
+expandTilde :: FilePath -> IO FilePath
+expandTilde = expandt True
+	where
+		expandt _ [] = return ""
+		expandt _ ('/':cs) = do
+			v <- expandt True cs
+			return ('/':v)
+		expandt True ('~':'/':cs) = do
+			h <- myHomeDir
+			return $ h  cs
+		expandt True ('~':cs) = do
+			let (name, rest) = findname "" cs
+			u <- getUserEntryForName name
+			return $ homeDirectory u  rest
+		expandt _ (c:cs) = do
+			v <- expandt False cs
+			return (c:v)
+		findname n [] = (n, "")
+		findname n (c:cs)
+			| c == '/' = (n, cs)
+			| otherwise = findname (n++[c]) cs
+
+seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
+seekUp want dir = do
+	ok <- want dir
+	if ok
+		then return $ Just dir
+		else case parentDir dir of
+			"" -> return Nothing
+			d -> seekUp want d
+
+isRepoTop :: FilePath -> IO Bool
+isRepoTop dir = do
+	r <- isRepo
+	b <- isBareRepo
+	return (r || b)
+	where
+		isRepo = gitSignature ".git" ".git/config"
+		isBareRepo = gitSignature "objects" "config"
+		gitSignature subdir file = liftM2 (&&)
+			(doesDirectoryExist (dir ++ "/" ++ subdir))
+			(doesFileExist (dir ++ "/" ++ file))
+
+newFrom :: RepoLocation -> Repo
+newFrom l = 
+	Repo {
+		location = l,
+		config = M.empty,
+		remotes = [],
+		remoteName = Nothing
+	}
diff --git a/Git/Types.hs b/Git/Types.hs
new file mode 100644
index 0000000000..250da5f5e5
--- /dev/null
+++ b/Git/Types.hs
@@ -0,0 +1,36 @@
+{- git data types
+ -
+ - Copyright 2010,2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Types where
+
+import Network.URI
+import qualified Data.Map as M
+
+{- There are two types of repositories; those on local disk and those
+ - accessed via an URL. -}
+data RepoLocation = Dir FilePath | Url URI | Unknown
+	deriving (Show, Eq)
+
+data Repo = Repo {
+	location :: RepoLocation,
+	config :: M.Map String String,
+	remotes :: [Repo],
+	-- remoteName holds the name used for this repo in remotes
+	remoteName :: Maybe String 
+} deriving (Show, Eq)
+
+{- A git ref. Can be a sha1, or a branch or tag name. -}
+newtype Ref = Ref String
+	deriving (Eq)
+
+instance Show Ref where
+	show (Ref v) = v
+
+{- Aliases for Ref. -}
+type Branch = Ref
+type Sha = Ref
+type Tag = Ref
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 7871638e44..a5b9609b6d 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -10,7 +10,8 @@ module GitAnnex where
 import System.Console.GetOpt
 
 import Common.Annex
-import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import CmdLine
 import Command
 import Types.TrustLevel
@@ -125,11 +126,11 @@ options = commonOptions ++
 		setprint0 v = Annex.changeState $ \s -> s { Annex.print0 = v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
-			newg <- inRepo $ Git.configStore v
+			newg <- inRepo $ Git.Config.store v
 			Annex.changeState $ \s -> s { Annex.repo = newg }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch args cmds options header Git.repoFromCwd
+run args = dispatch args cmds options header Git.Construct.fromCwd
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index e705bbb344..4d63d88e1d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -15,6 +15,8 @@ import System.Process
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import Config
 import Annex.Ssh
 import Remote.Helper.Special
@@ -163,8 +165,8 @@ storeBupUUID u buprepo = do
 				[Params $ "config annex.uuid " ++ v]
 					>>! error "ssh failed"
 		else liftIO $ do
-			r' <- Git.configRead r
-			let olduuid = Git.configGet "annex.uuid" "" r'
+			r' <- Git.Config.read r
+			let olduuid = Git.Config.get "annex.uuid" "" r'
 			when (olduuid == "") $
 				Git.run "config"
 					[Param "annex.uuid", Param v] r'
@@ -192,9 +194,9 @@ getBupUUID :: Git.Repo -> UUID -> Annex (UUID, Git.Repo)
 getBupUUID r u
 	| Git.repoIsUrl r = return (u, r)
 	| otherwise = liftIO $ do
-		ret <- try $ Git.configRead r
+		ret <- try $ Git.Config.read r
 		case ret of
-			Right r' -> return (toUUID $ Git.configGet "annex.uuid" "" r', r')
+			Right r' -> return (toUUID $ Git.Config.get "annex.uuid" "" r', r')
 			Left _ -> return (NoUUID, r)
 
 {- Converts a bup remote path spec into a Git.Repo. There are some
@@ -203,13 +205,13 @@ bup2GitRemote :: BupRepo -> IO Git.Repo
 bup2GitRemote "" = do
 	-- bup -r "" operates on ~/.bup
 	h <- myHomeDir
-	Git.repoFromAbsPath $ h  ".bup"
+	Git.Construct.fromAbsPath $ h  ".bup"
 bup2GitRemote r
 	| bupLocal r = 
 		if head r == '/'
-			then Git.repoFromAbsPath r
+			then Git.Construct.fromAbsPath r
 			else error "please specify an absolute path"
-	| otherwise = Git.repoFromUrl $ "ssh://" ++ host ++ slash dir
+	| otherwise = Git.Construct.fromUrl $ "ssh://" ++ host ++ slash dir
 		where
 			bits = split ":" r
 			host = head bits
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 2f9288e1b4..9d80f4c1cb 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -16,6 +16,8 @@ import Utility.RsyncFile
 import Annex.Ssh
 import Types.Remote
 import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import qualified Annex
 import Annex.UUID
 import qualified Annex.Content
@@ -44,7 +46,7 @@ list = do
 			case M.lookup (annexurl n) c of
 				Nothing -> return r
 				Just url -> Git.repoRemoteNameSet n <$>
-					inRepo (Git.genRemote url)
+					inRepo (Git.Construct.fromRemoteLocation url)
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r u _ = do
@@ -100,7 +102,7 @@ tryGitConfigRead r
 
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
-				Git.hConfigRead r
+				Git.Config.hRead r
 
 		geturlconfig = do
 			s <- Url.get (Git.repoLocation r ++ "/config")
@@ -108,7 +110,7 @@ tryGitConfigRead r
 				hPutStr h s
 				hClose h
 				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
-					Git.hConfigRead r
+					Git.Config.hRead r
 
 		store a = do
 			r' <- a
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 77478eb1d5..72c4842d88 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -12,6 +12,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Construct
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
@@ -23,7 +24,7 @@ findSpecialRemotes s = do
 	return $ map construct $ remotepairs m
 	where
 		remotepairs = M.toList . M.filterWithKey match
-		construct (k,_) = Git.repoRemoteNameFromKey k Git.repoFromUnknown
+		construct (k,_) = Git.repoRemoteNameFromKey k Git.Construct.fromUnknown
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
diff --git a/Remote/Web.hs b/Remote/Web.hs
index d5acd7d862..c4e9f8bd69 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -10,6 +10,7 @@ module Remote.Web (remote) where
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Construct
 import Config
 import Logs.Web
 import qualified Utility.Url as Url
@@ -26,7 +27,7 @@ remote = RemoteType {
 -- (If the web should cease to exist, remove this module and redistribute
 -- a new release to the survivors by carrier pigeon.)
 list :: Annex [Git.Repo]
-list = return [Git.repoRemoteNameSet "web" Git.repoFromUnknown]
+list = return [Git.repoRemoteNameSet "web" Git.Construct.fromUnknown]
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 9a9d2f0925..872dabc58b 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -9,7 +9,7 @@ import System.Environment
 import System.Console.GetOpt
 
 import Common.Annex
-import qualified Git
+import qualified Git.Construct
 import CmdLine
 import Command
 import Annex.UUID
@@ -80,7 +80,7 @@ builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
 	checkNotReadOnly cmd
 	dispatch (cmd : filterparams params) cmds options header $
-		Git.repoAbsPath dir >>= Git.repoFromAbsPath
+		Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath
 
 external :: [String] -> IO ()
 external params = do
diff --git a/git-annex.cabal b/git-annex.cabal
index 35b3e690f9..ae6a129b3b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111211
+Version: 3.20111212
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index edd9330c80..eeb6944018 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -9,6 +9,8 @@ import System.Environment
 
 import Common
 import qualified Git.UnionMerge
+import qualified Git.Config
+import qualified Git.Construct
 import qualified Git
 
 header :: String
@@ -38,7 +40,7 @@ parseArgs = do
 main :: IO ()
 main = do
 	[aref, bref, newref] <- map Git.Ref <$> parseArgs
-	g <- Git.configRead =<< Git.repoFromCwd
+	g <- Git.Config.read =<< Git.Construct.fromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
 	Git.UnionMerge.merge aref bref g
diff --git a/test.hs b/test.hs
index 91c11873da..14994865f8 100644
--- a/test.hs
+++ b/test.hs
@@ -25,6 +25,8 @@ import qualified Annex
 import qualified Annex.UUID
 import qualified Backend
 import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import qualified Locations
 import qualified Types.Backend
 import qualified Types
@@ -496,8 +498,8 @@ git_annex command params = do
 -- are not run; this should only be used for actions that query state.
 annexeval :: Types.Annex a -> IO a
 annexeval a = do
-	g <- Git.repoFromCwd
-	g' <- Git.configRead g
+	g <- Git.Construct.fromCwd
+	g' <- Git.Config.read g
 	s <- Annex.new g'
 	Annex.eval s a
 

From 25b2cc4148e4cc8f7435cdbcf4b124cc317c1305 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 13 Dec 2011 15:08:44 -0400
Subject: [PATCH 2687/2835] move commit to Git.Branch

---
 Annex/Branch.hs |  4 ++--
 Git.hs          | 18 ------------------
 Git/Branch.hs   | 17 +++++++++++++++++
 3 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index f9fa6cbb31..a2ecd50a70 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -71,7 +71,7 @@ getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
 			fromMaybe (error $ "failed to create " ++ show name)
 				<$> branchsha
 		go False = withIndex' True $ do
-			inRepo $ Git.commit "branch created" fullname []
+			inRepo $ Git.Branch.commit "branch created" fullname []
 		use sha = do
 			setIndexSha sha
 			return sha
@@ -190,7 +190,7 @@ commit message = whenM journalDirty $ lockJournal $ do
 commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex ()
 commitBranch branchref message parents = do
 	updateIndex branchref
-	committedref <- inRepo $ Git.commit message fullname parents
+	committedref <- inRepo $ Git.Branch.commit message fullname parents
 	setIndexSha committedref
 	parentrefs <- commitparents <$> catObject committedref
 	when (racedetected branchref parentrefs) $
diff --git a/Git.hs b/Git.hs
index cb7cc19c25..36b83c65b5 100644
--- a/Git.hs
+++ b/Git.hs
@@ -52,7 +52,6 @@ module Git (
 	useIndex,
 	getSha,
 	shaSize,
-	commit,
 	assertLocal,
 
 	prop_idempotent_deencode
@@ -354,23 +353,6 @@ getSha subcommand a = do
 shaSize :: Int
 shaSize = 40
 
-{- Commits the index into the specified branch (or other ref), 
- - with the specified parent refs, and returns the committed sha -}
-commit :: String -> Branch -> [Ref] -> Repo -> IO Sha
-commit message branch parentrefs repo = do
-	tree <- getSha "write-tree" $ asString $
-		pipeRead [Param "write-tree"] repo
-	sha <- getSha "commit-tree" $ asString $
-		ignorehandle $ pipeWriteRead
-			(map Param $ ["commit-tree", show tree] ++ ps)
-			(L.pack message) repo
-	run "update-ref" [Param $ show branch, Param $ show sha] repo
-	return sha
-	where
-		ignorehandle a = snd <$> a
-		asString a = L.unpack <$> a
-		ps = concatMap (\r -> ["-p", show r]) parentrefs
-
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
diff --git a/Git/Branch.hs b/Git/Branch.hs
index e69e96f288..8b0d1e5afc 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -58,3 +58,20 @@ fastForward branch (first:rest) repo = do
 				(True, False) -> findbest r rs -- better
 				(False, True) -> findbest c rs -- worse
 				(False, False) -> findbest c rs -- same
+
+{- Commits the index into the specified branch (or other ref), 
+ - with the specified parent refs, and returns the committed sha -}
+commit :: String -> Branch -> [Ref] -> Repo -> IO Sha
+commit message branch parentrefs repo = do
+	tree <- getSha "write-tree" $ asString $
+		pipeRead [Param "write-tree"] repo
+	sha <- getSha "commit-tree" $ asString $
+		ignorehandle $ pipeWriteRead
+			(map Param $ ["commit-tree", show tree] ++ ps)
+			(L.pack message) repo
+	run "update-ref" [Param $ show branch, Param $ show sha] repo
+	return sha
+	where
+		ignorehandle a = snd <$> a
+		asString a = L.unpack <$> a
+		ps = concatMap (\r -> ["-p", show r]) parentrefs

From 9db8ec210f8491de78cd7e83b94c50ead8049e72 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 13 Dec 2011 15:22:43 -0400
Subject: [PATCH 2688/2835] split out two more Git modules

---
 Backend.hs         |   3 +-
 Git.hs             | 113 +--------------------------------------------
 Git/CheckAttr.hs   |  44 ++++++++++++++++++
 Git/Filename.hs    |  84 +++++++++++++++++++++++++++++++++
 Git/LsTree.hs      |   3 +-
 Seek.hs            |   3 +-
 git-union-merge.hs |   3 +-
 test.hs            |   3 +-
 8 files changed, 139 insertions(+), 117 deletions(-)
 create mode 100644 Git/CheckAttr.hs
 create mode 100644 Git/Filename.hs

diff --git a/Backend.hs b/Backend.hs
index c7cb57440b..38f0629f48 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -22,6 +22,7 @@ import System.Posix.Files
 import Common.Annex
 import qualified Git
 import qualified Git.Config
+import qualified Git.CheckAttr
 import qualified Annex
 import Types.Key
 import qualified Types.Backend as B
@@ -103,7 +104,7 @@ chooseBackends :: [FilePath] -> Annex [BackendFile]
 chooseBackends fs = Annex.getState Annex.forcebackend >>= go
 	where
 		go Nothing = do
-			pairs <- inRepo $ Git.checkAttr "annex.backend" fs
+			pairs <- inRepo $ Git.CheckAttr.lookup "annex.backend" fs
 			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
 		go (Just _) = do
 			l <- orderedList
diff --git a/Git.hs b/Git.hs
index 36b83c65b5..0280acedcd 100644
--- a/Git.hs
+++ b/Git.hs
@@ -45,25 +45,16 @@ module Git (
 	repoRemoteName,
 	repoRemoteNameSet,
 	repoRemoteNameFromKey,
-	checkAttr,
-	decodeGitFile,
-	encodeGitFile,
 	reap,
 	useIndex,
 	getSha,
 	shaSize,
 	assertLocal,
-
-	prop_idempotent_deencode
 ) where
 
-import qualified Data.Map as M hiding (map, split)
+import qualified Data.Map as M
 import Network.URI
 import Data.Char
-import Data.Word (Word8)
-import Codec.Binary.UTF8.String (encode)
-import Text.Printf
-import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -360,105 +351,3 @@ configTrue s = map toLower s == "true"
 {- Access to raw config Map -}
 configMap :: Repo -> M.Map String String
 configMap = config
-
-{- Efficiently looks up a gitattributes value for each file in a list. -}
-checkAttr :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
-checkAttr attr files repo = do
-	-- git check-attr needs relative filenames input; it will choke
-	-- on some absolute filenames. This also means it will output
-	-- all relative filenames.
-	cwd <- getCurrentDirectory
-	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
-	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
-        _ <- forkProcess $ do
-		hClose fromh
-                hPutStr toh $ join "\0" relfiles
-                hClose toh
-                exitSuccess
-        hClose toh
-	(map topair . lines) <$> hGetContents fromh
-	where
-		params = gitCommandLine 
-				[ Param "check-attr"
-				, Param attr
-				, Params "-z --stdin"
-				] repo
-		topair l = (file, value)
-			where 
-				file = decodeGitFile $ join sep $ take end bits
-				value = bits !! end
-				end = length bits - 1
-				bits = split sep l
-				sep = ": " ++ attr ++ ": "
-
-{- Some git commands output encoded filenames. Decode that (annoyingly
- - complex) encoding. -}
-decodeGitFile :: String -> FilePath
-decodeGitFile [] = []
-decodeGitFile f@(c:s)
-	-- encoded strings will be inside double quotes
-	| c == '"' = unescape ("", middle)
-	| otherwise = f
-	where
-		e = '\\'
-		middle = init s
-		unescape (b, []) = b
-		-- look for escapes starting with '\'
-		unescape (b, v) = b ++ beginning ++ unescape (decode rest)
-			where
-				pair = span (/= e) v
-				beginning = fst pair
-				rest = snd pair
-		isescape x = x == e
-		-- \NNN is an octal encoded character
-		decode (x:n1:n2:n3:rest)
-			| isescape x && alloctal = (fromoctal, rest)
-				where
-					alloctal = isOctDigit n1 &&
-						isOctDigit n2 &&
-						isOctDigit n3
-					fromoctal = [chr $ readoctal [n1, n2, n3]]
-					readoctal o = read $ "0o" ++ o :: Int
-		-- \C is used for a few special characters
-		decode (x:nc:rest)
-			| isescape x = ([echar nc], rest)
-			where
-				echar 'a' = '\a'
-				echar 'b' = '\b'
-				echar 'f' = '\f'
-				echar 'n' = '\n'
-				echar 'r' = '\r'
-				echar 't' = '\t'
-				echar 'v' = '\v'
-				echar a = a
-		decode n = ("", n)
-
-{- Should not need to use this, except for testing decodeGitFile. -}
-encodeGitFile :: FilePath -> String
-encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""
-	where
-		e c = '\\' : [c]
-		echar '\a' = e 'a'
-		echar '\b' = e 'b'
-		echar '\f' = e 'f'
-		echar '\n' = e 'n'
-		echar '\r' = e 'r'
-		echar '\t' = e 't'
-		echar '\v' = e 'v'
-		echar '\\' = e '\\'
-		echar '"'  = e '"'
-		echar x
-			| ord x < 0x20 = e_num x -- low ascii
-			| ord x >= 256 = e_utf x
-			| ord x > 0x7E = e_num x -- high ascii
-			| otherwise = [x]        -- printable ascii
-			where 
-				showoctal i = '\\' : printf "%03o" i
-				e_num c = showoctal $ ord c
-				-- unicode character is decomposed to
-				-- Word8s and each is shown in octal
-				e_utf c = showoctal =<< (encode [c] :: [Word8])
-
-{- for quickcheck -}
-prop_idempotent_deencode :: String -> Bool
-prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
new file mode 100644
index 0000000000..e9269b1edb
--- /dev/null
+++ b/Git/CheckAttr.hs
@@ -0,0 +1,44 @@
+{- git check-attr interface
+ -
+ - Copyright 2010, 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.CheckAttr where
+
+import System.Exit
+
+import Common
+import Git
+import qualified Git.Filename
+
+{- Efficiently looks up a gitattributes value for each file in a list. -}
+lookup :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
+lookup attr files repo = do
+	-- git check-attr needs relative filenames input; it will choke
+	-- on some absolute filenames. This also means it will output
+	-- all relative filenames.
+	cwd <- getCurrentDirectory
+	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
+	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
+        _ <- forkProcess $ do
+		hClose fromh
+                hPutStr toh $ join "\0" relfiles
+                hClose toh
+                exitSuccess
+        hClose toh
+	(map topair . lines) <$> hGetContents fromh
+	where
+		params = gitCommandLine 
+				[ Param "check-attr"
+				, Param attr
+				, Params "-z --stdin"
+				] repo
+		topair l = (file, value)
+			where 
+				file = Git.Filename.decode $ join sep $ take end bits
+				value = bits !! end
+				end = length bits - 1
+				bits = split sep l
+				sep = ": " ++ attr ++ ": "
diff --git a/Git/Filename.hs b/Git/Filename.hs
new file mode 100644
index 0000000000..69f36d0861
--- /dev/null
+++ b/Git/Filename.hs
@@ -0,0 +1,84 @@
+{- Some git commands output encoded filenames, in a rather annoyingly complex
+ - C-style encoding.
+ -
+ - Copyright 2010, 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Filename where
+
+import qualified Codec.Binary.UTF8.String
+import Data.Char
+import Data.Word (Word8)
+import Text.Printf
+
+decode :: String -> FilePath
+decode [] = []
+decode f@(c:s)
+	-- encoded strings will be inside double quotes
+	| c == '"' = unescape ("", middle)
+	| otherwise = f
+	where
+		e = '\\'
+		middle = init s
+		unescape (b, []) = b
+		-- look for escapes starting with '\'
+		unescape (b, v) = b ++ beginning ++ unescape (handle rest)
+			where
+				pair = span (/= e) v
+				beginning = fst pair
+				rest = snd pair
+		isescape x = x == e
+		-- \NNN is an octal encoded character
+		handle (x:n1:n2:n3:rest)
+			| isescape x && alloctal = (fromoctal, rest)
+				where
+					alloctal = isOctDigit n1 &&
+						isOctDigit n2 &&
+						isOctDigit n3
+					fromoctal = [chr $ readoctal [n1, n2, n3]]
+					readoctal o = read $ "0o" ++ o :: Int
+		-- \C is used for a few special characters
+		handle (x:nc:rest)
+			| isescape x = ([echar nc], rest)
+			where
+				echar 'a' = '\a'
+				echar 'b' = '\b'
+				echar 'f' = '\f'
+				echar 'n' = '\n'
+				echar 'r' = '\r'
+				echar 't' = '\t'
+				echar 'v' = '\v'
+				echar a = a
+		handle n = ("", n)
+
+{- Should not need to use this, except for testing decode. -}
+encode :: FilePath -> String
+encode s = foldl (++) "\"" (map echar s) ++ "\""
+	where
+		e c = '\\' : [c]
+		echar '\a' = e 'a'
+		echar '\b' = e 'b'
+		echar '\f' = e 'f'
+		echar '\n' = e 'n'
+		echar '\r' = e 'r'
+		echar '\t' = e 't'
+		echar '\v' = e 'v'
+		echar '\\' = e '\\'
+		echar '"'  = e '"'
+		echar x
+			| ord x < 0x20 = e_num x -- low ascii
+			| ord x >= 256 = e_utf x
+			| ord x > 0x7E = e_num x -- high ascii
+			| otherwise = [x]        -- printable ascii
+			where 
+				showoctal i = '\\' : printf "%03o" i
+				e_num c = showoctal $ ord c
+				-- unicode character is decomposed to
+				-- Word8s and each is shown in octal
+				e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+
+{- for quickcheck -}
+prop_idempotent_deencode :: String -> Bool
+prop_idempotent_deencode s = s == decode (encode s)
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 8aa16a308b..342a125eb8 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -17,6 +17,7 @@ import System.Posix.Types
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
+import qualified Git.Filename
 import Utility.SafeCommand
 
 data TreeItem = TreeItem
@@ -38,7 +39,7 @@ parseLsTree l = TreeItem
 	{ mode = fst $ head $ readOct $ L.unpack m
 	, typeobj = L.unpack t
 	, sha = L.unpack s
-	, file = decodeGitFile $ L.unpack f
+	, file = Git.Filename.decode $ L.unpack f
 	}
 	where
 		-- l =  SP  SP  TAB 
diff --git a/Seek.hs b/Seek.hs
index 1430ebabd1..28c6ffc00c 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -18,6 +18,7 @@ import Backend
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
+import qualified Git.CheckAttr
 import qualified Limit
 
 seekHelper :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> [FilePath] -> Annex [FilePath]
@@ -31,7 +32,7 @@ withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
 withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
 	files <- seekHelper LsFiles.inRepo params
-	prepFilteredGen a fst $ inRepo $ Git.checkAttr attr files
+	prepFilteredGen a fst $ inRepo $ Git.CheckAttr.lookup attr files
 
 withNumCopies :: (Maybe Int -> FilePath -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
diff --git a/git-union-merge.hs b/git-union-merge.hs
index eeb6944018..f67414bdd3 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -11,6 +11,7 @@ import Common
 import qualified Git.UnionMerge
 import qualified Git.Config
 import qualified Git.Construct
+import qualified Git.Branch
 import qualified Git
 
 header :: String
@@ -44,5 +45,5 @@ main = do
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
 	Git.UnionMerge.merge aref bref g
-	_ <- Git.commit "union merge" newref [aref, bref] g
+	_ <- Git.Branch.commit "union merge" newref [aref, bref] g
 	cleanup g
diff --git a/test.hs b/test.hs
index 14994865f8..1ce9d103d1 100644
--- a/test.hs
+++ b/test.hs
@@ -27,6 +27,7 @@ import qualified Backend
 import qualified Git
 import qualified Git.Config
 import qualified Git.Construct
+import qualified Git.Filename
 import qualified Locations
 import qualified Types.Backend
 import qualified Types
@@ -69,7 +70,7 @@ propigate (Counts { errors = e , failures = f }, _)
 
 quickcheck :: Test
 quickcheck = TestLabel "quickcheck" $ TestList
-	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
+	[ qctest "prop_idempotent_deencode" Git.Filename.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape

From 3c851368750a5faf19ff3437854a60ae4c679c69 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 13 Dec 2011 15:27:38 -0400
Subject: [PATCH 2689/2835] remove dead code

---
 Git.hs | 31 -------------------------------
 1 file changed, 31 deletions(-)

diff --git a/Git.hs b/Git.hs
index 0280acedcd..b4cbd91aa0 100644
--- a/Git.hs
+++ b/Git.hs
@@ -21,7 +21,6 @@ module Git (
 	repoDescribe,
 	repoLocation,
 	workTree,
-	workTreeFile,
 	gitDir,
 	urlPath,
 	urlHost,
@@ -166,36 +165,6 @@ workTree r@(Repo { location = Url _ }) = urlPath r
 workTree (Repo { location = Dir d }) = d
 workTree Repo { location = Unknown } = undefined
 
-{- Given a relative or absolute filename inside a git repository's
- - workTree, calculates the name to use to refer to that file to git.
- -
- - This is complicated because the best choice can vary depending on
- - whether the cwd is in a subdirectory of the git repository, or not.
- -
- - For example, when adding a file "/tmp/repo/foo", it's best to refer
- - to it as "foo" if the cwd is outside the repository entirely
- - (this avoids a gotcha with using the full path name when /tmp/repo
- - is itself a symlink). But, if the cwd is "/tmp/repo/subdir",
- - it's best to refer to "../foo".
- -}
-workTreeFile :: FilePath -> Repo -> IO FilePath
-workTreeFile file repo@(Repo { location = Dir d }) = do
-	cwd <- getCurrentDirectory
-	let file' = absfile cwd
-	unless (inrepo file') $
-		error $ file ++ " is not located inside git repository " ++ absrepo
-	if inrepo $ addTrailingPathSeparator cwd
-		then return $ relPathDirToFile cwd file'
-		else return $ drop (length absrepo) file'
-	where
-		-- normalize both repo and file, so that repo
-		-- will be substring of file
-		absrepo = maybe bad addTrailingPathSeparator $ absNormPath "/" d
-		absfile c = fromMaybe file $ secureAbsNormPath c file
-		inrepo f = absrepo `isPrefixOf` f
-		bad = error $ "bad repo" ++ repoDescribe repo
-workTreeFile _ repo = assertLocal repo $ error "internal"
-
 {- Path of an URL repo. -}
 urlPath :: Repo -> String
 urlPath Repo { location = Url u } = uriPath u

From 490810ff9fba8e4711fef8025c655b2fa66b8801 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 13 Dec 2011 20:53:23 +0000
Subject: [PATCH 2690/2835] Added a comment

---
 ...mment_6_3660d45c5656f68924acbd23790024ee._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment b/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment
new file mode 100644
index 0000000000..fc66fbb8e1
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-12-13T20:53:23Z"
+ content="""
+It would be clearer to call \"git-annex-master\" \"synced/master\" (or really \"synced/$current_branch\"). That does highlight that this method of syncing is not particularly specific to git-annex.
+
+I think this would be annoying to those who do use a central bare repository, because of the unnecessary pushing and pulling to other repos, which could be expensive to do, especially if you have a lot of interconnected repos. So having a way to enable/disable it seems best.
+
+Maybe you should work up a patch to Command/Sync.hs, since I know you know haskell :)
+"""]]

From c92d407efdcff347376a4186abe84fa8f2240996 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Tue, 13 Dec 2011 22:01:13 +0000
Subject: [PATCH 2691/2835] add a link to git-union-merge

---
 doc/internals.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 68cc7c3cd9..b2fd1e5545 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -77,7 +77,7 @@ Example:
 	1287290776.765152s 1 e605dca6-446a-11e0-8b2a-002170d25c55
 	1287290767.478634s 0 26339d22-446b-11e0-9101-002170d25c55
 
-These files are designed to be auto-merged using git's union merge driver.
+These files are designed to be auto-merged using git's [[union merge driver|git-union-merge]].
 The timestamps allow the most recent information to be identified.
 
 ## `remote/web/aaa/bbb/*.log`

From 020c845058b926c608148759b2782e2539841fc3 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Tue, 13 Dec 2011 23:26:58 +0000
Subject: [PATCH 2692/2835]

---
 .../syncing_non-git_trees_with_git-annex.mdwn | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/forum/syncing_non-git_trees_with_git-annex.mdwn

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex.mdwn b/doc/forum/syncing_non-git_trees_with_git-annex.mdwn
new file mode 100644
index 0000000000..9973782610
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex.mdwn
@@ -0,0 +1,46 @@
+I have a bunch of directory trees with large data files scattered over various computers and disk drives - they contain photos, videos, music, and so on.  In many cases I initially copied one of these trees from one machine to another just as a cheap and dirty backup, and then made small modifications to both trees in ways I no longer remember. For example, I returned from a trip with a bunch of new photos, and then might have rotated some of them 90 degrees on one machine, and edited or renamed them on another.
+
+What I want to do now is use git-annex as a way of initially synchronising the trees, and then fully managing them on an ongoing basis.  Note that the trees are *not* yet git repositories.  In order to be able to detect straight-forward file renames, I believe that [[the SHA1 backend|tips/using_the_SHA1_backend]] probably makes the most sense.
+
+I've been playing around and arrived at the following setup procedure.  For the sake of discussion, I assume that we have two trees `a` and `b` which live in the same directory referred to by `$td`, and that all large files end with the `.avi` suffix.
+
+     # Setup git in 'a'.
+     cd $td/a
+     git init
+
+     # Setup git-annex in 'a'.
+     echo '* annex.backend=SHA1' > .gitattributes
+     git add .gitattributes
+     git commit -m'use SHA1 backend'
+     git annex init
+
+     # Annex all large files.
+     find -name \*.avi | xargs git annex add
+     git add .
+     git commit -m'Initial import'
+
+     # Setup git in 'b'.
+     cd $td/b
+     git clone -n $td/a new
+     mv new/.git .
+     rmdir new
+     git reset # reset git index to b's wd - hangover from cloning from 'a'
+
+     # Setup git-annex in 'b'.
+     # This merges a's (origin's) git-annex branch into the local git-annex branch.
+     git annex init
+
+     # Annex all large files - because we're using SHA1 backend, some
+     # should hash to the same keys as in 'a'.
+     find -name \*.avi | xargs git annex add
+     git add .
+     git commit -m'Changes in b tree'
+
+     git remote add a $td/a
+
+     # Now pull changes in 'b' back to 'a'.
+     cd $td/a
+     git remote add b $td/b
+     git pull b master
+
+This seems to work, but have I missed anything?

From 25a5f6664ed98a20bc59bc0ca3107dd8bf7bba0b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 14 Dec 2011 17:31:31 +0000
Subject: [PATCH 2693/2835] Added a comment

---
 ..._7f9593bdfd95e4a8814e6cc5c44619e6._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
new file mode 100644
index 0000000000..c1b9cfa370
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-14T17:31:31Z"
+ content="""
+This is an entirely reasonable way to go about it.
+
+However, doing it this way causes files in B to always \"win\" -- If the same filename is in both repositories, with differing content, the version added in B will superscede the version from A. If A has a file that is not in B, a git commit -a in B will commit a deletion of that file.
+
+I might do it your way and look at the changes in B before (or even after) committing them to see if files from A were deleted or changed.
+
+Or, I might just instead keep B in a separate subdirectory in the repository, set up like so:
+
+
+mv b old_b
+git clone a b
+cd b
+mv ../old_b .
+git annex add old_b --exclude --not '*.avi'
+

+
+Or, a third way would be to commit A to a branch like branchA and B to a separate branchB, and not merge the branches at all.
+"""]]

From 2b24e16a633575703a43e1fb991f34b290a1d7e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Dec 2011 13:32:13 -0400
Subject: [PATCH 2694/2835] typo

---
 .../comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
index c1b9cfa370..bdec508792 100644
--- a/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
@@ -17,7 +17,7 @@ mv b old_b
 git clone a b
 cd b
 mv ../old_b .
-git annex add old_b --exclude --not '*.avi'
+git annex add old_b --not --exclude '*.avi'
 

 
 Or, a third way would be to commit A to a branch like branchA and B to a separate branchB, and not merge the branches at all.

From 02f1bd2bf47d3ff49a222e9428ec27708ef55f64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Dec 2011 15:30:14 -0400
Subject: [PATCH 2695/2835] split more stuff out of Git.hs

---
 Annex/Branch.hs          |   3 +-
 Annex/Ssh.hs             |   5 +-
 Backend.hs               |   1 -
 Command/Map.hs           |  13 ++--
 Config.hs                |   2 +-
 Git.hs                   | 148 +++------------------------------------
 Git/Branch.hs            |   5 +-
 Git/CatFile.hs           |   5 +-
 Git/Construct.hs         |  27 +++++--
 Git/HashObject.hs        |   4 +-
 Git/Index.hs             |  24 +++++++
 Git/Ref.hs               |   6 +-
 Git/Sha.hs               |  27 +++++++
 Git/UnionMerge.hs        |   5 +-
 Git/Url.hs               |  70 ++++++++++++++++++
 Remote/Git.hs            |  17 ++---
 Remote/Helper/Special.hs |   6 +-
 Remote/Web.hs            |   4 +-
 git-union-merge.hs       |   3 +-
 test.hs                  |   1 -
 20 files changed, 197 insertions(+), 179 deletions(-)
 create mode 100644 Git/Index.hs
 create mode 100644 Git/Sha.hs
 create mode 100644 Git/Url.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a2ecd50a70..a22a4adcfe 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -28,6 +28,7 @@ import qualified Git.Ref
 import qualified Git.Branch
 import qualified Git.UnionMerge
 import qualified Git.HashObject
+import qualified Git.Index
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -249,7 +250,7 @@ withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
 	f <- fromRepo gitAnnexIndex
-	bracketIO (Git.useIndex f) id $ do
+	bracketIO (Git.Index.override f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
 			liftIO $ createDirectoryIfMissing True $ takeDirectory f
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 6893f94ef4..fe83aad001 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -10,6 +10,7 @@ module Annex.Ssh where
 import Control.Monad.State (liftIO)
 
 import qualified Git
+import qualified Git.Url
 import Utility.SafeCommand
 import Types
 import Config
@@ -22,10 +23,10 @@ sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam]
 sshToRepo repo sshcmd = do
 	s <- getConfig repo "ssh-options" ""
 	let sshoptions = map Param (words s)
-	let sshport = case Git.urlPort repo of
+	let sshport = case Git.Url.port repo of
 		Nothing -> []
 		Just p -> [Param "-p", Param (show p)]
-	let sshhost = Param $ Git.urlHostUser repo
+	let sshhost = Param $ Git.Url.hostuser repo
 	return $ sshoptions ++ sshport ++ [sshhost] ++ sshcmd
 
 {- Generates parameters to run a git-annex-shell command on a remote
diff --git a/Backend.hs b/Backend.hs
index 38f0629f48..4743bb202e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -20,7 +20,6 @@ import System.IO.Error (try)
 import System.Posix.Files
 
 import Common.Annex
-import qualified Git
 import qualified Git.Config
 import qualified Git.CheckAttr
 import qualified Annex
diff --git a/Command/Map.hs b/Command/Map.hs
index 815b142e72..ae8a694043 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -13,6 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Git
+import qualified Git.Url
 import qualified Git.Config
 import qualified Git.Construct
 import Annex.UUID
@@ -68,7 +69,7 @@ drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 
 hostname :: Git.Repo -> String
 hostname r
-	| Git.repoIsUrl r = Git.urlHost r
+	| Git.repoIsUrl r = Git.Url.host r
 	| otherwise = "localhost"
 
 basehostname :: Git.Repo -> String
@@ -82,7 +83,7 @@ repoName umap r
 	| otherwise = M.findWithDefault fallback repouuid umap
 	where
 		repouuid = getUncachedUUID r
-		fallback = fromMaybe "unknown" $ Git.repoRemoteName r
+		fallback = fromMaybe "unknown" $ Git.remoteName r
 
 {- A unique id for the node for a repo. Uses the annex.uuid if available. -}
 nodeId :: Git.Repo -> String
@@ -99,7 +100,7 @@ node umap fullinfo r = unlines $ n:edges
 			decorate $ Dot.graphNode (nodeId r) (repoName umap r)
 		edges = map (edge umap fullinfo r) (Git.remotes r)
 		decorate
-			| Git.configMap r == M.empty = unreachable
+			| Git.config r == M.empty = unreachable
 			| otherwise = reachable
 
 {- An edge between two repos. The second repo is a remote of the first. -}
@@ -116,7 +117,7 @@ edge umap fullinfo from to =
 		{- Only name an edge if the name is different than the name
 		 - that will be used for the destination node, and is
 		 - different from its hostname. (This reduces visual clutter.) -}
-		edgename = maybe Nothing calcname $ Git.repoRemoteName to
+		edgename = maybe Nothing calcname $ Git.remoteName to
 		calcname n
 			| n `elem` [repoName umap fullto, hostname fullto] = Nothing
 			| otherwise = Just n
@@ -141,7 +142,7 @@ spider' (r:rs) known
 		-- The remotes will be relative to r', and need to be
 		-- made absolute for later use.
 		remotes <- mapM (absRepo r') (Git.remotes r')
-		let r'' = Git.remotesAdd r' remotes
+		let r'' = r' { Git.remotes = remotes }
 
 		spider' (rs ++ remotes) (r'':known)
 
@@ -154,7 +155,7 @@ absRepo reference r
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
 same a b
-	| both Git.repoIsSsh = matching Git.urlAuthority && matching Git.workTree
+	| both Git.repoIsSsh = matching Git.Url.authority && matching Git.workTree
 	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
 	| neither Git.repoIsSsh = matching Git.workTree
 	| otherwise = False
diff --git a/Config.hs b/Config.hs
index 322dc8af82..07c9eedad7 100644
--- a/Config.hs
+++ b/Config.hs
@@ -31,7 +31,7 @@ getConfig r key def = do
 
 {- Looks up a per-remote config setting in git config. -}
 remoteConfig :: Git.Repo -> ConfigKey -> String
-remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
+remoteConfig r key = "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" ++ key
 
 {- Calculates cost for a remote. Either the default, or as configured 
  - by remote..annex-cost, or if remote..annex-cost-command
diff --git a/Git.hs b/Git.hs
index b4cbd91aa0..a3f2ad74cb 100644
--- a/Git.hs
+++ b/Git.hs
@@ -9,7 +9,7 @@
  -}
 
 module Git (
-	Repo,
+	Repo(..),
 	Ref(..),
 	Branch,
 	Sha,
@@ -22,13 +22,6 @@ module Git (
 	repoLocation,
 	workTree,
 	gitDir,
-	urlPath,
-	urlHost,
-	urlPort,
-	urlHostUser,
-	urlAuthority,
-	urlScheme,
-	configMap,
 	configTrue,
 	gitCommandLine,
 	run,
@@ -39,23 +32,14 @@ module Git (
 	pipeNullSplit,
 	pipeNullSplitB,
 	attributes,
-	remotes,
-	remotesAdd,
-	repoRemoteName,
-	repoRemoteNameSet,
-	repoRemoteNameFromKey,
 	reap,
-	useIndex,
-	getSha,
-	shaSize,
 	assertLocal,
 ) where
 
 import qualified Data.Map as M
-import Network.URI
 import Data.Char
-import System.Posix.Env (setEnv, unsetEnv, getEnv)
 import qualified Data.ByteString.Lazy.Char8 as L
+import Network.URI (uriPath, uriScheme)
 
 import Common
 import Git.Types
@@ -73,29 +57,6 @@ repoLocation Repo { location = Url url } = show url
 repoLocation Repo { location = Dir dir } = dir
 repoLocation Repo { location = Unknown } = undefined
 
-{- Constructs and returns an updated version of a repo with
- - different remotes list. -}
-remotesAdd :: Repo -> [Repo] -> Repo
-remotesAdd repo rs = repo { remotes = rs }
-
-{- Returns the name of the remote that corresponds to the repo, if
- - it is a remote. -}
-repoRemoteName :: Repo -> Maybe String
-repoRemoteName Repo { remoteName = Just name } = Just name
-repoRemoteName _ = Nothing
-
-{- Sets the name of a remote. -}
-repoRemoteNameSet :: String -> Repo -> Repo
-repoRemoteNameSet n r = r { remoteName = Just n }
-
-{- Sets the name of a remote based on the git config key, such as
-   "remote.foo.url". -}
-repoRemoteNameFromKey :: String -> Repo -> Repo
-repoRemoteNameFromKey k = repoRemoteNameSet basename
-	where
-		basename = join "." $ reverse $ drop 1 $
-				reverse $ drop 1 $ split "." k
-
 {- Some code needs to vary between URL and normal repos,
  - or bare and non-bare, these functions help with that. -}
 repoIsUrl :: Repo -> Bool
@@ -104,11 +65,13 @@ repoIsUrl _ = False
 
 repoIsSsh :: Repo -> Bool
 repoIsSsh Repo { location = Url url } 
-	| uriScheme url == "ssh:" = True
+	| scheme == "ssh:" = True
 	-- git treats these the same as ssh
-	| uriScheme url == "git+ssh:" = True
-	| uriScheme url == "ssh+git:" = True
+	| scheme == "git+ssh:" = True
+	| scheme == "ssh+git:" = True
 	| otherwise = False
+	where
+		scheme = uriScheme url
 repoIsSsh _ = False
 
 repoIsHttp :: Repo -> Bool
@@ -129,15 +92,8 @@ assertLocal :: Repo -> a -> a
 assertLocal repo action = 
 	if not $ repoIsUrl repo
 		then action
-		else error $ "acting on URL git repo " ++  repoDescribe repo ++ 
+		else error $ "acting on non-local git repo " ++  repoDescribe repo ++ 
 				" not supported"
-assertUrl :: Repo -> a -> a
-assertUrl repo action = 
-	if repoIsUrl repo
-		then action
-		else error $ "acting on local git repo " ++  repoDescribe repo ++ 
-				" not supported"
-
 configBare :: Repo -> Bool
 configBare repo = maybe unknown configTrue $ M.lookup "core.bare" $ config repo
 	where
@@ -161,61 +117,10 @@ gitDir repo
  -
  - Note that for URL repositories, this is the path on the remote host. -}
 workTree :: Repo -> FilePath
-workTree r@(Repo { location = Url _ }) = urlPath r
-workTree (Repo { location = Dir d }) = d
+workTree Repo { location = Url u } = uriPath u
+workTree Repo { location = Dir d } = d
 workTree Repo { location = Unknown } = undefined
 
-{- Path of an URL repo. -}
-urlPath :: Repo -> String
-urlPath Repo { location = Url u } = uriPath u
-urlPath repo = assertUrl repo $ error "internal"
-
-{- Scheme of an URL repo. -}
-urlScheme :: Repo -> String
-urlScheme Repo { location = Url u } = uriScheme u
-urlScheme repo = assertUrl repo $ error "internal"
-
-{- Work around a bug in the real uriRegName
- -  -}
-uriRegName' :: URIAuth -> String
-uriRegName' a = fixup $ uriRegName a
-	where
-		fixup x@('[':rest)
-			| rest !! len == ']' = take len rest
-			| otherwise = x
-			where
-				len  = length rest - 1
-		fixup x = x
-
-{- Hostname of an URL repo. -}
-urlHost :: Repo -> String
-urlHost = urlAuthPart uriRegName'
-
-{- Port of an URL repo, if it has a nonstandard one. -}
-urlPort :: Repo -> Maybe Integer
-urlPort r = 
-	case urlAuthPart uriPort r of
-		":" -> Nothing
-		(':':p) -> readMaybe p
-		_ -> Nothing
-
-{- Hostname of an URL repo, including any username (ie, "user@host") -}
-urlHostUser :: Repo -> String
-urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r
-
-{- The full authority portion an URL repo. (ie, "user@host:port") -}
-urlAuthority :: Repo -> String
-urlAuthority = urlAuthPart assemble
-	where
-		assemble a = uriUserInfo a ++ uriRegName' a ++ uriPort a
-
-{- Applies a function to extract part of the uriAuthority of an URL repo. -}
-urlAuthPart :: (URIAuth -> a) -> Repo -> a
-urlAuthPart a Repo { location = Url u } = a auth
-	where
-		auth = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
-urlAuthPart _ repo = assertUrl repo $ error "internal"
-
 {- Constructs a git command line operating on the specified repo. -}
 gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
 gitCommandLine params repo@(Repo { location = Dir _ } ) =
@@ -284,39 +189,6 @@ reap = do
 	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
 	maybe (return ()) (const reap) r
 
-{- Forces git to use the specified index file.
- - Returns an action that will reset back to the default
- - index file. -}
-useIndex :: FilePath -> IO (IO ())
-useIndex index = do
-	res <- getEnv var
-	setEnv var index True
-	return $ reset res
-	where
-		var = "GIT_INDEX_FILE"
-		reset (Just v) = setEnv var v True
-		reset _ = unsetEnv var
-
-{- Runs an action that causes a git subcommand to emit a sha, and strips
-   any trailing newline, returning the sha. -}
-getSha :: String -> IO String -> IO Sha
-getSha subcommand a = do
-	t <- a
-	let t' = if last t == '\n'
-		then init t
-		else t
-	when (length t' /= shaSize) $
-		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return $ Ref t'
-
-{- Size of a git sha. -}
-shaSize :: Int
-shaSize = 40
-
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
-
-{- Access to raw config Map -}
-configMap :: Repo -> M.Map String String
-configMap = config
diff --git a/Git/Branch.hs b/Git/Branch.hs
index 8b0d1e5afc..3e08e19c28 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -11,6 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
 import Git
+import Git.Sha
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
@@ -19,7 +20,7 @@ changed origbranch newbranch repo
 	| origbranch == newbranch = return False
 	| otherwise = not . L.null <$> diffs
 	where
-		diffs = Git.pipeRead
+		diffs = pipeRead
 			[ Param "log"
 			, Param (show origbranch ++ ".." ++ show newbranch)
 			, Params "--oneline -n1"
@@ -44,7 +45,7 @@ fastForward branch (first:rest) repo = do
 	where
 		no_ff = return False
 		do_ff to = do
-			Git.run "update-ref"
+			run "update-ref"
 				[Param $ show branch, Param $ show to] repo
 			return True
 		findbest c [] = return $ Just c
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index c1cafb8ba8..44c2a9f5ef 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -20,6 +20,7 @@ import qualified Data.ByteString.Char8 as S
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
+import Git.Sha
 import Utility.SafeCommand
 
 type CatFileHandle = (PipeHandle, Handle, Handle)
@@ -27,7 +28,7 @@ type CatFileHandle = (PipeHandle, Handle, Handle)
 {- Starts git cat-file running in batch mode in a repo and returns a handle. -}
 catFileStart :: Repo -> IO CatFileHandle
 catFileStart repo = hPipeBoth "git" $ toCommand $
-	Git.gitCommandLine [Param "cat-file", Param "--batch"] repo
+	gitCommandLine [Param "cat-file", Param "--batch"] repo
 
 {- Stops git cat-file. -}
 catFileStop :: CatFileHandle -> IO ()
@@ -49,7 +50,7 @@ catObject (_, from, to) object = do
 	header <- hGetLine from
 	case words header of
 		[sha, objtype, size]
-			| length sha == Git.shaSize &&
+			| length sha == shaSize &&
 			  validobjtype objtype -> handle size
 			| otherwise -> empty
 		_
diff --git a/Git/Construct.hs b/Git/Construct.hs
index 9149ab9ec0..a35a87cc77 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -11,6 +11,8 @@ module Git.Construct (
 	fromUrl,
 	fromUnknown,
 	localToUrl,
+	remoteNamed,
+	remoteNamedFromKey,
 	fromRemotes,
 	fromRemoteLocation,
 	repoAbsPath,
@@ -23,6 +25,7 @@ import Network.URI
 import Common
 import Git.Types
 import Git
+import qualified Git.Url as Url
 
 {- Finds the current git repository, which may be in a parent directory. -}
 fromCwd :: IO Repo
@@ -67,8 +70,8 @@ fromUrl url
 			bad = error $ "bad url " ++ url
 
 {- Creates a repo that has an unknown location. -}
-fromUnknown :: Repo
-fromUnknown = newFrom Unknown
+fromUnknown :: IO Repo
+fromUnknown = return $ newFrom Unknown
 
 {- Converts a local Repo into a remote repo, using the reference repo
  - which is assumed to be on the same host. -}
@@ -79,8 +82,8 @@ localToUrl reference r
 	| otherwise = r { location = Url $ fromJust $ parseURI absurl }
 	where
 		absurl =
-			urlScheme reference ++ "//" ++
-			urlAuthority reference ++
+			Url.scheme reference ++ "//" ++
+			Url.authority reference ++
 			workTree r
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
@@ -91,7 +94,21 @@ fromRemotes repo = mapM construct remotepairs
 		filterkeys f = filterconfig (\(k,_) -> f k)
 		remotepairs = filterkeys isremote
 		isremote k = startswith "remote." k && endswith ".url" k
-		construct (k,v) = repoRemoteNameFromKey k <$> fromRemoteLocation v repo
+		construct (k,v) = remoteNamedFromKey k $ fromRemoteLocation v repo
+
+{- Sets the name of a remote when constructing the Repo to represent it. -}
+remoteNamed :: String -> IO Repo -> IO Repo
+remoteNamed n constructor = do
+	r <- constructor
+	return $ r { remoteName = Just n }
+
+{- Sets the name of a remote based on the git config key, such as
+   "remote.foo.url". -}
+remoteNamedFromKey :: String -> IO Repo -> IO Repo
+remoteNamedFromKey k = remoteNamed basename
+	where
+		basename = join "." $ reverse $ drop 1 $
+				reverse $ drop 1 $ split "." k
 
 {- Constructs a new Repo for one of a Repo's remotes using a given
  - location (ie, an url). -}
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 99b96afcb9..60822f3f07 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -17,10 +17,10 @@ hashFiles paths repo = do
 	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo
 	_ <- forkProcess (feeder toh)
 	hClose toh
-	shas <- map Git.Ref . lines <$> hGetContentsStrict fromh
+	shas <- map Ref . lines <$> hGetContentsStrict fromh
 	return (shas, ender fromh pid)
 	where
-		git_hash_object = Git.gitCommandLine
+		git_hash_object = gitCommandLine
 			[Param "hash-object", Param "-w", Param "--stdin-paths"]
 		feeder toh = do
 			hPutStr toh $ unlines paths
diff --git a/Git/Index.hs b/Git/Index.hs
new file mode 100644
index 0000000000..aaf54e032e
--- /dev/null
+++ b/Git/Index.hs
@@ -0,0 +1,24 @@
+{- git index file stuff
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Index where
+
+import System.Posix.Env (setEnv, unsetEnv, getEnv)
+
+{- Forces git to use the specified index file.
+ -
+ - Returns an action that will reset back to the default
+ - index file. -}
+override :: FilePath -> IO (IO ())
+override index = do
+	res <- getEnv var
+	setEnv var index True
+	return $ reset res
+	where
+		var = "GIT_INDEX_FILE"
+		reset (Just v) = setEnv var v True
+		reset _ = unsetEnv var
diff --git a/Git/Ref.hs b/Git/Ref.hs
index 723bea6817..3b550cf5b9 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -37,11 +37,11 @@ sha branch repo = process . L.unpack <$> showref repo
 
 {- List of (refs, branches) matching a given ref spec.
  - Duplicate refs are filtered out. -}
-matching :: Ref -> Repo -> IO [(Git.Ref, Git.Branch)]
+matching :: Ref -> Repo -> IO [(Ref, Branch)]
 matching ref repo = do
-	r <- Git.pipeRead [Param "show-ref", Param $ show ref] repo
+	r <- pipeRead [Param "show-ref", Param $ show ref] repo
 	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
 	where
-		gen l = (Git.Ref $ head l, Git.Ref $ last l)
+		gen l = (Ref $ head l, Ref $ last l)
 		uref (a, _) (b, _) = a == b
 
diff --git a/Git/Sha.hs b/Git/Sha.hs
new file mode 100644
index 0000000000..475c2ba5f3
--- /dev/null
+++ b/Git/Sha.hs
@@ -0,0 +1,27 @@
+{- git SHA stuff
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Sha where
+
+import Common
+import Git.Types
+
+{- Runs an action that causes a git subcommand to emit a sha, and strips
+   any trailing newline, returning the sha. -}
+getSha :: String -> IO String -> IO Sha
+getSha subcommand a = do
+	t <- a
+	let t' = if last t == '\n'
+		then init t
+		else t
+	when (length t' /= shaSize) $
+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
+	return $ Ref t'
+
+{- Size of a git sha. -}
+shaSize :: Int
+shaSize = 40
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 0345af3994..a623e1cebb 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -20,6 +20,7 @@ import qualified Data.Set as S
 
 import Common
 import Git
+import Git.Sha
 import Git.CatFile
 
 type Streamer = (String -> IO ()) -> IO ()
@@ -27,7 +28,7 @@ type Streamer = (String -> IO ()) -> IO ()
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
  -
- - Should be run with a temporary index file configured by Git.useIndex.
+ - Should be run with a temporary index file configured by useIndex.
  -}
 merge :: Ref -> Ref -> Repo -> IO ()
 merge x y repo = do
@@ -53,7 +54,7 @@ update_index repo ls = stream_update_index repo [(`mapM_` ls)]
 {- Streams content into update-index. -}
 stream_update_index :: Repo -> [Streamer] -> IO ()
 stream_update_index repo as = do
-	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
 	forM_ as (stream h)
 	hClose h
 	forceSuccess p
diff --git a/Git/Url.hs b/Git/Url.hs
new file mode 100644
index 0000000000..6a893d92fe
--- /dev/null
+++ b/Git/Url.hs
@@ -0,0 +1,70 @@
+{- git repository urls
+ -
+ - Copyright 2010, 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Url (
+	scheme,
+	host,
+	port,
+	hostuser,
+	authority,
+) where
+
+import Network.URI hiding (scheme, authority)
+
+import Common
+import Git.Types
+import Git
+
+{- Scheme of an URL repo. -}
+scheme :: Repo -> String
+scheme Repo { location = Url u } = uriScheme u
+scheme repo = notUrl repo
+
+{- Work around a bug in the real uriRegName
+ -  -}
+uriRegName' :: URIAuth -> String
+uriRegName' a = fixup $ uriRegName a
+	where
+		fixup x@('[':rest)
+			| rest !! len == ']' = take len rest
+			| otherwise = x
+			where
+				len  = length rest - 1
+		fixup x = x
+
+{- Hostname of an URL repo. -}
+host :: Repo -> String
+host = authpart uriRegName'
+
+{- Port of an URL repo, if it has a nonstandard one. -}
+port :: Repo -> Maybe Integer
+port r = 
+	case authpart uriPort r of
+		":" -> Nothing
+		(':':p) -> readMaybe p
+		_ -> Nothing
+
+{- Hostname of an URL repo, including any username (ie, "user@host") -}
+hostuser :: Repo -> String
+hostuser r = authpart uriUserInfo r ++ authpart uriRegName' r
+
+{- The full authority portion an URL repo. (ie, "user@host:port") -}
+authority :: Repo -> String
+authority = authpart assemble
+	where
+		assemble a = uriUserInfo a ++ uriRegName' a ++ uriPort a
+
+{- Applies a function to extract part of the uriAuthority of an URL repo. -}
+authpart :: (URIAuth -> a) -> Repo -> a
+authpart a Repo { location = Url u } = a auth
+	where
+		auth = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
+authpart _ repo = notUrl repo
+
+notUrl :: Repo -> a
+notUrl repo = error $
+	"acting on local git repo " ++ repoDescribe repo ++ " not supported"
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 9d80f4c1cb..d848a21b33 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -37,16 +37,17 @@ remote = RemoteType {
 
 list :: Annex [Git.Repo]
 list = do
-	c <- fromRepo Git.configMap
+	c <- fromRepo Git.config
 	mapM (tweakurl c) =<< fromRepo Git.remotes
 	where
 		annexurl n = "remote." ++ n ++ ".annexurl"
 		tweakurl c r = do
-			let n = fromJust $ Git.repoRemoteName r
+			let n = fromJust $ Git.remoteName r
 			case M.lookup (annexurl n) c of
 				Nothing -> return r
-				Just url -> Git.repoRemoteNameSet n <$>
-					inRepo (Git.Construct.fromRemoteLocation url)
+				Just url -> inRepo $ \g ->
+					Git.Construct.remoteNamed n $
+						Git.Construct.fromRemoteLocation url g
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r u _ = do
@@ -84,7 +85,7 @@ gen r u _ = do
  - returns the updated repo. -}
 tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
-	| not $ M.null $ Git.configMap r = return r -- already read
+	| not $ M.null $ Git.config r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsHttp r = store $ safely geturlconfig
 	| Git.repoIsUrl r = return r
@@ -116,13 +117,13 @@ tryGitConfigRead r
 			r' <- a
 			g <- gitRepo
 			let l = Git.remotes g
-			let g' = Git.remotesAdd g $ exchange l r'
+			let g' = g { Git.remotes = exchange l r' }
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 			return r'
 
 		exchange [] _ = []
 		exchange (old:ls) new =
-			if Git.repoRemoteName old == Git.repoRemoteName new
+			if Git.remoteName old == Git.remoteName new
 				then new : exchange ls new
 				else old : exchange ls new
 
@@ -167,7 +168,7 @@ onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
 	-- Avoid re-reading the repository's configuration if it was
 	-- already read.
-	state <- if M.null $ Git.configMap r
+	state <- if M.null $ Git.config r
 		then Annex.new r
 		else return $ Annex.newState r
 	Annex.eval state $ do
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 72c4842d88..c374a16aaf 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -20,11 +20,11 @@ import qualified Git.Construct
  -}
 findSpecialRemotes :: String -> Annex [Git.Repo]
 findSpecialRemotes s = do
-	m <- fromRepo Git.configMap
-	return $ map construct $ remotepairs m
+	m <- fromRepo Git.config
+	liftIO $ mapM construct $ remotepairs m
 	where
 		remotepairs = M.toList . M.filterWithKey match
-		construct (k,_) = Git.repoRemoteNameFromKey k Git.Construct.fromUnknown
+		construct (k,_) = Git.Construct.remoteNamedFromKey k Git.Construct.fromUnknown
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
diff --git a/Remote/Web.hs b/Remote/Web.hs
index c4e9f8bd69..e31539f885 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -27,7 +27,9 @@ remote = RemoteType {
 -- (If the web should cease to exist, remove this module and redistribute
 -- a new release to the survivors by carrier pigeon.)
 list :: Annex [Git.Repo]
-list = return [Git.repoRemoteNameSet "web" Git.Construct.fromUnknown]
+list = do
+	r <- liftIO $ Git.Construct.remoteNamed "web" Git.Construct.fromUnknown
+	return [r]
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index f67414bdd3..6fd19c8dae 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -12,6 +12,7 @@ import qualified Git.UnionMerge
 import qualified Git.Config
 import qualified Git.Construct
 import qualified Git.Branch
+import qualified Git.Index
 import qualified Git
 
 header :: String
@@ -42,7 +43,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- map Git.Ref <$> parseArgs
 	g <- Git.Config.read =<< Git.Construct.fromCwd
-	_ <- Git.useIndex (tmpIndex g)
+	_ <- Git.Index.override (tmpIndex g)
 	setup g
 	Git.UnionMerge.merge aref bref g
 	_ <- Git.Branch.commit "union merge" newref [aref, bref] g
diff --git a/test.hs b/test.hs
index 1ce9d103d1..daa2661b6e 100644
--- a/test.hs
+++ b/test.hs
@@ -24,7 +24,6 @@ import qualified Utility.SafeCommand
 import qualified Annex
 import qualified Annex.UUID
 import qualified Backend
-import qualified Git
 import qualified Git.Config
 import qualified Git.Construct
 import qualified Git.Filename

From ef28b3fef7e236d8c27ce35308c0e37ece58d20c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Dec 2011 15:56:11 -0400
Subject: [PATCH 2696/2835] split out Git/Command.hs

---
 Annex/Branch.hs          |  5 ++-
 CmdLine.hs               |  3 +-
 Command/Sync.hs          | 11 +++---
 Command/Unannex.hs       |  6 +--
 Command/Uninit.hs        |  6 ++-
 Command/Unused.hs        |  3 +-
 Config.hs                |  3 +-
 Git.hs                   | 78 --------------------------------------
 Git/Branch.hs            |  1 +
 Git/CatFile.hs           |  1 +
 Git/CheckAttr.hs         |  1 +
 Git/Command.hs           | 82 ++++++++++++++++++++++++++++++++++++++++
 Git/HashObject.hs        |  1 +
 Git/LsFiles.hs           |  1 +
 Git/LsTree.hs            |  1 +
 Git/Queue.hs             |  1 +
 Git/Ref.hs               |  1 +
 Git/UnionMerge.hs        |  1 +
 Remote/Bup.hs            |  5 ++-
 Remote/Git.hs            |  3 +-
 Remote/Helper/Special.hs |  3 +-
 Upgrade/V2.hs            |  8 ++--
 22 files changed, 125 insertions(+), 100 deletions(-)
 create mode 100644 Git/Command.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a22a4adcfe..af1878479a 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -24,6 +24,7 @@ import Annex.Exception
 import Annex.BranchState
 import Annex.Journal
 import qualified Git
+import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.Branch
 import qualified Git.UnionMerge
@@ -67,7 +68,7 @@ getBranch :: Annex (Git.Ref)
 getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
 	where
 		go True = do
-			inRepo $ Git.run "branch"
+			inRepo $ Git.Command.run "branch"
 				[Param $ show name, Param $ show originname]
 			fromMaybe (error $ "failed to create " ++ show name)
 				<$> branchsha
@@ -221,7 +222,7 @@ commitBranch branchref message parents = do
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
-	bfiles <- inRepo $ Git.pipeNullSplit
+	bfiles <- inRepo $ Git.Command.pipeNullSplit
 		[Params "ls-tree --name-only -r -z", Param $ show fullname]
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
diff --git a/CmdLine.hs b/CmdLine.hs
index 672969c30a..ebcca25aaf 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -20,6 +20,7 @@ import Common.Annex
 import qualified Annex
 import qualified Annex.Queue
 import qualified Git
+import qualified Git.Command
 import Annex.Content
 import Command
 
@@ -101,5 +102,5 @@ startup = return True
 shutdown :: Annex Bool
 shutdown = do
 	saveState
-	liftIO Git.reap -- zombies from long-running git processes
+	liftIO Git.Command.reap -- zombies from long-running git processes
 	return True
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 987eb6138f..a25bcad8c1 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -10,7 +10,7 @@ module Command.Sync where
 import Common.Annex
 import Command
 import qualified Annex.Branch
-import qualified Git
+import qualified Git.Command
 import qualified Git.Config
 
 import qualified Data.ByteString.Lazy.Char8 as L
@@ -28,7 +28,8 @@ commit = do
 	next $ next $ do
 		showOutput
 		-- Commit will fail when the tree is clean, so ignore failure.
-		_ <- inRepo $ Git.runBool "commit" [Param "-a", Param "-m", Param "sync"]
+		_ <- inRepo $ Git.Command.runBool "commit"
+			[Param "-a", Param "-m", Param "sync"]
 		return True
 
 pull :: CommandStart
@@ -38,7 +39,7 @@ pull = do
 	next $ next $ do
 		showOutput
 		checkRemote remote
-		inRepo $ Git.runBool "pull" [Param remote]
+		inRepo $ Git.Command.runBool "pull" [Param remote]
 
 push :: CommandStart
 push = do
@@ -47,7 +48,7 @@ push = do
 	next $ next $ do
 		Annex.Branch.update
 		showOutput
-		inRepo $ Git.runBool "push" [Param remote, matchingbranches]
+		inRepo $ Git.Command.runBool "push" [Param remote, matchingbranches]
 	where
 		-- git push may be configured to not push matching
 		-- branches; this should ensure it always does.
@@ -61,7 +62,7 @@ defaultRemote = do
 
 currentBranch :: Annex String
 currentBranch = last . split "/" . L.unpack . head . L.lines <$>
-	inRepo (Git.pipeRead [Param "symbolic-ref", Param "HEAD"])
+	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
 
 checkRemote :: String -> Annex ()
 checkRemote remote = do
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index bed857b060..8a511bf4da 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -13,7 +13,7 @@ import qualified Annex
 import Utility.FileMode
 import Logs.Location
 import Annex.Content
-import qualified Git
+import qualified Git.Command
 import qualified Git.LsFiles as LsFiles
 
 def :: [Command]
@@ -34,14 +34,14 @@ cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
 	liftIO $ removeFile file
 	-- git rm deletes empty directory without --cached
-	inRepo $ Git.run "rm" [Params "--cached --quiet --", File file]
+	inRepo $ Git.Command.run "rm" [Params "--cached --quiet --", File file]
 	
 	-- If the file was already committed, it is now staged for removal.
 	-- Commit that removal now, to avoid later confusing the
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
 	whenM (not . null <$> inRepo (LsFiles.staged [file])) $ do
-		inRepo $ Git.run "commit" [
+		inRepo $ Git.Command.run "commit" [
 			Param "-m", Param "content removed from git annex",
 			Param "--", File file]
 
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 48f5b1ac14..fc6f0cc275 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,6 +12,7 @@ import qualified Data.ByteString.Lazy.Char8 as B
 import Common.Annex
 import Command
 import qualified Git
+import qualified Git.Command
 import qualified Annex
 import qualified Command.Unannex
 import Init
@@ -29,7 +30,7 @@ check = do
 		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	where
 		current_branch = Git.Ref . head . lines . B.unpack <$> revhead
-		revhead = inRepo $ Git.pipeRead 
+		revhead = inRepo $ Git.Command.pipeRead 
 			[Params "rev-parse --abbrev-ref HEAD"]
 
 seek :: [CommandSeek]
@@ -57,5 +58,6 @@ cleanup = do
 	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
 	saveState
-	inRepo $ Git.run "branch" [Param "-D", Param $ show Annex.Branch.name]
+	inRepo $ Git.Command.run "branch"
+		[Param "-D", Param $ show Annex.Branch.name]
 	liftIO exitSuccess
diff --git a/Command/Unused.hs b/Command/Unused.hs
index cd1cd16024..8a70ff3356 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -20,6 +20,7 @@ import Utility.TempFile
 import Logs.Location
 import qualified Annex
 import qualified Git
+import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.LsFiles as LsFiles
 import qualified Git.LsTree as LsTree
@@ -148,7 +149,7 @@ unusedKeys = do
 excludeReferenced :: [Key] -> Annex [Key]
 excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
-	c <- inRepo $ Git.pipeRead [Param "show-ref"]
+	c <- inRepo $ Git.Command.pipeRead [Param "show-ref"]
 	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
 		(S.fromList l)
 	where
diff --git a/Config.hs b/Config.hs
index 07c9eedad7..4cc4c18668 100644
--- a/Config.hs
+++ b/Config.hs
@@ -10,6 +10,7 @@ module Config where
 import Common.Annex
 import qualified Git
 import qualified Git.Config
+import qualified Git.Command
 import qualified Annex
 
 type ConfigKey = String
@@ -17,7 +18,7 @@ type ConfigKey = String
 {- Changes a git config setting in both internal state and .git/config -}
 setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
-	inRepo $ Git.run "config" [Param k, Param value]
+	inRepo $ Git.Command.run "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
 	newg <- inRepo Git.Config.read
 	Annex.changeState $ \s -> s { Annex.repo = newg }
diff --git a/Git.hs b/Git.hs
index a3f2ad74cb..9420810a67 100644
--- a/Git.hs
+++ b/Git.hs
@@ -23,22 +23,12 @@ module Git (
 	workTree,
 	gitDir,
 	configTrue,
-	gitCommandLine,
-	run,
-	runBool,
-	pipeRead,
-	pipeWrite,
-	pipeWriteRead,
-	pipeNullSplit,
-	pipeNullSplitB,
 	attributes,
-	reap,
 	assertLocal,
 ) where
 
 import qualified Data.Map as M
 import Data.Char
-import qualified Data.ByteString.Lazy.Char8 as L
 import Network.URI (uriPath, uriScheme)
 
 import Common
@@ -121,74 +111,6 @@ workTree Repo { location = Url u } = uriPath u
 workTree Repo { location = Dir d } = d
 workTree Repo { location = Unknown } = undefined
 
-{- Constructs a git command line operating on the specified repo. -}
-gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
-gitCommandLine params repo@(Repo { location = Dir _ } ) =
-	-- force use of specified repo via --git-dir and --work-tree
-	[ Param ("--git-dir=" ++ gitDir repo)
-	, Param ("--work-tree=" ++ workTree repo)
-	] ++ params
-gitCommandLine _ repo = assertLocal repo $ error "internal"
-
-{- Runs git in the specified repo. -}
-runBool :: String -> [CommandParam] -> Repo -> IO Bool
-runBool subcommand params repo = assertLocal repo $
-	boolSystem "git" $ gitCommandLine (Param subcommand : params) repo
-
-{- Runs git in the specified repo, throwing an error if it fails. -}
-run :: String -> [CommandParam] -> Repo -> IO ()
-run subcommand params repo = assertLocal repo $
-	runBool subcommand params repo
-		>>! error $ "git " ++ show params ++ " failed"
-
-{- Runs a git subcommand and returns its output, lazily. 
- -
- - Note that this leaves the git process running, and so zombies will
- - result unless reap is called.
- -}
-pipeRead :: [CommandParam] -> Repo -> IO L.ByteString
-pipeRead params repo = assertLocal repo $ do
-	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo
-	hSetBinaryMode h True
-	L.hGetContents h
-
-{- Runs a git subcommand, feeding it input.
- - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: [CommandParam] -> L.ByteString -> Repo -> IO PipeHandle
-pipeWrite params s repo = assertLocal repo $ do
-	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
-	L.hPut h s
-	hClose h
-	return p
-
-{- Runs a git subcommand, feeding it input, and returning its output.
- - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: [CommandParam] -> L.ByteString -> Repo -> IO (PipeHandle, L.ByteString)
-pipeWriteRead params s repo = assertLocal repo $ do
-	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
-	hSetBinaryMode from True
-	L.hPut to s
-	hClose to
-	c <- L.hGetContents from
-	return (p, c)
-
-{- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it. -}
-pipeNullSplit :: [CommandParam] -> Repo -> IO [String]
-pipeNullSplit params repo = map L.unpack <$> pipeNullSplitB params repo
-
-{- For when Strings are not needed. -}
-pipeNullSplitB ::[CommandParam] -> Repo -> IO [L.ByteString]
-pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
-	pipeRead params repo
-
-{- Reaps any zombie git processes. -}
-reap :: IO ()
-reap = do
-	-- throws an exception when there are no child processes
-	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
-	maybe (return ()) (const reap) r
-
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
diff --git a/Git/Branch.hs b/Git/Branch.hs
index 3e08e19c28..cce56dcfa4 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -12,6 +12,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import Common
 import Git
 import Git.Sha
+import Git.Command
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 44c2a9f5ef..2cef9d5b3d 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -21,6 +21,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
 import Git.Sha
+import Git.Command
 import Utility.SafeCommand
 
 type CatFileHandle = (PipeHandle, Handle, Handle)
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index e9269b1edb..1ea38beeac 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -11,6 +11,7 @@ import System.Exit
 
 import Common
 import Git
+import Git.Command
 import qualified Git.Filename
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
diff --git a/Git/Command.hs b/Git/Command.hs
new file mode 100644
index 0000000000..2350bb0ca3
--- /dev/null
+++ b/Git/Command.hs
@@ -0,0 +1,82 @@
+{- running git commands
+ -
+ - Copyright 2010, 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Command where
+
+import qualified Data.ByteString.Lazy.Char8 as L
+
+import Common
+import Git
+import Git.Types
+
+{- Constructs a git command line operating on the specified repo. -}
+gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
+gitCommandLine params repo@(Repo { location = Dir _ } ) =
+	-- force use of specified repo via --git-dir and --work-tree
+	[ Param ("--git-dir=" ++ gitDir repo)
+	, Param ("--work-tree=" ++ workTree repo)
+	] ++ params
+gitCommandLine _ repo = assertLocal repo $ error "internal"
+
+{- Runs git in the specified repo. -}
+runBool :: String -> [CommandParam] -> Repo -> IO Bool
+runBool subcommand params repo = assertLocal repo $
+	boolSystem "git" $ gitCommandLine (Param subcommand : params) repo
+
+{- Runs git in the specified repo, throwing an error if it fails. -}
+run :: String -> [CommandParam] -> Repo -> IO ()
+run subcommand params repo = assertLocal repo $
+	runBool subcommand params repo
+		>>! error $ "git " ++ show params ++ " failed"
+
+{- Runs a git subcommand and returns its output, lazily. 
+ -
+ - Note that this leaves the git process running, and so zombies will
+ - result unless reap is called.
+ -}
+pipeRead :: [CommandParam] -> Repo -> IO L.ByteString
+pipeRead params repo = assertLocal repo $ do
+	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo
+	hSetBinaryMode h True
+	L.hGetContents h
+
+{- Runs a git subcommand, feeding it input.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWrite :: [CommandParam] -> L.ByteString -> Repo -> IO PipeHandle
+pipeWrite params s repo = assertLocal repo $ do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
+	L.hPut h s
+	hClose h
+	return p
+
+{- Runs a git subcommand, feeding it input, and returning its output.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWriteRead :: [CommandParam] -> L.ByteString -> Repo -> IO (PipeHandle, L.ByteString)
+pipeWriteRead params s repo = assertLocal repo $ do
+	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
+	hSetBinaryMode from True
+	L.hPut to s
+	hClose to
+	c <- L.hGetContents from
+	return (p, c)
+
+{- Reads null terminated output of a git command (as enabled by the -z 
+ - parameter), and splits it. -}
+pipeNullSplit :: [CommandParam] -> Repo -> IO [String]
+pipeNullSplit params repo = map L.unpack <$> pipeNullSplitB params repo
+
+{- For when Strings are not needed. -}
+pipeNullSplitB ::[CommandParam] -> Repo -> IO [L.ByteString]
+pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
+	pipeRead params repo
+
+{- Reaps any zombie git processes. -}
+reap :: IO ()
+reap = do
+	-- throws an exception when there are no child processes
+	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
+	maybe (return ()) (const reap) r
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 60822f3f07..f5e6d50cdf 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -9,6 +9,7 @@ module Git.HashObject where
 
 import Common
 import Git
+import Git.Command
 
 {- Injects a set of files into git, returning the shas of the objects
  - and an IO action to call ones the the shas have been used. -}
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 85215fe040..0c71ed884b 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -16,6 +16,7 @@ module Git.LsFiles (
 ) where
 
 import Git
+import Git.Command
 import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 342a125eb8..919e9af83a 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -17,6 +17,7 @@ import System.Posix.Types
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
+import Git.Command
 import qualified Git.Filename
 import Utility.SafeCommand
 
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 70c766d044..73470b1f0e 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -22,6 +22,7 @@ import Control.Monad (forM_)
 import Utility.SafeCommand
 
 import Git
+import Git.Command
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
diff --git a/Git/Ref.hs b/Git/Ref.hs
index 3b550cf5b9..117ead8f22 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -11,6 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
 import Git
+import Git.Command
 
 {- Converts a fully qualified git ref into a user-visible version. -}
 describe :: Ref -> String
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a623e1cebb..a9a51007f8 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -22,6 +22,7 @@ import Common
 import Git
 import Git.Sha
 import Git.CatFile
+import Git.Command
 
 type Streamer = (String -> IO ()) -> IO ()
 
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 4d63d88e1d..8bd484b7dd 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -15,6 +15,7 @@ import System.Process
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Construct
 import Config
@@ -148,7 +149,7 @@ checkPresent r bupr k
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
 	| otherwise = liftIO $ catchMsgIO $
-		boolSystem "git" $ Git.gitCommandLine params bupr
+		boolSystem "git" $ Git.Command.gitCommandLine params bupr
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
@@ -168,7 +169,7 @@ storeBupUUID u buprepo = do
 			r' <- Git.Config.read r
 			let olduuid = Git.Config.get "annex.uuid" "" r'
 			when (olduuid == "") $
-				Git.run "config"
+				Git.Command.run "config"
 					[Param "annex.uuid", Param v] r'
 	where
 		v = fromUUID u
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d848a21b33..f27d170840 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -16,6 +16,7 @@ import Utility.RsyncFile
 import Annex.Ssh
 import Types.Remote
 import qualified Git
+import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Construct
 import qualified Annex
@@ -176,7 +177,7 @@ onLocal r a = do
 		-- for anything onLocal is used to do.
 		Annex.BranchState.disableUpdate
 		ret <- a
-		liftIO Git.reap
+		liftIO Git.Command.reap
 		return ret
 
 keyUrls :: Git.Repo -> Key -> [String]
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index c374a16aaf..3f6c9c155f 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -12,6 +12,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Command
 import qualified Git.Construct
 
 {- Special remotes don't have a configured url, so Git.Repo does not
@@ -33,7 +34,7 @@ gitConfigSpecialRemote u c k v = do
 	set ("annex-"++k) v
 	set ("annex-uuid") (fromUUID u)
 	where
-		set a b = inRepo $ Git.run "config"
+		set a b = inRepo $ Git.Command.run "config"
 			[Param (configsetting a), Param b]
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 3440d504ba..ffc2f60022 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -9,6 +9,7 @@ module Upgrade.V2 where
 
 import Common.Annex
 import qualified Git
+import qualified Git.Command
 import qualified Git.Ref
 import qualified Annex.Branch
 import Logs.Location
@@ -53,7 +54,7 @@ upgrade = do
 	showProgress
 
 	when e $ do
-		inRepo $ Git.run "rm" [Param "-r", Param "-f", Param "-q", File old]
+		inRepo $ Git.Command.run "rm" [Param "-r", Param "-f", Param "-q", File old]
 		unless bare $ inRepo gitAttributesUnWrite
 	showProgress
 
@@ -104,7 +105,8 @@ push = do
 			Annex.Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
-			inRepo $ Git.run "push" [Param "origin", Param $ show Annex.Branch.name]
+			inRepo $ Git.Command.run "push"
+				[Param "origin", Param $ show Annex.Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
 			-- know about the new branch
@@ -127,7 +129,7 @@ gitAttributesUnWrite repo = do
 		c <- readFileStrict attributes
 		liftIO $ viaTmp writeFile attributes $ unlines $
 			filter (`notElem` attrLines) $ lines c
-		Git.run "add" [File attributes] repo
+		Git.Command.run "add" [File attributes] repo
 
 stateDir :: FilePath
 stateDir = addTrailingPathSeparator ".git-annex"

From 09cd04277577381464eddceb42558ad300c49378 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 12:46:04 -0400
Subject: [PATCH 2697/2835] Properly handle multiline git config values.

A crash on parsing was fixed a while ago. This adds support for fully
correctly parsing multiline git config values, using git config --null.

Since git-annex-shell configlist uses normal git config output, I left in
support for that too; the two forms of config output can be easily
identified by the parser. Since configlist only prints the annex.uuid
config, there's no risk of multiline values there, so no need to change it.
---
 Command/Map.hs   |  2 +-
 Git/Config.hs    | 14 +++++++++++---
 Remote/Git.hs    |  2 +-
 debian/changelog |  1 +
 4 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index ae8a694043..15ca5e1496 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -203,7 +203,7 @@ tryScan r
 			liftIO $ pipedconfig "ssh" sshparams
 			where
 				sshcmd = cddir ++ " && " ++
-					"git config --list"
+					"git config --null --list"
 				dir = Git.workTree r
 				cddir
 					| "/~" `isPrefixOf` dir =
diff --git a/Git/Config.hs b/Git/Config.hs
index 5f0e3fdc21..1fe9484992 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -33,7 +33,8 @@ read repo@(Repo { location = Dir d }) = do
 	   been already read. Instead, chdir to the repo. -}
 	cwd <- getCurrentDirectory
 	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
-		pOpen ReadFromPipe "git" ["config", "--list"] $ hRead repo
+		pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
+			hRead repo
 read r = assertLocal r $ error "internal"
 
 {- Reads git config from a handle and populates a repo with it. -}
@@ -51,8 +52,15 @@ store s repo = do
 	rs <- Git.Construct.fromRemotes repo'
 	return $ repo' { remotes = rs }
 
-{- Parses git config --list output into a config map. -}
+{- Parses git config --list or git config --null --list output into a
+ - config map. -}
 parse :: String -> M.Map String String
-parse s = M.fromList $ map pair $ lines s
+parse [] = M.empty
+parse s
+	-- --list output will have an = in the first line
+	| '=' `elem` head ls = M.fromList $ map (separate (== '=')) ls
+	-- --null --list output separates keys from values with newlines
+	| otherwise = M.fromList $ map (separate (== '\n')) $ split "\0" s
 	where
 		pair = separate (== '=')
+		ls = lines s
diff --git a/Remote/Git.hs b/Remote/Git.hs
index f27d170840..e527fa4fee 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -111,7 +111,7 @@ tryGitConfigRead r
 			withTempFile "git-annex.tmp" $ \tmpfile h -> do
 				hPutStr h s
 				hClose h
-				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
+				pOpen ReadFromPipe "git" ["config", "--null", "--list", "--file", tmpfile] $
 					Git.Config.hRead r
 
 		store a = do
diff --git a/debian/changelog b/debian/changelog
index 5fc01f9ead..3c977e817d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
 
   * Union merge now finds the least expensive way to represent the merge.
   * reinject: Add a sanity check for using an annexed file as the source file.
+  * Properly handle multiline git config values.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From a8643ca44c49cc945f0ecb0e02c2946595415054 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 13:05:47 -0400
Subject: [PATCH 2698/2835] refactor

---
 Git/Config.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Git/Config.hs b/Git/Config.hs
index 1fe9484992..8d79639944 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -58,9 +58,9 @@ parse :: String -> M.Map String String
 parse [] = M.empty
 parse s
 	-- --list output will have an = in the first line
-	| '=' `elem` head ls = M.fromList $ map (separate (== '=')) ls
+	| '=' `elem` head ls = sep '=' ls
 	-- --null --list output separates keys from values with newlines
-	| otherwise = M.fromList $ map (separate (== '\n')) $ split "\0" s
+	| otherwise = sep '\n' $ split "\0" s
 	where
-		pair = separate (== '=')
 		ls = lines s
+		sep c = M.fromList . map (separate (== c))

From 52fe8a17f381a45288c6db7d2d92d3b32c47a472 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 13:12:17 -0400
Subject: [PATCH 2699/2835] remove leftover debug print

---
 Command/AddUrl.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 75ca740314..027c508bcb 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -65,7 +65,6 @@ url2file :: URI -> IO FilePath
 url2file url = do
 	whenM (doesFileExist file) $
 		error $ "already have this url in " ++ file
-	liftIO $ print file
 	return file
 	where
 		file = escape $ uriRegName auth ++ uriPath url ++ uriQuery url

From 38b02de1a84728ff087bc18f08c597c384eacc78 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 15:21:03 -0400
Subject: [PATCH 2700/2835] update

---
 doc/todo/add_-all_option.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/todo/add_-all_option.mdwn b/doc/todo/add_-all_option.mdwn
index cb4722fa78..e6fa0b339d 100644
--- a/doc/todo/add_-all_option.mdwn
+++ b/doc/todo/add_-all_option.mdwn
@@ -5,7 +5,8 @@ every keys with content not present).
 This would be useful when a repository has a history with deleted files
 whose content you want to keep (so you're not using `dropunused`).
 Or when you have a lot of branches and just want to be able to fsck
-every file referenced in any branch.
+every file referenced in any branch. It could also be useful (or even a
+good default) in a bare repository.
 
 A problem with the idea is that `.gitattributes` values for keys not
 currently in the tree would not be available (without horrific anounts of

From e7a555bf211c5b7a3c998dcd3d4f0e3f0ce65974 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 15:39:33 -0400
Subject: [PATCH 2701/2835] fix types

---
 Utility/BadPrelude.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 7921a7e9b7..8e4105ceea 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -12,11 +12,11 @@ head :: [a] -> a
 head = Prelude.head
 
 {- tail is also partial -}
-tail :: [a] -> a
+tail :: [a] -> [a]
 tail = Prelude.tail
 
 {- init too -}
-init :: [a] -> a
+init :: [a] -> [a]
 init = Prelude.init
 
 {- last too -}

From 111b6937ec1110feed024deee6fa95cdb78b9c95 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 15:57:47 -0400
Subject: [PATCH 2702/2835] avoid partial functions, and added check for
 correct sha content

---
 Git/Sha.hs | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/Git/Sha.hs b/Git/Sha.hs
index 475c2ba5f3..9b3a346505 100644
--- a/Git/Sha.hs
+++ b/Git/Sha.hs
@@ -10,17 +10,26 @@ module Git.Sha where
 import Common
 import Git.Types
 
-{- Runs an action that causes a git subcommand to emit a sha, and strips
+{- Runs an action that causes a git subcommand to emit a Sha, and strips
    any trailing newline, returning the sha. -}
 getSha :: String -> IO String -> IO Sha
-getSha subcommand a = do
-	t <- a
-	let t' = if last t == '\n'
-		then init t
-		else t
-	when (length t' /= shaSize) $
-		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return $ Ref t'
+getSha subcommand a = maybe bad return =<< extractSha <$> a
+	where
+		bad = error $ "failed to read sha from git " ++ subcommand
+
+{- Extracts the Sha from a string. There can be a trailing newline after
+ - it, but nothing else. -}
+extractSha :: String -> Maybe Sha
+extractSha s
+	| len == shaSize = val s
+	| len == shaSize + 1 && length s' == shaSize = val s'
+	| otherwise = Nothing
+	where
+		len = length s
+		s' = firstLine s
+		val v
+			| all (`elem` "1234567890ABCDEFabcdef") v = Just $ Ref v
+			| otherwise = Nothing
 
 {- Size of a git sha. -}
 shaSize :: Int

From eb132a854ec9098acbc2a3caac5fd09947879268 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:04:08 -0400
Subject: [PATCH 2703/2835] avoid partial head function

(although it was used safely)
---
 Git/Config.hs | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/Git/Config.hs b/Git/Config.hs
index 8d79639944..7b72eba5a3 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -5,14 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Git.Config (
-	get,
-	read,
-	hRead,
-	store
-) where
+module Git.Config where
 
-import Prelude hiding (read)
 import System.Posix.Directory
 import Control.Exception (bracket_)
 import qualified Data.Map as M
@@ -58,7 +52,7 @@ parse :: String -> M.Map String String
 parse [] = M.empty
 parse s
 	-- --list output will have an = in the first line
-	| '=' `elem` head ls = sep '=' ls
+	| all ('=' `elem`) (take 1 ls) = sep '=' ls
 	-- --null --list output separates keys from values with newlines
 	| otherwise = sep '\n' $ split "\0" s
 	where

From fbc3d32f7dca870bb68f16dcec0b601840313eb1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:58:04 -0400
Subject: [PATCH 2704/2835] avoid partial function, and parse git-ref output
 better

It's possible that a ref name might contain a space, this properly
preserves the space.
---
 Git/Ref.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Git/Ref.hs b/Git/Ref.hs
index 117ead8f22..0197ae7893 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -41,8 +41,8 @@ sha branch repo = process . L.unpack <$> showref repo
 matching :: Ref -> Repo -> IO [(Ref, Branch)]
 matching ref repo = do
 	r <- pipeRead [Param "show-ref", Param $ show ref] repo
-	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
+	return $ nubBy uniqref $ map (gen . L.unpack) (L.lines r)
 	where
-		gen l = (Ref $ head l, Ref $ last l)
-		uref (a, _) (b, _) = a == b
-
+		uniqref (a, _) (b, _) = a == b
+		gen l = let (r, b) = separate (== ' ') l in
+			(Ref r, Ref b)

From 0f9859ae51324ceb21471a09b423715d9fec7f23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:58:58 -0400
Subject: [PATCH 2705/2835] avoid partial function

---
 Locations.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Locations.hs b/Locations.hs
index 85fcb98887..73a2473b56 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -90,7 +90,8 @@ gitAnnexLocation key r
 		return $ inrepo ".git"  annexLocation key hashDirMixed
 	where
 		inrepo d = Git.workTree r  d
-		check locs = fromMaybe (head locs) <$> firstM doesFileExist locs
+		check locs@(l:_) = fromMaybe l <$> firstM doesFileExist locs
+		check [] = error "internal"
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath

From 817b1936e6dac3831721e8f07400181d87f365ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:59:18 -0400
Subject: [PATCH 2706/2835] add beginning, end

Safe versions of init and last
---
 Utility/BadPrelude.hs | 54 ++++++++++++++++++++++++++++++-------------
 1 file changed, 38 insertions(+), 16 deletions(-)

diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 8e4105ceea..49837b9273 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -7,22 +7,44 @@
 
 module Utility.BadPrelude where
 
-{- head is a partial function; head [] is an error -}
-head :: [a] -> a
-head = Prelude.head
-
-{- tail is also partial -}
-tail :: [a] -> [a]
-tail = Prelude.tail
-
-{- init too -}
-init :: [a] -> [a]
-init = Prelude.init
-
-{- last too -}
-last :: [a] -> a
-last = Prelude.last
-
 {- read should be avoided, as it throws an error -}
 read :: Read a => String -> a
 read = Prelude.read
+
+{- head is a partial function; head [] is an error
+ - Instead, use: take 1 -}
+head :: [a] -> a
+head = Prelude.head
+
+{- tail is also partial
+ - Instead, use: drop 1 -}
+tail :: [a] -> [a]
+tail = Prelude.tail
+
+{- init too
+ - Instead, use: beginning -}
+init :: [a] -> [a]
+init = Prelude.init
+
+{- last too
+ - Instead, use: end -}
+last :: [a] -> a
+last = Prelude.last
+
+{- All but the last element of a list.
+ - (Like init, but no error on an empty list.) -}
+beginning :: [a] -> [a]
+beginning [] = []
+beginning (x:xs) =  beginning' x xs
+	where
+		beginning' _ [] = []
+		beginning' y (z:zs) = y : beginning' z zs
+
+{- Like last, but no error on an empty list. -}
+end :: [a] -> [a]
+end [] = []
+end (x:xs) = end' x xs
+	where
+		end' y [] = [y]
+		end' _ (y:ys) = end' y ys
+

From b7e0d39abbc9a09c21c6f0103ad6c9f4547f81fe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:59:48 -0400
Subject: [PATCH 2707/2835] remove some partial functions

A few were too hard to get rid of, and safe since the code
does check for an empty line.
---
 Logs/UUIDBased.hs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 04b12887d7..b09d93f903 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -64,15 +64,15 @@ parseLog parser = M.fromListWith best . mapMaybe parse . lines
 			where
 				makepair v = Just (toUUID u, LogEntry ts v)
 				ws = words line
-				u = head ws
-				end = last ws
+				u = Prelude.head ws
+				t = Prelude.last ws
 				ts
-					| tskey `isPrefixOf` end =
-						pdate $ tail $ dropWhile (/= '=') end
+					| tskey `isPrefixOf` t =
+						pdate $ drop 1 $ dropWhile (/= '=') t
 					| otherwise = Unknown
 				info
 					| ts == Unknown = drop 1 ws
-					| otherwise = drop 1 $ init ws
+					| otherwise = drop 1 $ beginning ws
 				pdate s = case parseTime defaultTimeLocale "%s%Qs" s of
 					Nothing -> Unknown
 					Just d -> Date $ utcTimeToPOSIXSeconds d

From 95d2391f58ae240e7100f0d5488dd7246f71f3bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 18:11:42 -0400
Subject: [PATCH 2708/2835] more partial function removal

Left a few Prelude.head's in where it was checked not null and too hard to
remove, etc.
---
 Backend.hs            |  6 ++----
 Backend/SHA.hs        |  7 +++----
 CmdLine.hs            |  2 +-
 Command/InitRemote.hs | 25 +++++++++----------------
 Command/Map.hs        |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/Status.hs     |  2 +-
 Command/Sync.hs       |  4 +++-
 Command/Uninit.hs     |  2 +-
 Command/Unused.hs     | 10 +++++-----
 Common.hs             |  4 +++-
 Config.hs             |  9 ++-------
 Git/CheckAttr.hs      |  7 +++----
 Git/UnionMerge.hs     |  2 +-
 Logs/Location.hs      |  4 ++--
 Logs/Remote.hs        |  5 ++---
 Logs/Trust.hs         | 14 ++++++--------
 Logs/UUID.hs          |  4 ++--
 Remote.hs             |  2 +-
 Remote/Bup.hs         |  8 ++++----
 Remote/Directory.hs   |  2 +-
 Remote/Rsync.hs       |  2 +-
 Upgrade/V1.hs         | 12 ++++++------
 Utility/BadPrelude.hs | 14 ++++++++++++--
 24 files changed, 73 insertions(+), 78 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 4743bb202e..2f788fcd00 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -107,7 +107,7 @@ chooseBackends fs = Annex.getState Annex.forcebackend >>= go
 			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
 		go (Just _) = do
 			l <- orderedList
-			return $ map (\f -> (Just $ head l, f)) fs
+			return $ map (\f -> (Just $ Prelude.head l, f)) fs
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend Annex
@@ -115,8 +115,6 @@ lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: String -> Maybe (Backend Annex)
-maybeLookupBackendName s
-	| length matches == 1 = Just $ head matches
-	| otherwise = Nothing
+maybeLookupBackendName s = headMaybe matches
 	where
 		matches = filter (\b -> s == B.name b) list
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 7935b6d262..eca312944e 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -62,11 +62,10 @@ shaN :: SHASize -> FilePath -> Annex String
 shaN size file = do
 	showAction "checksum"
 	liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do
-		line <- hGetLine h
-		let bits = split " " line
-		if null bits
+		sha <- fst . separate (== ' ') <$> hGetLine h
+		if null sha
 			then error $ command ++ " parse error"
-			else return $ head bits
+			else return sha
 	where
 		command = fromJust $ shaCommand size
 
diff --git a/CmdLine.hs b/CmdLine.hs
index ebcca25aaf..7f708f15a1 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -51,7 +51,7 @@ parseCmd argv cmds options header = check $ getOpt Permute options argv
 		check (_, [], []) = err "missing command"
 		check (flags, name:rest, [])
 			| null matches = err $ "unknown command " ++ name
-			| otherwise = (flags, head matches, rest)
+			| otherwise = (flags, Prelude.head matches, rest)
 			where
 				matches = filter (\c -> name == cmdname c) cmds
 		check (_, _, errs) = err $ concat errs
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 600e17eb84..1e6bc2ef17 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -25,9 +25,13 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: [String] -> CommandStart
-start ws = do
-	when (null ws) needname
-
+start [] = do
+	names <- remoteNames
+	error $ "Specify a name for the remote. " ++
+		if null names
+			then ""
+			else "Either a new name, or one of these existing special remotes: " ++ join " " names
+start (name:ws) = do
 	(u, c) <- findByName name
 	let fullconfig = config `M.union` c	
 	t <- findType fullconfig
@@ -36,15 +40,7 @@ start ws = do
 	next $ perform t u $ M.union config c
 
 	where
-		name = head ws
-		config = Logs.Remote.keyValToConfig $ tail ws
-		needname = do
-			let err s = error $ "Specify a name for the remote. " ++ s
-			names <- remoteNames
-			if null names
-				then err ""
-				else err $ "Either a new name, or one of these existing special remotes: " ++ join " " names
-			
+		config = Logs.Remote.keyValToConfig ws
 
 perform :: R.RemoteType Annex -> UUID -> R.RemoteConfig -> CommandPerform
 perform t u c = do
@@ -67,11 +63,8 @@ findByName name = do
 			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
-findByName' n m
-	| null matches = Nothing
-	| otherwise = Just $ head matches
+findByName' n = headMaybe . filter (matching . snd) . M.toList
 	where
-		matches = filter (matching . snd) $ M.toList m
 		matching c = case M.lookup nameKey c of
 			Nothing -> False
 			Just n'
diff --git a/Command/Map.hs b/Command/Map.hs
index 15ca5e1496..da129c8f65 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -73,7 +73,7 @@ hostname r
 	| otherwise = "localhost"
 
 basehostname :: Git.Repo -> String
-basehostname r = head $ split "." $ hostname r
+basehostname r = Prelude.head $ split "." $ hostname r
 
 {- A name to display for a repo. Uses the name from uuid.log if available,
  - or the remote name if not. -}
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 30288fc162..8778743ff5 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -31,7 +31,7 @@ start b file (key, oldbackend) = do
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = head <$> Backend.orderedList
+		choosebackend Nothing = Prelude.head <$> Backend.orderedList
 		choosebackend (Just backend) = return backend
 
 {- Checks if a key is upgradable to a newer representation. -}
diff --git a/Command/Status.hs b/Command/Status.hs
index 09da41987b..736d897ef3 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -116,7 +116,7 @@ remote_list level desc = stat n $ nojson $ lift $ do
 	us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap)
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
-	return $ if null s then "0" else show (length rs) ++ "\n" ++ init s
+	return $ if null s then "0" else show (length rs) ++ "\n" ++ beginning s
 	where
 		n = desc ++ " repositories"
 
diff --git a/Command/Sync.hs b/Command/Sync.hs
index a25bcad8c1..36c4eeef06 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -12,6 +12,8 @@ import Command
 import qualified Annex.Branch
 import qualified Git.Command
 import qualified Git.Config
+import qualified Git.Ref
+import qualified Git
 
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -61,7 +63,7 @@ defaultRemote = do
 	fromRepo $ Git.Config.get ("branch." ++ branch ++ ".remote") "origin"
 
 currentBranch :: Annex String
-currentBranch = last . split "/" . L.unpack . head . L.lines <$>
+currentBranch = Git.Ref.describe . Git.Ref . firstLine . L.unpack <$>
 	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
 
 checkRemote :: String -> Annex ()
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index fc6f0cc275..21ad4c7df5 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -29,7 +29,7 @@ check = do
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	where
-		current_branch = Git.Ref . head . lines . B.unpack <$> revhead
+		current_branch = Git.Ref . Prelude.head . lines . B.unpack <$> revhead
 		revhead = inRepo $ Git.Command.pipeRead 
 			[Params "rev-parse --abbrev-ref HEAD"]
 
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 8a70ff3356..ef398b01e1 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -154,13 +154,13 @@ excludeReferenced l = do
 		(S.fromList l)
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
-		refs = map (Git.Ref .  last) .
-			nubBy cmpheads .
+		refs = map (Git.Ref .  snd) .
+			nubBy uniqref .
 			filter ourbranches .
-			map words . lines . L.unpack
-		cmpheads a b = head a == head b
+			map (separate (== ' ')) . lines . L.unpack
+		uniqref (a, _) (b, _) = a == b
 		ourbranchend = '/' : show Annex.Branch.name
-		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
+		ourbranches (_, b) = not $ ourbranchend `isSuffixOf` b
 		removewith [] s = return $ S.toList s
 		removewith (a:as) s
 			| s == S.empty = return [] -- optimisation
diff --git a/Common.hs b/Common.hs
index 7e8dd9a2a5..f3dd701b14 100644
--- a/Common.hs
+++ b/Common.hs
@@ -6,7 +6,7 @@ import Control.Monad.State as X (liftIO)
 import Control.Exception.Extensible as X (IOException)
 
 import Data.Maybe as X
-import Data.List as X
+import Data.List as X hiding (head, tail, init, last)
 import Data.String.Utils as X
 
 import System.Path as X
@@ -25,3 +25,5 @@ import Utility.SafeCommand as X
 import Utility.Path as X
 import Utility.Directory as X
 import Utility.Monad as X
+
+import Utility.BadPrelude as X
diff --git a/Config.hs b/Config.hs
index 4cc4c18668..aa88858738 100644
--- a/Config.hs
+++ b/Config.hs
@@ -40,15 +40,10 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" +
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
 	cmd <- getConfig r "cost-command" ""
-	safeparse <$> if not $ null cmd
+	(fromMaybe def . readMaybe) <$>
+		if not $ null cmd
 			then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd]
 			else getConfig r "cost" ""
-	where
-		safeparse v
-			| null ws = def
-			| otherwise = fromMaybe def $ readMaybe $ head ws
-			where
-				ws = words v
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 1ea38beeac..0d3e798a1e 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -36,10 +36,9 @@ lookup attr files repo = do
 				, Param attr
 				, Params "-z --stdin"
 				] repo
-		topair l = (file, value)
+		topair l = (Git.Filename.decode file, value)
 			where 
-				file = Git.Filename.decode $ join sep $ take end bits
-				value = bits !! end
-				end = length bits - 1
+				file = join sep $ beginning bits
+				value = end bits !! 0
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a9a51007f8..d5323af1d1 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -134,7 +134,7 @@ hashObject repo content = getSha subcmd $ do
 calcMerge :: [(Ref, [L.ByteString])] -> Either Ref [L.ByteString]
 calcMerge shacontents
 	| null reuseable = Right $ new
-	| otherwise = Left $ fst $ head reuseable
+	| otherwise = Left $ fst $ Prelude.head reuseable
 	where
 		reuseable = filter (\c -> sorteduniq (snd c) == new) shacontents
 		new = sorteduniq $ concat $ map snd shacontents
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 27b4d709e7..588962bc57 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -68,7 +68,7 @@ logFile key = hashDirLower key ++ keyFile key ++ ".log"
 {- Converts a log filename into a key. -}
 logFileKey :: FilePath -> Maybe Key
 logFileKey file
-	| end == ".log" = fileKey beginning
+	| ext == ".log" = fileKey base
 	| otherwise = Nothing
 	where
-		(beginning, end) = splitAt (length file - 4) file
+		(base, ext) = splitAt (length file - 4) file
diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index 8d15f3151e..d9b41d8c47 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -73,14 +73,13 @@ configUnEscape = unescape
 			| c == '&' = entity rest
 			| otherwise = c : unescape rest
 		entity s = if ok
-				then chr (read num) : unescape rest
+				then chr (Prelude.read num) : unescape rest
 				else '&' : unescape s
 			where
 				num = takeWhile isNumber s
 				r = drop (length num) s
 				rest = drop 1 r
-				ok = not (null num) && 
-					not (null r) && head r == ';'
+				ok = not (null num) && take 1 r == ";"
 
 {- for quickcheck -}
 prop_idempotent_configEscape :: String -> Bool
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 196666a84e..5d769bd247 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -54,18 +54,16 @@ trustMap = do
 		Just m -> return m
 		Nothing -> do
 			overrides <- M.fromList <$> Annex.getState Annex.forcetrust
-			m <- (M.union overrides . simpleMap . parseLog parseTrust) <$>
+			m <- (M.union overrides . simpleMap . parseLog (Just . parseTrust)) <$>
 				Annex.Branch.get trustLog
 			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
 			return m
 
-parseTrust :: String -> Maybe TrustLevel
-parseTrust s
-	| length w > 0 = Just $ parse $ head w
-	-- back-compat; the trust.log used to only list trusted repos
-	| otherwise = Just Trusted
+{- The trust.log used to only list trusted repos, without a field for the
+ - trust status, which is why this defaults to Trusted. -}
+parseTrust :: String -> TrustLevel
+parseTrust s = maybe Trusted parse $ headMaybe $ words s
 	where
-		w = words s
 		parse "1" = Trusted
 		parse "0" = UnTrusted
 		parse "X" = DeadTrusted
@@ -82,6 +80,6 @@ trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid@(UUID _) level = do
 	ts <- liftIO getPOSIXTime
 	Annex.Branch.change trustLog $
-		showLog showTrust . changeLog ts uuid level . parseLog parseTrust
+		showLog showTrust . changeLog ts uuid level . parseLog (Just . parseTrust)
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
 trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index b325c78b6f..18cbee61e4 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -57,9 +57,9 @@ fixBadUUID = M.fromList . map fixup . M.toList
 				kuuid = fromUUID k
 				isbad = not (isuuid kuuid) && isuuid lastword
 				ws = words $ value v
-				lastword = last ws
+				lastword = Prelude.last ws
 				fixeduuid = toUUID lastword
-				fixedvalue = unwords $ kuuid: init ws
+				fixedvalue = unwords $ kuuid: Prelude.init ws
 		-- For the fixed line to take precidence, it should be
 		-- slightly newer, but only slightly.
 		newertime (LogEntry (Date d) _) = d + minimumPOSIXTimeSlice
diff --git a/Remote.hs b/Remote.hs
index b1be60ec4a..aa86934143 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -103,7 +103,7 @@ byName' n = do
 	let match = filter matching allremotes
 	if null match
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
-		else return $ Right $ head match
+		else return $ Right $ Prelude.head match
 	where
 		matching r = n == name r || toUUID n == uuid r
 
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 8bd484b7dd..cbd5d584ac 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -209,20 +209,20 @@ bup2GitRemote "" = do
 	Git.Construct.fromAbsPath $ h  ".bup"
 bup2GitRemote r
 	| bupLocal r = 
-		if head r == '/'
+		if "/" `isPrefixOf` r
 			then Git.Construct.fromAbsPath r
 			else error "please specify an absolute path"
 	| otherwise = Git.Construct.fromUrl $ "ssh://" ++ host ++ slash dir
 		where
 			bits = split ":" r
-			host = head bits
+			host = Prelude.head bits
 			dir = join ":" $ drop 1 bits
 			-- "host:~user/dir" is not supported specially by bup;
 			-- "host:dir" is relative to the home directory;
 			-- "host:" goes in ~/.bup
 			slash d
-				| d == "" = "/~/.bup"
-				| head d == '/' = d
+				| null d = "/~/.bup"
+				| "/" `isPrefixOf` d = d
 				| otherwise = "/~/" ++ d
 
 bupLocal :: BupRepo -> Bool
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index a6077d813c..7f78b2f493 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -96,7 +96,7 @@ storeEncrypted d (cipher, enck) k = do
 
 storeHelper :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
 storeHelper d key a = do
-	let dest = head $ locations d key
+	let dest = Prelude.head $ locations d key
 	let dir = parentDir dest
 	createDirectoryIfMissing True dir
 	allowWrite dir
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 81107cb561..c281420773 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -188,7 +188,7 @@ rsyncRemote o params = do
    directories. -}
 rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool
 rsyncSend o k src = withRsyncScratchDir $ \tmp -> do
-	let dest = tmp  head (keyPaths k)
+	let dest = tmp  Prelude.head (keyPaths k)
 	liftIO $ createDirectoryIfMissing True $ parentDir dest
 	liftIO $ createLink src dest
 	rsyncRemote o
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 567cf8e5bf..80554dc3bc 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -146,20 +146,20 @@ oldlog2key l =
 readKey1 :: String -> Key
 readKey1 v = 
 	if mixup
-		then fromJust $ readKey $ join ":" $ tail bits
+		then fromJust $ readKey $ join ":" $ Prelude.tail bits
 		else Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
 	where
 		bits = split ":" v
-		b = head bits
+		b = Prelude.head bits
 		n = join ":" $ drop (if wormy then 3 else 1) bits
 		t = if wormy
-			then Just (read (bits !! 1) :: EpochTime)
+			then Just (Prelude.read (bits !! 1) :: EpochTime)
 			else Nothing
 		s = if wormy
-			then Just (read (bits !! 2) :: Integer)
+			then Just (Prelude.read (bits !! 2) :: Integer)
 			else Nothing
-		wormy = head bits == "WORM"
-		mixup = wormy && isUpper (head $ bits !! 1)
+		wormy = Prelude.head bits == "WORM"
+		mixup = wormy && isUpper (Prelude.head $ bits !! 1)
 
 showKey1 :: Key -> String
 showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } =
diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 49837b9273..47d38ae7ba 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -12,7 +12,7 @@ read :: Read a => String -> a
 read = Prelude.read
 
 {- head is a partial function; head [] is an error
- - Instead, use: take 1 -}
+ - Instead, use: take 1 or headMaybe -}
 head :: [a] -> a
 head = Prelude.head
 
@@ -27,10 +27,20 @@ init :: [a] -> [a]
 init = Prelude.init
 
 {- last too
- - Instead, use: end -}
+ - Instead, use: end or lastMaybe -}
 last :: [a] -> a
 last = Prelude.last
 
+{- Like head but Nothing on empty list. -}
+headMaybe :: [a] -> Maybe a
+headMaybe [] = Nothing
+headMaybe v = Just $ Prelude.head v
+
+{- Like last but Nothing on empty list. -}
+lastMaybe :: [a] -> Maybe a
+lastMaybe [] = Nothing
+lastMaybe v = Just $ Prelude.last v
+
 {- All but the last element of a list.
  - (Like init, but no error on an empty list.) -}
 beginning :: [a] -> [a]

From 9901fc04a0dd9972e8910b5039a9e51e43d2c732 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 18:23:07 -0400
Subject: [PATCH 2709/2835] move

---
 Utility/BadPrelude.hs | 13 ++++++++++++-
 Utility/Misc.hs       |  6 ------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 47d38ae7ba..825adfa025 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -7,7 +7,8 @@
 
 module Utility.BadPrelude where
 
-{- read should be avoided, as it throws an error -}
+{- read should be avoided, as it throws an error
+ - Instead, use: readMaybe -}
 read :: Read a => String -> a
 read = Prelude.read
 
@@ -31,6 +32,16 @@ init = Prelude.init
 last :: [a] -> a
 last = Prelude.last
 
+{- Attempts to read a value from a String.
+ -
+ - Ignores leading/trailing whitespace, and throws away any trailing
+ - text after the part that can be read.
+ -}
+readMaybe :: (Read a) => String -> Maybe a
+readMaybe s = case reads s of
+	((x,_):_) -> Just x
+	_ -> Nothing
+
 {- Like head but Nothing on empty list. -}
 headMaybe :: [a] -> Maybe a
 headMaybe [] = Nothing
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index e95ac4319d..1d3c0e6763 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -21,12 +21,6 @@ hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict = readFile >=> \s -> length s `seq` return s
 
-{- Attempts to read a value from a String. -}
-readMaybe :: (Read a) => String -> Maybe a
-readMaybe s = case reads s of
-	((x,_):_) -> Just x
-	_ -> Nothing
-
 {- Like break, but the character matching the condition is not included
  - in the second result list.
  -

From ec11908799201024f052203a08e95b75efa3c098 Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Fri, 16 Dec 2011 00:13:54 +0000
Subject: [PATCH 2710/2835]

---
 .../git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 0dad8856e8..8df3608dbe 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -99,3 +99,5 @@ And yes, only the hash *annex copy* is checking for exists on the remote side. -
 > here's an already decrypted cipher -- it must be the right one!
 > 
 > Problem reproduced here, and fixed. [[done]] --[[Joey]]
+
+THX Joey! -- [[gebi]]

From 718a278f973e8005ce4ba76717218abcabe5d0b1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 22:19:05 -0400
Subject: [PATCH 2711/2835] simplify

---
 Utility/BadPrelude.hs | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 825adfa025..04c9d9b0b1 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -56,16 +56,9 @@ lastMaybe v = Just $ Prelude.last v
  - (Like init, but no error on an empty list.) -}
 beginning :: [a] -> [a]
 beginning [] = []
-beginning (x:xs) =  beginning' x xs
-	where
-		beginning' _ [] = []
-		beginning' y (z:zs) = y : beginning' z zs
+beginning l = Prelude.init l
 
 {- Like last, but no error on an empty list. -}
 end :: [a] -> [a]
 end [] = []
-end (x:xs) = end' x xs
-	where
-		end' y [] = [y]
-		end' _ (y:ys) = end' y ys
-
+end l = [Prelude.last l]

From a3e4bf8c168c1b67960735bc244fc4d96c9406b5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm_-2XlXNyd6cCLI4n_jaBNqVUOWwJquko"
 
Date: Fri, 16 Dec 2011 09:23:21 +0000
Subject: [PATCH 2712/2835]

---
 ...sed_repository:_starting_from_nothing.mdwn | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 doc/tips/centralised_repository:_starting_from_nothing.mdwn

diff --git a/doc/tips/centralised_repository:_starting_from_nothing.mdwn b/doc/tips/centralised_repository:_starting_from_nothing.mdwn
new file mode 100644
index 0000000000..b12246d368
--- /dev/null
+++ b/doc/tips/centralised_repository:_starting_from_nothing.mdwn
@@ -0,0 +1,75 @@
+If you are starting from nothing (no existing `git` or `git-annex` repository) and want to use a server as a centralised repository, try the following steps.
+
+On the server where you'll hold the "master" repository:
+
+	server$ cd /one/git
+	server$ mkdir m
+	server$ cd m
+	server$ git init --bare
+	Initialized empty Git repository in /one/git/m/
+	server$ git annex init origin
+	init origin ok
+	server$ 
+
+Clone that to the laptop:
+
+	laptop$ cd /other
+	laptop$ git clone ssh://server//one/git/m
+	Cloning into 'm'...
+	Warning: No xauth data; using fake authentication data for X11 forwarding.
+	remote: Counting objects: 5, done.        
+	remote: Compressing objects: 100% (3/3), done.        
+	remote: Total 5 (delta 0), reused 0 (delta 0)        
+	Receiving objects: 100% (5/5), done.
+	warning: remote HEAD refers to nonexistent ref, unable to checkout.
+
+	laptop$ cd m
+	laptop$ git annex init laptop
+	init laptop ok
+        laptop$ 
+
+Merge the `git-annex` repository (this is the bit that is often
+overlooked!):
+
+	laptop$ git annex merge	
+	merge . (merging "origin/git-annex" into git-annex...)
+	ok
+	laptop$ 
+
+Add some content:
+
+	laptop$ git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	"kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg"
+	addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg (downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...) --2011-12-15 08:13:10--  http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	Resolving kitenet.net (kitenet.net)... 2001:41c8:125:49::10, 80.68.85.49
+	Connecting to kitenet.net (kitenet.net)|2001:41c8:125:49::10|:80... connected.
+	HTTP request sent, awaiting response... 200 OK
+	Length: 39362757 (38M) [audio/ogg]
+	Saving to: `/other/m/.git/annex/tmp/URL--http&c%%kitenet.net%~joey%screencasts%git-annex_coding_in_haskell.ogg'
+
+	100%[======================================>] 39,362,757  2.31M/s   in 17s     
+
+	2011-12-15 08:13:27 (2.21 MB/s) - `/other/m/.git/annex/tmp/URL--http&c%%kitenet.net%~joey%screencasts%git-annex_coding_in_haskell.ogg' saved [39362757/39362757]
+
+	(checksum...) ok
+	(Recording state in git...)
+	laptop$ git commit -m 'See Joey play.'
+	[master (root-commit) 106e923] See Joey play.
+	 1 files changed, 1 insertions(+), 0 deletions(-)
+	  create mode 120000 kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg
+	laptop$ 
+
+All fine, now push it back to the centralised master:
+
+	laptop$ git push
+	Counting objects: 20, done.
+	Delta compression using up to 4 threads.
+	Compressing objects: 100% (11/11), done.
+	Writing objects: 100% (18/18), 1.50 KiB, done.
+	Total 18 (delta 1), reused 1 (delta 0)
+	To ssh://server//one/git/m
+	   3ba1386..ad3bc9e  git-annex -> git-annex
+	laptop$ 
+
+You can add more "client" repositories by following the `laptop`
+sequence of operations.

From 87c1c103eacf153aeaa79be9fd896df147a76c10 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 16 Dec 2011 16:56:31 -0400
Subject: [PATCH 2713/2835] add back message

---
 Command/Reinject.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 906f7c5176..0648e90fca 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -26,7 +26,11 @@ start (src:dest:[])
 	| otherwise = do
 		ifAnnexed src
 			(error $ "cannot used annexed file as src: " ++ src)
-			(next $ whenAnnexed (perform src) dest)
+			go
+	where
+		go = do
+			showStart "reinject" dest
+			next $ whenAnnexed (perform src) dest
 start _ = error "specify a src file and a dest file"
 
 perform :: FilePath -> FilePath -> (Key, Backend Annex) -> CommandPerform

From 9698eecf0f7410e03300db772c528a465ef58dc1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmKPMUX0YHBjE93eBsEnacwZsddSDue3PY"
 
Date: Sun, 18 Dec 2011 09:35:40 +0000
Subject: [PATCH 2714/2835]

---
 ..._synchronize_2_directories___40__like_unison__41__.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn

diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn
new file mode 100644
index 0000000000..86e317da87
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn
@@ -0,0 +1,7 @@
+I would like to use git-annex to synchronize 2 directories in the same manner as unison.
+
+I'm starting with 2 directories. There is an overlap of the same set of files in each directory, but each directory also has additional files as well.
+
+I create a git annex in each directory but when I do a git pull it merges and produces conflicts on those files that are the same.
+
+What is the correct workflow for this type of scenario?

From 3f01795a26f80a42e0e6b7397e7b521d9dbbd7b3 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sun, 18 Dec 2011 11:59:11 +0000
Subject: [PATCH 2715/2835]

---
 doc/bugs/Remote_repo_and_set_operation_with_find.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/Remote_repo_and_set_operation_with_find.mdwn

diff --git a/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
new file mode 100644
index 0000000000..2153cca4b2
--- /dev/null
+++ b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
@@ -0,0 +1,3 @@
+Currently, git annex find lists files that are present in the current repository, possibly restricted to a subdirectory. But it does not easily seem possible to get this information about a remote repository.
+
+I would find it useful if this command understood flags that makes it tell me what is present somewhere else (maybe "--on remote") and combinations of the flags ("--on remote1 --and --not-on remote2" or "--on disk1 --or --on disk2").

From a16f1accbcaae32372ecb9ffd2a9cf0f4ec5dec7 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sun, 18 Dec 2011 12:08:52 +0000
Subject: [PATCH 2716/2835] Added a comment

---
 ...ent_7_33db51096f568c65b22b4be0b5538c0d._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
new file mode 100644
index 0000000000..ce9f4266ff
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 7"
+ date="2011-12-18T12:08:51Z"
+ content="""
+I agree on the naming suggestions, and that it does not suit everybody. Maybe I’ll think some more about it. The point is: I’m trying to make live easy for those who do not want to manually create some complicated setup, so if it needs configuration, it is already off that track. But turning the current behavior into something people have to configure is also not well received by the users.
+
+Given that \"git annex sync\" is a new command, maybe it is fine to have this as a default behavior, and offer an easy way out. The easy way out could be one of two flags that can be set for a repo (or a remote):
+* \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
+* \"unsynced\", which makes git annex sync skip the repo.
+
+Maybe central is enough.
+"""]]

From fa9d83f144d8c35d58f584fe2690ed575ce9e096 Mon Sep 17 00:00:00 2001
From: Joachim Breitner 
Date: Sun, 18 Dec 2011 13:11:41 +0100
Subject: [PATCH 2717/2835] fix syntax

---
 .../comment_7_33db51096f568c65b22b4be0b5538c0d._comment       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
index ce9f4266ff..02d68d245c 100644
--- a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
+++ b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
@@ -7,8 +7,8 @@
 I agree on the naming suggestions, and that it does not suit everybody. Maybe I’ll think some more about it. The point is: I’m trying to make live easy for those who do not want to manually create some complicated setup, so if it needs configuration, it is already off that track. But turning the current behavior into something people have to configure is also not well received by the users.
 
 Given that \"git annex sync\" is a new command, maybe it is fine to have this as a default behavior, and offer an easy way out. The easy way out could be one of two flags that can be set for a repo (or a remote):
-* \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
-* \"unsynced\", which makes git annex sync skip the repo.
+ * \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
+ * \"unsynced\", which makes git annex sync skip the repo.
 
 Maybe central is enough.
 """]]

From 0bba09cd5b553118a0e956859cb5ffc9bc79a92c Mon Sep 17 00:00:00 2001
From: Joachim Breitner 
Date: Sun, 18 Dec 2011 13:12:28 +0100
Subject: [PATCH 2718/2835] fix syntax

---
 .../comment_7_33db51096f568c65b22b4be0b5538c0d._comment      | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
index 02d68d245c..753a2af169 100644
--- a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
+++ b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
@@ -7,8 +7,9 @@
 I agree on the naming suggestions, and that it does not suit everybody. Maybe I’ll think some more about it. The point is: I’m trying to make live easy for those who do not want to manually create some complicated setup, so if it needs configuration, it is already off that track. But turning the current behavior into something people have to configure is also not well received by the users.
 
 Given that \"git annex sync\" is a new command, maybe it is fine to have this as a default behavior, and offer an easy way out. The easy way out could be one of two flags that can be set for a repo (or a remote):
- * \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
- * \"unsynced\", which makes git annex sync skip the repo.
+
+* \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
+* \"unsynced\", which makes git annex sync skip the repo.
 
 Maybe central is enough.
 """]]

From 6a9b3c2f22eb348307c53fa16b39f0807196873c Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sun, 18 Dec 2011 13:57:34 +0000
Subject: [PATCH 2719/2835] Added a comment

---
 ...comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment

diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment
new file mode 100644
index 0000000000..4682ea64f7
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 1"
+ date="2011-12-18T13:57:33Z"
+ content="""
+Are the files identical or different? I today did something like that with similar, but not identical directories containing media files, and git happily merged them. but there, same files had same content.
+
+Also, make sure you use the same backend. In my case, one of the machines runs Debian stable, so I use the WORM backend, not the SHA backend.
+"""]]

From e341cdf0da8ae9398f0f96370b0a64950c1ffc30 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Mon, 19 Dec 2011 12:45:19 +0000
Subject: [PATCH 2720/2835] Added a comment: extra level of indirection

---
 ..._f63de6fe2f7189c8c2908cc41c4bc963._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment b/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment
new file mode 100644
index 0000000000..7398ac5614
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="extra level of indirection"
+ date="2011-12-19T12:45:18Z"
+ content="""
+Surely this could be handled with an extra layer of indirection?
+
+git-annex would ensure that every directory containing annexed data contains a new symlink `.git-annex` which points to `$git_root/.git/annex`.  Then every symlink to an annexed object uses a relative symlink via this: `.git_annex/objects/xx/yy/ZZZZZZZZZZ`.  Even though this symlink is relative, moving it to a different directory would not break anything: if the move destination directory already contained other annexed data, it would also already contain `.git-annex` so git-annex wouldn't need to do anything.  And if it didn't, git-annex would simply create a new `.git-annex` symlink there.
+
+These `.git-annex` symlinks could either be added to `.gitignore`, or manually/automatically checked in to the current branch - I'm not sure which would be best.  There's also the option of using multiple levels of indirection:
+
+    foo/bar/baz/.git-annex -> ../.git-annex
+    foo/bar/.git-annex -> ../.git-annex
+    foo/.git-annex -> ../.git-annex
+    .git-annex -> .git/annex
+
+I'm not sure whether this would bring any advantages.  It might bring a performance hit due to the kernel having to traverse more symlinks, but without benchmarking it's difficult to say how much.  I'd expect it only to be an issue with a large number of deep directory trees.
+"""]]

From 9b32d02a241a547feb827465ee7693c1f832626b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Dec 2011 18:22:25 +0000
Subject: [PATCH 2721/2835] Added a comment

---
 ...comment_7_7f20d0b2f6ed1c34021a135438037306._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment b/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment
new file mode 100644
index 0000000000..0a045feb63
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-12-19T18:22:25Z"
+ content="""
+That seems an excellent idea, also eliminating the need for git annex fix after moving. 
+
+However, I think CVS and svn have taught us the pain associated with a version control system putting something in every subdirectory. Would this pain be worth avoiding the minor pain of needing git annex fix and sometimes being unable to follow renames?
+"""]]

From cddd521e1feb64a1264a3f2c222076a0a994abff Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Dec 2011 18:24:59 +0000
Subject: [PATCH 2722/2835] Added a comment

---
 ...comment_2_648946353c6d90c57351cce4010f1301._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment

diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment
new file mode 100644
index 0000000000..bdd4b25e43
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-19T18:24:59Z"
+ content="""
+I'd recommend using the SHA backend for this, the WORM backend would produce conflicts if the files' modification times changed. 
+
+[[syncing_non-git_trees_with_git-annex]] describes one way to do it.
+"""]]

From 0b38397c6dd49fb0abd399ecebf1298f148e87c3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Dec 2011 18:29:01 +0000
Subject: [PATCH 2723/2835] Added a comment

---
 ...mment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment b/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment
new file mode 100644
index 0000000000..d33a296ca1
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-12-19T18:29:01Z"
+ content="""
+I don't mind changing the behavior of git-annex sync, certianly..
+
+Looking thru git's documentation, I found some existing configuration that could be reused following your idea.
+There is a remote.name.skipDefaultUpdate and a remote.name.skipFetchAll. Though both have to do with fetches, not pushes.
+Another approach might be to use git's remote group stuff.
+"""]]

From f0f84dbe48764b043c3373dc0e6a014b3a400b49 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Dec 2011 14:31:33 -0400
Subject: [PATCH 2724/2835] close

---
 doc/bugs/Remote_repo_and_set_operation_with_find.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
index 2153cca4b2..3e1acd4a81 100644
--- a/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
+++ b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
@@ -1,3 +1,6 @@
 Currently, git annex find lists files that are present in the current repository, possibly restricted to a subdirectory. But it does not easily seem possible to get this information about a remote repository.
 
 I would find it useful if this command understood flags that makes it tell me what is present somewhere else (maybe "--on remote") and combinations of the flags ("--on remote1 --and --not-on remote2" or "--on disk1 --or --on disk2").
+
+> Almost. You're looking for `--in remote`, which was added 2 months ago.
+> [[done]] --[[Joey]] 

From 89fd3b2e4bf302dffbf37b885c490fa3c716ee1b Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Mon, 19 Dec 2011 22:56:26 +0000
Subject: [PATCH 2725/2835] Added a comment

---
 ...ent_9_ace319652f9c7546883b5152ddc82591._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment b/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment
new file mode 100644
index 0000000000..de656d6629
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 9"
+ date="2011-12-19T22:56:26Z"
+ content="""
+Another option that would please the naive user without hindering the more advanced user: \"git annex init\", by default, creates a synced/master branch. \"git annex sync\" will pull from every /sync/master branch it finds, and also push to any /sync/master branch it finds, but will not create any. So by default (at least for new users), this provides simple one-step syncing.
+
+Advanced users can disable this per-repo by just deleting the synced/master branch. Presumably the logic will be: Every repo that should not be pushed to, because it has access to some central repo, should not have a synced/master branch. Every other repo, including the (or one of the few) central repos, will have the branch.
+
+This is not the most expressive solution, as it does not allow configuring syncing between arbitrary pairs of repos, but it feels like a good compromise between that and simplicity and transparency.
+
+I think it's about time that I provide less talk and more code. I’ll see when I find the time :-)
+"""]]

From 0de4341c97735e90ea8d90fa7f32579c268d0f84 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Tue, 20 Dec 2011 12:00:18 +0000
Subject: [PATCH 2726/2835] Added a comment

---
 ..._6a00500b24ba53248c78e1ffc8d1a591._comment | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment b/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment
new file mode 100644
index 0000000000..d53022302d
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 8"
+ date="2011-12-20T12:00:11Z"
+ content="""
+Personally I'd rather have working rename detection but I agree it's not 100% ideal to be littering multiple directories like this, so perhaps you could make it optional, e.g. based on a git config setting?
+
+Here are a few more considerations, some in defence of the approach, some against it:
+
+* `.git-annex` is hidden; `CVS/` is not.
+* Unlike `CVS/` and `.svn/`, it's only a symlink, not a directory containing other files.
+* It doesn't contain any data specific to that directory and could easily be regenerated if deleted accidentally or otherwise.
+* If a whole directory containing `.git-annex` was moved within the repository:
+    * git-annex would need to fix up these symlinks if and only if it's moved to a different depth within the tree.
+    * However, if the multi-level indirection approach is used, `.git-annex` in any subdirectory is *always* a symlink to `../.git-annex` so instead you would need to check that all of the new ancestors contain this symlink too, and optionally remove any no longer needed symlinks.
+    * In either case, git-annex already goes to the trouble of fixing symlinks, and if anything, I *think* this approach would reduce the number of symlinks which need checking (right?)
+* find `$git_root/foo -follow`, `diff -r` etc. would traverse into `$git_root/.git/annex`
+
+This last point is the only downside to this approach I can think of which gives me any noticeable cause for concern.  However, people are already use to working around this from CVS and svn days, e.g. `diff -r -x .svn` so I don't think it's anywhere near bad enough to rule it out.
+"""]]

From 92de7c8dcd598e6319e81778261ce900c2efe534 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 20 Dec 2011 14:56:12 +0000
Subject: [PATCH 2727/2835] Added a comment

---
 .../comment_9_75e0973f6d573df615e01005ebcea87d._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment b/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment
new file mode 100644
index 0000000000..919455bdcc
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 9"
+ date="2011-12-20T14:56:12Z"
+ content="""
+Git can follow the rename fine if the file is committed before `git annex fix` (you can git commit -n to see this), so 
+making git-annex pre-commit generate a fixup commit before the staged commit would be one way. Or the other two ways I originally mentioned when writing down this minor issue. I like all those approaches better than .git-annex clutter.
+"""]]

From dc5ed8d3b677f3c45118f5d5a89ad528ede7b171 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 11:01:50 -0400
Subject: [PATCH 2728/2835] amusing name

This is both a partial Prelude that conflicts with the real one, and a
way to guard against the Prelude's partial functions.
---
 Common.hs                                    | 2 +-
 Utility/{BadPrelude.hs => PartialPrelude.hs} | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename Utility/{BadPrelude.hs => PartialPrelude.hs} (92%)

diff --git a/Common.hs b/Common.hs
index f3dd701b14..90895f08e8 100644
--- a/Common.hs
+++ b/Common.hs
@@ -26,4 +26,4 @@ import Utility.Path as X
 import Utility.Directory as X
 import Utility.Monad as X
 
-import Utility.BadPrelude as X
+import Utility.PartialPrelude as X
diff --git a/Utility/BadPrelude.hs b/Utility/PartialPrelude.hs
similarity index 92%
rename from Utility/BadPrelude.hs
rename to Utility/PartialPrelude.hs
index 04c9d9b0b1..ad857196d6 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/PartialPrelude.hs
@@ -1,11 +1,11 @@
-{- Some stuff from Prelude should not be used, as it tends to be a source
- - of bugs.
+{- Parts of the Prelude are partial functions, which are a common source of
+ - bugs.
  -
  - This exports functions that conflict with the prelude, which avoids
  - them being accidentially used.
  -}
 
-module Utility.BadPrelude where
+module Utility.PartialPrelude where
 
 {- read should be avoided, as it throws an error
  - Instead, use: readMaybe -}

From da0bdc1a57d44ecfc7040affed725db4528d359a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 12:23:49 -0400
Subject: [PATCH 2729/2835] Fix the hook special remote, which bitrotted a
 while ago.

---
 Remote/Hook.hs   | 7 +++----
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index ab84533b28..5c761f43b0 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -62,13 +62,12 @@ hookEnv k f = Just $ fileenv f ++ keyenv
 		env s v = ("ANNEX_" ++ s, v)
 		keyenv =
 			[ env "KEY" (show k)
-			, env "HASH_1" hash_1
-			, env "HASH_2" hash_2
+			, env "HASH_1" (hashbits !! 0)
+			, env "HASH_2" (hashbits !! 1)
 			]
 		fileenv Nothing = []
 		fileenv (Just file) =  [env "FILE" file]
-		[hash_1, hash_2, _rest] =
-			map takeDirectory $ splitPath $ hashDirMixed k
+		hashbits = map takeDirectory $ splitPath $ hashDirMixed k
 
 lookupHook :: String -> String -> Annex (Maybe String)
 lookupHook hooktype hook =do
diff --git a/debian/changelog b/debian/changelog
index 3c977e817d..e187d8f6fa 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Union merge now finds the least expensive way to represent the merge.
   * reinject: Add a sanity check for using an annexed file as the source file.
   * Properly handle multiline git config values.
+  * Fix the hook special remote, which bitrotted a while ago.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From ee3b5b2a4279292d55af43c772cdfd0c56420798 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 14:37:53 -0400
Subject: [PATCH 2730/2835] use Common in a few more modules

---
 Annex.hs        |  6 +++---
 Annex/Ssh.hs    |  4 +---
 Git/CatFile.hs  | 10 +++++-----
 Git/Filename.hs | 11 +++++------
 Git/Index.hs    |  2 ++
 Git/LsFiles.hs  | 14 +++++++-------
 Git/LsTree.hs   |  4 ++--
 Git/Queue.hs    | 10 +++++-----
 8 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index acc38a5db9..3021732f05 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -29,7 +29,7 @@ import Common
 import qualified Git
 import qualified Git.Config
 import Git.CatFile
-import Git.Queue
+import qualified Git.Queue
 import Types.Backend
 import qualified Types.Remote
 import Types.Crypto
@@ -57,7 +57,7 @@ data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [Backend Annex]
 	, remotes :: [Types.Remote.Remote Annex]
-	, repoqueue :: Queue
+	, repoqueue :: Git.Queue.Queue
 	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
@@ -80,7 +80,7 @@ newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
-	, repoqueue = Git.Queue.empty
+	, repoqueue = Git.Queue.new
 	, output = NormalOutput
 	, force = False
 	, fast = False
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index fe83aad001..81e488b41d 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -7,11 +7,9 @@
 
 module Annex.Ssh where
 
-import Control.Monad.State (liftIO)
-
+import Common
 import qualified Git
 import qualified Git.Url
-import Utility.SafeCommand
 import Types
 import Config
 import Annex.UUID
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 2cef9d5b3d..16f0b11b95 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -19,10 +19,10 @@ import System.IO
 import qualified Data.ByteString.Char8 as S
 import qualified Data.ByteString.Lazy.Char8 as L
 
+import Common
 import Git
 import Git.Sha
 import Git.Command
-import Utility.SafeCommand
 
 type CatFileHandle = (PipeHandle, Handle, Handle)
 
@@ -53,21 +53,21 @@ catObject (_, from, to) object = do
 		[sha, objtype, size]
 			| length sha == shaSize &&
 			  validobjtype objtype -> handle size
-			| otherwise -> empty
+			| otherwise -> dne
 		_
-			| header == show object ++ " missing" -> empty
+			| header == show object ++ " missing" -> dne
 			| otherwise -> error $ "unknown response from git cat-file " ++ header
 	where
 		handle size = case reads size of
 			[(bytes, "")] -> readcontent bytes
-			_ -> empty
+			_ -> dne
 		readcontent bytes = do
 			content <- S.hGet from bytes
 			c <- hGetChar from
 			when (c /= '\n') $
 				error "missing newline from git cat-file"
 			return $ L.fromChunks [content]
-		empty = return L.empty
+		dne = return L.empty
 		validobjtype t
 			| t == "blob" = True
 			| t == "commit" = True
diff --git a/Git/Filename.hs b/Git/Filename.hs
index 69f36d0861..35b5532507 100644
--- a/Git/Filename.hs
+++ b/Git/Filename.hs
@@ -13,22 +13,21 @@ import Data.Char
 import Data.Word (Word8)
 import Text.Printf
 
+import Common
+
 decode :: String -> FilePath
 decode [] = []
 decode f@(c:s)
 	-- encoded strings will be inside double quotes
-	| c == '"' = unescape ("", middle)
+	| c == '"' && end s == ['"'] = unescape ("", beginning s)
 	| otherwise = f
 	where
 		e = '\\'
-		middle = init s
 		unescape (b, []) = b
 		-- look for escapes starting with '\'
-		unescape (b, v) = b ++ beginning ++ unescape (handle rest)
+		unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair)
 			where
 				pair = span (/= e) v
-				beginning = fst pair
-				rest = snd pair
 		isescape x = x == e
 		-- \NNN is an octal encoded character
 		handle (x:n1:n2:n3:rest)
@@ -38,7 +37,7 @@ decode f@(c:s)
 						isOctDigit n2 &&
 						isOctDigit n3
 					fromoctal = [chr $ readoctal [n1, n2, n3]]
-					readoctal o = read $ "0o" ++ o :: Int
+					readoctal o = Prelude.read $ "0o" ++ o :: Int
 		-- \C is used for a few special characters
 		handle (x:nc:rest)
 			| isescape x = ([echar nc], rest)
diff --git a/Git/Index.hs b/Git/Index.hs
index aaf54e032e..e58f1edd67 100644
--- a/Git/Index.hs
+++ b/Git/Index.hs
@@ -9,6 +9,8 @@ module Git.Index where
 
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
+import Common
+
 {- Forces git to use the specified index file.
  -
  - Returns an action that will reset back to the default
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 0c71ed884b..0de86383d3 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -15,9 +15,9 @@ module Git.LsFiles (
 	typeChangedStaged,
 ) where
 
+import Common
 import Git
 import Git.Command
-import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: [FilePath] -> Repo -> IO [FilePath]
@@ -43,10 +43,10 @@ stagedNotDeleted :: [FilePath] -> Repo -> IO [FilePath]
 stagedNotDeleted = staged' [Param "--diff-filter=ACMRT"]
 
 staged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
-staged' middle l = pipeNullSplit $ start ++ middle ++ end
+staged' ps l = pipeNullSplit $ prefix ++ ps ++ suffix
 	where
-		start = [Params "diff --cached --name-only -z"]
-		end = Param "--" : map File l
+		prefix = [Params "diff --cached --name-only -z"]
+		suffix = Param "--" : map File l
 
 {- Returns a list of files that have unstaged changes. -}
 changedUnstaged :: [FilePath] -> Repo -> IO [FilePath]
@@ -65,7 +65,7 @@ typeChanged :: [FilePath] -> Repo -> IO [FilePath]
 typeChanged = typeChanged' []
 
 typeChanged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
-typeChanged' middle l = pipeNullSplit $ start ++ middle ++ end
+typeChanged' ps l = pipeNullSplit $ prefix ++ ps ++ suffix
 	where
-		start = [Params "diff --name-only --diff-filter=T -z"]
-		end = Param "--" : map File l
+		prefix = [Params "diff --name-only --diff-filter=T -z"]
+		suffix = Param "--" : map File l
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 919e9af83a..aae7f1263b 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -16,10 +16,10 @@ import Control.Applicative
 import System.Posix.Types
 import qualified Data.ByteString.Lazy.Char8 as L
 
+import Common
 import Git
 import Git.Command
 import qualified Git.Filename
-import Utility.SafeCommand
 
 data TreeItem = TreeItem
 	{ mode :: FileMode
@@ -37,7 +37,7 @@ lsTree t repo = map parseLsTree <$>
  - (The --long format is not currently supported.) -}
 parseLsTree :: L.ByteString -> TreeItem
 parseLsTree l = TreeItem 
-	{ mode = fst $ head $ readOct $ L.unpack m
+	{ mode = fst $ Prelude.head $ readOct $ L.unpack m
 	, typeobj = L.unpack t
 	, sha = L.unpack s
 	, file = Git.Filename.decode $ L.unpack f
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 73470b1f0e..25c5b073c7 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -7,7 +7,7 @@
 
 module Git.Queue (
 	Queue,
-	empty,
+	new,
 	add,
 	size,
 	full,
@@ -18,9 +18,9 @@ import qualified Data.Map as M
 import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
-import Control.Monad (forM_)
 import Utility.SafeCommand
 
+import Common
 import Git
 import Git.Command
 
@@ -50,8 +50,8 @@ maxSize :: Int
 maxSize = 10240
 
 {- Constructor for empty queue. -}
-empty :: Queue
-empty = Queue 0 M.empty
+new :: Queue
+new = Queue 0 M.empty
 
 {- Adds an action to a queue. -}
 add :: Queue -> String -> [CommandParam] -> [FilePath] -> Queue
@@ -76,7 +76,7 @@ full (Queue n _) = n > maxSize
 flush :: Queue -> Repo -> IO Queue
 flush (Queue _ m) repo = do
 	forM_ (M.toList m) $ uncurry $ runAction repo
-	return empty
+	return new
 
 {- Runs an Action on a list of files in a git repository.
  -

From 4eec36c816fd0160f9963515d17d63031b2cab44 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 15:13:36 -0400
Subject: [PATCH 2731/2835] added a test of the hook special remote

---
 test.hs | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/test.hs b/test.hs
index daa2661b6e..86423ba0f2 100644
--- a/test.hs
+++ b/test.hs
@@ -102,6 +102,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_fsck
 	, test_migrate
 	, test_unused
+	, test_hook_remote
 	]
 
 test_init :: Test
@@ -482,6 +483,32 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 			r <- Backend.lookupFile f
 			return $ fst $ fromJust r
 
+test_hook_remote :: Test
+test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
+	git_annex "initremote" (words "foo type=hook encryption=none hooktype=foo") @? "initremote failed"
+	createDirectory dir
+	git_config "annex.foo-store-hook" $
+		"cp $ANNEX_FILE " ++ loc
+	git_config "annex.foo-retrieve-hook" $
+		"cp " ++ loc ++ " $ANNEX_FILE"
+	git_config "annex.foo-remove-hook" $
+		"rm -f " ++ loc
+	git_config "annex.foo-checkpresent-hook" $
+		"if [ -e " ++ loc ++ " ]; then echo $ANNEX_KEY; fi"
+	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	annexed_present annexedfile
+	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to hook remote failed"
+	annexed_present annexedfile
+	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	annexed_notpresent annexedfile
+	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from hook remote failed"
+	annexed_present annexedfile
+	where
+		dir = "dir"
+		loc = dir ++ "/$ANNEX_KEY"
+		git_config k v = boolSystem "git" [Param "config", Param k, Param v]
+			@? "git config failed"
+
 -- This is equivilant to running git-annex, but it's all run in-process
 -- so test coverage collection works.
 git_annex :: String -> [String] -> IO Bool

From b6664d5c94a4cd63da10cadde487cb2ced265a23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 15:17:56 -0400
Subject: [PATCH 2732/2835] add tests of directory and rsync special remotes

---
 test.hs | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/test.hs b/test.hs
index 86423ba0f2..1474c975ef 100644
--- a/test.hs
+++ b/test.hs
@@ -103,6 +103,8 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_migrate
 	, test_unused
 	, test_hook_remote
+	, test_directory_remote
+	, test_rsync_remote
 	]
 
 test_init :: Test
@@ -503,12 +505,47 @@ test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from hook remote failed"
 	annexed_present annexedfile
+	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	not r @? "drop failed to fail"
+	annexed_present annexedfile
 	where
 		dir = "dir"
 		loc = dir ++ "/$ANNEX_KEY"
 		git_config k v = boolSystem "git" [Param "config", Param k, Param v]
 			@? "git config failed"
 
+test_directory_remote :: Test
+test_directory_remote = "git-annex directory remote" ~: intmpclonerepo $ do
+	createDirectory "dir"
+	git_annex "initremote" (words $ "foo type=directory encryption=none directory=dir") @? "initremote failed"
+	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	annexed_present annexedfile
+	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to directory remote failed"
+	annexed_present annexedfile
+	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	annexed_notpresent annexedfile
+	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from directory remote failed"
+	annexed_present annexedfile
+	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	not r @? "drop failed to fail"
+	annexed_present annexedfile
+
+test_rsync_remote :: Test
+test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
+	createDirectory "dir"
+	git_annex "initremote" (words $ "foo type=rsync encryption=none rsyncurl=dir") @? "initremote failed"
+	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	annexed_present annexedfile
+	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to rsync remote failed"
+	annexed_present annexedfile
+	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	annexed_notpresent annexedfile
+	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from rsync remote failed"
+	annexed_present annexedfile
+	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	not r @? "drop failed to fail"
+	annexed_present annexedfile
+
 -- This is equivilant to running git-annex, but it's all run in-process
 -- so test coverage collection works.
 git_annex :: String -> [String] -> IO Bool

From c4c8c80f49878aecf7b8fc8200fefb1962e23574 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 15:48:25 -0400
Subject: [PATCH 2733/2835] factor out -q

---
 test.hs | 214 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 107 insertions(+), 107 deletions(-)

diff --git a/test.hs b/test.hs
index 1474c975ef..86eb006a38 100644
--- a/test.hs
+++ b/test.hs
@@ -109,7 +109,7 @@ blackbox = TestLabel "blackbox" $ TestList
 
 test_init :: Test
 test_init = "git-annex init" ~: TestCase $ innewrepo $ do
-	git_annex "init" ["-q", reponame] @? "init failed"
+	git_annex "init" [reponame] @? "init failed"
 	where
 		reponame = "test repo"
 
@@ -120,38 +120,38 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 		-- annexed file that later tests will use
 		basic = TestCase $ inmainrepo $ do
 			writeFile annexedfile $ content annexedfile
-			git_annex "add" ["-q", annexedfile] @? "add failed"
+			git_annex "add" [annexedfile] @? "add failed"
 			annexed_present annexedfile
 			writeFile sha1annexedfile $ content sha1annexedfile
-			git_annex "add" ["-q", sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
+			git_annex "add" [sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
 			annexed_present sha1annexedfile
 			writeFile ingitfile $ content ingitfile
 			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
 			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
-			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
+			git_annex "add" [ingitfile] @? "add ingitfile should be no-op"
 			unannexed ingitfile
 		sha1dup = TestCase $ intmpclonerepo $ do
 			writeFile sha1annexedfiledup $ content sha1annexedfiledup
-			git_annex "add" ["-q", sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
+			git_annex "add" [sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
 			annexed_present sha1annexedfiledup
 			annexed_present sha1annexedfile
 		subdirs = TestCase $ intmpclonerepo $ do
 			createDirectory "dir"
 			writeFile "dir/foo" $ content annexedfile
-			git_annex "add" ["-q", "dir"] @? "add of subdir failed"
+			git_annex "add" ["dir"] @? "add of subdir failed"
 			createDirectory "dir2"
 			writeFile "dir2/foo" $ content annexedfile
 			changeWorkingDirectory "dir"
-			git_annex "add" ["-q", "../dir2"] @? "add of ../subdir failed"
+			git_annex "add" ["../dir2"] @? "add of ../subdir failed"
 
 test_reinject :: Test
 test_reinject = "git-annex reinject/fromkey" ~: TestCase $ intmpclonerepo $ do
-	git_annex "drop" ["-q", "--force", sha1annexedfile] @? "drop failed"
+	git_annex "drop" ["--force", sha1annexedfile] @? "drop failed"
 	writeFile tmp $ content sha1annexedfile
 	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
 	let key = show $ fromJust r
-	git_annex "reinject" ["-q", tmp, sha1annexedfile] @? "reinject failed"
-	git_annex "fromkey" ["-q", key, sha1annexedfiledup] @? "fromkey failed"
+	git_annex "reinject" [tmp, sha1annexedfile] @? "reinject failed"
+	git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed"
 	annexed_present sha1annexedfiledup
 	where
 		tmp = "tmpfile"
@@ -161,44 +161,44 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy]
 	where
 		nocopy = "no content" ~: intmpclonerepo $ do
 			annexed_notpresent annexedfile
-			git_annex "unannex" ["-q", annexedfile] @? "unannex failed with no copy"
+			git_annex "unannex" [annexedfile] @? "unannex failed with no copy"
 			annexed_notpresent annexedfile
 		withcopy = "with content" ~: intmpclonerepo $ do
-			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "get" [annexedfile] @? "get failed"
 			annexed_present annexedfile
-			git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed"
+			git_annex "unannex" [annexedfile, sha1annexedfile] @? "unannex failed"
 			unannexed annexedfile
-			git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file"
+			git_annex "unannex" [annexedfile] @? "unannex failed on non-annexed file"
 			unannexed annexedfile
-			git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op"
+			git_annex "unannex" [ingitfile] @? "unannex ingitfile should be no-op"
 			unannexed ingitfile
 
 test_drop :: Test
 test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do
-			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "get" [annexedfile] @? "get failed"
 			boolSystem "git" [Params "remote rm origin"]
 				@? "git remote rm origin failed"
-			r <- git_annex "drop" ["-q", annexedfile]
+			r <- git_annex "drop" [annexedfile]
 			not r @? "drop wrongly succeeded with no known copy of file"
 			annexed_present annexedfile
-			git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed"
+			git_annex "drop" ["--force", annexedfile] @? "drop --force failed"
 			annexed_notpresent annexedfile
-			git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed"
-			git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op"
+			git_annex "drop" [annexedfile] @? "drop of dropped file failed"
+			git_annex "drop" [ingitfile] @? "drop ingitfile should be no-op"
 			unannexed ingitfile
 		withremote = "with remote" ~: TestCase $ intmpclonerepo $ do
-			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "get" [annexedfile] @? "get failed"
 			annexed_present annexedfile
-			git_annex "drop" ["-q", annexedfile] @? "drop failed though origin has copy"
+			git_annex "drop" [annexedfile] @? "drop failed though origin has copy"
 			annexed_notpresent annexedfile
 			inmainrepo $ annexed_present annexedfile
 		untrustedremote = "untrusted remote" ~: TestCase $ intmpclonerepo $ do
-			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
-			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "untrust" ["origin"] @? "untrust of origin failed"
+			git_annex "get" [annexedfile] @? "get failed"
 			annexed_present annexedfile
-			r <- git_annex "drop" ["-q", annexedfile]
+			r <- git_annex "drop" [annexedfile]
 			not r @? "drop wrongly suceeded with only an untrusted copy of the file"
 			annexed_present annexedfile
 			inmainrepo $ annexed_present annexedfile
@@ -207,15 +207,15 @@ test_get :: Test
 test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do
 	inmainrepo $ annexed_present annexedfile
 	annexed_notpresent annexedfile
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	inmainrepo $ annexed_present annexedfile
 	annexed_present annexedfile
-	git_annex "get" ["-q", annexedfile] @? "get of file already here failed"
+	git_annex "get" [annexedfile] @? "get of file already here failed"
 	inmainrepo $ annexed_present annexedfile
 	annexed_present annexedfile
 	inmainrepo $ unannexed ingitfile
 	unannexed ingitfile
-	git_annex "get" ["-q", ingitfile] @? "get ingitfile should be no-op"
+	git_annex "get" [ingitfile] @? "get ingitfile should be no-op"
 	inmainrepo $ unannexed ingitfile
 	unannexed ingitfile
 
@@ -223,24 +223,24 @@ test_move :: Test
 test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file failed"
+	git_annex "move" ["--from", "origin", annexedfile] @? "move --from of file failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_notpresent annexedfile
-	git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file already here failed"
+	git_annex "move" ["--from", "origin", annexedfile] @? "move --from of file already here failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_notpresent annexedfile
-	git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file failed"
+	git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file failed"
 	inmainrepo $ annexed_present annexedfile
 	annexed_notpresent annexedfile
-	git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed"
+	git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file already there failed"
 	inmainrepo $ annexed_present annexedfile
 	annexed_notpresent annexedfile
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
-	git_annex "move" ["-q", "--to", "origin", ingitfile] @? "move of ingitfile should be no-op"
+	git_annex "move" ["--to", "origin", ingitfile] @? "move of ingitfile should be no-op"
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
-	git_annex "move" ["-q", "--from", "origin", ingitfile] @? "move of ingitfile should be no-op"
+	git_annex "move" ["--from", "origin", ingitfile] @? "move of ingitfile should be no-op"
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
 
@@ -248,24 +248,24 @@ test_copy :: Test
 test_copy = "git-annex copy" ~: TestCase $ intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file failed"
+	git_annex "copy" ["--from", "origin", annexedfile] @? "copy --from of file failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file already here failed"
+	git_annex "copy" ["--from", "origin", annexedfile] @? "copy --from of file already here failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "copy" ["-q", "--to", "origin", annexedfile] @? "copy --to of file already there failed"
+	git_annex "copy" ["--to", "origin", annexedfile] @? "copy --to of file already there failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed"
+	git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file already there failed"
 	annexed_notpresent annexedfile
 	inmainrepo $ annexed_present annexedfile
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
-	git_annex "copy" ["-q", "--to", "origin", ingitfile] @? "copy of ingitfile should be no-op"
+	git_annex "copy" ["--to", "origin", ingitfile] @? "copy of ingitfile should be no-op"
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
-	git_annex "copy" ["-q", "--from", "origin", ingitfile] @? "copy of ingitfile should be no-op"
+	git_annex "copy" ["--from", "origin", ingitfile] @? "copy of ingitfile should be no-op"
 	checkregularfile ingitfile
 	checkcontent ingitfile
 
@@ -273,36 +273,36 @@ test_lock :: Test
 test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do
 	-- regression test: unlock of not present file should skip it
 	annexed_notpresent annexedfile
-	r <- git_annex "unlock" ["-q", annexedfile]
+	r <- git_annex "unlock" [annexedfile]
 	not r @? "unlock failed to fail with not present file"
 	annexed_notpresent annexedfile
 
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "unlock" ["-q", annexedfile] @? "unlock failed"		
+	git_annex "unlock" [annexedfile] @? "unlock failed"		
 	unannexed annexedfile
 	-- write different content, to verify that lock
 	-- throws it away
 	changecontent annexedfile
 	writeFile annexedfile $ content annexedfile ++ "foo"
-	git_annex "lock" ["-q", annexedfile] @? "lock failed"
+	git_annex "lock" [annexedfile] @? "lock failed"
 	annexed_present annexedfile
-	git_annex "unlock" ["-q", annexedfile] @? "unlock failed"		
+	git_annex "unlock" [annexedfile] @? "unlock failed"		
 	unannexed annexedfile
 	changecontent annexedfile
-	git_annex "add" ["-q", annexedfile] @? "add of modified file failed"
+	git_annex "add" [annexedfile] @? "add of modified file failed"
 	runchecks [checklink, checkunwritable] annexedfile
 	c <- readFile annexedfile
 	assertEqual "content of modified file" c (changedcontent annexedfile)
-	r' <- git_annex "drop" ["-q", annexedfile]
+	r' <- git_annex "drop" [annexedfile]
 	not r' @? "drop wrongly succeeded with no known copy of modified file"
 
 test_edit :: Test
 test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 	where t precommit = TestCase $ intmpclonerepo $ do
-		git_annex "get" ["-q", annexedfile] @? "get of file failed"
+		git_annex "get" [annexedfile] @? "get of file failed"
 		annexed_present annexedfile
-		git_annex "edit" ["-q", annexedfile] @? "edit failed"
+		git_annex "edit" [annexedfile] @? "edit failed"
 		unannexed annexedfile
 		changecontent annexedfile
 		if precommit
@@ -311,7 +311,7 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 				-- staged, normally git commit does this
 				boolSystem "git" [Param "add", File annexedfile]
 					@? "git add of edited file failed"
-				git_annex "pre-commit" ["-q"]
+				git_annex "pre-commit" []
 					@? "pre-commit failed"
 			else do
 				boolSystem "git" [Params "commit -q -a -m contentchanged"]
@@ -319,22 +319,22 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
 		assertEqual "content of modified file" c (changedcontent annexedfile)
-		r <- git_annex "drop" ["-q", annexedfile]
+		r <- git_annex "drop" [annexedfile]
 		not r @? "drop wrongly succeeded with no known copy of modified file"
 
 test_fix :: Test
 test_fix = "git-annex fix" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
-	git_annex "fix" ["-q", annexedfile] @? "fix of not present failed"
+	git_annex "fix" [annexedfile] @? "fix of not present failed"
 	annexed_notpresent annexedfile
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "fix" ["-q", annexedfile] @? "fix of present file failed"
+	git_annex "fix" [annexedfile] @? "fix of present file failed"
 	annexed_present annexedfile
 	createDirectory subdir
 	boolSystem "git" [Param "mv", File annexedfile, File subdir]
 		@? "git mv failed"
-	git_annex "fix" ["-q", newfile] @? "fix of moved file failed"
+	git_annex "fix" [newfile] @? "fix of moved file failed"
 	runchecks [checklink, checkunwritable] newfile
 	c <- readFile newfile
 	assertEqual "content of moved file" c (content annexedfile)
@@ -344,17 +344,17 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 
 test_trust :: Test
 test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
-	git_annex "trust" ["-q", repo] @? "trust failed"
+	git_annex "trust" [repo] @? "trust failed"
 	trustcheck Logs.Trust.Trusted "trusted 1"
-	git_annex "trust" ["-q", repo] @? "trust of trusted failed"
+	git_annex "trust" [repo] @? "trust of trusted failed"
 	trustcheck Logs.Trust.Trusted "trusted 2"
-	git_annex "untrust" ["-q", repo] @? "untrust failed"
+	git_annex "untrust" [repo] @? "untrust failed"
 	trustcheck Logs.Trust.UnTrusted "untrusted 1"
-	git_annex "untrust" ["-q", repo] @? "untrust of untrusted failed"
+	git_annex "untrust" [repo] @? "untrust of untrusted failed"
 	trustcheck Logs.Trust.UnTrusted "untrusted 2"
-	git_annex "semitrust" ["-q", repo] @? "semitrust failed"
+	git_annex "semitrust" [repo] @? "semitrust failed"
 	trustcheck Logs.Trust.SemiTrusted "semitrusted 1"
-	git_annex "semitrust" ["-q", repo] @? "semitrust of semitrusted failed"
+	git_annex "semitrust" [repo] @? "semitrust of semitrusted failed"
 	trustcheck Logs.Trust.SemiTrusted "semitrusted 2"
 	where
 		repo = "origin"
@@ -369,36 +369,36 @@ test_fsck :: Test
 test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withremoteuntrusted]
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
-			git_annex "fsck" ["-q"] @? "fsck failed"
+			git_annex "fsck" [] @? "fsck failed"
 			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			fsck_should_fail "numcopies unsatisfied"
 			boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
 		withlocaluntrusted = TestCase $ intmpclonerepo $ do
-			git_annex "get" ["-q", annexedfile] @? "get failed"
-			git_annex "untrust" ["-q", "origin"] @? "untrust of origin repo failed"
-			git_annex "untrust" ["-q", "."] @? "untrust of current repo failed"
+			git_annex "get" [annexedfile] @? "get failed"
+			git_annex "untrust" ["origin"] @? "untrust of origin repo failed"
+			git_annex "untrust" ["."] @? "untrust of current repo failed"
 			fsck_should_fail "content only available in untrusted (current) repository"
-			git_annex "trust" ["-q", "."] @? "trust of current repo failed"
-			git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo"
+			git_annex "trust" ["."] @? "trust of current repo failed"
+			git_annex "fsck" [annexedfile] @? "fsck failed on file present in trusted repo"
 		withremoteuntrusted = TestCase $ intmpclonerepo $ do
 			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
-			git_annex "get" ["-q", annexedfile] @? "get failed"
-			git_annex "get" ["-q", sha1annexedfile] @? "get failed"
-			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
-			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
+			git_annex "get" [annexedfile] @? "get failed"
+			git_annex "get" [sha1annexedfile] @? "get failed"
+			git_annex "fsck" [] @? "fsck failed with numcopies=2 and 2 copies"
+			git_annex "untrust" ["origin"] @? "untrust of origin failed"
 			fsck_should_fail "content not replicated to enough non-untrusted repositories"
 
 		corrupt f = do
-			git_annex "get" ["-q", f] @? "get of file failed"
+			git_annex "get" [f] @? "get of file failed"
 			Utility.FileMode.allowWrite f
 			writeFile f (changedcontent f)
-			r <- git_annex "fsck" ["-q"]
+			r <- git_annex "fsck" []
 			not r @? "fsck failed to fail with corrupted file content"
-			git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f
+			git_annex "fsck" [] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f
 		fsck_should_fail m = do
-			r <- git_annex "fsck" ["-q"]
+			r <- git_annex "fsck" []
 			not r @? "fsck failed to fail with " ++ m
 
 test_migrate :: Test
@@ -406,23 +406,23 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 	where t usegitattributes = TestCase $ intmpclonerepo $ do
 		annexed_notpresent annexedfile
 		annexed_notpresent sha1annexedfile
-		git_annex "migrate" ["-q", annexedfile] @? "migrate of not present failed"
-		git_annex "migrate" ["-q", sha1annexedfile] @? "migrate of not present failed"
-		git_annex "get" ["-q", annexedfile] @? "get of file failed"
-		git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
+		git_annex "migrate" [annexedfile] @? "migrate of not present failed"
+		git_annex "migrate" [sha1annexedfile] @? "migrate of not present failed"
+		git_annex "get" [annexedfile] @? "get of file failed"
+		git_annex "get" [sha1annexedfile] @? "get of file failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
 		if usegitattributes
 			then do
 				writeFile ".gitattributes" $ "* annex.backend=SHA1"
-				git_annex "migrate" ["-q", sha1annexedfile]
+				git_annex "migrate" [sha1annexedfile]
 					@? "migrate sha1annexedfile failed"
-				git_annex "migrate" ["-q", annexedfile]
+				git_annex "migrate" [annexedfile]
 					@? "migrate annexedfile failed"
 			else do
-				git_annex "migrate" ["-q", sha1annexedfile, "--backend", "SHA1"]
+				git_annex "migrate" [sha1annexedfile, "--backend", "SHA1"]
 					@? "migrate sha1annexedfile failed"
-				git_annex "migrate" ["-q", annexedfile, "--backend", "SHA1"]
+				git_annex "migrate" [annexedfile, "--backend", "SHA1"]
 					@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
@@ -431,9 +431,9 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 
 		-- check that reversing a migration works
 		writeFile ".gitattributes" $ "* annex.backend=WORM"
-		git_annex "migrate" ["-q", sha1annexedfile]
+		git_annex "migrate" [sha1annexedfile]
 			@? "migrate sha1annexedfile failed"
-		git_annex "migrate" ["-q", annexedfile]
+		git_annex "migrate" [annexedfile]
 			@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
@@ -451,8 +451,8 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	-- keys have to be looked up before files are removed
 	annexedfilekey <- annexeval $ findkey annexedfile
 	sha1annexedfilekey <- annexeval $ findkey sha1annexedfile
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
-	git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
+	git_annex "get" [sha1annexedfile] @? "get of file failed"
 	checkunused []
 	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
 	checkunused []
@@ -466,17 +466,17 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also
-	git_annex "dropkey" ["-q", "--force", show annexedfilekey]
+	git_annex "dropkey" ["--force", show annexedfilekey]
 		@? "dropkey failed"
 	checkunused [sha1annexedfilekey]
 
-	git_annex "dropunused" ["-q", "1", "2"] @? "dropunused failed"
+	git_annex "dropunused" ["1", "2"] @? "dropunused failed"
 	checkunused []
-	git_annex "dropunused" ["-q", "10", "501"] @? "dropunused failed on bogus numbers"
+	git_annex "dropunused" ["10", "501"] @? "dropunused failed on bogus numbers"
 
 	where
 		checkunused expectedkeys = do
-			git_annex "unused" ["-q"] @? "unused failed"
+			git_annex "unused" [] @? "unused failed"
 			unusedmap <- annexeval $ Command.DropUnused.readUnusedLog ""
 			let unusedkeys = M.elems unusedmap
 			assertEqual "unused keys differ"
@@ -497,15 +497,15 @@ test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 		"rm -f " ++ loc
 	git_config "annex.foo-checkpresent-hook" $
 		"if [ -e " ++ loc ++ " ]; then echo $ANNEX_KEY; fi"
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to hook remote failed"
+	git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to hook remote failed"
 	annexed_present annexedfile
-	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
 	annexed_notpresent annexedfile
-	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from hook remote failed"
+	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from hook remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
 	not r @? "drop failed to fail"
 	annexed_present annexedfile
 	where
@@ -518,15 +518,15 @@ test_directory_remote :: Test
 test_directory_remote = "git-annex directory remote" ~: intmpclonerepo $ do
 	createDirectory "dir"
 	git_annex "initremote" (words $ "foo type=directory encryption=none directory=dir") @? "initremote failed"
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to directory remote failed"
+	git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to directory remote failed"
 	annexed_present annexedfile
-	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
 	annexed_notpresent annexedfile
-	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from directory remote failed"
+	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from directory remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
 	not r @? "drop failed to fail"
 	annexed_present annexedfile
 
@@ -534,15 +534,15 @@ test_rsync_remote :: Test
 test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	createDirectory "dir"
 	git_annex "initremote" (words $ "foo type=rsync encryption=none rsyncurl=dir") @? "initremote failed"
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to rsync remote failed"
+	git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to rsync remote failed"
 	annexed_present annexedfile
-	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
 	annexed_notpresent annexedfile
-	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from rsync remote failed"
+	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from rsync remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
 	not r @? "drop failed to fail"
 	annexed_present annexedfile
 
@@ -556,7 +556,7 @@ git_annex command params = do
 		Right _ -> return True
 		Left _ -> return False
 	where
-		run = GitAnnex.run (command:params)
+		run = GitAnnex.run (command:"-q":params)
 
 -- Runs an action in the current annex. Note that shutdown actions
 -- are not run; this should only be used for actions that query state.

From f30d545256a56c9c39f2893aff2f75aad9dba924 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:02:59 -0400
Subject: [PATCH 2734/2835] remove seemingly unneeded dependencies

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index cef79be88c..6d2a2f8193 100644
--- a/Makefile
+++ b/Makefile
@@ -64,7 +64,7 @@ install: all
 		rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \
 	fi
 
-test: $(bins)
+test:
 	@if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
@@ -74,7 +74,7 @@ test: $(bins)
 		fi; \
 	fi
 
-testcoverage: $(bins)
+testcoverage:
 	rm -f test.tix test
 	ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test
 	./test

From 3d188d8db95d2a7e92e9adc4d672fa119ac6ebc8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:03:09 -0400
Subject: [PATCH 2735/2835] fix comment

---
 Annex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex.hs b/Annex.hs
index 3021732f05..e5792fbcb2 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -115,7 +115,7 @@ getState = gets
 
 {- Applies a state mutation function to change the internal state. 
  -
- - Example: changeState $ \s -> s { quiet = True }
+ - Example: changeState $ \s -> s { output = QuietOutput }
  -}
 changeState :: (AnnexState -> AnnexState) -> Annex ()
 changeState = modify

From 9f65410fd823c5e98037bcb4be282892feedf53c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:03:14 -0400
Subject: [PATCH 2736/2835] run annex monad in quiet mode when testing

This cleans up the merging messages.
---
 test.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index 86eb006a38..5c43c3d4d9 100644
--- a/test.hs
+++ b/test.hs
@@ -565,7 +565,7 @@ annexeval a = do
 	g <- Git.Construct.fromCwd
 	g' <- Git.Config.read g
 	s <- Annex.new g'
-	Annex.eval s a
+	Annex.eval s { Annex.output = Annex.QuietOutput } a
 
 innewrepo :: Assertion -> Assertion
 innewrepo a = withgitrepo $ \r -> indir r a

From 7dff46eeec44e34075f9506ce076e6ec0298c178 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:06:15 -0400
Subject: [PATCH 2737/2835] cleanup backend used when reversing migration

WORM is no longer the starting backend, so let's really reverse, getting
back to SHA256.
---
 test.hs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test.hs b/test.hs
index 5c43c3d4d9..d3d7609d8a 100644
--- a/test.hs
+++ b/test.hs
@@ -430,15 +430,15 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 		checkbackend sha1annexedfile backendSHA1
 
 		-- check that reversing a migration works
-		writeFile ".gitattributes" $ "* annex.backend=WORM"
+		writeFile ".gitattributes" $ "* annex.backend=SHA256"
 		git_annex "migrate" [sha1annexedfile]
 			@? "migrate sha1annexedfile failed"
 		git_annex "migrate" [annexedfile]
 			@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
-		checkbackend annexedfile backendWORM
-		checkbackend sha1annexedfile backendWORM
+		checkbackend annexedfile backendSHA256
+		checkbackend sha1annexedfile backendSHA256
 		
 		where
 			checkbackend file expected = do
@@ -761,8 +761,8 @@ changedcontent f = (content f) ++ " (modified)"
 backendSHA1 :: Types.Backend Types.Annex
 backendSHA1 = backend_ "SHA1"
 
-backendWORM :: Types.Backend Types.Annex
-backendWORM = backend_ "WORM"
+backendSHA256 :: Types.Backend Types.Annex
+backendSHA256 = backend_ "SHA256"
 
 backend_ :: String -> Types.Backend Types.Annex
 backend_ name = Backend.lookupBackendName name

From 9cfa7a969c7449ef48fe97daadf8ff89e8a103b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:07:34 -0400
Subject: [PATCH 2738/2835] note

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 6d2a2f8193..93586762a9 100644
--- a/Makefile
+++ b/Makefile
@@ -81,6 +81,7 @@ testcoverage:
 	@echo ""
 	@hpc report test --exclude=Main --exclude=QC
 	@hpc markup test --exclude=Main --exclude=QC --destdir=.hpc >/dev/null
+	@echo "(See .hpc/ for test coverage details.)"
 
 # If ikiwiki is available, build static html docs suitable for being
 # shipped in the software package.

From 9308a60bb2457568bf600901f6dad5c96d6b53ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:26:39 -0400
Subject: [PATCH 2739/2835] add basic tests for some of the newer commands

For most this just makes sure they run successfully, and improves test
coverage statistics; it's hard to test the output of eg, find and status.
---
 test.hs | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index d3d7609d8a..a3935cf66e 100644
--- a/test.hs
+++ b/test.hs
@@ -102,6 +102,12 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_fsck
 	, test_migrate
 	, test_unused
+	, test_addurl
+	, test_describe
+	, test_find
+	, test_merge
+	, test_status
+	, test_sync
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
@@ -343,7 +349,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 		newfile = subdir ++ "/" ++ annexedfile
 
 test_trust :: Test
-test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
+test_trust = "git-annex trust/untrust/semitrust/dead" ~: intmpclonerepo $ do
 	git_annex "trust" [repo] @? "trust failed"
 	trustcheck Logs.Trust.Trusted "trusted 1"
 	git_annex "trust" [repo] @? "trust of trusted failed"
@@ -352,6 +358,10 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 	trustcheck Logs.Trust.UnTrusted "untrusted 1"
 	git_annex "untrust" [repo] @? "untrust of untrusted failed"
 	trustcheck Logs.Trust.UnTrusted "untrusted 2"
+	git_annex "dead" [repo] @? "dead failed"
+	trustcheck Logs.Trust.DeadTrusted "deadtrusted 1"
+	git_annex "dead" [repo] @? "dead of dead failed"
+	trustcheck Logs.Trust.DeadTrusted "deadtrusted 2"
 	git_annex "semitrust" [repo] @? "semitrust failed"
 	trustcheck Logs.Trust.SemiTrusted "semitrusted 1"
 	git_annex "semitrust" [repo] @? "semitrust of semitrusted failed"
@@ -485,6 +495,37 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 			r <- Backend.lookupFile f
 			return $ fst $ fromJust r
 
+test_addurl :: Test
+test_addurl = "git-annex addurl" ~: intmpclonerepo $ do
+	annexed_notpresent annexedfile
+	-- can't check download; test suite should not access network,
+	-- and starting up a web server seems excessive
+	git_annex "addurl" ["--fast", "http://example.com/nosuchfile"] @? "addurl failed"
+
+test_describe :: Test
+test_describe = "git-annex describe" ~: intmpclonerepo $ do
+	git_annex "describe" [".", "this repo"] @? "describe 1 failed"
+	git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed"
+
+test_find :: Test
+test_find = "git-annex find" ~: intmpclonerepo $ do
+	annexed_notpresent annexedfile
+	git_annex "find" [] @? "find failed"
+	git_annex "get" [] @? "get failed"
+	git_annex "find" [] @? "find failed"
+
+test_merge :: Test
+test_merge = "git-annex merge" ~: intmpclonerepo $ do
+	git_annex "merge" [] @? "merge failed"
+
+test_status :: Test
+test_status = "git-annex status" ~: intmpclonerepo $ do
+	git_annex "status" [] @? "status failed"
+
+test_sync :: Test
+test_sync = "git-annex sync" ~: intmpclonerepo $ do
+	git_annex "sync" [] @? "sync failed"
+
 test_hook_remote :: Test
 test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 	git_annex "initremote" (words "foo type=hook encryption=none hooktype=foo") @? "initremote failed"

From 1c28237e0c7a920823354d4a10381b300aeb95e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:31:59 -0400
Subject: [PATCH 2740/2835] map: --fast disables use of dot to display map

Generally useful, and allows the test suite to test it.
---
 Command/Map.hs     | 13 +++++++++----
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  5 +++--
 test.hs            | 12 ++++++++++++
 4 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index da129c8f65..0f32e1130d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -16,6 +16,7 @@ import qualified Git
 import qualified Git.Url
 import qualified Git.Config
 import qualified Git.Construct
+import qualified Annex
 import Annex.UUID
 import Logs.UUID
 import Logs.Trust
@@ -40,10 +41,14 @@ start = do
 	trusted <- trustGet Trusted
 
 	liftIO $ writeFile file (drawMap rs umap trusted)
-	showLongNote $ "running: dot -Tx11 " ++ file
-	showOutput
-	r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file]
-	next $ next $ return r
+	next $ next $ do
+		fast <- Annex.getState Annex.fast
+		if fast
+			then return True
+			else do
+				showLongNote $ "running: dot -Tx11 " ++ file
+				showOutput
+				liftIO $ boolSystem "dot" [Param "-Tx11", File file]
 	where
 		file = "map.dot"
 
diff --git a/debian/changelog b/debian/changelog
index e187d8f6fa..600687d94b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * reinject: Add a sanity check for using an annexed file as the source file.
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
+  * map: --fast disables use of dot to display map
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d7a51663fd..b3d671bb89 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -268,9 +268,10 @@ subdirectories).
   Helps you keep track of your repositories, and the connections between them,
   by going out and looking at all the ones it can get to, and generating a
   Graphviz file displaying it all. If the `dot` command is available, it is
-  used to display the file to your screen (using x11 backend).
+  used to display the file to your screen (using x11 backend). (To disable
+  this display, specify --fast)
 
-  Note that this only connects to hosts that the host it's run on can
+  This command only connects to hosts that the host it's run on can
   directly connect to. It does not try to tunnel through intermediate hosts.
   So it might not show all connections between the repositories in the network.
 
diff --git a/test.hs b/test.hs
index a3935cf66e..aa2cd8be75 100644
--- a/test.hs
+++ b/test.hs
@@ -108,6 +108,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_merge
 	, test_status
 	, test_sync
+	, test_map
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
@@ -526,6 +527,17 @@ test_sync :: Test
 test_sync = "git-annex sync" ~: intmpclonerepo $ do
 	git_annex "sync" [] @? "sync failed"
 
+test_map :: Test
+test_map = "git-annex map" ~: intmpclonerepo $ do
+	-- set descriptions, that will be looked for in the map
+	git_annex "describe" [".", "this repo"] @? "describe 1 failed"
+	git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed"
+	-- --fast avoids it running graphviz, not a build dependency
+	git_annex "map" ["--fast"] @? "map failed"
+	doesFileExist "map.dot" @? "map.dot not generated"
+	c <- readFile "map.dot"
+	not ("this repo" `isInfixOf` c && "origin repo" `isInfixOf` c) @? "map.dot bad content"
+
 test_hook_remote :: Test
 test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 	git_annex "initremote" (words "foo type=hook encryption=none hooktype=foo") @? "initremote failed"

From cc88abd0ad84c0a196edbc7a67cec493f6f532b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 17:31:25 -0400
Subject: [PATCH 2741/2835] Test suite improvements. Current top-level test
 coverage: 68%

Been higher before, but a lot of new code has been added.
---
 debian/changelog |  1 +
 test.hs          | 62 +++++++++++++++++++++++++++++++++---------------
 2 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 600687d94b..e3e8079dcf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
+  * Test suite improvements. Current top-level test coverage: 68%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index aa2cd8be75..9ba0e5dd91 100644
--- a/test.hs
+++ b/test.hs
@@ -107,8 +107,12 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_find
 	, test_merge
 	, test_status
+	, test_version
 	, test_sync
 	, test_map
+	, test_uninit
+	, test_upgrade
+	, test_whereis
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
@@ -187,8 +191,7 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 			git_annex "get" [annexedfile] @? "get failed"
 			boolSystem "git" [Params "remote rm origin"]
 				@? "git remote rm origin failed"
-			r <- git_annex "drop" [annexedfile]
-			not r @? "drop wrongly succeeded with no known copy of file"
+			not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of file"
 			annexed_present annexedfile
 			git_annex "drop" ["--force", annexedfile] @? "drop --force failed"
 			annexed_notpresent annexedfile
@@ -205,8 +208,7 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 			git_annex "untrust" ["origin"] @? "untrust of origin failed"
 			git_annex "get" [annexedfile] @? "get failed"
 			annexed_present annexedfile
-			r <- git_annex "drop" [annexedfile]
-			not r @? "drop wrongly suceeded with only an untrusted copy of the file"
+			not <$> git_annex "drop" [annexedfile] @? "drop wrongly suceeded with only an untrusted copy of the file"
 			annexed_present annexedfile
 			inmainrepo $ annexed_present annexedfile
 
@@ -280,8 +282,7 @@ test_lock :: Test
 test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do
 	-- regression test: unlock of not present file should skip it
 	annexed_notpresent annexedfile
-	r <- git_annex "unlock" [annexedfile]
-	not r @? "unlock failed to fail with not present file"
+	not <$> git_annex "unlock" [annexedfile] @? "unlock failed to fail with not present file"
 	annexed_notpresent annexedfile
 
 	git_annex "get" [annexedfile] @? "get of file failed"
@@ -326,8 +327,7 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
 		assertEqual "content of modified file" c (changedcontent annexedfile)
-		r <- git_annex "drop" [annexedfile]
-		not r @? "drop wrongly succeeded with no known copy of modified file"
+		not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of modified file"
 
 test_fix :: Test
 test_fix = "git-annex fix" ~: intmpclonerepo $ do
@@ -405,12 +405,10 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			git_annex "get" [f] @? "get of file failed"
 			Utility.FileMode.allowWrite f
 			writeFile f (changedcontent f)
-			r <- git_annex "fsck" []
-			not r @? "fsck failed to fail with corrupted file content"
+			not <$> git_annex "fsck" [] @? "fsck failed to fail with corrupted file content"
 			git_annex "fsck" [] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f
 		fsck_should_fail m = do
-			r <- git_annex "fsck" []
-			not r @? "fsck failed to fail with " ++ m
+			not <$> git_annex "fsck" [] @? "fsck failed to fail with " ++ m
 
 test_migrate :: Test
 test_migrate = "git-annex migrate" ~: TestList [t False, t True]
@@ -523,6 +521,10 @@ test_status :: Test
 test_status = "git-annex status" ~: intmpclonerepo $ do
 	git_annex "status" [] @? "status failed"
 
+test_version :: Test
+test_version = "git-annex version" ~: intmpclonerepo $ do
+	git_annex "version" [] @? "version failed"
+
 test_sync :: Test
 test_sync = "git-annex sync" ~: intmpclonerepo $ do
 	git_annex "sync" [] @? "sync failed"
@@ -536,7 +538,32 @@ test_map = "git-annex map" ~: intmpclonerepo $ do
 	git_annex "map" ["--fast"] @? "map failed"
 	doesFileExist "map.dot" @? "map.dot not generated"
 	c <- readFile "map.dot"
-	not ("this repo" `isInfixOf` c && "origin repo" `isInfixOf` c) @? "map.dot bad content"
+	("this repo" `isInfixOf` c && "origin repo" `isInfixOf` c) @? ("map.dot bad content: " ++ c)
+
+test_uninit :: Test
+test_uninit = "git-annex uninit" ~: intmpclonerepo $ do
+	git_annex "get" [] @? "get failed"
+	annexed_present annexedfile
+	boolSystem "git" [Params "checkout git-annex"] @? "git checkout git-annex"
+	not <$> git_annex "uninit" [] @? "uninit failed to fail when git-annex branch was checked out"
+	boolSystem "git" [Params "checkout master"] @? "git checkout master"
+	git_annex "unannex" [] @? "unannex failed"
+	checkregularfile annexedfile
+	doesDirectoryExist ".git" @? ".git vanished in uninit"
+
+test_upgrade :: Test
+test_upgrade = "git-annex upgrade" ~: intmpclonerepo $ do
+	git_annex "upgrade" [] @? "upgrade from same version failed"
+
+test_whereis :: Test
+test_whereis = "git-annex whereis" ~: intmpclonerepo $ do
+	annexed_notpresent annexedfile
+	git_annex "whereis" [annexedfile] @? "whereis on non-present file failed"
+	git_annex "untrust" ["origin"] @? "untrust failed"
+	not <$> git_annex "whereis" [annexedfile] @? "whereis on non-present file only present in untrusted repo failed to fail"
+	git_annex "get" [annexedfile] @? "get failed"
+	annexed_present annexedfile
+	git_annex "whereis" [annexedfile] @? "whereis on present file failed"
 
 test_hook_remote :: Test
 test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
@@ -558,8 +585,7 @@ test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from hook remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
-	not r @? "drop failed to fail"
+	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 	where
 		dir = "dir"
@@ -579,8 +605,7 @@ test_directory_remote = "git-annex directory remote" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from directory remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
-	not r @? "drop failed to fail"
+	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 
 test_rsync_remote :: Test
@@ -595,8 +620,7 @@ test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from rsync remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
-	not r @? "drop failed to fail"
+	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 
 -- This is equivilant to running git-annex, but it's all run in-process

From 58a89c243cd7beeaff8fb8b45bf20d934d6882ed Mon Sep 17 00:00:00 2001
From: Ian 
Date: Tue, 20 Dec 2011 22:36:03 +0000
Subject: [PATCH 2742/2835]

---
 doc/forum/A_really_stupid_question.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/A_really_stupid_question.mdwn

diff --git a/doc/forum/A_really_stupid_question.mdwn b/doc/forum/A_really_stupid_question.mdwn
new file mode 100644
index 0000000000..38c7bcb56a
--- /dev/null
+++ b/doc/forum/A_really_stupid_question.mdwn
@@ -0,0 +1,3 @@
+Sorry, but all this wiki and the manpage seem to gloss over the most obvious question:
+
+What happens when you commit conflicting edits in different repositories?

From a2e482b59037e783729cf0d96a4d58553db0dfb4 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 20 Dec 2011 23:07:25 +0000
Subject: [PATCH 2743/2835] Added a comment: Good question!

---
 ..._40e02556de0b00b94f245a0196b5a89f._comment | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment

diff --git a/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment b/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment
new file mode 100644
index 0000000000..2a400db3b0
--- /dev/null
+++ b/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="Good question!"
+ date="2011-12-20T23:07:25Z"
+ content="""
+You get a regular git merge conflict, which can be resolved in any of the regular ways, except that conflicting files are just symlinks.
+
+Example:
+
+
+$ git pull
+...
+Auto-merging myfile
+CONFLICT (add/add): Merge conflict in myfile
+Automatic merge failed; fix conflicts and then commit the result.
+$ git status
+# On branch master
+# Your branch and 'origin/master' have diverged,
+# and have 1 and 1 different commit(s) each, respectively.
+#
+# Unmerged paths:
+#   (use \"git add/rm ...\" as appropriate to mark resolution)
+#
+#	both added:         myfile
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+$ git add myfile
+$ git commit -m \"took local version of the conflicting file\"
+

+"""]]

From 815d1318d7a54836f65ba4b72703808959223bb2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 18:00:09 -0400
Subject: [PATCH 2744/2835] comment

---
 Utility/Url.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Utility/Url.hs b/Utility/Url.hs
index 617fe3f4d0..7ce43d9bd1 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -33,8 +33,12 @@ exists url =
 				_ -> return False
 
 {- Used to download large files, such as the contents of keys.
+ -
  - Uses wget or curl program for its progress bar. (Wget has a better one,
- - so is preferred.) -}
+ - so is preferred.) Which program to use is determined at run time; it
+ - would not be appropriate to test at configure time and build support
+ - for only one in.
+ -}
 download :: URLString -> FilePath -> IO Bool
 download url file = do
 	e <- inPath "wget"

From 8e2f74f7ab188b72b1053140f28f1f4a6a792675 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 18:14:20 -0400
Subject: [PATCH 2745/2835] update

---
 Utility/Url.hs | 4 ++++
 configure.hs   | 1 +
 2 files changed, 5 insertions(+)

diff --git a/Utility/Url.hs b/Utility/Url.hs
index 7ce43d9bd1..f215a1ebdb 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -7,6 +7,7 @@
 
 module Utility.Url (
 	exists,
+	canDownload,
 	download,
 	get
 ) where
@@ -32,6 +33,9 @@ exists url =
 				(2,_,_) -> return True
 				_ -> return False
 
+canDownload :: IO Bool
+canDownload = (||) <$> inPath "wget" <*> inPath "curl"
+
 {- Used to download large files, such as the contents of keys.
  -
  - Uses wget or curl program for its progress bar. (Wget has a better one,
diff --git a/configure.hs b/configure.hs
index 0d96b39554..bf25de5079 100644
--- a/configure.hs
+++ b/configure.hs
@@ -19,6 +19,7 @@ tests =
 	, TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null"
 	, TestCase "curl" $ testCmd "curl" "curl --version >/dev/null"
+	, TestCase "wget" $ testCmd "wget" "wget --version >/dev/null"
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
 	] ++ shaTestCases [1, 256, 512, 224, 384]

From bb84f6e4bd57b15b9e83e2baf1b678d66d5009be Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 21:30:00 -0400
Subject: [PATCH 2746/2835] make gpg code more generic

---
 Crypto.hs | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/Crypto.hs b/Crypto.hs
index 24bb79ba0d..70ee6183be 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -71,7 +71,7 @@ genCipher c = do
 	random <- genrandom
 	encryptCipher (Cipher random) ks
 	where
-		genrandom = gpgRead
+		genrandom = gpgReadStrict
 			-- Armor the random data, to avoid newlines,
 			-- since gpg only reads ciphers up to the first
 			-- newline.
@@ -150,12 +150,12 @@ encryptKey c k = Key
 {- Runs an action, passing it a handle from which it can 
  - stream encrypted content. -}
 withEncryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"]
+withEncryptedHandle = gpgPassphraseHandle [Params "--symmetric --force-mdc"] . cipherPassphrase
 
 {- Runs an action, passing it a handle from which it can
  - stream decrypted content. -}
 withDecryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-withDecryptedHandle = gpgCipherHandle [Param "--decrypt"]
+withDecryptedHandle = gpgPassphraseHandle [Param "--decrypt"] . cipherPassphrase
 
 {- Streams encrypted content to an action. -}
 withEncryptedContent :: Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
@@ -180,11 +180,14 @@ gpgParams params = do
 		-- be quiet, even about checking the trustdb
 		defaults = ["--quiet", "--trust-model", "always"]
 
-gpgRead :: [CommandParam] -> IO String
-gpgRead params = do
+{- Runs gpg with some params and returns its stdout, strictly. -}
+gpgReadStrict :: [CommandParam] -> IO String
+gpgReadStrict params = do
 	params' <- gpgParams params
 	pOpen ReadFromPipe "gpg" params' hGetContentsStrict
 
+{- Runs gpg, piping an input value to it, and returninging its stdout,
+ - strictly. -}
 gpgPipeStrict :: [CommandParam] -> String -> IO String
 gpgPipeStrict params input = do
 	params' <- gpgParams params
@@ -194,23 +197,24 @@ gpgPipeStrict params input = do
 	forceSuccess pid
 	return output
 
-{- Runs gpg with a cipher and some parameters, feeding it an input,
- - and passing a handle to its output to an action.
+{- Runs gpg with some parameters, first feeding it a passphrase via
+ - --passphrase-fd, then feeding it an input, and passing a handle
+ - to its output to an action.
  -
  - Note that to avoid deadlock with the cleanup stage,
  - the action must fully consume gpg's input before returning. -}
-gpgCipherHandle :: [CommandParam] -> Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-gpgCipherHandle params c a b = do
+gpgPassphraseHandle :: [CommandParam] -> String -> IO L.ByteString -> (Handle -> IO a) -> IO a
+gpgPassphraseHandle params passphrase a b = do
 	-- pipe the passphrase into gpg on a fd
 	(frompipe, topipe) <- createPipe
 	_ <- forkIO $ do
 		toh <- fdToHandle topipe
-		hPutStrLn toh $ cipherPassphrase c
+		hPutStrLn toh passphrase
 		hClose toh
-	let Fd passphrasefd = frompipe
-	let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd]
+	let Fd pfd = frompipe
+	let passphrasefd = [Param "--passphrase-fd", Param $ show pfd]
 
-	params' <- gpgParams $ passphrase ++ params
+	params' <- gpgParams $ passphrasefd ++ params
 	(pid, fromh, toh) <- hPipeBoth "gpg" params'
 	pid2 <- forkProcess $ do
 		L.hPut toh =<< a
@@ -226,7 +230,7 @@ gpgCipherHandle params c a b = do
 	return ret
 
 configKeyIds :: RemoteConfig -> IO KeyIds
-configKeyIds c = parse <$> gpgRead params
+configKeyIds c = parse <$> gpgReadStrict params
 	where
 		params = [Params "--with-colons --list-public-keys",
 			Param $ configGet c "encryption"]

From c11cfea35555ae3bab429c283d8c7571d285d4b1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 21:47:56 -0400
Subject: [PATCH 2747/2835] split out Utility.Gpg with the generic gpg
 interface, from Crypto

---
 Crypto.hs       | 84 ++++-----------------------------------------
 Types/Crypto.hs | 11 +++---
 Utility/Gpg.hs  | 91 +++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 105 insertions(+), 81 deletions(-)
 create mode 100644 Utility/Gpg.hs

diff --git a/Crypto.hs b/Crypto.hs
index 70ee6183be..cb1ca40d14 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -30,14 +30,10 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.ByteString.Lazy.UTF8 (fromString)
 import Data.Digest.Pure.SHA
-import System.Posix.Types
 import Control.Applicative
-import Control.Concurrent
-import Control.Exception (finally)
-import System.Exit
-import System.Environment
 
 import Common.Annex
+import qualified Utility.Gpg as Gpg
 import Types.Key
 import Types.Remote
 import Utility.Base64
@@ -71,7 +67,7 @@ genCipher c = do
 	random <- genrandom
 	encryptCipher (Cipher random) ks
 	where
-		genrandom = gpgReadStrict
+		genrandom = Gpg.readStrict
 			-- Armor the random data, to avoid newlines,
 			-- since gpg only reads ciphers up to the first
 			-- newline.
@@ -119,7 +115,7 @@ extractCipher c =
 encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher
 encryptCipher (Cipher c) (KeyIds ks) = do
 	let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids
-	encipher <- gpgPipeStrict (encrypt++recipients ks') c
+	encipher <- Gpg.pipeStrict (encrypt++recipients ks') c
 	return $ EncryptedCipher encipher (KeyIds ks')
 	where
 		encrypt = [ Params "--encrypt" ]
@@ -132,7 +128,7 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
 decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
 decryptCipher _ (EncryptedCipher encipher _) = 
-	Cipher <$> gpgPipeStrict decrypt encipher
+	Cipher <$> Gpg.pipeStrict decrypt encipher
 	where
 		decrypt = [ Param "--decrypt" ]
 
@@ -150,12 +146,12 @@ encryptKey c k = Key
 {- Runs an action, passing it a handle from which it can 
  - stream encrypted content. -}
 withEncryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-withEncryptedHandle = gpgPassphraseHandle [Params "--symmetric --force-mdc"] . cipherPassphrase
+withEncryptedHandle = Gpg.passphraseHandle [Params "--symmetric --force-mdc"] . cipherPassphrase
 
 {- Runs an action, passing it a handle from which it can
  - stream decrypted content. -}
 withDecryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-withDecryptedHandle = gpgPassphraseHandle [Param "--decrypt"] . cipherPassphrase
+withDecryptedHandle = Gpg.passphraseHandle [Param "--decrypt"] . cipherPassphrase
 
 {- Streams encrypted content to an action. -}
 withEncryptedContent :: Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
@@ -169,74 +165,8 @@ pass :: (Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a)
       -> Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 pass to c i a = to c i $ \h -> a =<< L.hGetContents h
 
-gpgParams :: [CommandParam] -> IO [String]
-gpgParams params = do
-	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
-	-- gpg output about password prompts.
-	e <- catchDefaultIO (getEnv "GPG_AGENT_INFO") ""
-	let batch = if null e then [] else ["--batch"]
-	return $ batch ++ defaults ++ toCommand params
-	where
-		-- be quiet, even about checking the trustdb
-		defaults = ["--quiet", "--trust-model", "always"]
-
-{- Runs gpg with some params and returns its stdout, strictly. -}
-gpgReadStrict :: [CommandParam] -> IO String
-gpgReadStrict params = do
-	params' <- gpgParams params
-	pOpen ReadFromPipe "gpg" params' hGetContentsStrict
-
-{- Runs gpg, piping an input value to it, and returninging its stdout,
- - strictly. -}
-gpgPipeStrict :: [CommandParam] -> String -> IO String
-gpgPipeStrict params input = do
-	params' <- gpgParams params
-	(pid, fromh, toh) <- hPipeBoth "gpg" params'
-	_ <- forkIO $ finally (hPutStr toh input) (hClose toh)
-	output <- hGetContentsStrict fromh
-	forceSuccess pid
-	return output
-
-{- Runs gpg with some parameters, first feeding it a passphrase via
- - --passphrase-fd, then feeding it an input, and passing a handle
- - to its output to an action.
- -
- - Note that to avoid deadlock with the cleanup stage,
- - the action must fully consume gpg's input before returning. -}
-gpgPassphraseHandle :: [CommandParam] -> String -> IO L.ByteString -> (Handle -> IO a) -> IO a
-gpgPassphraseHandle params passphrase a b = do
-	-- pipe the passphrase into gpg on a fd
-	(frompipe, topipe) <- createPipe
-	_ <- forkIO $ do
-		toh <- fdToHandle topipe
-		hPutStrLn toh passphrase
-		hClose toh
-	let Fd pfd = frompipe
-	let passphrasefd = [Param "--passphrase-fd", Param $ show pfd]
-
-	params' <- gpgParams $ passphrasefd ++ params
-	(pid, fromh, toh) <- hPipeBoth "gpg" params'
-	pid2 <- forkProcess $ do
-		L.hPut toh =<< a
-		hClose toh
-		exitSuccess
-	hClose toh
-	ret <- b fromh
-
-	-- cleanup
-	forceSuccess pid
-	_ <- getProcessStatus True False pid2
-	closeFd frompipe
-	return ret
-
 configKeyIds :: RemoteConfig -> IO KeyIds
-configKeyIds c = parse <$> gpgReadStrict params
-	where
-		params = [Params "--with-colons --list-public-keys",
-			Param $ configGet c "encryption"]
-		parse = KeyIds . map keyIdField . filter pubKey . lines
-		pubKey = isPrefixOf "pub:"
-		keyIdField s = split ":" s !! 4
+configKeyIds c = Gpg.findPubKeys $ configGet c "encryption"
 
 configGet :: RemoteConfig -> String -> String
 configGet c key = fromMaybe missing $ M.lookup key c
diff --git a/Types/Crypto.hs b/Types/Crypto.hs
index 29a4cd099c..686bf5c1a6 100644
--- a/Types/Crypto.hs
+++ b/Types/Crypto.hs
@@ -5,13 +5,16 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Types.Crypto where
+module Types.Crypto (
+	Cipher(..),
+	EncryptedCipher(..),
+	KeyIds(..),
+) where
+
+import Utility.Gpg (KeyIds(..))
 
 -- XXX ideally, this would be a locked memory region
 newtype Cipher = Cipher String
 
 data EncryptedCipher = EncryptedCipher String KeyIds
 	deriving (Ord, Eq)
-
-newtype KeyIds = KeyIds [String]
-	deriving (Ord, Eq)
diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
new file mode 100644
index 0000000000..c74c2bfd04
--- /dev/null
+++ b/Utility/Gpg.hs
@@ -0,0 +1,91 @@
+{- gpg interface
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Gpg where
+
+import qualified Data.ByteString.Lazy.Char8 as L
+import System.Posix.Types
+import Control.Applicative
+import Control.Concurrent
+import Control.Exception (finally)
+import System.Exit
+import System.Environment
+
+import Common
+
+newtype KeyIds = KeyIds [String]
+        deriving (Ord, Eq)
+
+stdParams :: [CommandParam] -> IO [String]
+stdParams params = do
+	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
+	-- gpg output about password prompts.
+	e <- catchDefaultIO (getEnv "GPG_AGENT_INFO") ""
+	let batch = if null e then [] else ["--batch"]
+	return $ batch ++ defaults ++ toCommand params
+	where
+		-- be quiet, even about checking the trustdb
+		defaults = ["--quiet", "--trust-model", "always"]
+
+{- Runs gpg with some params and returns its stdout, strictly. -}
+readStrict :: [CommandParam] -> IO String
+readStrict params = do
+	params' <- stdParams params
+	pOpen ReadFromPipe "gpg" params' hGetContentsStrict
+
+{- Runs gpg, piping an input value to it, and returninging its stdout,
+ - strictly. -}
+pipeStrict :: [CommandParam] -> String -> IO String
+pipeStrict params input = do
+	params' <- stdParams params
+	(pid, fromh, toh) <- hPipeBoth "gpg" params'
+	_ <- forkIO $ finally (hPutStr toh input) (hClose toh)
+	output <- hGetContentsStrict fromh
+	forceSuccess pid
+	return output
+
+{- Runs gpg with some parameters, first feeding it a passphrase via
+ - --passphrase-fd, then feeding it an input, and passing a handle
+ - to its output to an action.
+ -
+ - Note that to avoid deadlock with the cleanup stage,
+ - the action must fully consume gpg's input before returning. -}
+passphraseHandle :: [CommandParam] -> String -> IO L.ByteString -> (Handle -> IO a) -> IO a
+passphraseHandle params passphrase a b = do
+	-- pipe the passphrase into gpg on a fd
+	(frompipe, topipe) <- createPipe
+	_ <- forkIO $ do
+		toh <- fdToHandle topipe
+		hPutStrLn toh passphrase
+		hClose toh
+	let Fd pfd = frompipe
+	let passphrasefd = [Param "--passphrase-fd", Param $ show pfd]
+
+	params' <- stdParams $ passphrasefd ++ params
+	(pid, fromh, toh) <- hPipeBoth "gpg" params'
+	pid2 <- forkProcess $ do
+		L.hPut toh =<< a
+		hClose toh
+		exitSuccess
+	hClose toh
+	ret <- b fromh
+
+	-- cleanup
+	forceSuccess pid
+	_ <- getProcessStatus True False pid2
+	closeFd frompipe
+	return ret
+
+{- Finds gpg public keys matching some string. (Could be an email address,
+ - a key id, or a name. -}
+findPubKeys :: String -> IO KeyIds
+findPubKeys for = KeyIds . parse <$> readStrict params
+	where
+		params = [Params "--with-colons --list-public-keys", Param for]
+		parse = map keyIdField . filter pubKey . lines
+		pubKey = isPrefixOf "pub:"
+		keyIdField s = split ":" s !! 4

From 82a145df91ca93a55020172076297e79ff6c52e5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 23:20:36 -0400
Subject: [PATCH 2748/2835] test encrypted special remote

This involved adding a test harness to run gpg with a dummy key, and lots
of fun.
---
 Utility/Gpg.hs   | 113 ++++++++++++++++++++++++++++++++++++++++++++---
 debian/changelog |   2 +-
 test.hs          |  29 ++++++++++++
 3 files changed, 138 insertions(+), 6 deletions(-)

diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
index c74c2bfd04..f3a1ac0bb5 100644
--- a/Utility/Gpg.hs
+++ b/Utility/Gpg.hs
@@ -11,9 +11,9 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import System.Posix.Types
 import Control.Applicative
 import Control.Concurrent
-import Control.Exception (finally)
+import Control.Exception (finally, bracket)
 import System.Exit
-import System.Environment
+import System.Posix.Env (setEnv, unsetEnv, getEnv)	
 
 import Common
 
@@ -24,8 +24,8 @@ stdParams :: [CommandParam] -> IO [String]
 stdParams params = do
 	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
 	-- gpg output about password prompts.
-	e <- catchDefaultIO (getEnv "GPG_AGENT_INFO") ""
-	let batch = if null e then [] else ["--batch"]
+	e <- getEnv "GPG_AGENT_INFO"
+	let batch = if isNothing e then [] else ["--batch"]
 	return $ batch ++ defaults ++ toCommand params
 	where
 		-- be quiet, even about checking the trustdb
@@ -37,7 +37,7 @@ readStrict params = do
 	params' <- stdParams params
 	pOpen ReadFromPipe "gpg" params' hGetContentsStrict
 
-{- Runs gpg, piping an input value to it, and returninging its stdout,
+{- Runs gpg, piping an input value to it, and returning its stdout,
  - strictly. -}
 pipeStrict :: [CommandParam] -> String -> IO String
 pipeStrict params input = do
@@ -89,3 +89,106 @@ findPubKeys for = KeyIds . parse <$> readStrict params
 		parse = map keyIdField . filter pubKey . lines
 		pubKey = isPrefixOf "pub:"
 		keyIdField s = split ":" s !! 4
+
+
+
+{- A test key. This is provided pre-generated since generating a new gpg
+ - key is too much work (requires too much entropy) for a test suite to
+ - do.
+ -
+ - This key was generated with no exipiration date, and a small keysize. 
+ - It has an empty passphrase. -}
+testKeyId :: String
+testKeyId = "129D6E0AC537B9C7"
+testKey :: String
+testKey = keyBlock True
+	[ "mI0ETvFAZgEEAKnqwWgZqznMhi1RQExem2H8t3OyKDxaNN3rBN8T6LWGGqAYV4wT"
+	, "r8In5tfsnz64bKpE1Qi68JURFwYmthgUL9N48tbODU8t3xzijdjLOSaTyqkH1ik6"
+	, "EyulfKN63xLne9i4F9XqNwpiZzukXYbNfHkDA2yb0M6g4UFKLY/fNzGXABEBAAG0"
+	, "W2luc2VjdXJlIHRlc3Qga2V5ICh0aGlzIGlzIGEgdGVzdCBrZXksIGRvIG5vdCB1"
+	, "c2UgZm9yIGFjdHVhbCBlbmNyeXB0aW9uKSA8dGVzdEBleGFtcGxlLmNvbT6IuAQT"
+	, "AQgAIgUCTvFAZgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQEp1uCsU3"
+	, "uceQ9wP/YMd1f0+/eLLcwGXNBvGqyVhUOfAKknO1bMzGbqTsq9g60qegy/cldqee"
+	, "xVxNfy0VN//JeMfgdcb8+RgJYLoaMrTy9CcsUcFPxtwN9tcLmsM0V2/fNmmFBO9t"
+	, "v75iH+zeFbNg0/FbPkHiN6Mjw7P2gXYKQXgTvQZBWaphk8oQlBm4jQRO8UBmAQQA"
+	, "vdi50M/WRCkOLt2RsUve8V8brMWYTJBJTTWoHUeRr82v4NCdX7OE1BsoVK8cy/1Q"
+	, "Y+gLOH9PqinuGGNWRmPV2Ju/RYn5H7sdewXA8E80xWhc4phHRMJ8Jjhg/GVPamkJ"
+	, "8B5zeKF0jcLFl7cuVdOyQakhoeDWJd0CyfW837nmPtMAEQEAAYifBBgBCAAJBQJO"
+	, "8UBmAhsMAAoJEBKdbgrFN7nHclAEAKBShuP/toH03atDUQTbGE34CA4yEC9BVghi"
+	, "7kviOZlOz2s8xAfp/8AYsrECx1kgbXcA7JD902eNyp7NzXsdJX0zJwHqiuZW0XlD"
+	, "T8ZJu4qrYRYgl/790WPESZ+ValvHD/fqkR38RF4tfxvyoMhhp0roGmJY33GASIG/"
+	, "+gQkDF9/"
+	, "=1k11"
+	]
+testSecretKey :: String
+testSecretKey = keyBlock False
+	[ "lQHYBE7xQGYBBACp6sFoGas5zIYtUUBMXpth/Ldzsig8WjTd6wTfE+i1hhqgGFeM"
+	, "E6/CJ+bX7J8+uGyqRNUIuvCVERcGJrYYFC/TePLWzg1PLd8c4o3Yyzkmk8qpB9Yp"
+	, "OhMrpXyjet8S53vYuBfV6jcKYmc7pF2GzXx5AwNsm9DOoOFBSi2P3zcxlwARAQAB"
+	, "AAP+PlRboxy7Z0XjuG70N6+CrzSddQbW5KCwgPFrxYsPk7sAPFcBkmRMVlv9vZpS"
+	, "phbP4bvDK+MrSntM51g+9uE802yhPhSWdmEbImiWfV2ucEhlLjD8gw7JDex9XZ0a"
+	, "EbTOV56wOsILuedX/jF/6i6IQzy5YmuMeo+ip1XQIsIN+80CAMyXepOBJgHw/gBD"
+	, "VdXh/l//vUkQQlhInQYwgkKbr0POCTdr8DM1qdKLcUD9Q1khgNRp0vZGGz+5xsrc"
+	, "KaODUlMCANSczLJcYWa8yPqB3S14yTe7qmtDiOS362+SeVUwQA7eQ06PcHLPsN+p"
+	, "NtWoHRfYazxrs+g0JvmoQOYdj4xSQy0CAMq7H/l6aeG1n8tpyMxqE7OvBOsvzdu5"
+	, "XS7I1AnwllVFgvTadVvqgf7b+hdYd91doeHDUGqSYO78UG1GgaBHJdylqrRbaW5z"
+	, "ZWN1cmUgdGVzdCBrZXkgKHRoaXMgaXMgYSB0ZXN0IGtleSwgZG8gbm90IHVzZSBm"
+	, "b3IgYWN0dWFsIGVuY3J5cHRpb24pIDx0ZXN0QGV4YW1wbGUuY29tPoi4BBMBCAAi"
+	, "BQJO8UBmAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRASnW4KxTe5x5D3"
+	, "A/9gx3V/T794stzAZc0G8arJWFQ58AqSc7VszMZupOyr2DrSp6DL9yV2p57FXE1/"
+	, "LRU3/8l4x+B1xvz5GAlguhoytPL0JyxRwU/G3A321wuawzRXb982aYUE722/vmIf"
+	, "7N4Vs2DT8Vs+QeI3oyPDs/aBdgpBeBO9BkFZqmGTyhCUGZ0B2ARO8UBmAQQAvdi5"
+	, "0M/WRCkOLt2RsUve8V8brMWYTJBJTTWoHUeRr82v4NCdX7OE1BsoVK8cy/1QY+gL"
+	, "OH9PqinuGGNWRmPV2Ju/RYn5H7sdewXA8E80xWhc4phHRMJ8Jjhg/GVPamkJ8B5z"
+	, "eKF0jcLFl7cuVdOyQakhoeDWJd0CyfW837nmPtMAEQEAAQAD/RaVtFFTkF1udun7"
+	, "YOwzJvQXCO9OWHZvSdEeG4BUNdAwy4YWu0oZzKkBDBS6+lWILqqb/c28U4leUJ1l"
+	, "H+viz5svN9BWWyj/UpI00uwUo9JaIqalemwfLx6vsh69b54L1B4exLZHYGLvy/B3"
+	, "5T6bT0gpOE+53BRtKcJaOh/McQeJAgDTOCBU5weWOf6Bhqnw3Vr/gRfxntAz2okN"
+	, "gqz/h79mWbCc/lHKoYQSsrCdMiwziHSjXwvehUrdWE/AcomtW0vbAgDmGJqJ2fNr"
+	, "HvdsGx4Ld/BxyiZbCURJLUQ5CwzfHGIvBu9PMT8zM26NOSncaXRjxDna2Ggh8Uum"
+	, "ANEwbnhxFwZpAf9L9RLYIMTtAqwBjfXJg/lHcc2R+VP0hL5c8zFz+S+w7bRqINwL"
+	, "ff1JstKuHT2nJnu0ustK66by8YI3T0hDFFahnNCInwQYAQgACQUCTvFAZgIbDAAK"
+	, "CRASnW4KxTe5x3JQBACgUobj/7aB9N2rQ1EE2xhN+AgOMhAvQVYIYu5L4jmZTs9r"
+	, "PMQH6f/AGLKxAsdZIG13AOyQ/dNnjcqezc17HSV9MycB6ormVtF5Q0/GSbuKq2EW"
+	, "IJf+/dFjxEmflWpbxw/36pEd/EReLX8b8qDIYadK6BpiWN9xgEiBv/oEJAxffw=="
+	, "=LDsg"
+	]
+keyBlock :: Bool -> [String] -> String
+keyBlock public ls = unlines
+	[ "-----BEGIN PGP "++t++" KEY BLOCK-----"
+	, "Version: GnuPG v1.4.11 (GNU/Linux)"
+	, ""
+	, unlines ls
+	, "-----END PGP "++t++" KEY BLOCK-----"
+	]
+	where
+		t
+			| public = "PUBLIC"
+			| otherwise = "PRIVATE"
+
+{- Runs an action using gpg in a test harness, in which gpg does
+ - not use ~/.gpg/, but a directory with the test key set up to be used. -}
+testHarness :: IO a -> IO a
+testHarness a = do
+	orig <- getEnv var
+	bracket setup (cleanup orig) (const a)
+	where
+		var = "GNUPGHOME"		
+
+		setup = do
+			base <- getTemporaryDirectory
+			dir <- mktmpdir $ base  "gpgtmpXXXXXX"
+			setEnv var dir True
+			_ <- pipeStrict [Params "--import -q"] $ unlines
+				[testSecretKey, testKey]
+			return dir
+		
+		cleanup orig tmpdir = removeDirectoryRecursive tmpdir >> reset orig
+                reset (Just v) = setEnv var v True
+                reset _ = unsetEnv var
+
+{- Tests the test harness. -}
+testTestHarness :: IO Bool
+testTestHarness = do
+	keys <- testHarness $ findPubKeys testKeyId
+	return $ KeyIds [testKeyId] == keys
diff --git a/debian/changelog b/debian/changelog
index e3e8079dcf..e6fa9c16f7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
-  * Test suite improvements. Current top-level test coverage: 68%
+  * Test suite improvements. Current top-level test coverage: 70%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index 9ba0e5dd91..7c47443880 100644
--- a/test.hs
+++ b/test.hs
@@ -42,6 +42,8 @@ import qualified Config
 import qualified Crypto
 import qualified Utility.Path
 import qualified Utility.FileMode
+import qualified Utility.Gpg
+import qualified Build.SysConfig
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -116,6 +118,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
+	, test_crypto
 	]
 
 test_init :: Test
@@ -623,6 +626,32 @@ test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 
+test_crypto :: Test
+test_crypto = "git-annex crypto" ~: intmpclonerepo $
+	-- gpg is not a build dependency, so only test when it's available
+	when Build.SysConfig.gpg $ do
+		Utility.Gpg.testTestHarness @? "test harness self-test failed"
+		Utility.Gpg.testHarness $ do
+			createDirectory "dir"
+			let initremote = git_annex "initremote"
+				[ "foo"
+				, "type=directory"
+				, "encryption=" ++ Utility.Gpg.testKeyId
+				, "directory=dir"
+				]
+			initremote @? "initremote failed"
+			initremote @? "initremote failed when run twice in a row"
+			git_annex "get" [annexedfile] @? "get of file failed"
+			annexed_present annexedfile
+			git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed"
+			annexed_present annexedfile
+			git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
+			annexed_notpresent annexedfile
+			git_annex "move" [annexedfile, "--from", "foo"] @? "move --from encrypted remote failed"
+			annexed_present annexedfile
+			not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
+			annexed_present annexedfile	
+
 -- This is equivilant to running git-annex, but it's all run in-process
 -- so test coverage collection works.
 git_annex :: String -> [String] -> IO Bool

From a99af6338e9487d2212d4e82aa1afa4873cbfca8 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Wed, 21 Dec 2011 16:06:26 +0000
Subject: [PATCH 2749/2835] Added a comment

---
 ...ent_3_1aa89725b5196e40a16edeeb5ccfa371._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment b/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment
new file mode 100644
index 0000000000..0ead32dad2
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 3"
+ date="2011-12-21T16:06:25Z"
+ content="""
+hmmmm - I'm still not sure I get this.
+
+If I'm using a whole bunch of distributed annexs with no central repo, then I can not do a `git pull remote` without either specifying the branch to use or changing default tracked remote via `git branch --set-upstream`.  The former like you note doesn't pull the git-annex branch down the latter only works one-at-a-time.
+
+The docs read to me as though I ought to be able to do a `git pull remote ; git annex get .` using anyone of my distributed annexs.
+
+Am I doing something wrong?  Or is the above correct?
+"""]]

From c61f3d7b7b61583a61a97b2a7f3adbf4233cd93e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 02:32:40 -0400
Subject: [PATCH 2750/2835] test coverage improvements

---
 debian/changelog |  2 +-
 test.hs          | 47 ++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e6fa9c16f7..33a9cd575f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
-  * Test suite improvements. Current top-level test coverage: 70%
+  * Test suite improvements. Current top-level test coverage: 72%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index 7c47443880..eaa3fb8b64 100644
--- a/test.hs
+++ b/test.hs
@@ -17,6 +17,7 @@ import System.Posix.Env
 import qualified Control.Exception.Extensible as E
 import qualified Data.Map as M
 import System.IO.HVFS (SystemFS(..))
+import Text.JSON
 
 import Common
 
@@ -512,9 +513,16 @@ test_describe = "git-annex describe" ~: intmpclonerepo $ do
 test_find :: Test
 test_find = "git-annex find" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
-	git_annex "find" [] @? "find failed"
-	git_annex "get" [] @? "get failed"
-	git_annex "find" [] @? "find failed"
+	git_annex_expectoutput "find" [] []
+	git_annex "get" [annexedfile] @? "get failed"
+	annexed_present annexedfile
+	annexed_notpresent sha1annexedfile
+	git_annex_expectoutput "find" [] [annexedfile]
+	git_annex_expectoutput "find" ["--exclude", annexedfile, "--and", "--exclude", sha1annexedfile] []
+	git_annex_expectoutput "find" ["--not", "--in", "origin"] []
+	git_annex_expectoutput "find" ["--copies", "1", "--and", "--not", "--copies", "2"] [sha1annexedfile]
+	git_annex_expectoutput "find" ["--inbackend", "SHA1"] [sha1annexedfile]
+	git_annex_expectoutput "find" ["--inbackend", "WORM"] []
 
 test_merge :: Test
 test_merge = "git-annex merge" ~: intmpclonerepo $ do
@@ -522,7 +530,10 @@ test_merge = "git-annex merge" ~: intmpclonerepo $ do
 
 test_status :: Test
 test_status = "git-annex status" ~: intmpclonerepo $ do
-	git_annex "status" [] @? "status failed"
+	json <- git_annex_output "status" ["--json"]
+	case Text.JSON.decodeStrict json :: Text.JSON.Result (JSObject JSValue) of
+		Ok _ -> return ()
+		Error e -> assertFailure e
 
 test_version :: Test
 test_version = "git-annex version" ~: intmpclonerepo $ do
@@ -550,9 +561,10 @@ test_uninit = "git-annex uninit" ~: intmpclonerepo $ do
 	boolSystem "git" [Params "checkout git-annex"] @? "git checkout git-annex"
 	not <$> git_annex "uninit" [] @? "uninit failed to fail when git-annex branch was checked out"
 	boolSystem "git" [Params "checkout master"] @? "git checkout master"
-	git_annex "unannex" [] @? "unannex failed"
+	_ <- git_annex "uninit" [] -- exit status not checked; does abnormal exit
 	checkregularfile annexedfile
 	doesDirectoryExist ".git" @? ".git vanished in uninit"
+	not <$> doesDirectoryExist ".git/annex" @? ".git/annex still present after uninit"
 
 test_upgrade :: Test
 test_upgrade = "git-annex upgrade" ~: intmpclonerepo $ do
@@ -664,6 +676,31 @@ git_annex command params = do
 	where
 		run = GitAnnex.run (command:"-q":params)
 
+{- Runs git-annex and returns its output. -}
+git_annex_output :: String -> [String] -> IO String
+git_annex_output command params = do
+	(frompipe, topipe) <- createPipe
+	pid <- forkProcess $ do
+		_ <- dupTo topipe stdOutput
+		closeFd frompipe
+		_ <- git_annex command params
+		exitSuccess
+	-- XXX since the above is a separate process, code coverage stats are
+	-- not gathered for things run in it.
+	closeFd topipe
+	fromh <- fdToHandle frompipe
+	got <- hGetContentsStrict fromh
+	hClose fromh
+	_ <- getProcessStatus True False pid
+	-- XXX hack Run same command again, to get code coverage.
+	_ <- git_annex command params
+	return got
+
+git_annex_expectoutput :: String -> [String] -> [String] -> IO ()
+git_annex_expectoutput command params expected = do
+	got <- lines <$> git_annex_output command params
+	got == expected @? ("unexpected value running " ++ command ++ " " ++ show params ++ " -- got: " ++ show got ++ " expected: " ++ show expected)
+
 -- Runs an action in the current annex. Note that shutdown actions
 -- are not run; this should only be used for actions that query state.
 annexeval :: Types.Annex a -> IO a

From 8cdcd78b210756f184188a583c263569d87f08b4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 13:50:33 -0400
Subject: [PATCH 2751/2835] test bup special remote (74% coverage)

---
 debian/changelog |  2 +-
 test.hs          | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 33a9cd575f..59c6392fe7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
-  * Test suite improvements. Current top-level test coverage: 72%
+  * Test suite improvements. Current top-level test coverage: 74%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index eaa3fb8b64..0cd820a2b9 100644
--- a/test.hs
+++ b/test.hs
@@ -119,6 +119,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
+	, test_bup_remote
 	, test_crypto
 	]
 
@@ -638,6 +639,24 @@ test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 
+test_bup_remote :: Test
+test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ checkbup $ do
+	dir <- absPath "dir" -- bup special remote needs an absolute path
+	createDirectory dir
+	git_annex "initremote" (words $ "foo type=bup encryption=none buprepo="++dir) @? "initremote failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
+	annexed_present annexedfile
+	git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to bup remote failed"
+	annexed_present annexedfile
+	git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
+	annexed_notpresent annexedfile
+	git_annex "copy" [annexedfile, "--from", "foo"] @? "copy --from bup remote failed"
+	annexed_present annexedfile
+	not <$> git_annex "move" [annexedfile, "--from", "foo"] @? "move --from bup remote failed to fail"
+	annexed_present annexedfile
+	where
+		checkbup = whenM (inPath "bup")
+
 test_crypto :: Test
 test_crypto = "git-annex crypto" ~: intmpclonerepo $
 	-- gpg is not a build dependency, so only test when it's available

From 57235315c4dcab9d2271adcca94aba610b5ed0c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 14:10:36 -0400
Subject: [PATCH 2752/2835] improve

---
 test.hs | 74 +++++++++++++++++++++++++++++++++------------------------
 1 file changed, 43 insertions(+), 31 deletions(-)

diff --git a/test.hs b/test.hs
index 0cd820a2b9..1f595eaf23 100644
--- a/test.hs
+++ b/test.hs
@@ -141,6 +141,12 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			writeFile sha1annexedfile $ content sha1annexedfile
 			git_annex "add" [sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
 			annexed_present sha1annexedfile
+			checkbackend sha1annexedfile backendSHA1
+			writeFile wormannexedfile $ content wormannexedfile
+			git_annex "add" [wormannexedfile, "--backend=WORM"] @? "add with WORM failed"
+			annexed_present wormannexedfile
+			checkbackend wormannexedfile backendWORM
+			boolSystem "git" [Params "rm --force -q", File wormannexedfile] @? "git rm failed"
 			writeFile ingitfile $ content ingitfile
 			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
 			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
@@ -455,10 +461,6 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 		checkbackend sha1annexedfile backendSHA256
 		
 		where
-			checkbackend file expected = do
-				r <- annexeval $ Backend.lookupFile file
-				let b = snd $ fromJust r
-				assertEqual ("backend for " ++ file) expected b
 
 test_unused :: Test
 test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
@@ -640,7 +642,7 @@ test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	annexed_present annexedfile
 
 test_bup_remote :: Test
-test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ checkbup $ do
+test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ when Build.SysConfig.bup $ do
 	dir <- absPath "dir" -- bup special remote needs an absolute path
 	createDirectory dir
 	git_annex "initremote" (words $ "foo type=bup encryption=none buprepo="++dir) @? "initremote failed"
@@ -654,34 +656,31 @@ test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ checkbup $ do
 	annexed_present annexedfile
 	not <$> git_annex "move" [annexedfile, "--from", "foo"] @? "move --from bup remote failed to fail"
 	annexed_present annexedfile
-	where
-		checkbup = whenM (inPath "bup")
 
+-- gpg is not a build dependency, so only test when it's available
 test_crypto :: Test
-test_crypto = "git-annex crypto" ~: intmpclonerepo $
-	-- gpg is not a build dependency, so only test when it's available
-	when Build.SysConfig.gpg $ do
-		Utility.Gpg.testTestHarness @? "test harness self-test failed"
-		Utility.Gpg.testHarness $ do
-			createDirectory "dir"
-			let initremote = git_annex "initremote"
-				[ "foo"
-				, "type=directory"
-				, "encryption=" ++ Utility.Gpg.testKeyId
-				, "directory=dir"
-				]
-			initremote @? "initremote failed"
-			initremote @? "initremote failed when run twice in a row"
-			git_annex "get" [annexedfile] @? "get of file failed"
-			annexed_present annexedfile
-			git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed"
-			annexed_present annexedfile
-			git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
-			annexed_notpresent annexedfile
-			git_annex "move" [annexedfile, "--from", "foo"] @? "move --from encrypted remote failed"
-			annexed_present annexedfile
-			not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
-			annexed_present annexedfile	
+test_crypto = "git-annex crypto" ~: intmpclonerepo $ when Build.SysConfig.gpg $ do
+	Utility.Gpg.testTestHarness @? "test harness self-test failed"
+	Utility.Gpg.testHarness $ do
+		createDirectory "dir"
+		let initremote = git_annex "initremote"
+			[ "foo"
+			, "type=directory"
+			, "encryption=" ++ Utility.Gpg.testKeyId
+			, "directory=dir"
+			]
+		initremote @? "initremote failed"
+		initremote @? "initremote failed when run twice in a row"
+		git_annex "get" [annexedfile] @? "get of file failed"
+		annexed_present annexedfile
+		git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed"
+		annexed_present annexedfile
+		git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
+		annexed_notpresent annexedfile
+		git_annex "move" [annexedfile, "--from", "foo"] @? "move --from encrypted remote failed"
+		annexed_present annexedfile
+		not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
+		annexed_present annexedfile	
 
 -- This is equivilant to running git-annex, but it's all run in-process
 -- so test coverage collection works.
@@ -844,6 +843,12 @@ checklocationlog f expected = do
 				expected (thisuuid `elem` uuids)
 		_ -> assertFailure $ f ++ " failed to look up key"
 
+checkbackend :: FilePath -> Types.Backend Types.Annex -> Assertion			
+checkbackend file expected = do
+	r <- annexeval $ Backend.lookupFile file
+	let b = snd $ fromJust r
+	assertEqual ("backend for " ++ file) expected b
+
 inlocationlog :: FilePath -> Assertion
 inlocationlog f = checklocationlog f True
 
@@ -897,6 +902,9 @@ tmprepodir = tmpdir ++ "/tmprepo"
 annexedfile :: String
 annexedfile = "foo"
 
+wormannexedfile :: String
+wormannexedfile = "apple"
+
 sha1annexedfile :: String
 sha1annexedfile = "sha1foo"
 
@@ -912,6 +920,7 @@ content f
 	| f == ingitfile = "normal file content"
 	| f == sha1annexedfile ="sha1 annexed file content"
 	| f == sha1annexedfiledup = content sha1annexedfile
+	| f == wormannexedfile = "worm annexed file content"
 	| otherwise = "unknown file " ++ f
 
 changecontent :: FilePath -> IO ()
@@ -926,5 +935,8 @@ backendSHA1 = backend_ "SHA1"
 backendSHA256 :: Types.Backend Types.Annex
 backendSHA256 = backend_ "SHA256"
 
+backendWORM :: Types.Backend Types.Annex
+backendWORM = backend_ "WORM"
+
 backend_ :: String -> Types.Backend Types.Annex
 backend_ name = Backend.lookupBackendName name

From a76b13b848612595a254025f3eb415c2395a92de Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 14:20:41 -0400
Subject: [PATCH 2753/2835] test fsck in bare repos (75%)

---
 debian/changelog |  2 +-
 test.hs          | 20 +++++++++++++-------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 59c6392fe7..62c7d40f23 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
-  * Test suite improvements. Current top-level test coverage: 74%
+  * Test suite improvements. Current top-level test coverage: 75%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index 1f595eaf23..645da588e8 100644
--- a/test.hs
+++ b/test.hs
@@ -388,7 +388,7 @@ test_trust = "git-annex trust/untrust/semitrust/dead" ~: intmpclonerepo $ do
 			assertBool msg present
 
 test_fsck :: Test
-test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withremoteuntrusted]
+test_fsck = "git-annex fsck" ~: TestList [basicfsck, barefsck, withlocaluntrusted, withremoteuntrusted]
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
 			git_annex "fsck" [] @? "fsck failed"
@@ -397,6 +397,8 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
+		barefsck = TestCase $ intmpbareclonerepo $ do
+			git_annex "fsck" [] @? "fsck failed"
 		withlocaluntrusted = TestCase $ intmpclonerepo $ do
 			git_annex "get" [annexedfile] @? "get failed"
 			git_annex "untrust" ["origin"] @? "untrust of origin repo failed"
@@ -735,10 +737,13 @@ inmainrepo :: Assertion -> Assertion
 inmainrepo a = indir mainrepodir a
 
 intmpclonerepo :: Assertion -> Assertion
-intmpclonerepo a = withtmpclonerepo $ \r -> indir r a
+intmpclonerepo a = withtmpclonerepo False $ \r -> indir r a
 
-withtmpclonerepo :: (FilePath -> Assertion) -> Assertion
-withtmpclonerepo = bracket (clonerepo mainrepodir tmprepodir) cleanup
+intmpbareclonerepo :: Assertion -> Assertion
+intmpbareclonerepo a = withtmpclonerepo True $ \r -> indir r a
+
+withtmpclonerepo :: Bool -> (FilePath -> Assertion) -> Assertion
+withtmpclonerepo bare = bracket (clonerepo mainrepodir tmprepodir bare) cleanup
 
 withgitrepo :: (FilePath -> Assertion) -> Assertion
 withgitrepo = bracket (setuprepo mainrepodir) return
@@ -766,11 +771,12 @@ setuprepo dir = do
 	return dir
 
 -- clones are always done as local clones; we cannot test ssh clones
-clonerepo :: FilePath -> FilePath -> IO FilePath
-clonerepo old new = do
+clonerepo :: FilePath -> FilePath -> Bool -> IO FilePath
+clonerepo old new bare = do
 	cleanup new
 	ensuretmpdir
-	boolSystem "git" [Params "clone -q", File old, File new] @? "git clone failed"
+	let b = if bare then " --bare" else ""
+	boolSystem "git" [Params ("clone -q" ++ b), File old, File new] @? "git clone failed"
 	indir new $ git_annex "init" ["-q", new] @? "git annex init failed"
 	return new
 	

From 20482712d0d909f60707db7cde634e460e17058b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 16:56:48 -0400
Subject: [PATCH 2754/2835] Improve deletion of files from rsync special
 remotes. Closes: #652849

Rsync is only run once, with include / exclude rules used to specify
exactly what to delete. This is faster, and avoids ugly error messages
from rsync, and doesn't fail if the content already got deleted somehow.
---
 Remote/Rsync.hs  | 39 +++++++++++++++++++++++----------------
 debian/changelog |  1 +
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index c281420773..68566c52a5 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -92,11 +92,6 @@ rsyncUrls o k = map use annexHashes
 		use h = rsyncUrl o  h k  rsyncEscape o (f  f)
                 f = keyFile k
 
-rsyncUrlDirs :: RsyncOpts -> Key -> [String]
-rsyncUrlDirs o k = map use annexHashes
-	where
-		use h = rsyncUrl o  h k  rsyncEscape o (keyFile k)
-
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
 
@@ -125,17 +120,29 @@ retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
 		else return res
 
 remove :: RsyncOpts -> Key -> Annex Bool
-remove o k = untilTrue (rsyncUrlDirs o k) $ \d ->
-	withRsyncScratchDir $ \tmp -> liftIO $ do
-		{- Send an empty directory to rysnc as the
-		 - parent directory of the file to remove. -}
-		let dummy = tmp  keyFile k
-		createDirectoryIfMissing True dummy
-		rsync $ rsyncOptions o ++
-			[ Params "--quiet --delete --recursive"
-			, partialParams
-			, Param $ addTrailingPathSeparator dummy
-			, Param d
+remove o k = withRsyncScratchDir $ \tmp -> liftIO $ do
+	{- Send an empty directory to rysnc to make it delete. -}
+	let dummy = tmp  keyFile k
+	createDirectoryIfMissing True dummy
+	rsync $ rsyncOptions o ++
+		map (\s -> Param $ "--include=" ++ s) includes ++
+		[ Param "--exclude=*" -- exclude everything else
+		, Params "--quiet --delete --recursive"
+		, partialParams
+		, Param $ addTrailingPathSeparator dummy
+		, Param $ rsyncUrl o
+		]
+	where
+		{- Specify include rules to match the directories where the
+		 - content could be. Note that the parent directories have
+		 - to also be explicitly included, due to how rsync
+		 - traverses directories. -}
+		includes = concatMap use annexHashes
+		use h = let dir = h k in
+			[ parentDir dir
+			, dir
+			-- match content directory and anything in it
+			, dir  keyFile k  "***"
 			]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool)
diff --git a/debian/changelog b/debian/changelog
index 62c7d40f23..564e144038 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
   * Test suite improvements. Current top-level test coverage: 75%
+  * Improve deletion of files from rsync special remotes. Closes: #652849
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From 97bef4af733ac8c42adfe6809ba6ae7269530473 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 22 Dec 2011 12:31:36 +0000
Subject: [PATCH 2755/2835] Added a comment: List the duplicate filenames, then
 let the user decide what to do

---
 ..._f120d1e83c1a447f2ecce302fc69cf74._comment | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment
new file mode 100644
index 0000000000..a218ee3d51
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="List the duplicate filenames, then let the user decide what to do"
+ date="2011-12-22T12:31:29Z"
+ content="""
+I have the same use case as Asheesh but I want to be able to see which filenames point to the same objects and then decide which of the duplicates to drop myself.  I think
+
+    git annex drop --by-contents
+
+would be the wrong approach because how does git-annex know which ones to drop?  There's too much potential for error.
+
+Instead it would be great to have something like
+
+    git annex finddups
+
+While it's easy enough to knock up a bit of shell or Perl to achieve this, that relies on knowledge of the annex symlink structure, so I think really it belongs inside git-annex.
+
+If this command gave output similar to the excellent `fastdup` utility:
+
+    Scanning for files... 672 files in 10.439 seconds
+    Comparing 2 sets of files...
+
+    2 files (70.71 MB/ea)
+            /home/adam/media/flat/tour/flat-tour.3gp
+            /home/adam/videos/tour.3gp
+
+    Found 1 duplicate of 1 file (70.71 MB wasted)
+    Scanned 672 files (1.96 GB) in 11.415 seconds
+
+then you could do stuff like
+
+    git annex finddups | grep /home/adam/media/flat | xargs rm
+
+"""]]

From 6808b08c1ab096eb8c7c986379ec7a24d70434e3 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 22 Dec 2011 15:43:52 +0000
Subject: [PATCH 2756/2835] Added a comment: Here's a Perl version

---
 ...comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment
new file mode 100644
index 0000000000..e48a4a9b38
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="Here's a Perl version"
+ date="2011-12-22T15:43:51Z"
+ content="""
+https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups
+
+but it would be better in git-annex itself ...
+"""]]

From 30cf6ce81ca8ff99f5284c5b991e546eb1da72ae Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 22 Dec 2011 16:39:24 +0000
Subject: [PATCH 2757/2835] Added a comment

---
 ...ent_6_f24541ada1c86d755acba7e9fa7cff24._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
new file mode 100644
index 0000000000..93d3d41f43
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-12-22T16:39:24Z"
+ content="""
+My main concern with putting this in git-annex is that finding duplicates necessarily involves storing a list of every key and file in the repository, and git-annex is very carefully built to avoid things that require non-constant memory use, so that it can scale to very big repositories. (The only exception is the `unused` command, and reducing its memory usage is a continuing goal.)
+
+So I would rather come at this from a different angle.. like providing a way to output a list of files and their associated keys, which the user can then use in their own shell pipelines to find duplicate keys:
+
+      git annex find --include '*' --format=\"%f %k\n\" | sort foo --key 2 | uniq --all-repeated --skip-fields=1
+
+(Making that properly handle filenames with spaces is left as an exercise for the reader..)
+"""]]

From 6bffe509d7f1ec60168522585925a43dbfffbd36 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 13:53:06 -0400
Subject: [PATCH 2758/2835] Add --include, which is the same as --not
 --exclude.

---
 Git/Index.hs       |  2 --
 GitAnnex.hs        |  2 ++
 Limit.hs           | 10 ++++++++--
 debian/changelog   |  1 +
 doc/git-annex.mdwn | 22 ++++++++++++++++------
 test.hs            |  1 +
 6 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/Git/Index.hs b/Git/Index.hs
index e58f1edd67..aaf54e032e 100644
--- a/Git/Index.hs
+++ b/Git/Index.hs
@@ -9,8 +9,6 @@ module Git.Index where
 
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
-import Common
-
 {- Forces git to use the specified index file.
  -
  - Returns an action that will reset back to the default
diff --git a/GitAnnex.hs b/GitAnnex.hs
index a5b9609b6d..40ebed0d69 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -112,6 +112,8 @@ options = commonOptions ++
 		"terminate filename with null"
 	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
+	, Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob)
+		"don't skip files matching the glob pattern"
 	, Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote)
 		"skip files not present in a remote"
 	, Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber)
diff --git a/Limit.hs b/Limit.hs
index c68c3bdd8b..26e5d689c9 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -55,10 +55,16 @@ addLimit :: (FilePath -> Annex Bool) -> Annex ()
 addLimit = add . Utility.Matcher.Operation
 
 {- Add a limit to skip files that do not match the glob. -}
+addInclude :: String -> Annex ()
+addInclude glob = addLimit $ return . matchglob glob
+
+{- Add a limit to skip files that match the glob. -}
 addExclude :: String -> Annex ()
-addExclude glob = addLimit $ return . notExcluded
+addExclude glob = addLimit $ return . not . matchglob glob
+
+matchglob :: String -> FilePath -> Bool
+matchglob glob f = isJust $ match cregex f []
 	where
-		notExcluded f = isNothing $ match cregex f []
 		cregex = compile regex []
 		regex = '^':wildToRegex glob
 
diff --git a/debian/changelog b/debian/changelog
index 564e144038..13bca33260 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * map: --fast disables use of dot to display map
   * Test suite improvements. Current top-level test coverage: 75%
   * Improve deletion of files from rsync special remotes. Closes: #652849
+  * Add --include, which is the same as --not --exclude.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index b3d671bb89..f4eef5c4c7 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -241,12 +241,13 @@ subdirectories).
 
 * find [path ...]
 
-  Outputs a list of annexed files whose content is currently present.
-  Or, if a file matching option is specified, outputs a list of all
-  matching files, whether or not their content is currently present.
+  Outputs a list of annexed files in the specified path. With no path,
+  finds files in the current directory and its subdirectories.
 
-  With no parameters, defaults to finding all files in the current directory
-  and its subdirectories.
+  By default, only lists annexed files whose content is currently present.
+  This can be changed by specifying file matching options. To list all
+  annexed files, present or not, specify --include "*". To list all
+  annexed files whose content is not present, specify --not --in "."
 
   To output filenames terminated with nulls, for use with xargs -0,
   specify --print0.
@@ -447,7 +448,16 @@ file contents are present at either of two repositories.
 * --exclude=glob
 
   Skips files matching the glob pattern. The glob is matched relative to
-  the current directory. For example: --exclude='*.mp3' --exclude='subdir/*'
+  the current directory. For example:
+
+	--exclude='*.mp3' --exclude='subdir/*'
+
+* --include=glob
+
+  Skips files not matching the glob pattern.  (Same as --not --exclude.)
+  For example, to include only mp3 and ogg files:
+
+	--include='*.mp3' --or --include='*.ogg'
 
 * --in=repository
 
diff --git a/test.hs b/test.hs
index 645da588e8..b4823fda8d 100644
--- a/test.hs
+++ b/test.hs
@@ -524,6 +524,7 @@ test_find = "git-annex find" ~: intmpclonerepo $ do
 	annexed_notpresent sha1annexedfile
 	git_annex_expectoutput "find" [] [annexedfile]
 	git_annex_expectoutput "find" ["--exclude", annexedfile, "--and", "--exclude", sha1annexedfile] []
+	git_annex_expectoutput "find" ["--include", annexedfile] [annexedfile]
 	git_annex_expectoutput "find" ["--not", "--in", "origin"] []
 	git_annex_expectoutput "find" ["--copies", "1", "--and", "--not", "--copies", "2"] [sha1annexedfile]
 	git_annex_expectoutput "find" ["--inbackend", "SHA1"] [sha1annexedfile]

From 7892397020fe345886ef9aa84a6c1580ebee5348 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 14:50:20 -0400
Subject: [PATCH 2759/2835] improve output

---
 Command/Unannex.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 8a511bf4da..66611cbd74 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -41,8 +41,10 @@ cleanup file key = do
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
 	whenM (not . null <$> inRepo (LsFiles.staged [file])) $ do
+		showOutput
 		inRepo $ Git.Command.run "commit" [
-			Param "-m", Param "content removed from git annex",
+			Param "-q",
+			Params "-m", Param "content removed from git annex",
 			Param "--", File file]
 
 	fast <- Annex.getState Annex.fast

From 5a275a3f5d79977170a70b3e3ee16e38ac32916f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 14:59:25 -0400
Subject: [PATCH 2760/2835] Can now be built with older git versions (before
 1.7.7); the resulting binary should only be used with old git.

Remove git old version check from configure, and use the git version
it was built against in the git check-attr code.
---
 Git/CheckAttr.hs | 34 ++++++++++++++++++++++++++++------
 Git/Version.hs   | 38 ++++++++++++++++++++++++++++++++++++++
 configure.hs     | 23 ++++-------------------
 debian/changelog |  2 ++
 4 files changed, 72 insertions(+), 25 deletions(-)
 create mode 100644 Git/Version.hs

diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 0d3e798a1e..eedaf66420 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -13,32 +13,54 @@ import Common
 import Git
 import Git.Command
 import qualified Git.Filename
+import qualified Git.Version
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 lookup :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
 lookup attr files repo = do
-	-- git check-attr needs relative filenames input; it will choke
-	-- on some absolute filenames. This also means it will output
-	-- all relative filenames.
 	cwd <- getCurrentDirectory
-	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
         _ <- forkProcess $ do
 		hClose fromh
-                hPutStr toh $ join "\0" relfiles
+                hPutStr toh $ join "\0" $ input cwd
                 hClose toh
                 exitSuccess
         hClose toh
-	(map topair . lines) <$> hGetContents fromh
+	output cwd . lines <$> hGetContents fromh
 	where
 		params = gitCommandLine 
 				[ Param "check-attr"
 				, Param attr
 				, Params "-z --stdin"
 				] repo
+
+		{- Before git 1.7.7, git check-attr worked best with
+		 - absolute filenames; using them worked around some bugs
+		 - with relative filenames.
+		 - 
+		 - With newer git, git check-attr chokes on some absolute
+		 - filenames, and the bugs that necessitated them were fixed,
+		 - so use relative filenames. -}
+		oldgit = Git.Version.older "1.7.7"
+		input cwd
+			| oldgit = map (absPathFrom cwd) files
+			| otherwise = map (relPathDirToFile cwd . absPathFrom cwd) files
+		output cwd
+			| oldgit = map (torel cwd . topair)
+			| otherwise = map topair
+
 		topair l = (Git.Filename.decode file, value)
 			where 
 				file = join sep $ beginning bits
 				value = end bits !! 0
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
+
+		torel cwd (file, value) = (relfile, value)
+			where
+				relfile
+					| startswith cwd' file = drop (length cwd') file
+					| otherwise = relPathDirToFile top' file
+				top = workTree repo
+				cwd' = cwd ++ "/"
+				top' = top ++ "/"
diff --git a/Git/Version.hs b/Git/Version.hs
new file mode 100644
index 0000000000..c8bc121d66
--- /dev/null
+++ b/Git/Version.hs
@@ -0,0 +1,38 @@
+{- git version checking
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Version where
+
+import Common
+import qualified Build.SysConfig
+
+{- Using the version it was configured for avoids running git to check its
+ - version, at the cost that upgrading git won't be noticed.
+ - This is only acceptable because it's rare that git's version influences
+ - code's behavior. -}
+version :: String
+version = Build.SysConfig.gitversion
+
+older :: String -> Bool
+older v = normalize version < normalize v
+
+{- To compare dotted versions like 1.7.7 and 1.8, they are normalized to
+ - a somewhat arbitrary integer representation. -}
+normalize :: String -> Integer
+normalize = sum . mult 1 . reverse .
+		extend precision . take precision .
+		map readi . split "."
+	where
+		extend n l = l ++ replicate (n - length l) 0
+		mult _ [] = []
+		mult n (x:xs) = (n*x) : mult (n*10^width) xs
+		readi :: String -> Integer
+		readi s = case reads s of
+			((x,_):_) -> x
+			_ -> 0
+		precision = 10 -- number of segments of the version to compare
+		width = length "yyyymmddhhmmss" -- maximum width of a segment
diff --git a/configure.hs b/configure.hs
index bf25de5079..3b3626dd22 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,7 +2,6 @@
 
 import System.Directory
 import Data.List
-import Data.String.Utils
 import System.Cmd.Utils
 
 import Build.TestConfig
@@ -11,7 +10,7 @@ tests :: [TestCase]
 tests =
 	[ TestCase "version" getVersion
 	, TestCase "git" $ requireCmd "git" "git --version >/dev/null"
-	, TestCase "git version" checkGitVersion
+	, TestCase "git version" getGitVersion
 	, testCp "cp_a" "-a"
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
@@ -58,25 +57,11 @@ getVersionString = do
 	where
 		middle = drop 1 . init
 
-{- Checks for a new enough version of git. -}
-checkGitVersion :: Test
-checkGitVersion = do
+getGitVersion :: Test
+getGitVersion = do
 	(_, s) <- pipeFrom "git" ["--version"]
 	let version = last $ words $ head $ lines s
-	if dotted version < dotted need
-		then error $ "git version " ++ version ++ " too old; need " ++ need
-		else return $ Config "gitversion" (StringConfig version)
-	where
-		-- for git-check-attr behavior change
-		need = "1.7.7"
-		dotted = sum . mult 1 . reverse . extend 10 . map readi . split "." 
-		extend n l = l ++ replicate (n - length l) 0
-		mult _ [] = []
-		mult n (x:xs) = (n*x) : mult (n*100) xs
-		readi :: String -> Integer
-		readi s = case reads s of
-			((x,_):_) -> x
-			_ -> 0
+	return $ Config "gitversion" (StringConfig version)
 
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
diff --git a/debian/changelog b/debian/changelog
index 13bca33260..6c5b6effb9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,8 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Test suite improvements. Current top-level test coverage: 75%
   * Improve deletion of files from rsync special remotes. Closes: #652849
   * Add --include, which is the same as --not --exclude.
+  * Can now be built with older git versions (before 1.7.7); the resulting
+    binary should only be used with old git.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From 1071a3cf02a2be9b8fd7ec2222e80f6090c4b053 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlrnOT_XLcNNtmIwVdAJCJYu1BwAAOYtBI"
 
Date: Thu, 22 Dec 2011 19:21:50 +0000
Subject: [PATCH 2761/2835]

---
 .../link_file_to_remote_repo_feature.mdwn     | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 doc/todo/link_file_to_remote_repo_feature.mdwn

diff --git a/doc/todo/link_file_to_remote_repo_feature.mdwn b/doc/todo/link_file_to_remote_repo_feature.mdwn
new file mode 100644
index 0000000000..4bbd2b1a8f
--- /dev/null
+++ b/doc/todo/link_file_to_remote_repo_feature.mdwn
@@ -0,0 +1,48 @@
+I have two repos, using SHA1 backend and both using git.
+The first one is a laptop, the second one is a usb drive.
+
+When I drop a file on the laptop repo, the file is not available on that repo until I run *git annex get*
+But when the usb drive is plugged in the file is actually available.
+
+How about adding a feature to link some/all files to the remote repo?
+
+e.g. 
+We have *railscasts/196-nested-model-form-part-1.mp4* file added to git, and only available on the usb drive:
+
+      $ git annex whereis 196-nested-model-form-part-1.mp4 
+         whereis 196-nested-model-form-part-1.mp4 (1 copy) 
+  	   a7b7d7a4-2a8a-11e1-aebc-d3c589296e81 -- origin (Portable usb drive)
+
+I can see the link with:
+
+    $ cd railscasts
+    $ ls -ls 196*
+    8 lrwxr-xr-x  1 framallo  staff  193 Dec 20 05:49 196-nested-model-form-part-1.mp4 -> ../.git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e
+
+I save this in a variable just to make the example more clear:
+
+    ID=".git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e"
+
+The file doesn't exist on the local repo:
+
+    $ ls ../$ID
+    ls: ../$ID: No such file or directory
+
+however I can create a link to access that file on the remote repo.
+First I create a needed dir:
+
+    $ mkdir ../.git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/
+
+Then I link to the remote file:
+
+    $ ln -s /mnt/usb_drive/repo_folder/$ID ../$ID
+
+now I can open the file in the laptop repo.
+
+
+I think it could be easy to implement. Maybe It's a naive approach, but looks apealing.
+Checking if it's a real file or a link shouldn't impact on performance.
+The limitation is that it would work only with remote repos on local dirs
+
+Also allows you to have one directory structure like AFS or other distributed FS. If the file is not local I go to the remote server. 
+Which is great for apps like Picasa, Itunes, and friends that depends on the file location.

From 2a7246864eb2a43f041eb35ffe58166ca2c59229 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlrnOT_XLcNNtmIwVdAJCJYu1BwAAOYtBI"
 
Date: Thu, 22 Dec 2011 19:33:11 +0000
Subject: [PATCH 2762/2835]

---
 doc/forum/vlc_and_git-annex.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/vlc_and_git-annex.mdwn

diff --git a/doc/forum/vlc_and_git-annex.mdwn b/doc/forum/vlc_and_git-annex.mdwn
new file mode 100644
index 0000000000..cb07f8183c
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex.mdwn
@@ -0,0 +1,11 @@
+I used to save movies with the srt subtitle files next to them. 
+
+Usually vlc finds it because it's on the same directory than the movie file, however with git annex the link is located on another folder.
+So after adding movies to git, the subtitles doesn't load anymore.
+
+couldn't find a quick fix. I'm thinking a bash script, but wanted to discuss it here with all annex users.
+
+I know It's out of annex scope, but I think a movie archive is a great scenario for git-annex.
+most of my HD is filled up with movies from the camcorder, screencast, etc... 
+And we usually don't modify those files
+

From 58956c2a9071193ac1faeaab6551a4dba12e22fd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 15:38:15 -0400
Subject: [PATCH 2763/2835] cleanup

---
 test.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test.hs b/test.hs
index b4823fda8d..75d169105b 100644
--- a/test.hs
+++ b/test.hs
@@ -461,8 +461,6 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 		annexed_present sha1annexedfile
 		checkbackend annexedfile backendSHA256
 		checkbackend sha1annexedfile backendSHA256
-		
-		where
 
 test_unused :: Test
 test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do

From 38ad1065c9b26d98ba63185ba53407410e058455 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 22 Dec 2011 20:04:21 +0000
Subject: [PATCH 2764/2835] Added a comment

---
 ..._c39f1bb7c61a89b238c61bee1c049767._comment | 54 +++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment
new file mode 100644
index 0000000000..a337002804
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment
@@ -0,0 +1,54 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 7"
+ date="2011-12-22T20:04:14Z"
+ content="""
+> My main concern with putting this in git-annex is that finding
+> duplicates necessarily involves storing a list of every key and file
+> in the repository
+
+Only if you want to search the *whole* repository for duplicates, and if
+you do, then you're necessarily going to have to chew up memory in
+some process anyway, so what difference whether it's git-annex or
+(say) a Perl wrapper?
+
+> and git-annex is very carefully built to avoid things that require
+> non-constant memory use, so that it can scale to very big
+> repositories.
+
+That's a worthy goal, but if everything could be implemented with an
+O(1) memory footprint then we'd be in much more pleasant world :-)
+Even O(n) isn't that bad ...
+
+That aside, I like your `--format=\"%f %k\n\"` idea a lot.  That opens
+up the \"black box\" of `.git/annex/objects` and makes nice things
+possible, as your pipeline already demonstrates.  However, I'm not
+sure why you think `git annex find | sort | uniq` would be more
+efficient.  Not only does the sort require the very thing you were
+trying to avoid (i.e. the whole list in memory), but it's also 
+O(n log n) which is significantly slower than my O(n) Perl script 
+linked above.
+
+More considerations about this pipeline:
+
+* Doesn't it only include locally available files?  Ideally it should
+  spot duplicates even when the backing blob is not available locally.
+* What's the point of `--include '*'` ?  Doesn't `git annex find` 
+  with no arguments already include all files, modulo the requirement
+  above that they're locally available?
+* Any user using this `git annex find | ...` approach is likely to
+  run up against its limitations sooner rather than later, because
+  they're already used to the plethora of options `find(1)` provides.
+  Rather than reinventing the wheel, is there some way `git annex find`
+  could harness the power of `find(1)` ?
+
+Those considerations aside, a combined approach would be to implement
+
+    git annex find --format=...
+
+and then alter my Perl wrapper to `popen(2)` from that rather than using
+`File::Find`.  But I doubt you would want to ship Perl wrappers in the
+distribution, so if you don't provide a Haskell equivalent then users
+who can't code are left high and dry.
+"""]]

From c02605919b3d3cd30cb2f8e14ba5545e0ffaf205 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 22 Dec 2011 20:15:23 +0000
Subject: [PATCH 2765/2835] Added a comment: How much memory would it actually
 use anyway?

---
 .../comment_8_221ed2e53420278072a6d879c6f251d1._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment
new file mode 100644
index 0000000000..5ac292afeb
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="How much memory would it actually use anyway?"
+ date="2011-12-22T20:15:22Z"
+ content="""
+Another thought - an SHA1 digest is 20 bytes.  That means you can fit over 50 million keys into 1GB of RAM.  Granted you also need memory to store the values (pathnames) which in many cases will be longer, and some users may also choose more expensive backends than SHA1 ... but even so, it seems to me that you are at risk of throwing the baby out with the bath water.  
+"""]]

From cf496f09ab3777cc4ffe601e876b432dc320bd12 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 17:59:14 -0400
Subject: [PATCH 2766/2835] add a text formatter

This is built for speed; a format string is parsed once, generating a
Format, that can be applied repeatedly to different sets of variables
to generate output.
---
 Utility/Format.hs | 94 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 Utility/Format.hs

diff --git a/Utility/Format.hs b/Utility/Format.hs
new file mode 100644
index 0000000000..a49d95ff87
--- /dev/null
+++ b/Utility/Format.hs
@@ -0,0 +1,94 @@
+{- Formatted string handling.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Format (gen, format) where
+
+import Text.Printf (printf)
+import Data.String.Utils (replace)
+import Data.Char (isAlphaNum)
+import qualified Data.Map as M
+import Data.Maybe
+
+import Utility.PartialPrelude
+
+type FormatString = String
+
+{- A format consists of a list of fragments, with other text suffixed to
+ - the end. -}
+data Format = Format { spans :: [Frag], suffix :: String }
+	deriving (Show)
+
+{- A fragment is a variable (which may be padded), prefixed by some text. -}
+data Frag = Frag { prefix :: String, varname :: String, pad :: Int }
+	deriving (Show)
+
+newFormat :: Format
+newFormat = Format [] ""
+
+{- Expands a Format using some variables, generating a formatted string.
+ - This can be repeatedly called, efficiently. -}
+format :: Format -> M.Map String String -> String
+format f vars = concat $ concat $ reverse $ [suffix f] : go (spans f) []
+	where
+		go [] c = c
+		go (s:rest) c = go rest $ [prefix s, val s]:c
+		val (Frag { varname = var, pad = p }) =
+			justify p $ fromMaybe "" $ M.lookup var vars
+		justify p v
+			| p > 0 = take (p - length v) spaces ++ v
+			| p < 0 = v ++ take (-1 * (length v + p)) spaces
+			| otherwise = v
+		spaces = repeat ' '
+
+{- Generates a Format that can be used to expand variables in a
+ - format string, such as "${foo} ${bar}\n"
+ -
+ - To handle \n etc, printf is used, first escaping %, to
+ - avoid it needing any printf arguments.
+ -
+ - Left padding is enabled by "${var;width}"
+ - Right padding is enabled by "${var;-width}"
+ -
+ - (This is the same type of format string used by dpkg-query.)
+ -}
+gen :: FormatString -> Format
+gen = scan newFormat . printf . escapeprintf
+	where
+		escapeprintf = replace "%" "%%"
+		-- The Format is built up with fields reversed, for
+		-- efficiency.
+		finalize f v = f
+			{ suffix = (reverse $ suffix f) ++ v
+			, spans = (reverse $ spans f)
+			}
+		scan f (a:b:cs)
+			| a == '$' && b == '{' = invar f [] cs
+			| otherwise = scan f { suffix = a:suffix f } (b:cs)
+		scan f v = finalize f v
+		invar f var [] = finalize f $ novar var
+		invar f var (c:cs)
+			| c == '}' = foundvar f var 0 cs
+			| isAlphaNum c = invar f (c:var) cs
+			| c == ';' = inpad "" f var cs
+			| otherwise = scan f { suffix = (reverse $ novar $ c:var) ++ suffix f } cs
+		inpad p f var (c:cs)
+			| c == '}' = foundvar f var (readpad $ reverse p) cs
+			| otherwise = inpad (c:p) f var cs
+		inpad p f var [] = finalize f $ novar $ p++";"++var
+		readpad = fromMaybe 0 . readMaybe
+		novar v = "${" ++ reverse v
+		foundvar f v p cs = scan f' cs
+			where
+				f' = f
+					{ suffix = ""
+					, spans = newspan:spans f
+					}
+				newspan = Frag
+					{ prefix = reverse $ suffix f
+					, varname = reverse v
+					, pad = p
+					}

From 06bafae9e016b4d65c023546516e51bd32b08649 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 18:31:44 -0400
Subject: [PATCH 2767/2835] Format strings can be specified using the new
 --find option, to control what is output by git annex find.

---
 Annex.hs           |  5 +++--
 Command/Find.hs    | 23 +++++++++++++++++++----
 GitAnnex.hs        | 10 +++++++---
 Options.hs         |  2 ++
 Utility/Format.hs  |  2 +-
 debian/changelog   |  2 ++
 doc/git-annex.mdwn | 14 +++++++++++++-
 7 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index e5792fbcb2..e82ffc5d1f 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -37,6 +37,7 @@ import Types.BranchState
 import Types.TrustLevel
 import Types.UUID
 import qualified Utility.Matcher
+import qualified Utility.Format
 import qualified Data.Map as M
 
 -- git-annex's monad
@@ -62,7 +63,7 @@ data AnnexState = AnnexState
 	, force :: Bool
 	, fast :: Bool
 	, auto :: Bool
-	, print0 :: Bool
+	, format :: Maybe Utility.Format.Format
 	, branchstate :: BranchState
 	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
@@ -85,7 +86,7 @@ newState gitrepo = AnnexState
 	, force = False
 	, fast = False
 	, auto = False
-	, print0 = False
+	, format = Nothing
 	, branchstate = startBranchState
 	, catfilehandle = Nothing
 	, forcebackend = Nothing
diff --git a/Command/Find.hs b/Command/Find.hs
index 47058fa257..6050ff7bbb 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,11 +7,16 @@
 
 module Command.Find where
 
+import qualified Data.Map as M
+
 import Common.Annex
 import Command
 import Annex.Content
 import Limit
 import qualified Annex
+import qualified Utility.Format
+import Utility.DataUnits
+import Types.Key
 
 def :: [Command]
 def = [command "find" paramPaths seek "lists available files"]
@@ -24,8 +29,18 @@ start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
 	whenM (liftM2 (||) (inAnnex key) limited) $ do
-		print0 <- Annex.getState Annex.print0
-		if print0
-			then liftIO $ putStr (file ++ "\0")
-			else liftIO $ putStrLn file
+		f <- Annex.getState Annex.format
+		case f of
+			Nothing -> liftIO $ putStrLn file
+			Just formatter -> liftIO $ putStr $
+				Utility.Format.format formatter vars
 	stop
+	where
+		vars = M.fromList
+			[ ("file", file)
+			, ("key", show key)
+			, ("backend", keyBackendName key)
+			, ("bytesize", size show)
+			, ("humansize", size $ roughSize storageUnits True)
+			]
+		size c = maybe "unknown" c $ keySize key
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 40ebed0d69..7243d69cb0 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -18,6 +18,7 @@ import Types.TrustLevel
 import qualified Annex
 import qualified Remote
 import qualified Limit
+import qualified Utility.Format
 
 import qualified Command.Add
 import qualified Command.Unannex
@@ -108,8 +109,10 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	, Option [] ["print0"] (NoArg (setprint0 True))
-		"terminate filename with null"
+	, Option [] ["print0"] (NoArg setprint0)
+		"terminate output with null"
+	, Option [] ["format"] (ReqArg setformat paramFormat)
+		"control format of output"
 	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
 	, Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob)
@@ -125,7 +128,8 @@ options = commonOptions ++
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
-		setprint0 v = Annex.changeState $ \s -> s { Annex.print0 = v }
+		setformat v = Annex.changeState $ \s -> s { Annex.format = Just $ Utility.Format.gen v }
+		setprint0 = setformat "${file}\0"
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
 			newg <- inRepo $ Git.Config.store v
diff --git a/Options.hs b/Options.hs
index a8c165a816..cce750316e 100644
--- a/Options.hs
+++ b/Options.hs
@@ -82,6 +82,8 @@ paramUUID :: String
 paramUUID = "UUID"
 paramType :: String
 paramType = "TYPE"
+paramFormat :: String
+paramFormat = "FORMAT"
 paramKeyValue :: String
 paramKeyValue = "K=V"
 paramNothing :: String
diff --git a/Utility/Format.hs b/Utility/Format.hs
index a49d95ff87..cde63f57cd 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.Format (gen, format) where
+module Utility.Format (Format, gen, format) where
 
 import Text.Printf (printf)
 import Data.String.Utils (replace)
diff --git a/debian/changelog b/debian/changelog
index 6c5b6effb9..a27611f553 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,8 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Add --include, which is the same as --not --exclude.
   * Can now be built with older git versions (before 1.7.7); the resulting
     binary should only be used with old git.
+  * Format strings can be specified using the new --find option, to control
+    what is output by git annex find.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index f4eef5c4c7..70e54a91f4 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -250,7 +250,11 @@ subdirectories).
   annexed files whose content is not present, specify --not --in "."
 
   To output filenames terminated with nulls, for use with xargs -0,
-  specify --print0.
+  specify --print0. Or, a custom output formatting can be specified using
+  --format. The default output format is the same as --format='${file}\n'
+
+  These variables are available for use in formats: file, key, backend,
+  bytesize, humansize
 
 * whereis [path ...]
 
@@ -428,6 +432,14 @@ subdirectories).
   are in the annex, their backend is known and this option is not
   necessary.
 
+* --format=value
+
+  Specifies a custom output format. The value is a format string, 
+  in which '${var}' is expanded to the value of a variable. To right-align
+  a variable with whitespace, use '${var;width}' ; to left-align
+  a variable, use '${var;-width}'. Also, '\n' is a newline, '\0' is a NULL,
+  etc.
+
 * -c name=value
 
   Used to override git configuration settings. May be specified multiple times.

From a0872a8ec34894689489dd0cf47887d8609b9f47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 19:56:31 -0400
Subject: [PATCH 2768/2835] better data type

---
 Utility/Format.hs  | 86 ++++++++++++++++++++--------------------------
 doc/git-annex.mdwn |  4 +--
 2 files changed, 40 insertions(+), 50 deletions(-)

diff --git a/Utility/Format.hs b/Utility/Format.hs
index cde63f57cd..5a74da96b2 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -17,78 +17,68 @@ import Utility.PartialPrelude
 
 type FormatString = String
 
-{- A format consists of a list of fragments, with other text suffixed to
- - the end. -}
-data Format = Format { spans :: [Frag], suffix :: String }
+{- A format consists of a list of fragments. -}
+type Format = [Frag]
+
+{- A fragment is either a constant string, or a variable, with a padding. -}
+data Frag = Const String | Var String Padding
 	deriving (Show)
 
-{- A fragment is a variable (which may be padded), prefixed by some text. -}
-data Frag = Frag { prefix :: String, varname :: String, pad :: Int }
-	deriving (Show)
+{- Positive padding is right justification; negative padding is left
+ - justification. -}
+type Padding = Int
 
-newFormat :: Format
-newFormat = Format [] ""
+empty :: Frag -> Bool
+empty (Const "") = True
+empty _ = False
 
 {- Expands a Format using some variables, generating a formatted string.
  - This can be repeatedly called, efficiently. -}
 format :: Format -> M.Map String String -> String
-format f vars = concat $ concat $ reverse $ [suffix f] : go (spans f) []
+format f vars = concatMap expand f
 	where
-		go [] c = c
-		go (s:rest) c = go rest $ [prefix s, val s]:c
-		val (Frag { varname = var, pad = p }) =
-			justify p $ fromMaybe "" $ M.lookup var vars
-		justify p v
-			| p > 0 = take (p - length v) spaces ++ v
-			| p < 0 = v ++ take (-1 * (length v + p)) spaces
-			| otherwise = v
+		expand (Const s) = s
+		expand (Var name padding) = justify padding $
+			fromMaybe "" $ M.lookup name vars
+		justify p s
+			| p > 0 = take (p - length s) spaces ++ s
+			| p < 0 = s ++ take (-1 * (length s + p)) spaces
+			| otherwise = s
 		spaces = repeat ' '
 
 {- Generates a Format that can be used to expand variables in a
- - format string, such as "${foo} ${bar}\n"
- -
- - To handle \n etc, printf is used, first escaping %, to
- - avoid it needing any printf arguments.
- -
- - Left padding is enabled by "${var;width}"
- - Right padding is enabled by "${var;-width}"
+ - format string, such as "${foo} ${bar;10} ${baz;-10}\n"
  -
  - (This is the same type of format string used by dpkg-query.)
  -}
 gen :: FormatString -> Format
-gen = scan newFormat . printf . escapeprintf
+gen = finalize . scan []
 	where
-		escapeprintf = replace "%" "%%"
-		-- The Format is built up with fields reversed, for
-		-- efficiency.
-		finalize f v = f
-			{ suffix = (reverse $ suffix f) ++ v
-			, spans = (reverse $ spans f)
-			}
+		-- The Format is built up in reverse, for efficiency,
+		-- To finalize it, fix the reversing and do some
+		-- optimisations, including fusing adjacent Consts.
+		finalize = filter (not . empty) . fuse []
+		fuse f [] = f
+		fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs
+		fuse f (v:vs) = fuse (v:f) vs
+
 		scan f (a:b:cs)
 			| a == '$' && b == '{' = invar f [] cs
-			| otherwise = scan f { suffix = a:suffix f } (b:cs)
-		scan f v = finalize f v
-		invar f var [] = finalize f $ novar var
+			| otherwise = scan (Const [a] : f ) (b:cs)
+		scan f v = Const v : f
+
+		invar f var [] = Const (novar var) : f
 		invar f var (c:cs)
 			| c == '}' = foundvar f var 0 cs
 			| isAlphaNum c = invar f (c:var) cs
 			| c == ';' = inpad "" f var cs
-			| otherwise = scan f { suffix = (reverse $ novar $ c:var) ++ suffix f } cs
+			| otherwise = scan ((Const $ reverse $ novar $ c:var):f) cs
+
 		inpad p f var (c:cs)
 			| c == '}' = foundvar f var (readpad $ reverse p) cs
 			| otherwise = inpad (c:p) f var cs
-		inpad p f var [] = finalize f $ novar $ p++";"++var
+		inpad p f var [] = Const (novar $ p++";"++var) : f
 		readpad = fromMaybe 0 . readMaybe
+
 		novar v = "${" ++ reverse v
-		foundvar f v p cs = scan f' cs
-			where
-				f' = f
-					{ suffix = ""
-					, spans = newspan:spans f
-					}
-				newspan = Frag
-					{ prefix = reverse $ suffix f
-					, varname = reverse v
-					, pad = p
-					}
+		foundvar f v p cs = scan (Var (reverse v) p : f) cs
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 70e54a91f4..050a56980f 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -435,8 +435,8 @@ subdirectories).
 * --format=value
 
   Specifies a custom output format. The value is a format string, 
-  in which '${var}' is expanded to the value of a variable. To right-align
-  a variable with whitespace, use '${var;width}' ; to left-align
+  in which '${var}' is expanded to the value of a variable. To right-justify
+  a variable with whitespace, use '${var;width}' ; to left-justify
   a variable, use '${var;-width}'. Also, '\n' is a newline, '\0' is a NULL,
   etc.
 

From cba3ce08dfaa3318aa80b414e4d6d4f40d843d15 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 20:14:35 -0400
Subject: [PATCH 2769/2835] handle C-style escapes in Format

I was happily able to repurpose some code from Git.Filename to handle this.

I remember writing that code... a whole afternoon at a coffee shop, after
which I felt I'd struggled with Haskell and git, and sorta lost, in needing
to write this nasty peice of code. But was also pleased at the use of a
pair of functions and quickcheck that allowed me to get it 100% right.
So, turns out I not only got it right, but the code wasn't as special-purpose
as I'd feared. Yay!
---
 Git/Filename.hs   | 61 ++-----------------------------
 Utility/Format.hs | 93 ++++++++++++++++++++++++++++++++++++++++++-----
 test.hs           |  4 +-
 3 files changed, 90 insertions(+), 68 deletions(-)

diff --git a/Git/Filename.hs b/Git/Filename.hs
index 35b5532507..5e076d3b5a 100644
--- a/Git/Filename.hs
+++ b/Git/Filename.hs
@@ -8,10 +8,7 @@
 
 module Git.Filename where
 
-import qualified Codec.Binary.UTF8.String
-import Data.Char
-import Data.Word (Word8)
-import Text.Printf
+import Utility.Format (decode_c, encode_c)
 
 import Common
 
@@ -19,64 +16,12 @@ decode :: String -> FilePath
 decode [] = []
 decode f@(c:s)
 	-- encoded strings will be inside double quotes
-	| c == '"' && end s == ['"'] = unescape ("", beginning s)
+	| c == '"' && end s == ['"'] = decode_c $ beginning s
 	| otherwise = f
-	where
-		e = '\\'
-		unescape (b, []) = b
-		-- look for escapes starting with '\'
-		unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair)
-			where
-				pair = span (/= e) v
-		isescape x = x == e
-		-- \NNN is an octal encoded character
-		handle (x:n1:n2:n3:rest)
-			| isescape x && alloctal = (fromoctal, rest)
-				where
-					alloctal = isOctDigit n1 &&
-						isOctDigit n2 &&
-						isOctDigit n3
-					fromoctal = [chr $ readoctal [n1, n2, n3]]
-					readoctal o = Prelude.read $ "0o" ++ o :: Int
-		-- \C is used for a few special characters
-		handle (x:nc:rest)
-			| isescape x = ([echar nc], rest)
-			where
-				echar 'a' = '\a'
-				echar 'b' = '\b'
-				echar 'f' = '\f'
-				echar 'n' = '\n'
-				echar 'r' = '\r'
-				echar 't' = '\t'
-				echar 'v' = '\v'
-				echar a = a
-		handle n = ("", n)
 
 {- Should not need to use this, except for testing decode. -}
 encode :: FilePath -> String
-encode s = foldl (++) "\"" (map echar s) ++ "\""
-	where
-		e c = '\\' : [c]
-		echar '\a' = e 'a'
-		echar '\b' = e 'b'
-		echar '\f' = e 'f'
-		echar '\n' = e 'n'
-		echar '\r' = e 'r'
-		echar '\t' = e 't'
-		echar '\v' = e 'v'
-		echar '\\' = e '\\'
-		echar '"'  = e '"'
-		echar x
-			| ord x < 0x20 = e_num x -- low ascii
-			| ord x >= 256 = e_utf x
-			| ord x > 0x7E = e_num x -- high ascii
-			| otherwise = [x]        -- printable ascii
-			where 
-				showoctal i = '\\' : printf "%03o" i
-				e_num c = showoctal $ ord c
-				-- unicode character is decomposed to
-				-- Word8s and each is shown in octal
-				e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+encode s = "\"" ++ encode_c s ++ "\""
 
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
diff --git a/Utility/Format.hs b/Utility/Format.hs
index 5a74da96b2..804dbff4c8 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -1,17 +1,25 @@
 {- Formatted string handling.
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2010, 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.Format (Format, gen, format) where
+module Utility.Format (
+	Format,
+	gen,
+	format,
+	decode_c,
+	encode_c,
+	prop_idempotent_deencode
+) where
 
 import Text.Printf (printf)
-import Data.String.Utils (replace)
-import Data.Char (isAlphaNum)
+import Data.Char (isAlphaNum, isOctDigit, chr, ord)
+import Data.Maybe (fromMaybe)
+import Data.Word (Word8)
+import qualified Codec.Binary.UTF8.String
 import qualified Data.Map as M
-import Data.Maybe
 
 import Utility.PartialPrelude
 
@@ -52,12 +60,11 @@ format f vars = concatMap expand f
  - (This is the same type of format string used by dpkg-query.)
  -}
 gen :: FormatString -> Format
-gen = finalize . scan []
+gen = filter (not . empty) . fuse [] . scan [] . decode_c
 	where
 		-- The Format is built up in reverse, for efficiency,
-		-- To finalize it, fix the reversing and do some
-		-- optimisations, including fusing adjacent Consts.
-		finalize = filter (not . empty) . fuse []
+		-- and can have adjacent Consts. Fusing it fixes both
+		-- problems.
 		fuse f [] = f
 		fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs
 		fuse f (v:vs) = fuse (v:f) vs
@@ -82,3 +89,71 @@ gen = finalize . scan []
 
 		novar v = "${" ++ reverse v
 		foundvar f v p cs = scan (Var (reverse v) p : f) cs
+
+
+{- Decodes a C-style encoding, where \n is a newline, \NNN is an octal
+ - encoded character, etc.
+ -}
+decode_c :: FormatString -> FormatString
+decode_c [] = []
+decode_c s = unescape ("", s)
+	where
+		e = '\\'
+		unescape (b, []) = b
+		-- look for escapes starting with '\'
+		unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair)
+			where
+				pair = span (/= e) v
+		isescape x = x == e
+		-- \NNN is an octal encoded character
+		handle (x:n1:n2:n3:rest)
+			| isescape x && alloctal = (fromoctal, rest)
+				where
+					alloctal = isOctDigit n1 &&
+						isOctDigit n2 &&
+						isOctDigit n3
+					fromoctal = [chr $ readoctal [n1, n2, n3]]
+					readoctal o = Prelude.read $ "0o" ++ o :: Int
+		-- \C is used for a few special characters
+		handle (x:nc:rest)
+			| isescape x = ([echar nc], rest)
+			where
+				echar 'a' = '\a'
+				echar 'b' = '\b'
+				echar 'f' = '\f'
+				echar 'n' = '\n'
+				echar 'r' = '\r'
+				echar 't' = '\t'
+				echar 'v' = '\v'
+				echar a = a
+		handle n = ("", n)
+
+{- Should not need to use this, except for testing decode_c. -}
+encode_c :: FormatString -> FormatString
+encode_c s = concatMap echar s
+	where
+		e c = '\\' : [c]
+		echar '\a' = e 'a'
+		echar '\b' = e 'b'
+		echar '\f' = e 'f'
+		echar '\n' = e 'n'
+		echar '\r' = e 'r'
+		echar '\t' = e 't'
+		echar '\v' = e 'v'
+		echar '\\' = e '\\'
+		echar '"'  = e '"'
+		echar x
+			| ord x < 0x20 = e_num x -- low ascii
+			| ord x >= 256 = e_utf x
+			| ord x > 0x7E = e_num x -- high ascii
+			| otherwise = [x]        -- printable ascii
+			where 
+				showoctal i = '\\' : printf "%03o" i
+				e_num c = showoctal $ ord c
+				-- unicode character is decomposed to
+				-- Word8s and each is shown in octal
+				e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+
+{- for quickcheck -}
+prop_idempotent_deencode :: String -> Bool
+prop_idempotent_deencode s = s == decode_c (encode_c s)
diff --git a/test.hs b/test.hs
index 75d169105b..a2fa98e4df 100644
--- a/test.hs
+++ b/test.hs
@@ -45,6 +45,7 @@ import qualified Utility.Path
 import qualified Utility.FileMode
 import qualified Utility.Gpg
 import qualified Build.SysConfig
+import qualified Utility.Format
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -72,7 +73,8 @@ propigate (Counts { errors = e , failures = f }, _)
 
 quickcheck :: Test
 quickcheck = TestLabel "quickcheck" $ TestList
-	[ qctest "prop_idempotent_deencode" Git.Filename.prop_idempotent_deencode
+	[ qctest "prop_idempotent_deencode_git" Git.Filename.prop_idempotent_deencode
+	, qctest "prop_idempotent_deencode" Utility.Format.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape

From eb7ef6264ee57f69520960ae818ad0b94e5873c2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 20:18:33 -0400
Subject: [PATCH 2770/2835] fix

---
 doc/git-annex.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 050a56980f..7ad3fac69e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -251,7 +251,7 @@ subdirectories).
 
   To output filenames terminated with nulls, for use with xargs -0,
   specify --print0. Or, a custom output formatting can be specified using
-  --format. The default output format is the same as --format='${file}\n'
+  --format. The default output format is the same as --format='${file}\\n'
 
   These variables are available for use in formats: file, key, backend,
   bytesize, humansize
@@ -437,7 +437,7 @@ subdirectories).
   Specifies a custom output format. The value is a format string, 
   in which '${var}' is expanded to the value of a variable. To right-justify
   a variable with whitespace, use '${var;width}' ; to left-justify
-  a variable, use '${var;-width}'. Also, '\n' is a newline, '\0' is a NULL,
+  a variable, use '${var;-width}'. Also, '\\n' is a newline, '\\000' is a NULL,
   etc.
 
 * -c name=value

From db964e358f16d19e9a2e6124d6be8a9c87fdd88b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 20:21:42 -0400
Subject: [PATCH 2771/2835] reorg

---
 Utility/Format.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Utility/Format.hs b/Utility/Format.hs
index 804dbff4c8..d167087011 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -36,10 +36,6 @@ data Frag = Const String | Var String Padding
  - justification. -}
 type Padding = Int
 
-empty :: Frag -> Bool
-empty (Const "") = True
-empty _ = False
-
 {- Expands a Format using some variables, generating a formatted string.
  - This can be repeatedly called, efficiently. -}
 format :: Format -> M.Map String String -> String
@@ -63,7 +59,7 @@ gen :: FormatString -> Format
 gen = filter (not . empty) . fuse [] . scan [] . decode_c
 	where
 		-- The Format is built up in reverse, for efficiency,
-		-- and can have adjacent Consts. Fusing it fixes both
+		-- and can have many adjacent Consts. Fusing it fixes both
 		-- problems.
 		fuse f [] = f
 		fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs
@@ -90,6 +86,9 @@ gen = filter (not . empty) . fuse [] . scan [] . decode_c
 		novar v = "${" ++ reverse v
 		foundvar f v p cs = scan (Var (reverse v) p : f) cs
 
+empty :: Frag -> Bool
+empty (Const "") = True
+empty _ = False
 
 {- Decodes a C-style encoding, where \n is a newline, \NNN is an octal
  - encoded character, etc.

From 13a0c292b3bc72917cb8ce89e96f805602e81904 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 20:29:24 -0400
Subject: [PATCH 2772/2835] update example to actually work with new --format
 option

---
 ...__git_annex__34___command_that_will_skip_duplicates.mdwn | 2 ++
 .../comment_6_f24541ada1c86d755acba7e9fa7cff24._comment     | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index eda17aea66..ca18afc578 100644
--- a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -24,3 +24,5 @@ I want this because I have copies of various of mine (photos, in particular) sca
 (Another way to do this would be to "git annex add" them all, and then use a "git annex remove-duplicates" that could prompt me about which files are duplicates of each other, and then I could pipe that command's output into xargs git rm.)
 
 (As I write this, I realize it's possible to parse the destination of the symlink in a way that does this..)
+
+> 
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
index 93d3d41f43..5d8ac8e61b 100644
--- a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
@@ -8,7 +8,9 @@ My main concern with putting this in git-annex is that finding duplicates necess
 
 So I would rather come at this from a different angle.. like providing a way to output a list of files and their associated keys, which the user can then use in their own shell pipelines to find duplicate keys:
 
-      git annex find --include '*' --format=\"%f %k\n\" | sort foo --key 2 | uniq --all-repeated --skip-fields=1
+      git annex find --include '*' --format='${file} ${key}\n' | sort --key 2 | uniq --all-repeated --skip-fields=1
 
-(Making that properly handle filenames with spaces is left as an exercise for the reader..)
+Which is implemented now!
+
+(Making that pipeline properly handle filenames with spaces is left as an exercise for the reader..)
 """]]

From 7227dd8f21f24c2ccadd38e1a3dec7b888a23e92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 21:23:11 -0400
Subject: [PATCH 2773/2835] add escape_var hack

Makes it easy to find files with duplicate contents, anyway.. :)
---
 Utility/Format.hs                             | 48 ++++++++++++-------
 doc/git-annex.mdwn                            |  6 ++-
 doc/tips/finding_duplicate_files.mdwn         | 21 ++++++++
 ...4___command_that_will_skip_duplicates.mdwn |  2 +-
 4 files changed, 58 insertions(+), 19 deletions(-)
 create mode 100644 doc/tips/finding_duplicate_files.mdwn

diff --git a/Utility/Format.hs b/Utility/Format.hs
index d167087011..c0ba465806 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -18,6 +18,8 @@ import Text.Printf (printf)
 import Data.Char (isAlphaNum, isOctDigit, chr, ord)
 import Data.Maybe (fromMaybe)
 import Data.Word (Word8)
+import Data.List (isPrefixOf)
+import Data.String.Utils (replace)
 import qualified Codec.Binary.UTF8.String
 import qualified Data.Map as M
 
@@ -42,8 +44,16 @@ format :: Format -> M.Map String String -> String
 format f vars = concatMap expand f
 	where
 		expand (Const s) = s
-		expand (Var name padding) = justify padding $
-			fromMaybe "" $ M.lookup name vars
+		expand (Var name padding) = justify padding $ getvar name
+		getvar name
+			| "escaped_" `isPrefixOf` name = 
+				-- escape whitespace too
+				replace " " (e_asc ' ') $
+				replace "\t" (e_asc '\t') $
+				encode_c $
+				getvar' $ drop (length "escaped_") name
+			| otherwise = getvar' name
+		getvar' name = fromMaybe "" $ M.lookup name vars
 		justify p s
 			| p > 0 = take (p - length s) spaces ++ s
 			| p < 0 = s ++ take (-1 * (length s + p)) spaces
@@ -73,9 +83,9 @@ gen = filter (not . empty) . fuse [] . scan [] . decode_c
 		invar f var [] = Const (novar var) : f
 		invar f var (c:cs)
 			| c == '}' = foundvar f var 0 cs
-			| isAlphaNum c = invar f (c:var) cs
+			| isAlphaNum c || c == '_' = invar f (c:var) cs
 			| c == ';' = inpad "" f var cs
-			| otherwise = scan ((Const $ reverse $ novar $ c:var):f) cs
+			| otherwise = scan ((Const $ novar $ c:var):f) cs
 
 		inpad p f var (c:cs)
 			| c == '}' = foundvar f var (readpad $ reverse p) cs
@@ -127,7 +137,7 @@ decode_c s = unescape ("", s)
 				echar a = a
 		handle n = ("", n)
 
-{- Should not need to use this, except for testing decode_c. -}
+{- Inverse of decode_c. -}
 encode_c :: FormatString -> FormatString
 encode_c s = concatMap echar s
 	where
@@ -141,17 +151,23 @@ encode_c s = concatMap echar s
 		echar '\v' = e 'v'
 		echar '\\' = e '\\'
 		echar '"'  = e '"'
-		echar x
-			| ord x < 0x20 = e_num x -- low ascii
-			| ord x >= 256 = e_utf x
-			| ord x > 0x7E = e_num x -- high ascii
-			| otherwise = [x]        -- printable ascii
-			where 
-				showoctal i = '\\' : printf "%03o" i
-				e_num c = showoctal $ ord c
-				-- unicode character is decomposed to
-				-- Word8s and each is shown in octal
-				e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+		echar c
+			| ord c < 0x20 = e_asc c -- low ascii
+			| ord c >= 256 = e_utf c
+			| ord c > 0x7E = e_asc c -- high ascii
+			| otherwise = [c]        -- printable ascii
+
+-- unicode character is decomposed to individual Word8s,
+-- and each is shown in octal
+e_utf :: Char -> String
+e_utf c = showoctal . toInteger =<<
+	(Codec.Binary.UTF8.String.encode [c] :: [Word8])
+		
+e_asc :: Char -> String
+e_asc c = showoctal $ toInteger $ ord c
+
+showoctal :: Integer -> String				
+showoctal i = '\\' : printf "%03o" i
 
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 7ad3fac69e..2d0d2597ea 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -437,8 +437,10 @@ subdirectories).
   Specifies a custom output format. The value is a format string, 
   in which '${var}' is expanded to the value of a variable. To right-justify
   a variable with whitespace, use '${var;width}' ; to left-justify
-  a variable, use '${var;-width}'. Also, '\\n' is a newline, '\\000' is a NULL,
-  etc.
+  a variable, use '${var;-width}'; to escape unusual characters in a variable,
+  use '${escaped_var}'
+
+  Also, '\\n' is a newline, '\\000' is a NULL, etc.
 
 * -c name=value
 
diff --git a/doc/tips/finding_duplicate_files.mdwn b/doc/tips/finding_duplicate_files.mdwn
new file mode 100644
index 0000000000..94fc85400e
--- /dev/null
+++ b/doc/tips/finding_duplicate_files.mdwn
@@ -0,0 +1,21 @@
+Maybe you had a lot of files scattered around on different drives, and you
+added them all into a single git-annex repository. Some of the files are
+surely duplicates of others.
+
+While git-annex stores the file contents efficiently, it would still
+help in cleaning up this mess if you could find, and perhaps remove
+the duplicate files.
+
+Here's a command line that will show duplicate sets of files grouped together:
+
+	git annex find --include '*' --format='${file} ${escaped_key}\n' | \
+		sort -k2 | uniq --all-repeated=separate -f1 | \
+		sed 's/ [^ ]*$//'
+
+Here's a command line that will remove one of each duplicate set of files:
+
+	git annex find --include '*' --format='${file} ${escaped_key}\n' | \
+		sort -k2 | uniq --repeated -f1 | sed 's/ [^ ]*$//' | \
+		xargs -d '\n' git rm
+
+--[[Joey]] 
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index ca18afc578..9336535788 100644
--- a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -25,4 +25,4 @@ I want this because I have copies of various of mine (photos, in particular) sca
 
 (As I write this, I realize it's possible to parse the destination of the symlink in a way that does this..)
 
-> 
+> [[done]]; see [[tips/finding_duplicate_files]] --[[Joey]]

From fdf02986cf2c7db74cc8ca8ebd2786922652ff9b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 22:03:18 -0400
Subject: [PATCH 2774/2835] find --json

---
 Command/Find.hs  | 16 +++++++++-------
 Messages.hs      | 10 +++++++++-
 Messages/JSON.hs |  9 ++++++++-
 debian/changelog |  3 ++-
 4 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/Command/Find.hs b/Command/Find.hs
index 6050ff7bbb..91386bbd08 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -28,15 +28,17 @@ start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
-	whenM (liftM2 (||) (inAnnex key) limited) $ do
-		f <- Annex.getState Annex.format
-		case f of
-			Nothing -> liftIO $ putStrLn file
-			Just formatter -> liftIO $ putStr $
-				Utility.Format.format formatter vars
+	whenM (liftM2 (||) (inAnnex key) limited) $
+		unlessM (showFullJSON vars) $ do
+			f <- Annex.getState Annex.format
+			case f of
+				Nothing -> liftIO $ putStrLn file
+				Just formatter -> liftIO $ putStr $
+					Utility.Format.format formatter $
+						M.fromList vars
 	stop
 	where
-		vars = M.fromList
+		vars =
 			[ ("file", file)
 			, ("key", show key)
 			, ("backend", keyBackendName key)
diff --git a/Messages.hs b/Messages.hs
index a7f14f4853..1294e44f69 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -20,6 +20,7 @@ module Messages (
 	warning,
 	indent,
 	maybeShowJSON,
+	showFullJSON,
 	showCustom,
 	showHeader,
 	showRaw,
@@ -90,10 +91,17 @@ warning' w = do
 indent :: String -> String
 indent = join "\n" . map (\l -> "  " ++ l) . lines
 
-{- Shows a JSON value only when in json mode. -}
+{- Shows a JSON fragment only when in json mode. -}
 maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
 maybeShowJSON v = handle (JSON.add v) q
 
+{- Shows a complete JSON value, only when in json mode. -}
+showFullJSON :: JSON a => [(String, a)] -> Annex Bool
+showFullJSON v = Annex.getState Annex.output >>= liftIO . go
+	where
+		go Annex.JSONOutput = JSON.complete v >> return True
+		go _ = return False
+
 {- Performs an action that outputs nonstandard/customized output, and
  - in JSON mode wraps its output in JSON.start and JSON.end, so it's
  - a complete JSON document.
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
index a325ef130b..f7a031e381 100644
--- a/Messages/JSON.hs
+++ b/Messages/JSON.hs
@@ -9,7 +9,8 @@ module Messages.JSON (
 	start,
 	end,
 	note,
-	add
+	add,
+	complete
 ) where
 
 import Text.JSON
@@ -31,3 +32,9 @@ note s = add [("note", s)]
 
 add :: JSON a => [(String, a)] -> IO ()
 add v = putStr $ Stream.add v
+
+complete :: JSON a => [(String, a)] -> IO ()
+complete v = putStr $ concat
+	[ Stream.start v
+	, Stream.end
+	]
diff --git a/debian/changelog b/debian/changelog
index a27611f553..33d196fa32 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,8 +10,9 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Add --include, which is the same as --not --exclude.
   * Can now be built with older git versions (before 1.7.7); the resulting
     binary should only be used with old git.
-  * Format strings can be specified using the new --find option, to control
+  * Format strings can be specified using the new --format option, to control
     what is output by git annex find.
+  * Support git annex find --json
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From f015ef5fde9184b6756ee74c2be1bb39ae5f54ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 00:36:25 -0400
Subject: [PATCH 2775/2835] cleanup

---
 Utility/Format.hs | 79 +++++++++++++++++++++++------------------------
 1 file changed, 39 insertions(+), 40 deletions(-)

diff --git a/Utility/Format.hs b/Utility/Format.hs
index c0ba465806..2c2042cc22 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -15,11 +15,10 @@ module Utility.Format (
 ) where
 
 import Text.Printf (printf)
-import Data.Char (isAlphaNum, isOctDigit, chr, ord)
+import Data.Char (isAlphaNum, isOctDigit, isSpace, chr, ord)
 import Data.Maybe (fromMaybe)
 import Data.Word (Word8)
 import Data.List (isPrefixOf)
-import Data.String.Utils (replace)
 import qualified Codec.Binary.UTF8.String
 import qualified Data.Map as M
 
@@ -30,13 +29,13 @@ type FormatString = String
 {- A format consists of a list of fragments. -}
 type Format = [Frag]
 
-{- A fragment is either a constant string, or a variable, with a padding. -}
-data Frag = Const String | Var String Padding
+{- A fragment is either a constant string,
+ - or a variable, with a justification. -}
+data Frag = Const String | Var String Justify
 	deriving (Show)
 
-{- Positive padding is right justification; negative padding is left
- - justification. -}
-type Padding = Int
+data Justify = LeftJustified Int | RightJustified Int | UnJustified
+	deriving (Show)
 
 {- Expands a Format using some variables, generating a formatted string.
  - This can be repeatedly called, efficiently. -}
@@ -44,20 +43,16 @@ format :: Format -> M.Map String String -> String
 format f vars = concatMap expand f
 	where
 		expand (Const s) = s
-		expand (Var name padding) = justify padding $ getvar name
-		getvar name
-			| "escaped_" `isPrefixOf` name = 
-				-- escape whitespace too
-				replace " " (e_asc ' ') $
-				replace "\t" (e_asc '\t') $
-				encode_c $
-				getvar' $ drop (length "escaped_") name
-			| otherwise = getvar' name
-		getvar' name = fromMaybe "" $ M.lookup name vars
-		justify p s
-			| p > 0 = take (p - length s) spaces ++ s
-			| p < 0 = s ++ take (-1 * (length s + p)) spaces
-			| otherwise = s
+		expand (Var name j)
+			| "escaped_" `isPrefixOf` name =
+				justify j $ encode_c_strict $
+					getvar $ drop (length "escaped_") name
+			| otherwise = justify j $ getvar name
+		getvar name = fromMaybe "" $ M.lookup name vars
+		justify UnJustified s        = s
+		justify (LeftJustified i) s  = s ++ pad i s
+		justify (RightJustified i) s = pad i s ++ s
+		pad i s = take (i - length s) spaces
 		spaces = repeat ' '
 
 {- Generates a Format that can be used to expand variables in a
@@ -82,17 +77,20 @@ gen = filter (not . empty) . fuse [] . scan [] . decode_c
 
 		invar f var [] = Const (novar var) : f
 		invar f var (c:cs)
-			| c == '}' = foundvar f var 0 cs
+			| c == '}' = foundvar f var UnJustified cs
 			| isAlphaNum c || c == '_' = invar f (c:var) cs
 			| c == ';' = inpad "" f var cs
 			| otherwise = scan ((Const $ novar $ c:var):f) cs
 
 		inpad p f var (c:cs)
-			| c == '}' = foundvar f var (readpad $ reverse p) cs
+			| c == '}' = foundvar f var (readjustify $ reverse p) cs
 			| otherwise = inpad (c:p) f var cs
 		inpad p f var [] = Const (novar $ p++";"++var) : f
-		readpad = fromMaybe 0 . readMaybe
-
+		readjustify = getjustify . fromMaybe 0 . readMaybe
+		getjustify i
+			| i == 0 = UnJustified
+			| i < 0 = LeftJustified (-1 * i)
+			| otherwise = RightJustified i
 		novar v = "${" ++ reverse v
 		foundvar f v p cs = scan (Var (reverse v) p : f) cs
 
@@ -139,7 +137,14 @@ decode_c s = unescape ("", s)
 
 {- Inverse of decode_c. -}
 encode_c :: FormatString -> FormatString
-encode_c s = concatMap echar s
+encode_c = encode_c' (const False)
+
+{- Encodes more strictly, including whitespace. -}
+encode_c_strict :: FormatString -> FormatString
+encode_c_strict = encode_c' isSpace
+
+encode_c' :: (Char -> Bool) -> FormatString -> FormatString
+encode_c' p = concatMap echar
 	where
 		e c = '\\' : [c]
 		echar '\a' = e 'a'
@@ -153,21 +158,15 @@ encode_c s = concatMap echar s
 		echar '"'  = e '"'
 		echar c
 			| ord c < 0x20 = e_asc c -- low ascii
-			| ord c >= 256 = e_utf c
+			| ord c >= 256 = e_utf c -- unicode
 			| ord c > 0x7E = e_asc c -- high ascii
-			| otherwise = [c]        -- printable ascii
-
--- unicode character is decomposed to individual Word8s,
--- and each is shown in octal
-e_utf :: Char -> String
-e_utf c = showoctal . toInteger =<<
-	(Codec.Binary.UTF8.String.encode [c] :: [Word8])
-		
-e_asc :: Char -> String
-e_asc c = showoctal $ toInteger $ ord c
-
-showoctal :: Integer -> String				
-showoctal i = '\\' : printf "%03o" i
+			| p c          = e_asc c -- unprintable ascii
+			| otherwise    = [c]     -- printable ascii
+		-- unicode character is decomposed to individual Word8s,
+		-- and each is shown in octal
+		e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+		e_asc c = showoctal $ ord c
+		showoctal i = '\\' : printf "%03o" i
 
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool

From 77ffd070adda29cafb4a20076eeda3bba4805e07 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 13:31:34 +0000
Subject: [PATCH 2776/2835] Added a comment

---
 ...comment_1_c0b7682a2b6f3078457b85683c825baf._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment

diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment
new file mode 100644
index 0000000000..e627ead47c
--- /dev/null
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 1"
+ date="2011-12-23T13:31:33Z"
+ content="""
+ControlPersist is awesome - thanks!
+
+Here's [an alternative, git-specific approach](http://thread.gmane.org/gmane.comp.version-control.home-dir/502).
+"""]]

From abba5d3e827b5d31766b95b2e2003aa821f289fc Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 14:04:46 +0000
Subject: [PATCH 2777/2835] Added a comment: I think Matt is right.

---
 ..._646f2077edcabc000a7d9cb75a93cf55._comment | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment b/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment
new file mode 100644
index 0000000000..6ba1796939
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="I think Matt is right."
+ date="2011-12-23T14:04:44Z"
+ content="""
+I got bitten by this too.  It seems that the user is expected to fetch
+remote git-annex branches themselves, but this is not documented
+anywhere.
+
+The man page says of \"git annex merge\":
+
+    Automatically merges any changes from remotes into the git-annex
+    branch.
+
+I am not a git newbie, but even so I had incorrectly assumed that git
+annex merge would take care of pulling the git-annex branch from the
+remote prior to merging, thereby ensuring all versions of the
+git-annex branch would be merged, and that the location tracking data
+would be synced across all peer repositories.
+
+My master branches do not track any specific upstream branch, because
+I am operating in a decentralized fashion.  Therefore the error
+message caused by `git pull $remote` succeeded in encouraging me to
+instead use `git pull $remote master`, and this excludes the git-annex
+branch from the fetch.  Even worse, a git newbie might realise this
+and be tempted to do `git pull $remote git-annex`.
+
+Therefore I think it needs to be explicitly documented that
+
+    git fetch $remote
+    git merge $remote/master
+
+is required when the local branch doesn't track an upstream branch.
+Or maybe a `--fetch` option could be added to `git annex merge` to
+perform the fetch from all remotes before running the merge(s).
+"""]]

From 538665f4779825427f7f42d6245e83032459951b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 16:07:39 +0000
Subject: [PATCH 2778/2835] Added a comment

---
 ...ent_9_aecfa896c97b9448f235bce18a40621d._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment
new file mode 100644
index 0000000000..82c6921ebb
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 9"
+ date="2011-12-23T16:07:39Z"
+ content="""
+Adam, to answer a lot of points breifly..
+
+* --include='*' makes find list files whether their contents are present or not
+* Your perl script is not O(n). Inserting into perl hash tables has overhead of minimum O(n log n). Not counting the overhead of resizing hash tables, the grevious slowdown if the bucket size is overcome by data (it probably falls back to a linked list or something then), and the overhead of traversing the hash tables to get data out.
+* I think that git-annex's set of file matching options is coming along nicely, and new ones can easily be added, so see no need to pull in unix find(1).
+* Your memory size calculations ignore the overhead of a hash table or other data structure to store the data in, which will tend to be more than the actual data size it's storing.  I estimate your 50 million number is off by at least one order of magnitude, and more likely two; in any case I don't want git-annex to use 1 gb of ram.
+* Little known fact: sort(1) will use a temp file as a buffer if too much memory is needed to hold the data to sort. It's also written in the most efficient language possible and has been ruthlessly optimised for 30 years, so I would be very surprised if it was not the best choice.
+"""]]

From e2b2c67f05945db9bf65bc6cc490244ffa5cf0e8 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 16:16:19 +0000
Subject: [PATCH 2779/2835] Added a comment

---
 ...comment_1_9c9ab8ce463cf74418aa2f385955f165._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment

diff --git a/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment b/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment
new file mode 100644
index 0000000000..700b3808de
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-23T16:16:19Z"
+ content="""
+From what you say, it seems that vlc is following the symlink to the movie content, and then looking for subtitles next to the file the symlink points to. It would have to explicitly realpath the symlink to have this behavior, and this sounds like a misfeature.. perhaps you could point out to the vlc people the mistake in doing so?
+
+There's a simple use-case where this behavior is obviously wrong, without involving git-annex. Suppose I have a movie, and one version of subtitles for it, in directory `foo`. I want to modify the subtitles, so I make a new directory `bar`, symlink the large movie file from `foo` to save space, and copy over and edit the subtitles from `foo`. Now I run vlc in `bar` to test my new subtitles. If it ignores the locally present subtitles and goes off looking for the ones in `bar`, I say this is broken behavior.
+"""]]

From 0df51acc529c7dfc5beae7e6031d00a6064c64a0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 16:20:43 +0000
Subject: [PATCH 2780/2835]

---
 doc/todo/link_file_to_remote_repo_feature.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/todo/link_file_to_remote_repo_feature.mdwn b/doc/todo/link_file_to_remote_repo_feature.mdwn
index 4bbd2b1a8f..d6b41e8059 100644
--- a/doc/todo/link_file_to_remote_repo_feature.mdwn
+++ b/doc/todo/link_file_to_remote_repo_feature.mdwn
@@ -46,3 +46,7 @@ The limitation is that it would work only with remote repos on local dirs
 
 Also allows you to have one directory structure like AFS or other distributed FS. If the file is not local I go to the remote server. 
 Which is great for apps like Picasa, Itunes, and friends that depends on the file location.
+
+> This is a duplicate of [[union_mounting]]. So closing it: [[done]]. 
+> 
+> It's a good idea, but making sure git-annex correctly handles these links in all cases is a subtle problem that has not yet been tackled. --[[Joey]]

From 9e6ff9e73425e53a8aac884fd97e3ea8f56bc74c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 12:48:59 -0400
Subject: [PATCH 2781/2835] improve wording to not imply a pull is done by
 merge

---
 doc/git-annex.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 2d0d2597ea..39fe5d99ea 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -218,10 +218,10 @@ subdirectories).
 
 * merge
 
-  Automatically merges any changes from remotes into the git-annex branch.
-  While git-annex mostly handles keeping the git-annex branch merged
-  automatically, if you find you are unable to push the git-annex branch
-  due non-fast-forward, this will fix it.
+  Automatically merges remote tracking branches */git-annex into
+  the git-annex branch. While git-annex mostly handles keeping the
+  git-annex branch merged automatically, if you find you are unable
+  to push the git-annex branch due non-fast-forward, this will fix it.
 
 * fix [path ...]
 

From 304930f1fc952b8f2dbe019049d5e02d8eb582dc Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 16:50:26 +0000
Subject: [PATCH 2782/2835] Added a comment

---
 ...comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment b/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment
new file mode 100644
index 0000000000..c01f241202
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-12-23T16:50:26Z"
+ content="""
+My goal for `git-annex merge` is that users should not need to know about it, so it should not be doing expensive pulls.
+
+I hope that `git annex sync` will grow some useful features to support fully distributed git usage, as being discussed in [[pure_git-annex_only_workflow]]. I still use centralized git to avoid these problems myself.
+"""]]

From a0227e81f9c82afc12ac1bd1cecd63cc0894d751 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 12:55:11 -0400
Subject: [PATCH 2783/2835] put in explicit fetch ; merge in walkthrough for
 now

and link to centralized repository tutorial
---
 doc/walkthrough/getting_file_content.mdwn | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index 71f95f79e0..5ad97bc325 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -6,14 +6,18 @@ We can use this to copy everything in the laptop's annex to the
 USB drive.
 
 	# cd /media/usb/annex
-	# git pull laptop
+	# git fetch laptop
+	# git merge laptop/master
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok
 
-Notice that you had to git pull from laptop first, this lets git-annex know
-what has changed in laptop, and so it knows about the files present there and
-can get them. The alternate approach is to set up a central bare repository,
-and always push changes to it after committing them, then in the above,
-you can just pull from the central repository to get synced up to all
-repositories.
+Notice that you had to git fetch and merge from laptop first, this lets
+git-annex know what has changed in laptop, and so it knows about the files
+present there and can get them.
+
+The alternate approach is to set up a
+[[central bare repository|tips/centralized_git_repository_tutorial]], and
+always push changes to it after committing them, then in the above,
+you can just pull from the central repository to get synced up to
+all repositories.

From 6fcf76bcd162a31b7099bc6ae624b11015ea52b4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 12:57:52 -0400
Subject: [PATCH 2784/2835] slight simplification

---
 doc/walkthrough/getting_file_content.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index 5ad97bc325..fac6aa2a78 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -6,8 +6,7 @@ We can use this to copy everything in the laptop's annex to the
 USB drive.
 
 	# cd /media/usb/annex
-	# git fetch laptop
-	# git merge laptop/master
+	# git fetch laptop; git merge laptop
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok

From 3b1c80d7950216ea90f706d975ff663cb92e52c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 12:58:39 -0400
Subject: [PATCH 2785/2835] fix

---
 doc/walkthrough/getting_file_content.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index fac6aa2a78..cdb4a72e0a 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -6,7 +6,7 @@ We can use this to copy everything in the laptop's annex to the
 USB drive.
 
 	# cd /media/usb/annex
-	# git fetch laptop; git merge laptop
+	# git fetch laptop; git merge laptop/master
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok

From fe65981de87cea18c18240b541ac54b36534390e Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 17:14:03 +0000
Subject: [PATCH 2786/2835] Added a comment

---
 .../comment_6_3925d1aa56bce9380f712e238d63080f._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment b/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment
new file mode 100644
index 0000000000..f4b5ebec20
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 6"
+ date="2011-12-23T17:14:03Z"
+ content="""
+Extending `git annex sync` would be nice, although auto-commit does not suit every use case, so it would be better not to couple one to the other.
+"""]]

From b05c08b5c1ad0deacec8943128e1eec8362471a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 13:19:28 -0400
Subject: [PATCH 2787/2835] reorder less expensive terminal first

Out of general principles, it did not seem to actually speed it
up appreciably. (I suspect ghc is being smart.)
---
 Command/Find.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Find.hs b/Command/Find.hs
index 91386bbd08..1961e6b748 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -28,7 +28,7 @@ start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
-	whenM (liftM2 (||) (inAnnex key) limited) $
+	whenM (liftM2 (||) limited (inAnnex key)) $
 		unlessM (showFullJSON vars) $ do
 			f <- Annex.getState Annex.format
 			case f of

From d3e80eabe8bc58fa232c49e10f442c94c0107d79 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 17:22:12 +0000
Subject: [PATCH 2788/2835] Added a comment

---
 ..._d78d79fb2f3713aa69f45d2691cf8dfe._comment | 68 +++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment
new file mode 100644
index 0000000000..5dbb66cf66
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment
@@ -0,0 +1,68 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 10"
+ date="2011-12-23T17:22:11Z"
+ content="""
+> Your perl script is not O(n). Inserting into perl hash tables has
+> overhead of minimum O(n log n).
+
+What's your source for this assertion?  I would expect an amortized
+average of `O(1)` per insertion, i.e. `O(n)` for full population.
+
+> Not counting the overhead of resizing hash tables,
+> the grevious slowdown if the bucket size is overcome by data (it
+> probably falls back to a linked list or something then), and the
+> overhead of traversing the hash tables to get data out.
+
+None of which necessarily change the algorithmic complexity.  However
+real benchmarks are far more useful here than complexity analysis, and
+[the dangers of premature optimization](http://c2.com/cgi/wiki?PrematureOptimization) 
+should not be forgotten.
+
+> Your memory size calculations ignore the overhead of a hash table or
+> other data structure to store the data in, which will tend to be
+> more than the actual data size it's storing. I estimate your 50
+> million number is off by at least one order of magnitude, and more
+> likely two;
+
+Sure, I was aware of that, but my point still stands.  Even 500k keys
+per 1GB of RAM does not sound expensive to me.
+
+> in any case I don't want git-annex to use 1 gb of ram.
+
+Why not?  What's the maximum it should use?  512MB?  256MB?
+32MB?  I don't see the sense in the author of a program 
+dictating thresholds which are entirely dependent on the context
+in which the program is *run*, not the context in which it's *written*.
+That's why systems have files such as `/etc/security/limits.conf`.
+
+You said you want git-annex to scale to enormous repositories.  If you
+impose an arbitrary memory restriction such as the above, that means
+avoiding implementing *any* kind of functionality which requires `O(n)`
+memory or worse.  Isn't it reasonable to assume that many users use
+git-annex on repositories which are *not* enormous?  Even when they do
+work with enormous repositories, just like with any other program,
+they would naturally expect certain operations to take longer or
+become impractical without sufficient RAM.  That's why I say that this
+restriction amounts to throwing out the baby with the bathwater.
+It just means that those who need the functionality would have to
+reimplement it themselves, assuming they are able, which is likely
+to result in more wheel reinventions.  I've already shared
+[my implementation](https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups)
+but how many people are likely to find it, let alone get it working?
+
+> Little known fact: sort(1) will use a temp file as a buffer if too
+> much memory is needed to hold the data to sort.
+
+Interesting.  Presumably you are referring to some undocumented
+behaviour, rather than `--batch-size` which only applies when merging
+multiple files, and not when only sorting STDIN.
+
+> It's also written in the most efficient language possible and has
+> been ruthlessly optimised for 30 years, so I would be very surprised
+> if it was not the best choice.
+
+It's the best choice for sorting.  But sorting purely to detect
+duplicates is a dismally bad choice.
+"""]]

From 74e86b6da20ae7d7465a30627e29eef81b3244ce Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 17:24:59 +0000
Subject: [PATCH 2789/2835] Added a comment

---
 .../comment_7_24c45ee981b18bc78325c768242e635d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment b/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment
new file mode 100644
index 0000000000..dad2c0af21
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 7"
+ date="2011-12-23T17:24:58Z"
+ content="""
+P.S. I see you already [fixed the docs](http://source.git-annex.branchable.com/?p=source.git;a=commitdiff;h=a0227e81f9c82afc12ac1bd1cecd63cc0894d751) - thanks! :)
+"""]]

From d5889f827105d31131a69d0feb2e2574d8be8b9e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 17:52:21 +0000
Subject: [PATCH 2790/2835] Added a comment

---
 .../comment_11_4316d9d74312112dc4c823077af7febe._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment
new file mode 100644
index 0000000000..286487eee5
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 11"
+ date="2011-12-23T17:52:21Z"
+ content="""
+I don't think that [[tips/finding_duplicate_files]] is hard to find, and the multiple different ways it shows to deal with the duplicate files shows the flexability of the unix pipeline approach.
+"""]]

From cbaf13e587e340debf22aa1de505f4ebb15f583c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 18:02:24 +0000
Subject: [PATCH 2791/2835] Added a comment

---
 .../comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment
new file mode 100644
index 0000000000..909beed837
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 12"
+ date="2011-12-23T18:02:24Z"
+ content="""
+BTW, sort -S '90%' benchmarks consistently 2x as fast as perl's hashes all the way up to 1 million files. Of course the pipeline approach allows you to swap in perl or whatever else is best for you at scale.
+"""]]

From 1940e52793ff20cf1c48f78fef95985434ed7dee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 14:08:04 -0400
Subject: [PATCH 2792/2835] skip repos without a description set when --in=""
 is specified

Picking one of probably several remotes with no description set was not
useful behavior.
---
 Remote.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Remote.hs b/Remote.hs
index aa86934143..10bf9d7694 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -112,6 +112,7 @@ byName' n = do
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID -- special case for current repo
+nameToUUID "" = error "no remote specified"
 nameToUUID n = byName' n >>= go
 	where
 		go (Right r) = return $ uuid r

From 131d560ba66405badf85e2450529d222f15e1e21 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 18:43:05 +0000
Subject: [PATCH 2793/2835] Added a comment

---
 .../comment_2_037f94c1deeac873dbdb36cd4c927e45._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment

diff --git a/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment b/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment
new file mode 100644
index 0000000000..3c69f5fe46
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-23T18:43:05Z"
+ content="""
+Since subtitle files are typically pretty small, a workaround is to simply check them into git directly, and only use git-annex for the movies. (Or `git annex unannex` the ones you've already annexed.)
+"""]]

From bfa5509dbdb3615584023cc6106fbc42937fa7ac Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 19:16:51 +0000
Subject: [PATCH 2794/2835] Added a comment: Cool

---
 .../comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment

diff --git a/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment b/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment
new file mode 100644
index 0000000000..d1bd4475e5
--- /dev/null
+++ b/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="Cool"
+ date="2011-12-23T19:16:50Z"
+ content="""
+Very nice :)  Just for reference, here's [my Perl implementation](https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups).  As per [this discussion](http://git-annex.branchable.com/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/#comment-fb15d5829a52cd05bcbd5dc53edaffb2) it would be interesting to benchmark these two approaches and see if one is substantially more efficient than the other w.r.t. CPU and memory usage.
+"""]]

From 56e4acfa2450ed94194a3ab5284a81e432e24c1d Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 19:19:53 +0000
Subject: [PATCH 2795/2835] Added a comment

---
 .../comment_1_b0d22822017646775869ce1292e676f4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment

diff --git a/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment b/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment
new file mode 100644
index 0000000000..22857af3e8
--- /dev/null
+++ b/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-23T19:19:53Z"
+ content="""
+See also: [[centralized_git_repository_tutorial]]
+"""]]

From 4fb31142287b05291ba66258477d28670c0d3562 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmKPMUX0YHBjE93eBsEnacwZsddSDue3PY"
 
Date: Fri, 23 Dec 2011 22:04:09 +0000
Subject: [PATCH 2796/2835] Added a comment

---
 ...t_2_49f15478781a0ad5e46e75319070335c._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment
new file mode 100644
index 0000000000..94b5c2ec11
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmKPMUX0YHBjE93eBsEnacwZsddSDue3PY"
+ nickname="Oliver"
+ subject="comment 2"
+ date="2011-12-23T22:04:08Z"
+ content="""
+As joey points out the problem is B overwrites A, so that any files in A that aren't in B will be removed. But the suggestion to keep B in a separate subdirectory in the repository means I'll end up with duplicates of files in both A and B. What I want is to have the merged superset of all files from both A and B with only one copy of identical files. 
+
+The problem is that unique symlinks in A/master are deleted when B/master is merged in. To add back the deleted files after the merge you can do this:
+
+    git checkout master~1 deleted_file_name                                                              #checkout a single deleted file called deleted_file_name
+    git diff master~1 master --numstat --name-only --diff-filter=D                                       #get the names of all files deleted between master and master~1
+    git diff master~1 master --numstat --name-only --diff-filter=D | xargs git checkout master~1         #checkout all deleted files between master and master~1
+
+Once the first merge has been done after set up, you can continue to make changes to A and B and future merges won't require accounting for deleted files in this way.
+"""]]

From 010780dc5534461afdba6e5c29b7f8bbc8fdf04a Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 24 Dec 2011 00:13:43 +0000
Subject: [PATCH 2797/2835] add a page for openSUSE install instructions

---
 doc/install.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 6f892e37a2..cc26ee91d5 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -5,6 +5,7 @@
 * [[Ubuntu]]
 * [[Fedora]]
 * [[FreeBSD]]
+* [[openSUSE]]
 
 ## Using cabal
 

From 1202e605d7b036693112e24148f89188d4f61aec Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 24 Dec 2011 00:45:43 +0000
Subject: [PATCH 2798/2835]

---
 doc/install/openSUSE.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/install/openSUSE.mdwn

diff --git a/doc/install/openSUSE.mdwn b/doc/install/openSUSE.mdwn
new file mode 100644
index 0000000000..762db1f2d2
--- /dev/null
+++ b/doc/install/openSUSE.mdwn
@@ -0,0 +1,4 @@
+Unfortunately there is currently no git-annex rpm available for openSUSE; however it is possible to build it via cabal or from source as described on the [[install]] page.  Fulfilling the dependencies listed on that page should not be a problem, except for obtaining a suitable version of the Haskell library.
+
+The last [official release of Haskell for openSUSE](https://build.opensuse.org/project/show?project=devel:languages:haskell) is quite old, and may not satisfy the dependencies needed by git-annex.  Fortunately [searching the openSUSE build service](http://software.opensuse.org/search?q=cabal&baseproject=openSUSE%3A11.4&lang=en&include_home=true&exclude_debug=true) reveals that Peter Trommler has built a [newer Haskell suite](https://build.opensuse.org/project/show?project=home%3Aptrommler%3Adevel%3Alanguages%3Ahaskell) based on ghc 7.2.
+To install this, simply visit the previous URL and click on the relevant "1-Click Install" link.

From f78b0cb4e77be51344b2eb6d1f29b62192748e7b Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 24 Dec 2011 00:48:37 +0000
Subject: [PATCH 2799/2835]

---
 doc/install/openSUSE.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/openSUSE.mdwn b/doc/install/openSUSE.mdwn
index 762db1f2d2..0383cbbf2a 100644
--- a/doc/install/openSUSE.mdwn
+++ b/doc/install/openSUSE.mdwn
@@ -1,4 +1,4 @@
 Unfortunately there is currently no git-annex rpm available for openSUSE; however it is possible to build it via cabal or from source as described on the [[install]] page.  Fulfilling the dependencies listed on that page should not be a problem, except for obtaining a suitable version of the Haskell library.
 
 The last [official release of Haskell for openSUSE](https://build.opensuse.org/project/show?project=devel:languages:haskell) is quite old, and may not satisfy the dependencies needed by git-annex.  Fortunately [searching the openSUSE build service](http://software.opensuse.org/search?q=cabal&baseproject=openSUSE%3A11.4&lang=en&include_home=true&exclude_debug=true) reveals that Peter Trommler has built a [newer Haskell suite](https://build.opensuse.org/project/show?project=home%3Aptrommler%3Adevel%3Alanguages%3Ahaskell) based on ghc 7.2.
-To install this, simply visit the previous URL and click on the relevant "1-Click Install" link.
+To install this, simply click on the relevant "1-Click Install" link in the openSUSE build service search results.

From 4eafd43cce00942625b24adb50917e9074ae7e4b Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 24 Dec 2011 01:05:08 +0000
Subject: [PATCH 2800/2835] Added a comment: Any update on this?

---
 ..._1_3c7e3f021c2c94277eecf9c8af6cec5f._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_1_3c7e3f021c2c94277eecf9c8af6cec5f._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_1_3c7e3f021c2c94277eecf9c8af6cec5f._comment b/doc/bugs/problems_with_utf8_names/comment_1_3c7e3f021c2c94277eecf9c8af6cec5f._comment
new file mode 100644
index 0000000000..692b5d5378
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_1_3c7e3f021c2c94277eecf9c8af6cec5f._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="Any update on this?"
+ date="2011-12-24T01:05:07Z"
+ content="""
+I just noticed this issue, and was wondering what the current status is.
+
+    % ls -l 04\ -\ Orixás.mp3
+    -rw-r--r-- 1 adam users 8377816 Jul 12  2007 04 - Orixás.mp3
+    % echo 04\ -\ Orixás.mp3 | od -c
+    0000000   0   4       -       O   r   i   x 303 241   s   .   m   p   3
+    0000020  \n
+    0000021
+    % git annex add 04\ -\ Orixás.mp3
+    git-annex: /home/adam/music/RotC/transcribe/04 - Orixás.mp3: getSymbolicLinkStatus: does not exist (No such file or directory)
+"""]]

From ccddaf2f5c024be02efd44494ed11d09b2102872 Mon Sep 17 00:00:00 2001
From: "http://www.openid.albertlash.com/openid/"
 
Date: Sat, 24 Dec 2011 06:08:46 +0000
Subject: [PATCH 2801/2835] Added a comment

---
 ...nt_1_9a2a2a8eac9af97e0c984ad105763a73._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment b/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment
new file mode 100644
index 0000000000..807180660b
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://www.openid.albertlash.com/openid/"
+ ip="71.178.29.218"
+ subject="comment 1"
+ date="2011-12-24T06:08:45Z"
+ content="""
+Looks like you are missing a closing double quote on the line:
+
+
+echo '$GL_ADC_PATH = \"/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+
+right after /;
+
+I got this working by the way - great stuff.
+"""]]

From ffe9973ed5163e88d93b61ecc3496e11e6a0084e Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sat, 24 Dec 2011 12:49:41 +0000
Subject: [PATCH 2802/2835] Added a comment

---
 ...comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment b/doc/bugs/problems_with_utf8_names/comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment
new file mode 100644
index 0000000000..a45706e4a3
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 2"
+ date="2011-12-24T12:49:40Z"
+ content="""
+This (rather longish) thread discusses the current situation, the planned changes for 7.2 and the various issues: http://haskell.org/pipermail/glasgow-haskell-users/2011-November/021115.html
+
+The summary seems to be: From 7.2 on, getDirectoryContents _will_ return proper Strings, i.e. where a Char represents a Unicode code point, and not a Word8, which will fix the problem of outputting them.
+"""]]

From ab1435310c36ba3309c075b544bc8b3bf8e7ffa8 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sat, 24 Dec 2011 12:51:43 +0000
Subject: [PATCH 2803/2835] Added a comment

---
 .../comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment b/doc/bugs/problems_with_utf8_names/comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment
new file mode 100644
index 0000000000..9fef2eb1f2
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 3"
+ date="2011-12-24T12:51:43Z"
+ content="""
+An alternative that is available from ghc 7.4 on is a pure ByteString based unix API: http://thread.gmane.org/gmane.comp.lang.haskell.libraries/16556
+"""]]

From c87a81404656da4f5d3607687c3cb5da32295af3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 24 Dec 2011 16:49:13 +0000
Subject: [PATCH 2804/2835] Added a comment

---
 ...comment_4_93bee35f5fa7744834994bc7a253a6f9._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_4_93bee35f5fa7744834994bc7a253a6f9._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_4_93bee35f5fa7744834994bc7a253a6f9._comment b/doc/bugs/problems_with_utf8_names/comment_4_93bee35f5fa7744834994bc7a253a6f9._comment
new file mode 100644
index 0000000000..5e11af6abe
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_4_93bee35f5fa7744834994bc7a253a6f9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-12-24T16:49:13Z"
+ content="""
+Adam, this bug was fixed a long time ago, first using option #2 above, but later switching to option #3 -- git-annex treats filenames as opaque binary blobs and never decodes them in any encoding; haskell's normal encoding support for stdio is disabled. 
+
+And it never resulted in a failure like you show. I cannot reproduce your problem, but it is a different bug, please open a new bug report.
+"""]]

From 0c107c98de1c3ffa9dea3d9ccc3029984a3d2ffa Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 24 Dec 2011 16:54:31 +0000
Subject: [PATCH 2805/2835] Added a comment

---
 .../comment_2_d8efea4ab9576555fadbb47666ecefa9._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment b/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment
new file mode 100644
index 0000000000..007a009ea1
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-24T16:54:31Z"
+ content="""
+I've fixed the typo (anyone can edit pages in this wiki FWIW.)
+"""]]

From 82cc5d3d3120a00cd7f823794f2242fcd3b42817 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 14:14:00 -0400
Subject: [PATCH 2806/2835] slightly improve typography

---
 doc/git-annex.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 39fe5d99ea..8096005ce2 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -247,7 +247,7 @@ subdirectories).
   By default, only lists annexed files whose content is currently present.
   This can be changed by specifying file matching options. To list all
   annexed files, present or not, specify --include "*". To list all
-  annexed files whose content is not present, specify --not --in "."
+  annexed files whose content is not present, specify --not --in="."
 
   To output filenames terminated with nulls, for use with xargs -0,
   specify --print0. Or, a custom output formatting can be specified using
@@ -454,7 +454,7 @@ limit which files git-annex acts on.
 Arbitrarily complicated expressions can be built using these options.
 For example:
 
-	--exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+	--exclude '*.mp3' --and --not -( --in=usbdrive --or --in=archive -)
 
 The above example prevents git-annex from working on mp3 files whose
 file contents are present at either of two repositories.

From ffefe742d8cc2655e29fa356c82326e867e27aa9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 24 Dec 2011 12:51:03 -0400
Subject: [PATCH 2807/2835] fix typo

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index a51d6539b6..ac0a1f6b6c 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -21,7 +21,7 @@ gitolite installation, just with different paths.
 Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
 
 
-echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;"' >>~gitolite/.gitolite.rc
 

 
 Make the ADC directory, and a "ua" subdirectory.

From 899b20f89c1ba7d6ac1a09ea83c7f37052c43166 Mon Sep 17 00:00:00 2001
From: "http://www.openid.albertlash.com/openid/"
 
Date: Sat, 24 Dec 2011 17:05:38 +0000
Subject: [PATCH 2808/2835]

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index ac0a1f6b6c..92048c4429 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -75,3 +75,15 @@ sent 2606 bytes  received 31 bytes  1758.00 bytes/sec
 total size is 2502  speedup is 0.95
 ok
 

+
+
+### Troubleshooting
+
+I got an error like this when setting up gitolite *after* setting up a local git repo and git annex:
+
+
+git-annex-shell: First run: git-annex init
+Command ssh ["git@git.example.com","git-annex-shell 'configlist' '/~/myrepo.git'"] failed; exit code 1
+

+
+because I forgot to "git pull --all" after adding the new gitolite remote.

From 2101ef20d152891176401447f76758935313d7de Mon Sep 17 00:00:00 2001
From: "http://www.openid.albertlash.com/openid/"
 
Date: Sun, 25 Dec 2011 01:55:45 +0000
Subject: [PATCH 2809/2835]

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index 92048c4429..0d89a255be 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -86,4 +86,4 @@ git-annex-shell: First run: git-annex init
 Command ssh ["git@git.example.com","git-annex-shell 'configlist' '/~/myrepo.git'"] failed; exit code 1
 

 
-because I forgot to "git pull --all" after adding the new gitolite remote.
+because I forgot to "git push --all" after adding the new gitolite remote.

From b9d328b245dc973b232a1c6de6025526f1d03b7e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 01:24:07 -0400
Subject: [PATCH 2810/2835] add git-post-fetch hook support

I spent 10 hours today writing code to feed and parse the output of this
hook in C. I had 10 minutes so thought I'd do the same in Haskell,
as a way to get all those pointers, explicit memory management, impure
code, and lack of abstractions out of my head.

Later, if the hook is actually accepted into git, git-annex will be able
to use it to run the git-annex merge, as well as perhaps enable a mode
where pulling from remotes known to git-annex automatically merges
their master branch into the local master.
---
 Git/PostFetch.hs | 56 ++++++++++++++++++++++++++++++++++++++++++++++++
 Git/Sha.hs       |  5 ++++-
 2 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 Git/PostFetch.hs

diff --git a/Git/PostFetch.hs b/Git/PostFetch.hs
new file mode 100644
index 0000000000..2079ea2624
--- /dev/null
+++ b/Git/PostFetch.hs
@@ -0,0 +1,56 @@
+{- git post-fetch hook support
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.PostFetch where
+
+import Common
+import Git
+import Git.Types
+import Git.Sha
+
+{- Each line fed to the post-fetch hook should represent a ref that is
+ - being updated. To avoid breaking if the format changes, unparsable
+ - lines are stored as-is. -}
+data FetchedRef = Unparsable String | FetchedRef
+	{ sha :: Sha
+	, merge :: Bool
+	, remote :: Ref
+	, local :: Ref
+	}
+	deriving (Show)
+
+input :: IO [FetchedRef]
+input = map parseLine . lines <$> getContents
+
+output :: [FetchedRef] -> IO ()
+output = mapM_ $ putStrLn . genLine
+
+parseLine :: String -> FetchedRef
+parseLine line = go $ words line
+	where
+		go [s, m, r, l]
+			| not $ isSha s = Unparsable line
+			| m == "merge" = parsed True
+			| m == "not-for-merge" = parsed False
+			| otherwise = Unparsable line
+			where
+				parsed v = FetchedRef
+					{ sha = Ref s
+					, merge = v
+					, remote = Ref r
+					, local = Ref l
+					}
+		go _ = Unparsable line
+
+genLine :: FetchedRef -> String
+genLine (Unparsable l) = l
+genLine r = unwords
+	[ show $ sha r
+	, if merge r then "merge" else "not-for-merge"
+	, show $ remote r
+	, show $ local r
+	]
diff --git a/Git/Sha.hs b/Git/Sha.hs
index 9b3a346505..cdf9853cfa 100644
--- a/Git/Sha.hs
+++ b/Git/Sha.hs
@@ -28,9 +28,12 @@ extractSha s
 		len = length s
 		s' = firstLine s
 		val v
-			| all (`elem` "1234567890ABCDEFabcdef") v = Just $ Ref v
+			| isSha v = Just $ Ref v
 			| otherwise = Nothing
 
+isSha :: String -> Bool
+isSha v = all (`elem` "1234567890ABCDEFabcdef") v && length v == shaSize
+
 {- Size of a git sha. -}
 shaSize :: Int
 shaSize = 40

From d50c48693fa599fe2ed69f7c816c5c2dd3e8be2b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 01:49:04 -0400
Subject: [PATCH 2811/2835] add a runHook enforcing a nice invariant

---
 Git/PostFetch.hs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/Git/PostFetch.hs b/Git/PostFetch.hs
index 2079ea2624..a031164b68 100644
--- a/Git/PostFetch.hs
+++ b/Git/PostFetch.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Git.PostFetch where
+module Git.PostFetch (runHook, FetchedRef(..)) where
 
 import Common
 import Git
@@ -13,7 +13,9 @@ import Git.Types
 import Git.Sha
 
 {- Each line fed to the post-fetch hook should represent a ref that is
- - being updated. To avoid breaking if the format changes, unparsable
+ - being updated. It's important that the hook always outputs every line
+ - that is fed into it (possibly modified), otherwise incoming refs will
+ - not be stored. So to avoid breaking if the format changes, unparsable
  - lines are stored as-is. -}
 data FetchedRef = Unparsable String | FetchedRef
 	{ sha :: Sha
@@ -23,6 +25,15 @@ data FetchedRef = Unparsable String | FetchedRef
 	}
 	deriving (Show)
 
+{- Runs the hook, allowing lines to be mutated and even produce more
+ - than one output line, but never be discarded. Unparsable lines are
+ - passed through unchanged. -}
+runHook :: (FetchedRef -> IO [FetchedRef]) -> IO ()
+runHook mutate = input >>= mapM callmutate >>= output . concat
+	where
+		callmutate u@(Unparsable _) = return [u]
+		callmutate f = catchDefaultIO (mutate f) [f]
+
 input :: IO [FetchedRef]
 input = map parseLine . lines <$> getContents
 

From 1b1b1c32b99a77f3bf82e18d7902b19cd40ca457 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 01:59:50 -0400
Subject: [PATCH 2812/2835] better types

---
 Git/PostFetch.hs | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/Git/PostFetch.hs b/Git/PostFetch.hs
index a031164b68..a33b8ea834 100644
--- a/Git/PostFetch.hs
+++ b/Git/PostFetch.hs
@@ -12,12 +12,7 @@ import Git
 import Git.Types
 import Git.Sha
 
-{- Each line fed to the post-fetch hook should represent a ref that is
- - being updated. It's important that the hook always outputs every line
- - that is fed into it (possibly modified), otherwise incoming refs will
- - not be stored. So to avoid breaking if the format changes, unparsable
- - lines are stored as-is. -}
-data FetchedRef = Unparsable String | FetchedRef
+data FetchedRef = FetchedRef
 	{ sha :: Sha
 	, merge :: Bool
 	, remote :: Ref
@@ -25,41 +20,48 @@ data FetchedRef = Unparsable String | FetchedRef
 	}
 	deriving (Show)
 
+{- Each line fed to the post-fetch hook should represent a ref that is
+ - being updated. It's important that the hook always outputs every line
+ - that is fed into it (possibly modified), otherwise incoming refs will
+ - not be stored. So to avoid breaking if the format changes, unparsable
+ - lines are stored as-is. -}
+type HookLine = Either String FetchedRef
+
 {- Runs the hook, allowing lines to be mutated and even produce more
  - than one output line, but never be discarded. Unparsable lines are
  - passed through unchanged. -}
 runHook :: (FetchedRef -> IO [FetchedRef]) -> IO ()
-runHook mutate = input >>= mapM callmutate >>= output . concat
+runHook mutate = output . concat =<< mapM go =<< input
 	where
-		callmutate u@(Unparsable _) = return [u]
-		callmutate f = catchDefaultIO (mutate f) [f]
+		go u@(Left _) = return [u]
+		go (Right r) = map Right <$> catchDefaultIO (mutate r) [r]
 
-input :: IO [FetchedRef]
+input :: IO [HookLine]
 input = map parseLine . lines <$> getContents
 
-output :: [FetchedRef] -> IO ()
+output :: [HookLine] -> IO ()
 output = mapM_ $ putStrLn . genLine
 
-parseLine :: String -> FetchedRef
+parseLine :: String -> HookLine
 parseLine line = go $ words line
 	where
 		go [s, m, r, l]
-			| not $ isSha s = Unparsable line
+			| not $ isSha s = Left line
 			| m == "merge" = parsed True
 			| m == "not-for-merge" = parsed False
-			| otherwise = Unparsable line
+			| otherwise = Left line
 			where
-				parsed v = FetchedRef
+				parsed v = Right $ FetchedRef
 					{ sha = Ref s
 					, merge = v
 					, remote = Ref r
 					, local = Ref l
 					}
-		go _ = Unparsable line
+		go _ = Left line
 
-genLine :: FetchedRef -> String
-genLine (Unparsable l) = l
-genLine r = unwords
+genLine :: HookLine -> String
+genLine (Left l) = l
+genLine (Right r) = unwords
 	[ show $ sha r
 	, if merge r then "merge" else "not-for-merge"
 	, show $ remote r

From cacdf58e9c5f03c6aa4ecd97a0a8395102a5d82e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 03:02:57 -0400
Subject: [PATCH 2813/2835] refactor

---
 Git/PostFetch.hs | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/Git/PostFetch.hs b/Git/PostFetch.hs
index a33b8ea834..c4ed5568ac 100644
--- a/Git/PostFetch.hs
+++ b/Git/PostFetch.hs
@@ -5,11 +5,10 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Git.PostFetch (runHook, FetchedRef(..)) where
+module Git.PostFetch (runHook, runHookUnsafe, FetchedRef(..)) where
 
 import Common
 import Git
-import Git.Types
 import Git.Sha
 
 data FetchedRef = FetchedRef
@@ -27,15 +26,25 @@ data FetchedRef = FetchedRef
  - lines are stored as-is. -}
 type HookLine = Either String FetchedRef
 
-{- Runs the hook, allowing lines to be mutated and even produce more
- - than one output line, but never be discarded. Unparsable lines are
- - passed through unchanged. -}
-runHook :: (FetchedRef -> IO [FetchedRef]) -> IO ()
-runHook mutate = output . concat =<< mapM go =<< input
+{- Runs the hook, allowing lines to be mutated, but never be discarded.
+ - Unparsable lines are passed through unchanged. -}
+runHook :: (FetchedRef -> IO FetchedRef) -> IO ()
+runHook mutate = runHook' go id
+	where
+		go u@(Left _) = return u
+		go (Right r) = Right <$> catchDefaultIO (mutate r) r
+
+{- Runs the hook, allowing lines to be mutated, discarded, or produce
+ - multiple output lines. Unparsable lines are passed through unchanged. -}
+runHookUnsafe :: (FetchedRef -> IO [FetchedRef]) -> IO ()
+runHookUnsafe mutate = runHook' go concat
 	where
 		go u@(Left _) = return [u]
 		go (Right r) = map Right <$> catchDefaultIO (mutate r) [r]
 
+runHook' :: (HookLine -> IO b) -> ([b] -> [HookLine]) -> IO ()
+runHook' mutate reduce = output . reduce =<< mapM mutate =<< input
+
 input :: IO [HookLine]
 input = map parseLine . lines <$> getContents
 

From bd86f83b8459675c5611ceb2221332b97015608f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 03:03:49 -0400
Subject: [PATCH 2814/2835] refactor

---
 Git/PostFetch.hs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Git/PostFetch.hs b/Git/PostFetch.hs
index c4ed5568ac..965aefce9c 100644
--- a/Git/PostFetch.hs
+++ b/Git/PostFetch.hs
@@ -23,11 +23,10 @@ data FetchedRef = FetchedRef
  - being updated. It's important that the hook always outputs every line
  - that is fed into it (possibly modified), otherwise incoming refs will
  - not be stored. So to avoid breaking if the format changes, unparsable
- - lines are stored as-is. -}
+ - lines are passed through unchanged. -}
 type HookLine = Either String FetchedRef
 
-{- Runs the hook, allowing lines to be mutated, but never be discarded.
- - Unparsable lines are passed through unchanged. -}
+{- Runs the hook, allowing lines to be mutated, but never be discarded. -}
 runHook :: (FetchedRef -> IO FetchedRef) -> IO ()
 runHook mutate = runHook' go id
 	where
@@ -35,7 +34,7 @@ runHook mutate = runHook' go id
 		go (Right r) = Right <$> catchDefaultIO (mutate r) r
 
 {- Runs the hook, allowing lines to be mutated, discarded, or produce
- - multiple output lines. Unparsable lines are passed through unchanged. -}
+ - multiple output lines. -}
 runHookUnsafe :: (FetchedRef -> IO [FetchedRef]) -> IO ()
 runHookUnsafe mutate = runHook' go concat
 	where

From 717caac52ec80a62591f54bf582cc8d407344276 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 13:08:52 -0400
Subject: [PATCH 2815/2835] hook renamed

---
 Git/{PostFetch.hs => TweakFetch.hs} | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
 rename Git/{PostFetch.hs => TweakFetch.hs} (91%)

diff --git a/Git/PostFetch.hs b/Git/TweakFetch.hs
similarity index 91%
rename from Git/PostFetch.hs
rename to Git/TweakFetch.hs
index 965aefce9c..8e527829df 100644
--- a/Git/PostFetch.hs
+++ b/Git/TweakFetch.hs
@@ -1,11 +1,11 @@
-{- git post-fetch hook support
+{- git tweak-fetch hook support
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Git.PostFetch (runHook, runHookUnsafe, FetchedRef(..)) where
+module Git.TweakFetch (runHook, runHookUnsafe, FetchedRef(..)) where
 
 import Common
 import Git
@@ -19,7 +19,7 @@ data FetchedRef = FetchedRef
 	}
 	deriving (Show)
 
-{- Each line fed to the post-fetch hook should represent a ref that is
+{- Each line fed to the tweak-fetch hook should represent a ref that is
  - being updated. It's important that the hook always outputs every line
  - that is fed into it (possibly modified), otherwise incoming refs will
  - not be stored. So to avoid breaking if the format changes, unparsable

From 2b97f5381a5417e3df77820bf4144e9a2b00c6fa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 14:25:37 -0400
Subject: [PATCH 2816/2835] add tweak-fetch command, for use in the tweak-fetch
 hook

tweak-fetch is a new git hook I have developed (not yet accepted into
git, but looking bright). Amoung other things, the hook can be used to
observe what is being fetched, notice remote git-annex branches that might
be updated, and merge them into the git-annex branch.

This will solve problems where users do a git pull, immediately followed
by a push, and it refuses to push because their git-annex branch is
diverged, and they neither ran git annex merge by hand, nor ran other
git-annex commands that auto-merge.

The tweak-fetch is written by git annex init. Of course, existing
repositories won't have it, which is ok, because git-annex still
automatically does a merge if changed branches have appeared. Indeed,
it will always need to do that check, as long as it needs to support
support git-annex branches that might be updated by other means.

Eventually though, I will want to ensure all repositories have the
tweak-fetch hook. Perhaps a minor verison upgrade to ensure it is added?

A subtlety of the hook is that when it's run, the remote tracking refs
have not yet been updated. So Annex.Branch.updateTo has to be careful to
only use the sha1 that was fetched, not the branch name. The branch
name is only used in the commit message.

The other tricky thing is that git tweak-fetch hook should *only*
output lines in a specific format, and git will be unhappy if it also
outputs status messages, etc. So those messages are sent to stderr.
---
 Annex/Branch.hs       | 23 ++++++++++++------
 Command/TweakFetch.hs | 34 ++++++++++++++++++++++++++
 Git/TweakFetch.hs     | 32 +++++++++++++------------
 GitAnnex.hs           |  2 ++
 Init.hs               | 55 +++++++++++++++++++++++--------------------
 doc/git-annex.mdwn    |  5 ++++
 6 files changed, 103 insertions(+), 48 deletions(-)
 create mode 100644 Command/TweakFetch.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index af1878479a..5f678b9d30 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -11,6 +11,7 @@ module Annex.Branch (
 	hasSibling,
 	create,
 	update,
+	updateTo,
 	get,
 	change,
 	commit,
@@ -81,10 +82,19 @@ getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
 
 {- Ensures that the branch and index are is up-to-date; should be
  - called before data is read from it. Runs only once per git-annex run.
+ -}
+update :: Annex ()
+update = runUpdateOnce $ updateTo =<< siblingBranches
+
+{- Merges the specified Refs into the index, if they have any changes not
+ - already in it. The Branch names are only used in the commit message;
+ - it's even possible that the provided Branches have not been updated to
+ - point to the Refs yet.
  -
  - Before refs are merged into the index, it's important to first stage the
  - journal into the index. Otherwise, any changes in the journal would
  - later get staged, and might overwrite changes made during the merge.
+ - If no Refs are provided, the journal is still staged and committed.
  -
  - (It would be cleaner to handle the merge by updating the journal, not the
  - index, with changes from the branches.)
@@ -92,13 +102,13 @@ getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
  - The branch is fast-forwarded if possible, otherwise a merge commit is
  - made.
  -}
-update :: Annex ()
-update = runUpdateOnce $ do
+updateTo :: [(Git.Ref, Git.Branch)] -> Annex ()
+updateTo pairs = do
 	-- ensure branch exists, and get its current ref
 	branchref <- getBranch
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
-	(refs, branches) <- unzip <$> newerSiblings
+	(refs, branches) <- unzip <$> filterM isnewer pairs
 	if (not dirty && null refs)
 		then updateIndex branchref
 		else withIndex $ lockJournal $ do
@@ -110,7 +120,7 @@ update = runUpdateOnce $ do
 					" into " ++ show name
 			unless (null branches) $ do
 				showSideAction merge_desc
-				mergeIndex branches
+				mergeIndex refs
 			ff <- if dirty
 				then return False
 				else inRepo $ Git.Branch.fastForward fullname refs
@@ -120,8 +130,7 @@ update = runUpdateOnce $ do
 					(nub $ fullname:refs)
 			invalidateCache
 	where
-		newerSiblings = filterM isnewer =<< siblingBranches
-		isnewer (_, b) = inRepo $ Git.Branch.changed fullname b
+		isnewer (r, _) = inRepo $ Git.Branch.changed fullname r
 
 {- Gets the content of a file on the branch, or content from the journal, or
  - staged in the index.
@@ -238,7 +247,7 @@ genIndex :: Git.Repo -> IO ()
 genIndex g = Git.UnionMerge.stream_update_index g
 	[Git.UnionMerge.ls_tree fullname g]
 
-{- Merges the specified branches into the index.
+{- Merges the specified refs into the index.
  - Any changes staged in the index will be preserved. -}
 mergeIndex :: [Git.Ref] -> Annex ()
 mergeIndex branches = do
diff --git a/Command/TweakFetch.hs b/Command/TweakFetch.hs
new file mode 100644
index 0000000000..077041b576
--- /dev/null
+++ b/Command/TweakFetch.hs
@@ -0,0 +1,34 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.TweakFetch where
+
+import Common
+import Command
+import qualified Git.TweakFetch
+import qualified Annex.Branch
+
+def :: [Command]
+def = [command "tweak-fetch" paramNothing seek "run by git tweak-fetch hook"]
+
+seek :: [CommandSeek]
+seek = [ withNothing start]
+
+start :: CommandStart
+start = do
+	-- First, pass the hook's input through to its output, unchanged.
+	fetched <- liftIO $ Git.TweakFetch.runHook return
+
+	-- If one of the fetched refs is going to be stored on a git-annex
+	-- tracking branch, then merge in the new sha for that ref.
+	let tomerge = filter siblings fetched
+	unless (null tomerge) $ Annex.Branch.updateTo $ map topairs tomerge
+	stop
+	where
+		siblings f = suffix `isSuffixOf` (show $ Git.TweakFetch.local f)
+		suffix = "/" ++ show Annex.Branch.name
+		topairs f = (Git.TweakFetch.sha f, Git.TweakFetch.local f)
diff --git a/Git/TweakFetch.hs b/Git/TweakFetch.hs
index 8e527829df..41cc0499bd 100644
--- a/Git/TweakFetch.hs
+++ b/Git/TweakFetch.hs
@@ -5,7 +5,10 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Git.TweakFetch (runHook, runHookUnsafe, FetchedRef(..)) where
+module Git.TweakFetch (runHook, FetchedRef(..)) where
+
+import Data.Either (rights)
+import System.Posix.IO
 
 import Common
 import Git
@@ -26,24 +29,23 @@ data FetchedRef = FetchedRef
  - lines are passed through unchanged. -}
 type HookLine = Either String FetchedRef
 
-{- Runs the hook, allowing lines to be mutated, but never be discarded. -}
-runHook :: (FetchedRef -> IO FetchedRef) -> IO ()
-runHook mutate = runHook' go id
+{- Runs the hook, allowing lines to be mutated, but never be discarded.
+ - Returns same FetchedRefs that are output by the hook, for further use. -}
+runHook :: (FetchedRef -> IO FetchedRef) -> IO [FetchedRef]
+runHook mutate = do
+	ls <- mapM go =<< input
+	output ls
+
+	-- Nothing more should be output to stdout; only hook output
+	-- is accepted by git. Redirect stdout to stderr.
+	hFlush stdout
+	_ <- liftIO $ dupTo stdError stdOutput
+
+	return $ rights ls
 	where
 		go u@(Left _) = return u
 		go (Right r) = Right <$> catchDefaultIO (mutate r) r
 
-{- Runs the hook, allowing lines to be mutated, discarded, or produce
- - multiple output lines. -}
-runHookUnsafe :: (FetchedRef -> IO [FetchedRef]) -> IO ()
-runHookUnsafe mutate = runHook' go concat
-	where
-		go u@(Left _) = return [u]
-		go (Right r) = map Right <$> catchDefaultIO (mutate r) [r]
-
-runHook' :: (HookLine -> IO b) -> ([b] -> [HookLine]) -> IO ()
-runHook' mutate reduce = output . reduce =<< mapM mutate =<< input
-
 input :: IO [HookLine]
 input = map parseLine . lines <$> getContents
 
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 7243d69cb0..43daf7367b 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -39,6 +39,7 @@ import qualified Command.DropUnused
 import qualified Command.Unlock
 import qualified Command.Lock
 import qualified Command.PreCommit
+import qualified Command.TweakFetch
 import qualified Command.Find
 import qualified Command.Whereis
 import qualified Command.Merge
@@ -73,6 +74,7 @@ cmds = concat
 	, Command.Unannex.def
 	, Command.Uninit.def
 	, Command.PreCommit.def
+	, Command.TweakFetch.def
 	, Command.Trust.def
 	, Command.Untrust.def
 	, Command.Semitrust.def
diff --git a/Init.hs b/Init.hs
index c8deadf3b4..47ac9e3d35 100644
--- a/Init.hs
+++ b/Init.hs
@@ -24,12 +24,12 @@ initialize mdescription = do
 	prepUUID
 	Annex.Branch.create
 	setVersion
-	gitPreCommitHookWrite
+	gitHooksWrite
 	u <- getUUID
 	maybe (recordUUID u) (describeUUID u) mdescription
 
 uninitialize :: Annex ()
-uninitialize = gitPreCommitHookUnWrite
+uninitialize = gitHooksUnWrite
 
 {- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
@@ -44,37 +44,40 @@ ensureInitialized = getVersion >>= maybe needsinit checkVersion
 				then initialize Nothing
 				else error "First run: git-annex init"
 
-{- set up a git pre-commit hook, if one is not already present -}
-gitPreCommitHookWrite :: Annex ()
-gitPreCommitHookWrite = unlessBare $ do
-	hook <- preCommitHook
-	exists <- liftIO $ doesFileExist hook
+{- set up git hooks, if not already present -}
+gitHooksWrite :: Annex ()
+gitHooksWrite = unlessBare $ forM_ hooks $ \(hook, content) -> do
+	file <- hookFile hook
+	exists <- liftIO $ doesFileExist file
 	if exists
-		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
+		then warning $ hook ++ " hook (" ++ file ++ ") already exists, not configuring"
 		else liftIO $ do
-			viaTmp writeFile hook preCommitScript
-			p <- getPermissions hook
-			setPermissions hook $ p {executable = True}
+			viaTmp writeFile file content
+			p <- getPermissions file
+			setPermissions file $ p {executable = True}
 
-gitPreCommitHookUnWrite :: Annex ()
-gitPreCommitHookUnWrite = unlessBare $ do
-	hook <- preCommitHook
-	whenM (liftIO $ doesFileExist hook) $ do
-		c <- liftIO $ readFile hook
-		if c == preCommitScript
-			then liftIO $ removeFile hook
-			else warning $ "pre-commit hook (" ++ hook ++ 
+gitHooksUnWrite :: Annex ()
+gitHooksUnWrite = unlessBare $ forM_ hooks $ \(hook, content) -> do
+	file <- hookFile hook
+	whenM (liftIO $ doesFileExist file) $ do
+		c <- liftIO $ readFile file
+		if c == content
+			then liftIO $ removeFile file
+			else warning $ hook ++ " hook (" ++ file ++ 
 				") contents modified; not deleting." ++
 				" Edit it to remove call to git annex."
 
 unlessBare :: Annex () -> Annex ()
 unlessBare = unlessM $ fromRepo $ Git.repoIsLocalBare
 
-preCommitHook :: Annex FilePath
-preCommitHook = () <$> fromRepo Git.gitDir <*> pure "hooks/pre-commit"
+hookFile :: FilePath -> Annex FilePath
+hookFile f = () <$> fromRepo Git.gitDir <*> pure ("hooks/" ++ f)
 
-preCommitScript :: String
-preCommitScript = 
-	"#!/bin/sh\n" ++
-	"# automatically configured by git-annex\n" ++ 
-	"git annex pre-commit .\n"
+hooks :: [(String, String)]
+hooks = [ ("pre-commit", hookscript "git annex pre-commit .")
+	, ("tweak-fetch", hookscript "git annex tweak-fetch")
+	]
+	where
+		hookscript s = "#!/bin/sh\n" ++
+			"# automatically configured by git-annex\n" ++
+			s ++ "\n";
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 8096005ce2..b7fd1b5217 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -344,6 +344,11 @@ subdirectories).
   This is meant to be called from git's pre-commit hook. `git annex init`
   automatically creates a pre-commit hook using this.
 
+* tweak-fetch
+
+  This is meant ot be called from git's tweak-fetch hook. `git annex init`
+  automatically creates a tweak-fetch hook using this.
+
 * fromkey key file
 
   This plumbing-level command can be used to manually set up a file

From d74f8db80842385223c62ea11c76c16fa1f7e6a0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 14:38:55 -0400
Subject: [PATCH 2817/2835] de-emphasize git annex merge

---
 doc/git-annex.mdwn                                        | 5 +++--
 .../centralised_repository:_starting_from_nothing.mdwn    | 8 --------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index b7fd1b5217..e0f0d78703 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -220,8 +220,9 @@ subdirectories).
 
   Automatically merges remote tracking branches */git-annex into
   the git-annex branch. While git-annex mostly handles keeping the
-  git-annex branch merged automatically, if you find you are unable
-  to push the git-annex branch due non-fast-forward, this will fix it.
+  git-annex branch merged automatically, via the tweak-fetch hook and other
+  means, if you find you are unable to push the git-annex branch due
+  non-fast-forward, this will fix it.
 
 * fix [path ...]
 
diff --git a/doc/tips/centralised_repository:_starting_from_nothing.mdwn b/doc/tips/centralised_repository:_starting_from_nothing.mdwn
index b12246d368..899068485f 100644
--- a/doc/tips/centralised_repository:_starting_from_nothing.mdwn
+++ b/doc/tips/centralised_repository:_starting_from_nothing.mdwn
@@ -28,14 +28,6 @@ Clone that to the laptop:
 	init laptop ok
         laptop$ 
 
-Merge the `git-annex` repository (this is the bit that is often
-overlooked!):
-
-	laptop$ git annex merge	
-	merge . (merging "origin/git-annex" into git-annex...)
-	ok
-	laptop$ 
-
 Add some content:
 
 	laptop$ git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg

From 6d74830afde16bc3d0b385a89c4a1e705c3903cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 26 Dec 2011 14:41:20 -0400
Subject: [PATCH 2818/2835] note

---
 doc/todo/optimise_git-annex_merge.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/todo/optimise_git-annex_merge.mdwn b/doc/todo/optimise_git-annex_merge.mdwn
index 91d18ebd77..8b9999eb88 100644
--- a/doc/todo/optimise_git-annex_merge.mdwn
+++ b/doc/todo/optimise_git-annex_merge.mdwn
@@ -21,3 +21,13 @@ There is already a Git.CatFile library that can do this easily. --[[Joey]]
 Merging used to use memory proportional to the size of the diff. It now
 streams data, running in constant space. This probably sped it up a lot,
 as there's much less allocation and GC action. --[[Joey]] 
+
+----
+
+Another option is to stop doing the automatic merging. Once the git
+tweak-fetch hook is widely available, the automatic mergcing won't be
+needed when pulling from remotes. The only remaining use would be if
+a sibling git-annex branch appeared in some other way, and git annex merge
+could be manually run in such an unlikely case. Making this change
+would require a hard dependency on an appropriate version of git, and a
+version number bump to ensure the tweak-fetch hooks are set up. --[[Joey]]

From 5652fa5cc4d1bd8cdbdb889b509f4d8ef59af20c Mon Sep 17 00:00:00 2001
From: Antoine Martin 
Date: Mon, 12 Aug 2024 02:04:32 -0400
Subject: [PATCH 2819/2835] .forgejo: initial ci

---
 .forgejo/workflows/generate-tarball.yml  | 81 ++++++++++++++++++++++++
 .forgejo/workflows/mirror-repository.yml | 38 +++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 .forgejo/workflows/generate-tarball.yml
 create mode 100644 .forgejo/workflows/mirror-repository.yml

diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-tarball.yml
new file mode 100644
index 0000000000..6a1407653e
--- /dev/null
+++ b/.forgejo/workflows/generate-tarball.yml
@@ -0,0 +1,81 @@
+on:
+  workflow_dispatch:
+    inputs:
+      ref_name:
+        description: 'Tag or commit'
+        required: true
+        type: string
+
+jobs:
+  cabal-config-edge:
+    name: Generate cabal config for edge
+    runs-on: x86_64
+    container:
+      image: alpine:edge
+    env:
+      CI_PROJECT_NAME: git-annex
+      CI_ALPINE_TARGET_RELEASE: edge
+    steps:
+      - name: Environment setup
+        run: apk add nodejs git cabal patch
+      - name: Repo pull
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 500
+          ref: ${{ inputs.ref_name }}
+      - name: Config generation
+        run: |
+          wget 'https://lab.ilot.io/mirrors/git-annex/-/raw/gitlab-ci/ghc-9.8.patch'
+          patch -p1 -i ghc-9.8.patch
+          HOME="${{ github.workspace}}"/cabal_cache cabal update
+          HOME="${{ github.workspace}}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted"
+          mv cabal.project.freeze git-annex-${{ input.ref_name }}-$CI_ALPINE_TARGET_RELEASE.config
+      - name: Package upload
+        uses: forgejo/upload-artifact@v3
+        with:
+          name: cabal-config-${{ input.ref_name }}
+          path: git-annex${{ input.ref_name }}*
+  cabal-config-v320:
+    name: Generate cabal config for edge
+    runs-on: x86_64
+    container:
+      image: alpine:
+    env:
+      CI_PROJECT_NAME: git-annex
+      CI_ALPINE_TARGET_RELEASE: v3.20
+    steps:
+      - name: Environment setup
+        run: apk add nodejs git cabal patch
+      - name: Repo pull
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 500
+          ref: ${{ inputs.ref_name }}
+      - name: Config generation
+        run: |
+          wget 'https://lab.ilot.io/mirrors/git-annex/-/raw/gitlab-ci/ghc-9.8.patch'
+          patch -p1 -i ghc-9.8.patch
+          HOME="${{ github.workspace }}"/cabal_cache cabal update
+          HOME="${{ github.workspace }}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted"
+          mv cabal.project.freeze git-annex-${{ input.ref_name }}-$CI_ALPINE_TARGET_RELEASE.config
+      - name: Package upload
+        uses: forgejo/upload-artifact@v3
+        with:
+          name: cabal-config-${{ input.ref_name }}
+          path: git-annex${{ input.ref_name }}*
+  upload-tarball:
+    name: Upload to generic repo
+    runs-on: x86_64
+    needs: [build-tarball]
+    container:
+      image: alpine:3.20
+    steps:
+      - name: Environment setup
+        run: apk add nodejs curl findutils
+      - name: Package download
+        uses: forgejo/download-artifact@v3
+      - name: Package deployment 
+        run: |
+          find .
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./tarball/git-annex-${{ inputs.ref_name }}.tar.gz ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-${{ inputs.ref_name}}.tar.gz
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./tarball/git-annex-${{ inputs.ref_name }}.tar.gz.sha512sum ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-${{ inputs.ref_name}}.tar.gz.sha512sum
diff --git a/.forgejo/workflows/mirror-repository.yml b/.forgejo/workflows/mirror-repository.yml
new file mode 100644
index 0000000000..791fe23d0e
--- /dev/null
+++ b/.forgejo/workflows/mirror-repository.yml
@@ -0,0 +1,38 @@
+on:
+  workflow_dispatch:
+
+  schedule:
+    - cron: '@hourly'
+
+  push:
+    branches:
+      - 'master'
+
+jobs:
+  mirror:
+    name: Pull from upstream
+    runs-on: x86_64
+    container:
+      image: 'docker.io/node:20-bookworm'
+    steps:
+      - name: git mirror branches {v*/,}master & tags
+        run: |
+          git init --bare .
+          git remote add origin https://git.joeyh.name/git/git-annex.git 
+          git fetch origin refs/heads/master:refs/mirror/master
+          git ls-remote origin refs/heads/v*/master | while read sha full_ref ; do
+            ref=${full_ref#refs/heads/}
+            git fetch origin $full_ref:refs/mirror/$ref
+            tag=${ref%/master}
+            git fetch origin refs/tags/${tag}*:refs/mirror-tags/${tag}*
+          done
+          if test "${{ vars.TEST }}" != true ; then
+            token=${{ secrets.CODE_FORGEJO_TOKEN }}
+            destination=${{ vars.DESTINATION }}
+            owner=${{ vars.OWNER }}
+          else
+            token=${{ secrets.TEST_FORGEJO_TOKEN }}
+            destination=${{ vars.TEST_DESTINATION }}
+            owner=${{ vars.TEST_OWNER }}
+          fi
+          git push --force https://any:$token@$destination/$owner/git-annex refs/mirror/*:refs/heads/* refs/mirror-tags/*:refs/tags/* --tags   

From 6a68875d6a8ba3d4e5dd9096678f1e03295be407 Mon Sep 17 00:00:00 2001
From: Antoine Martin 
Date: Mon, 12 Aug 2024 02:06:16 -0400
Subject: [PATCH 2820/2835] .forgejo: typo

---
 .forgejo/workflows/generate-tarball.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-tarball.yml
index 6a1407653e..939af686ec 100644
--- a/.forgejo/workflows/generate-tarball.yml
+++ b/.forgejo/workflows/generate-tarball.yml
@@ -39,7 +39,7 @@ jobs:
     name: Generate cabal config for edge
     runs-on: x86_64
     container:
-      image: alpine:
+      image: alpine:3.20
     env:
       CI_PROJECT_NAME: git-annex
       CI_ALPINE_TARGET_RELEASE: v3.20
@@ -68,7 +68,7 @@ jobs:
     runs-on: x86_64
     needs: [build-tarball]
     container:
-      image: alpine:3.20
+      image: alpine:latest
     steps:
       - name: Environment setup
         run: apk add nodejs curl findutils

From f834ede6b50fc3078b1725d291ec6dbd520003ce Mon Sep 17 00:00:00 2001
From: Antoine Martin 
Date: Mon, 12 Aug 2024 02:08:37 -0400
Subject: [PATCH 2821/2835] .forgejo: fix dependency

---
 .forgejo/workflows/generate-tarball.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-tarball.yml
index 939af686ec..5a33bea5f0 100644
--- a/.forgejo/workflows/generate-tarball.yml
+++ b/.forgejo/workflows/generate-tarball.yml
@@ -34,7 +34,7 @@ jobs:
         uses: forgejo/upload-artifact@v3
         with:
           name: cabal-config-${{ input.ref_name }}
-          path: git-annex${{ input.ref_name }}*
+          path: git-annex*
   cabal-config-v320:
     name: Generate cabal config for edge
     runs-on: x86_64
@@ -62,11 +62,11 @@ jobs:
         uses: forgejo/upload-artifact@v3
         with:
           name: cabal-config-${{ input.ref_name }}
-          path: git-annex${{ input.ref_name }}*
+          path: git-annex*
   upload-tarball:
     name: Upload to generic repo
     runs-on: x86_64
-    needs: [build-tarball]
+    needs: [cabal-config-edge,cabal-config-v320]
     container:
       image: alpine:latest
     steps:

From 629f4181d9791da770756ac3fcccc00ad3e3776d Mon Sep 17 00:00:00 2001
From: Antoine Martin 
Date: Mon, 12 Aug 2024 02:11:26 -0400
Subject: [PATCH 2822/2835] .forgejo: change title

---
 .forgejo/workflows/generate-tarball.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-tarball.yml
index 5a33bea5f0..527eb371c4 100644
--- a/.forgejo/workflows/generate-tarball.yml
+++ b/.forgejo/workflows/generate-tarball.yml
@@ -33,7 +33,7 @@ jobs:
       - name: Package upload
         uses: forgejo/upload-artifact@v3
         with:
-          name: cabal-config-${{ input.ref_name }}
+          name: cabalconfigedge
           path: git-annex*
   cabal-config-v320:
     name: Generate cabal config for edge
@@ -61,7 +61,7 @@ jobs:
       - name: Package upload
         uses: forgejo/upload-artifact@v3
         with:
-          name: cabal-config-${{ input.ref_name }}
+          name: cabalconfig320
           path: git-annex*
   upload-tarball:
     name: Upload to generic repo

From d5f3fc8626c05aabf4d64f8084ed983feb1dc892 Mon Sep 17 00:00:00 2001
From: Antoine Martin 
Date: Mon, 12 Aug 2024 02:17:18 -0400
Subject: [PATCH 2823/2835] forgejo: debug conf location

---
 .forgejo/workflows/generate-tarball.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-tarball.yml
index 527eb371c4..24a46e5746 100644
--- a/.forgejo/workflows/generate-tarball.yml
+++ b/.forgejo/workflows/generate-tarball.yml
@@ -29,6 +29,7 @@ jobs:
           patch -p1 -i ghc-9.8.patch
           HOME="${{ github.workspace}}"/cabal_cache cabal update
           HOME="${{ github.workspace}}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted"
+          find .
           mv cabal.project.freeze git-annex-${{ input.ref_name }}-$CI_ALPINE_TARGET_RELEASE.config
       - name: Package upload
         uses: forgejo/upload-artifact@v3
@@ -58,6 +59,7 @@ jobs:
           HOME="${{ github.workspace }}"/cabal_cache cabal update
           HOME="${{ github.workspace }}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted"
           mv cabal.project.freeze git-annex-${{ input.ref_name }}-$CI_ALPINE_TARGET_RELEASE.config
+          find .
       - name: Package upload
         uses: forgejo/upload-artifact@v3
         with:
@@ -77,5 +79,5 @@ jobs:
       - name: Package deployment 
         run: |
           find .
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./tarball/git-annex-${{ inputs.ref_name }}.tar.gz ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-${{ inputs.ref_name}}.tar.gz
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./tarball/git-annex-${{ inputs.ref_name }}.tar.gz.sha512sum ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-${{ inputs.ref_name}}.tar.gz.sha512sum
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-edge.cabal
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig320/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-

From a8cdba442d16b256ef446a6c225cd2ab848049dd Mon Sep 17 00:00:00 2001
From: Antoine Martin 
Date: Mon, 12 Aug 2024 02:18:51 -0400
Subject: [PATCH 2824/2835] forgejo: debug conf location

---
 .forgejo/workflows/generate-tarball.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-tarball.yml
index 24a46e5746..2101ce18e5 100644
--- a/.forgejo/workflows/generate-tarball.yml
+++ b/.forgejo/workflows/generate-tarball.yml
@@ -17,7 +17,7 @@ jobs:
       CI_ALPINE_TARGET_RELEASE: edge
     steps:
       - name: Environment setup
-        run: apk add nodejs git cabal patch
+        run: apk add nodejs git cabal patch wget
       - name: Repo pull
         uses: actions/checkout@v4
         with:
@@ -46,7 +46,7 @@ jobs:
       CI_ALPINE_TARGET_RELEASE: v3.20
     steps:
       - name: Environment setup
-        run: apk add nodejs git cabal patch
+        run: apk add nodejs git cabal patch wget
       - name: Repo pull
         uses: actions/checkout@v4
         with:

From 7e0dac877202d3fca7d162cf33e2b1f945a90a0d Mon Sep 17 00:00:00 2001
From: forge 
Date: Thu, 5 Sep 2024 21:34:14 +0000
Subject: [PATCH 2825/2835] Add missing v3.20 name

---
 .forgejo/workflows/generate-tarball.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-tarball.yml
index 2101ce18e5..a343ba60b0 100644
--- a/.forgejo/workflows/generate-tarball.yml
+++ b/.forgejo/workflows/generate-tarball.yml
@@ -80,4 +80,4 @@ jobs:
         run: |
           find .
           curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-edge.cabal
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig320/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig320/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-v3.20.cabal

From a5268f5c32b9dceba52b900c5e5aa6f4c1c55753 Mon Sep 17 00:00:00 2001
From: forge 
Date: Thu, 5 Sep 2024 21:38:20 +0000
Subject: [PATCH 2826/2835] Put version in file name

---
 .forgejo/workflows/generate-tarball.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-tarball.yml
index a343ba60b0..afc91a2380 100644
--- a/.forgejo/workflows/generate-tarball.yml
+++ b/.forgejo/workflows/generate-tarball.yml
@@ -79,5 +79,5 @@ jobs:
       - name: Package deployment 
         run: |
           find .
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-edge.cabal
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig320/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-v3.20.cabal
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-${{ inputs.ref_name }}-edge.cabal
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig320/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-${{ inputs.ref_name }}-v320.cabal

From 020a43fd908d424909324658da6cacb17d5dd031 Mon Sep 17 00:00:00 2001
From: Antoine Martin 
Date: Mon, 9 Sep 2024 19:24:23 -0400
Subject: [PATCH 2827/2835] Generate lockfile automatically on new tag

---
 .forgejo/patches/ghc-9.8.patch                | 18 ++++++
 ...rate-tarball.yml => generate-lockfile.yml} | 40 ++++++------
 .forgejo/workflows/mirror-repository.yml      | 62 +++++++++++--------
 3 files changed, 76 insertions(+), 44 deletions(-)
 create mode 100644 .forgejo/patches/ghc-9.8.patch
 rename .forgejo/workflows/{generate-tarball.yml => generate-lockfile.yml} (69%)

diff --git a/.forgejo/patches/ghc-9.8.patch b/.forgejo/patches/ghc-9.8.patch
new file mode 100644
index 0000000000..85796d787d
--- /dev/null
+++ b/.forgejo/patches/ghc-9.8.patch
@@ -0,0 +1,18 @@
+Support ghc-9.8 by widening a lot of constraints.
+
+This patch can be removed once upstream supports ghc 9.8 offically.
+
+diff -uprN git-annex-10.20240227.orig/cabal.project git-annex-10.20240227/cabal.project
+--- git-annex-10.20240227.orig/cabal.project	1970-01-01 01:00:00.000000000 +0100
++++ git-annex-10.20240227/cabal.project	2024-04-28 13:30:14.061706299 +0200
+@@ -0,0 +1,10 @@
++packages: *.cabal
++
++allow-newer: dav
++allow-newer: haskeline:filepath
++allow-newer: haskeline:directory
++allow-newer: xml-hamlet
++allow-newer: aws:filepath
++allow-newer: dbus:network
++allow-newer: dbus:filepath
++allow-newer: microstache:filepath
diff --git a/.forgejo/workflows/generate-tarball.yml b/.forgejo/workflows/generate-lockfile.yml
similarity index 69%
rename from .forgejo/workflows/generate-tarball.yml
rename to .forgejo/workflows/generate-lockfile.yml
index afc91a2380..0d98abd451 100644
--- a/.forgejo/workflows/generate-tarball.yml
+++ b/.forgejo/workflows/generate-lockfile.yml
@@ -6,6 +6,10 @@ on:
         required: true
         type: string
 
+  push:
+    tags:
+      - '*'
+
 jobs:
   cabal-config-edge:
     name: Generate cabal config for edge
@@ -13,58 +17,52 @@ jobs:
     container:
       image: alpine:edge
     env:
-      CI_PROJECT_NAME: git-annex
       CI_ALPINE_TARGET_RELEASE: edge
     steps:
       - name: Environment setup
-        run: apk add nodejs git cabal patch wget
+        run: apk add nodejs git cabal patch
       - name: Repo pull
         uses: actions/checkout@v4
         with:
-          fetch-depth: 500
+          fetch-depth: 1
           ref: ${{ inputs.ref_name }}
       - name: Config generation
         run: |
-          wget 'https://lab.ilot.io/mirrors/git-annex/-/raw/gitlab-ci/ghc-9.8.patch'
-          patch -p1 -i ghc-9.8.patch
+          patch -p1 -i .forgejo/patches/ghc-9.8.patch
           HOME="${{ github.workspace}}"/cabal_cache cabal update
           HOME="${{ github.workspace}}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted"
-          find .
-          mv cabal.project.freeze git-annex-${{ input.ref_name }}-$CI_ALPINE_TARGET_RELEASE.config
+          mv cabal.project.freeze git-annex.config
       - name: Package upload
         uses: forgejo/upload-artifact@v3
         with:
           name: cabalconfigedge
-          path: git-annex*
+          path: git-annex*.config
   cabal-config-v320:
     name: Generate cabal config for edge
     runs-on: x86_64
     container:
       image: alpine:3.20
     env:
-      CI_PROJECT_NAME: git-annex
       CI_ALPINE_TARGET_RELEASE: v3.20
     steps:
       - name: Environment setup
-        run: apk add nodejs git cabal patch wget
+        run: apk add nodejs git cabal patch
       - name: Repo pull
         uses: actions/checkout@v4
         with:
-          fetch-depth: 500
+          fetch-depth: 1
           ref: ${{ inputs.ref_name }}
       - name: Config generation
         run: |
-          wget 'https://lab.ilot.io/mirrors/git-annex/-/raw/gitlab-ci/ghc-9.8.patch'
-          patch -p1 -i ghc-9.8.patch
+          patch -p1 -i .forgejo/patches/ghc-9.8.patch
           HOME="${{ github.workspace }}"/cabal_cache cabal update
           HOME="${{ github.workspace }}"/cabal_cache cabal v2-freeze --shadow-installed-packages --strong-flags --flags="+assistant +webapp +pairing +production +torrentparser +magicmime +benchmark -debuglocks +dbus +networkbsd +gitlfs +httpclientrestricted"
-          mv cabal.project.freeze git-annex-${{ input.ref_name }}-$CI_ALPINE_TARGET_RELEASE.config
-          find .
+          mv cabal.project.freeze git-annex.config
       - name: Package upload
         uses: forgejo/upload-artifact@v3
         with:
           name: cabalconfig320
-          path: git-annex*
+          path: git-annex*.config
   upload-tarball:
     name: Upload to generic repo
     runs-on: x86_64
@@ -78,6 +76,10 @@ jobs:
         uses: forgejo/download-artifact@v3
       - name: Package deployment 
         run: |
-          find .
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-${{ inputs.ref_name }}-edge.cabal
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig320/git-annex.cabal ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/${{ inputs.ref_name }}/git-annex-${{ inputs.ref_name }}-v320.cabal
+          if test $GITHUB_REF_NAME == "ci" ; then
+            CI_REF_NAME=${{ inputs.ref_name }}
+          else
+            CI_REF_NAME=$GITHUB_REF_NAME
+          fi
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-edge.cabal
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig320/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-v320.cabal
diff --git a/.forgejo/workflows/mirror-repository.yml b/.forgejo/workflows/mirror-repository.yml
index 791fe23d0e..b820421db5 100644
--- a/.forgejo/workflows/mirror-repository.yml
+++ b/.forgejo/workflows/mirror-repository.yml
@@ -4,35 +4,47 @@ on:
   schedule:
     - cron: '@hourly'
 
-  push:
-    branches:
-      - 'master'
-
 jobs:
   mirror:
     name: Pull from upstream
     runs-on: x86_64
     container:
-      image: 'docker.io/node:20-bookworm'
+      image: alpine:latest
+    env:
+      upstream: https://git.joeyh.name/git/git-annex.git 
+      tags: '10.2024*'
     steps:
-      - name: git mirror branches {v*/,}master & tags
+      - name: Environment setup
+        run: apk add grep git sed coreutils bash nodejs
+      - name: Fetch destination
+        uses: actions/checkout@v4
+        with:
+          fetch_depth: 1
+          ref: ci
+          token: ${{ secrets.CODE_FORGEJO_TOKEN }}
+      - name: Missing tag detecting
         run: |
-          git init --bare .
-          git remote add origin https://git.joeyh.name/git/git-annex.git 
-          git fetch origin refs/heads/master:refs/mirror/master
-          git ls-remote origin refs/heads/v*/master | while read sha full_ref ; do
-            ref=${full_ref#refs/heads/}
-            git fetch origin $full_ref:refs/mirror/$ref
-            tag=${ref%/master}
-            git fetch origin refs/tags/${tag}*:refs/mirror-tags/${tag}*
-          done
-          if test "${{ vars.TEST }}" != true ; then
-            token=${{ secrets.CODE_FORGEJO_TOKEN }}
-            destination=${{ vars.DESTINATION }}
-            owner=${{ vars.OWNER }}
-          else
-            token=${{ secrets.TEST_FORGEJO_TOKEN }}
-            destination=${{ vars.TEST_DESTINATION }}
-            owner=${{ vars.TEST_OWNER }}
-          fi
-          git push --force https://any:$token@$destination/$owner/git-annex refs/mirror/*:refs/heads/* refs/mirror-tags/*:refs/tags/* --tags   
+          git ls-remote $upstream "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' > upstream_tags
+          git ls-remote ${{ github.server_url}}/${{ github.repository }} "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' > destination_tags
+          cat upstream_tags destination_tags | tr ' ' '\n' | sort | uniq -u > missing_tags
+          echo "Missing tags:"
+          cat missing_tags
+      - name: Missing tag fetch
+        run: |
+          git remote add upstream $upstream
+          while read tag; do
+            git fetch upstream tag $tag --no-tags
+          done < missing_tags
+      - name: Packaging workflow injection
+        run: |
+          while read tag; do
+            git checkout $tag
+            git tag -d $tag
+            git checkout ci -- ./.forgejo
+            git config user.name "forgejo-actions[bot]"
+            git config user.email "dev@ayakael.net"
+            git commit -m 'Inject custom workflow'
+            git tag -a $tag -m $tag
+          done < missing_tags
+      - name: Push to destination
+        run: git push --force origin refs/tags/*:refs/tags/* --tags   

From 5266539427dd62a7493a6dadadb6c5083f2e67a2 Mon Sep 17 00:00:00 2001
From: forge 
Date: Fri, 18 Oct 2024 11:01:17 +0000
Subject: [PATCH 2828/2835] Use comm instead of uniq to compute missing tags

---
 .forgejo/workflows/mirror-repository.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.forgejo/workflows/mirror-repository.yml b/.forgejo/workflows/mirror-repository.yml
index b820421db5..91dc5b9452 100644
--- a/.forgejo/workflows/mirror-repository.yml
+++ b/.forgejo/workflows/mirror-repository.yml
@@ -24,9 +24,9 @@ jobs:
           token: ${{ secrets.CODE_FORGEJO_TOKEN }}
       - name: Missing tag detecting
         run: |
-          git ls-remote $upstream "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' > upstream_tags
-          git ls-remote ${{ github.server_url}}/${{ github.repository }} "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' > destination_tags
-          cat upstream_tags destination_tags | tr ' ' '\n' | sort | uniq -u > missing_tags
+          git ls-remote $upstream "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' | sort > upstream_tags
+          git ls-remote ${{ github.server_url}}/${{ github.repository }} "refs/tags/$tags" | grep -v '{' | sed 's|.*/||' | sort > destination_tags
+          comm -23 upstream_tags destination_tags > missing_tags
           echo "Missing tags:"
           cat missing_tags
       - name: Missing tag fetch

From 3b3f367a640bec1cf30209ebf8ef9b82bd744735 Mon Sep 17 00:00:00 2001
From: forge 
Date: Sun, 5 Jan 2025 17:31:08 +0000
Subject: [PATCH 2829/2835] Generate cabal config for v3.21

---
 .forgejo/workflows/generate-lockfile.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.forgejo/workflows/generate-lockfile.yml b/.forgejo/workflows/generate-lockfile.yml
index 0d98abd451..9c4547cef3 100644
--- a/.forgejo/workflows/generate-lockfile.yml
+++ b/.forgejo/workflows/generate-lockfile.yml
@@ -37,13 +37,13 @@ jobs:
         with:
           name: cabalconfigedge
           path: git-annex*.config
-  cabal-config-v320:
-    name: Generate cabal config for edge
+  cabal-config-v321:
+    name: Generate cabal config for v3.21
     runs-on: x86_64
     container:
-      image: alpine:3.20
+      image: alpine:3.21
     env:
-      CI_ALPINE_TARGET_RELEASE: v3.20
+      CI_ALPINE_TARGET_RELEASE: v3.21
     steps:
       - name: Environment setup
         run: apk add nodejs git cabal patch
@@ -61,12 +61,12 @@ jobs:
       - name: Package upload
         uses: forgejo/upload-artifact@v3
         with:
-          name: cabalconfig320
+          name: cabalconfig321
           path: git-annex*.config
   upload-tarball:
     name: Upload to generic repo
     runs-on: x86_64
-    needs: [cabal-config-edge,cabal-config-v320]
+    needs: [cabal-config-edge,cabal-config-v321]
     container:
       image: alpine:latest
     steps:
@@ -82,4 +82,4 @@ jobs:
             CI_REF_NAME=$GITHUB_REF_NAME
           fi
           curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-edge.cabal
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig320/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-v320.cabal
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig321/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-v321.cabal

From 9fc588e1eefbe39ff02e627c609524e190d63a2a Mon Sep 17 00:00:00 2001
From: forge 
Date: Sun, 5 Jan 2025 17:32:42 +0000
Subject: [PATCH 2830/2835] Sync 10.2025* tags

---
 .forgejo/workflows/mirror-repository.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.forgejo/workflows/mirror-repository.yml b/.forgejo/workflows/mirror-repository.yml
index 91dc5b9452..f44c4668cf 100644
--- a/.forgejo/workflows/mirror-repository.yml
+++ b/.forgejo/workflows/mirror-repository.yml
@@ -12,7 +12,7 @@ jobs:
       image: alpine:latest
     env:
       upstream: https://git.joeyh.name/git/git-annex.git 
-      tags: '10.2024*'
+      tags: '10.2025*'
     steps:
       - name: Environment setup
         run: apk add grep git sed coreutils bash nodejs

From 222c727007c23c8dd28e4bea0b60518b24f39b80 Mon Sep 17 00:00:00 2001
From: forge 
Date: Thu, 22 May 2025 03:06:29 +0000
Subject: [PATCH 2831/2835] ci: upgrade in env setup

---
 .forgejo/workflows/generate-lockfile.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.forgejo/workflows/generate-lockfile.yml b/.forgejo/workflows/generate-lockfile.yml
index 9c4547cef3..663cec452c 100644
--- a/.forgejo/workflows/generate-lockfile.yml
+++ b/.forgejo/workflows/generate-lockfile.yml
@@ -20,7 +20,9 @@ jobs:
       CI_ALPINE_TARGET_RELEASE: edge
     steps:
       - name: Environment setup
-        run: apk add nodejs git cabal patch
+        run: |
+          apk upgrade -a
+          apk add nodejs git cabal patch
       - name: Repo pull
         uses: actions/checkout@v4
         with:
@@ -46,7 +48,9 @@ jobs:
       CI_ALPINE_TARGET_RELEASE: v3.21
     steps:
       - name: Environment setup
-        run: apk add nodejs git cabal patch
+        run: |
+          apk upgrade -a
+          apk add nodejs git cabal patch
       - name: Repo pull
         uses: actions/checkout@v4
         with:

From 4f5641710b61a130c1a589a2952470570be2445f Mon Sep 17 00:00:00 2001
From: forge 
Date: Sun, 8 Jun 2025 23:48:25 +0000
Subject: [PATCH 2832/2835] Move to v3.22

---
 .forgejo/workflows/generate-lockfile.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.forgejo/workflows/generate-lockfile.yml b/.forgejo/workflows/generate-lockfile.yml
index 663cec452c..6c9d347fbb 100644
--- a/.forgejo/workflows/generate-lockfile.yml
+++ b/.forgejo/workflows/generate-lockfile.yml
@@ -40,12 +40,12 @@ jobs:
           name: cabalconfigedge
           path: git-annex*.config
   cabal-config-v321:
-    name: Generate cabal config for v3.21
+    name: Generate cabal config for v3.22
     runs-on: x86_64
     container:
-      image: alpine:3.21
+      image: alpine:3.22
     env:
-      CI_ALPINE_TARGET_RELEASE: v3.21
+      CI_ALPINE_TARGET_RELEASE: v3.22
     steps:
       - name: Environment setup
         run: |
@@ -70,7 +70,7 @@ jobs:
   upload-tarball:
     name: Upload to generic repo
     runs-on: x86_64
-    needs: [cabal-config-edge,cabal-config-v321]
+    needs: [cabal-config-edge,cabal-config-v322]
     container:
       image: alpine:latest
     steps:

From 9d019ac36d9c6ee31e342127d492b96092ba550c Mon Sep 17 00:00:00 2001
From: forge 
Date: Sun, 8 Jun 2025 23:53:25 +0000
Subject: [PATCH 2833/2835] Fix workflow

---
 .forgejo/workflows/generate-lockfile.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.forgejo/workflows/generate-lockfile.yml b/.forgejo/workflows/generate-lockfile.yml
index 6c9d347fbb..ff6e24d1bb 100644
--- a/.forgejo/workflows/generate-lockfile.yml
+++ b/.forgejo/workflows/generate-lockfile.yml
@@ -65,7 +65,7 @@ jobs:
       - name: Package upload
         uses: forgejo/upload-artifact@v3
         with:
-          name: cabalconfig321
+          name: cabalconfig322
           path: git-annex*.config
   upload-tarball:
     name: Upload to generic repo

From 130d8b30d2a99f59ed1289538631d375e409abe2 Mon Sep 17 00:00:00 2001
From: forge 
Date: Sun, 8 Jun 2025 23:57:21 +0000
Subject: [PATCH 2834/2835] Fix workflow

---
 .forgejo/workflows/generate-lockfile.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.forgejo/workflows/generate-lockfile.yml b/.forgejo/workflows/generate-lockfile.yml
index ff6e24d1bb..a567f3b03a 100644
--- a/.forgejo/workflows/generate-lockfile.yml
+++ b/.forgejo/workflows/generate-lockfile.yml
@@ -86,4 +86,4 @@ jobs:
             CI_REF_NAME=$GITHUB_REF_NAME
           fi
           curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfigedge/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-edge.cabal
-          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig321/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-v321.cabal
+          curl --user ${{ vars.CODE_FORGEJO_USER }}:${{ secrets.CODE_FORGEJO_TOKEN }} --upload-file ./cabalconfig322/git-annex.config ${{ github.server_url }}/api/packages/mirrors/generic/git-annex/$CI_REF_NAME/git-annex-$CI_REF_NAME-v322.cabal

From 1e6080647b03aa40acffde75e8f20cd255c3ca92 Mon Sep 17 00:00:00 2001
From: forge 
Date: Sun, 8 Jun 2025 23:59:34 +0000
Subject: [PATCH 2835/2835] Fix workflow

---
 .forgejo/workflows/generate-lockfile.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.forgejo/workflows/generate-lockfile.yml b/.forgejo/workflows/generate-lockfile.yml
index a567f3b03a..8dbb579e67 100644
--- a/.forgejo/workflows/generate-lockfile.yml
+++ b/.forgejo/workflows/generate-lockfile.yml
@@ -39,7 +39,7 @@ jobs:
         with:
           name: cabalconfigedge
           path: git-annex*.config
-  cabal-config-v321:
+  cabal-config-v322:
     name: Generate cabal config for v3.22
     runs-on: x86_64
     container: